S61: Reorganizing files to optimize docker caching layers (#920)

* First batch of changes to Dockerfiles. Testing, but want to track changes * combining run commands * second batch of changes. making layers more efficient * avoiding tzdata prompts * testing a way to obtain faster build times * reverting.
Shared-Reality-Lab · Dec 16, 2024 · 25eccce · 25eccce
1 parent 9593698
commit 25eccce
Show file tree

Hide file tree

Showing 24 changed files with 112 additions and 134 deletions.
diff --git a/handlers/hello-handler/Dockerfile b/handlers/hello-handler/Dockerfile
@@ -8,8 +8,8 @@ COPY /handlers/hello-handler/package*.json ./
 RUN npm ci
 COPY /handlers/hello-handler/*.json ./
 COPY /handlers/hello-handler/.eslintrc.js ./
-COPY /handlers/hello-handler/src ./src
 COPY /schemas src/schemas
+COPY /handlers/hello-handler/src ./src
 RUN npm run build
 
 ENV NODE_ENV=production

diff --git a/orchestrator/Dockerfile b/orchestrator/Dockerfile
@@ -1,25 +1,24 @@
 FROM node:alpine as builder
 
-
 WORKDIR /usr/src/app
 COPY orchestrator/*.js* ./
 RUN npm ci
-COPY orchestrator/src ./src
+
 COPY schemas src/schemas
+COPY orchestrator/src ./src
 RUN npm run build && npm prune --production
 
 FROM node:alpine as final
 
-RUN apk add memcached
+RUN apk add memcached supercronic
 # Set up for logging
 WORKDIR /var/log/IMAGE
-RUN apk add supercronic
+
 RUN chmod o+wx .
 
 WORKDIR /usr/src/app
 COPY orchestrator/clean-cron ./clean-cron
-RUN chown node:node clean-cron
-RUN chmod 644 clean-cron
+RUN chown node:node clean-cron && chmod 644 clean-cron
 
 COPY --from=builder --chown=node:node /usr/src/app/*.js* ./
 COPY --from=builder --chown=node:node /usr/src/app/dist ./dist

diff --git a/preprocessors/action-recognition/Dockerfile b/preprocessors/action-recognition/Dockerfile
@@ -7,24 +7,21 @@ ENV PYTHONUNBUFFERED=TRUE
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y \
-    wget \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN adduser --disabled-password python
+    apt-get install --no-install-recommends wget -y && \
+    rm -rf /var/lib/apt/lists/* && \
+    adduser --disabled-password python
 
 WORKDIR /app
 
 RUN wget https://image.a11y.mcgill.ca/models/action_recognition/model.pth
 
 ENV PATH="/home/python/.local/bin:${PATH}"
 
-RUN pip3 install --no-cache-dir --upgrade pip
 COPY --chown=python:python /preprocessors/action-recognition/requirements.txt /app/requirements.txt
-RUN pip3 install --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --no-cache-dir --upgrade pip && \
+    pip3 install --no-cache-dir -r /app/requirements.txt
 
 COPY --chown=python:python /schemas /app/schemas
-
 COPY --chown=python:python /preprocessors/action-recognition /app
 
 EXPOSE 5000

diff --git a/preprocessors/autour/Dockerfile b/preprocessors/autour/Dockerfile
@@ -4,15 +4,11 @@ RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
 
-RUN pip3 install --upgrade pip
-
 COPY /preprocessors/autour/requirements.txt /app/requirements.txt
-
-
-RUN pip3 install -r requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
-
 COPY /preprocessors/autour/ /app
 
 EXPOSE 5000

diff --git a/preprocessors/celebrity-detector/Dockerfile b/preprocessors/celebrity-detector/Dockerfile
@@ -8,14 +8,12 @@ RUN apt-get update && apt-get install -y python3-opencv wget && rm -rf /var/lib/
 
 COPY /preprocessors/celebrity-detector/requirements.txt /app/requirements.txt
 
-RUN pip3 install --upgrade pip
-RUN pip install -r /app/requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
-WORKDIR /app
 COPY /preprocessors/celebrity-detector /app
 
-
 EXPOSE 5000
 ENV FLASK_APP=celebrity-detector.py
 USER python

diff --git a/preprocessors/clothes-detector/Dockerfile b/preprocessors/clothes-detector/Dockerfile
@@ -1,37 +1,38 @@
 FROM pytorch/pytorch:2.4.1-cuda11.8-cudnn9-runtime
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
 
-#install system dependencies
+#install system dependencies in one layer
 RUN apt-get update && \
-    apt-get install -y python3-opencv wget && \
-    apt-get install -y build-essential libfreetype6-dev libpng-dev libharfbuzz-dev libcairo2-dev python3-dev && \
+    apt-get install -y \
+    python3-opencv wget build-essential \
+    libfreetype6-dev libpng-dev libharfbuzz-dev \
+    libcairo2-dev python3-dev p7zip p7zip-full \
+    unace zip unzip && \
+    apt-get -y autoremove && \
+    apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 COPY /preprocessors/clothes-detector/requirements.txt /app/requirements.txt
-# COPY ./requirements.txt /app/requirements.txt
-
-RUN  apt-get update -y && \
-     apt-get upgrade -y && \
-     apt-get dist-upgrade -y && \
-     apt-get -y autoremove && \
-     apt-get clean
-RUN apt-get install -y p7zip p7zip-full unace zip unzip
-
 RUN pip3 install --upgrade pip && \
     pip3 install -r /app/requirements.txt
-
-RUN wget https://image.a11y.mcgill.ca/models/clothesDetector/yolo.zip && \
-    unzip yolo.zip
+# COPY ./requirements.txt /app/requirements.txt
 
 COPY /schemas /app/schemas
 # COPY ./schemas /app/schemas
+
+# good practice to remove the archive 
+RUN wget https://image.a11y.mcgill.ca/models/clothesDetector/yolo.zip && \
+    unzip yolo.zip && \
+    rm yolo.zip 
+
 COPY /preprocessors/clothes-detector /app
 # COPY ./ /app
 
-
 EXPOSE 5000
 ENV FLASK_APP=clothes.py
 USER python

diff --git a/preprocessors/collage-detector/Dockerfile b/preprocessors/collage-detector/Dockerfile
@@ -1,15 +1,14 @@
 FROM python:3
 
-COPY /preprocessors/collage-detector/requirements.txt /app/requirements.txt
-
-RUN apt-get update && apt-get install -y python3-opencv wget && rm -rf /var/lib/apt/lists/*
-
 RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
 
-RUN pip3 install --upgrade pip
-RUN pip install -r /app/requirements.txt
+RUN apt-get update && apt-get install -y python3-opencv wget && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+COPY /preprocessors/collage-detector/requirements.txt /app/requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
 COPY /preprocessors/collage-detector/ /app

diff --git a/preprocessors/content-categoriser/Dockerfile b/preprocessors/content-categoriser/Dockerfile
@@ -1,13 +1,13 @@
 FROM python:alpine3.20
 
-COPY /preprocessors/content-categoriser/requirements.txt /app/requirements.txt
-
 RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
 
-RUN pip3 install --upgrade pip
-RUN pip3 install -r /app/requirements.txt
+COPY /preprocessors/content-categoriser/requirements.txt /app/requirements.txt
+
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
 COPY /preprocessors/content-categoriser/ /app

diff --git a/preprocessors/depth-map-gen/Dockerfile b/preprocessors/depth-map-gen/Dockerfile
@@ -1,24 +1,30 @@
 FROM pytorch/pytorch:1.13.0-cuda11.6-cudnn8-runtime
 
-COPY /preprocessors/depth-map-gen/requirements.txt /app/requirements.txt
-
+#system dependencies
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3-opencv wget git && rm -rf /var/lib/apt/lists/*
 
+#user/environment setup
 RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
 ENV PYTHONPATH "${PYTHONPATH}:./AdelaiDepth/LeReS/Minist_Test/"
+ENV FLASK_APP=depth-map-generator.py
+ENV TORCH_HOME=/app
 
-RUN git clone https://github.com/aim-uofa/AdelaiDepth
-RUN wget -O ./res101.pth https://huggingface.co/ffranchina/LeReS/resolve/main/res101.pth
-RUN pip3 install --upgrade pip
-RUN pip3 install -r /app/requirements.txt
+#python dependencies
+COPY /preprocessors/depth-map-gen/requirements.txt /app/requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
 
+#schema
 COPY /schemas /app/schemas
+
+#model download, app code
+RUN git clone https://github.com/aim-uofa/AdelaiDepth && \
+    wget -O ./res101.pth https://huggingface.co/ffranchina/LeReS/resolve/main/res101.pth
 COPY /preprocessors/depth-map-gen/ /app
 
 EXPOSE 5000
-ENV FLASK_APP=depth-map-generator.py
-ENV TORCH_HOME=/app
+
 USER python
 CMD [ "gunicorn", "depth-map-generator:app", "-b", "0.0.0.0:5000", "--capture-output", "--log-level=debug" ]
diff --git a/preprocessors/graphic-caption/Dockerfile b/preprocessors/graphic-caption/Dockerfile
@@ -1,13 +1,12 @@
 FROM python:alpine3.20
 
-COPY /preprocessors/graphic-caption/requirements.txt /app/requirements.txt
-
 RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
 
-RUN pip3 install --upgrade pip
-RUN pip3 install -r /app/requirements.txt
+COPY /preprocessors/graphic-caption/requirements.txt /app/requirements.txt
+RUN pip3 install --upgrade pip && \ 
+    pip3 install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
 COPY /preprocessors/graphic-caption/ /app

diff --git a/preprocessors/graphic-tagger/Dockerfile b/preprocessors/graphic-tagger/Dockerfile
@@ -3,18 +3,15 @@ FROM python:3.9-alpine
 RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
+ENV FLASK_APP=azure_api.py
 
 COPY /preprocessors/graphic-tagger/requirements.txt /app/requirements.txt
-
-RUN pip3 install --upgrade pip
-RUN pip install -r /app/requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
-WORKDIR /app
 COPY /preprocessors/graphic-tagger /app
 
-
 EXPOSE 5000
-ENV FLASK_APP=azure_api.py
 USER python
 CMD [ "gunicorn", "azure_api:app", "-b", "0.0.0.0:5000", "--capture-output", "--log-level=debug" ]
diff --git a/preprocessors/grouping/Dockerfile b/preprocessors/grouping/Dockerfile
@@ -2,21 +2,19 @@ FROM python:3.8-alpine
 
 RUN apk add --no-cache curl && \
     adduser --disabled-password python
+
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
 
-RUN pip3 install --upgrade pip
-
 COPY /preprocessors/grouping/requirements.txt /app/requirements.txt
-RUN pip3 install -r requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
-
 COPY /preprocessors/grouping/ /app
 
 EXPOSE 5000
 
-
 ENV FLASK_APP=grouping.py
 USER python
 CMD [ "gunicorn", "grouping:app", "-b", "0.0.0.0:5000", "--capture-output", "--log-level=debug" ]
diff --git a/preprocessors/hello-preprocessor/Dockerfile b/preprocessors/hello-preprocessor/Dockerfile
@@ -6,8 +6,8 @@ WORKDIR /usr/src/app
 # Docker images are onions
 COPY /preprocessors/hello-preprocessor/package*.json ./
 RUN npm ci
-COPY /preprocessors/hello-preprocessor/ .
 COPY /schemas src/schemas
+COPY /preprocessors/hello-preprocessor/ .
 RUN npm run build
 
 ENV NODE_ENV=production

diff --git a/preprocessors/line-charts/Dockerfile b/preprocessors/line-charts/Dockerfile
@@ -3,19 +3,17 @@ FROM python:3.10
 RUN adduser --disabled-password python
 WORKDIR /app
 ENV PATH="/home/python/.local/bin:${PATH}"
-
-RUN pip3 install --upgrade pip
+ENV FLASK_APP=charts.py
 
 COPY /preprocessors/line-charts/requirements.txt /app/requirements.txt
 
-RUN pip3 install -r requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
-
 COPY /preprocessors/line-charts/ /app
 
 EXPOSE 5000
 
-ENV FLASK_APP=charts.py
 USER python
 CMD [ "gunicorn", "charts:app", "-b", "0.0.0.0:5000", "--capture-output", "--log-level=debug" ]
diff --git a/preprocessors/mmsemseg/Dockerfile b/preprocessors/mmsemseg/Dockerfile
@@ -21,12 +21,11 @@ RUN apt-get update && \
     openjdk-11-jre-headless \
     wget \
     # MMDet Requirements
-    ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 \
-    curl && \
+    ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 curl && \
     rm -rf /var/lib/apt/lists/*
 
 ENV PATH="/opt/conda/bin:$PATH"
-RUN export FORCE_CUDA=1
+RUN export FORCE_CUDA=1 
 
 # MMLAB
 ARG PYTORCH
@@ -39,8 +38,8 @@ RUN adduser --disabled-password python
 ENV PATH="/home/python/.local/bin:${PATH}"
 
 COPY /preprocessors/mmsemseg/requirements.txt /app/requirements.txt
-RUN pip install --upgrade pip
-RUN pip install -r /app/requirements.txt
+RUN pip install --upgrade pip && \
+    pip install -r /app/requirements.txt
 
 COPY /schemas /app/schemas
 COPY /preprocessors/mmsemseg/ /app

diff --git a/preprocessors/ner/Dockerfile b/preprocessors/ner/Dockerfile
@@ -2,24 +2,26 @@ FROM ubuntu:20.04
 
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
 
-RUN apt-get update -y \
-        && apt-get install openjdk-8-jdk -y \
-        && apt-get install python3-pip -y \
-        && export JAVA_HOME \
-        && apt-get clean \
-        && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    openjdk-8-jdk \
+    python3-pip \
+    gcc \
+    git \
+    && apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 RUN adduser --disabled-password python
 ENV PATH="/home/python/.local/bin:${PATH}"
 
+COPY /preprocessors/ner/requirements.txt /app/requirements.txt
+RUN pip3 install --upgrade pip && \
+    pip3 install -r /app/requirements.txt
+
 COPY /schemas /app/schemas
 COPY /preprocessors/ner/ /app
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc git && rm -rf /var/lib/apt/lists/*
-
-RUN pip3 install -r requirements.txt
-
 EXPOSE 5000
 ENV FLASK_APP=ner.py