chore: reduce dockerfile size (#300)

- Reduce aarch64 image size from 1.75GB to 1.16GB - Reduce amd64 image size from 1.9GB to 1.5GB --------- Signed-off-by: Avik Basu <[email protected]>
numaproj · Sep 27, 2023 · 2d99849 · 2d99849
1 parent fdec237
commit 2d99849
Show file tree

Hide file tree

Showing 3 changed files with 252 additions and 34 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -13,4 +13,30 @@ CONTRIBUTING.md
 LICENSE
 mkdocs.yml
 README.md
-USERS.md
+USERS.md
+.idea
+.git
+.DS_Store
+.codecov.yml
+.pytest_cache
+**/__pycache__/
+**/*.py[cod]
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environment
+.env
+venv/
diff --git a/Dockerfile b/Dockerfile
@@ -1,54 +1,52 @@
 ####################################################################################################
-# builder: install needed dependencies
+# builder: install needed dependencies and setup virtual environment
 ####################################################################################################
 
 ARG PYTHON_VERSION=3.11
 ARG POETRY_VERSION=1.6
+ARG INSTALL_EXTRAS
 FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
 
-ENV PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PIP_NO_CACHE_DIR=off \
-  PIP_DISABLE_PIP_VERSION_CHECK=on \
-  PIP_DEFAULT_TIMEOUT=100 \
-  POETRY_VERSION=${POETRY_VERSION} \
-  POETRY_HOME="/opt/poetry" \
-  POETRY_VIRTUALENVS_IN_PROJECT=true \
-  POETRY_NO_INTERACTION=1 \
-  PYSETUP_PATH="/opt/pysetup" \
-  VENV_PATH="/opt/pysetup/.venv"
-
-ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache \
+    POETRY_VERSION=${POETRY_VERSION} \
+    POETRY_HOME="/opt/poetry" \
+    PATH="$POETRY_HOME/bin:$PATH"
 
 RUN apt-get update \
-    && apt-get install --no-install-recommends -y \
-        curl \
-        build-essential \
-        dumb-init \
+    && apt-get install --no-install-recommends -y build-essential dumb-init \
     && apt-get clean && rm -rf /var/lib/apt/lists/* \
-    && pip install --no-cache --upgrade pip \
-    && curl -sSL https://install.python-poetry.org | python3 -
+    && pip install --no-cache-dir poetry
+
+WORKDIR /app
+COPY poetry.lock pyproject.toml ./
+
+RUN poetry install --without dev --no-root --extras "${INSTALL_EXTRAS}"  \
+    && poetry run pip install --no-cache-dir "torch>=2.0,<3.0" --index-url https://download.pytorch.org/whl/cpu \
+    && poetry run pip install --no-cache-dir "lightning[pytorch]" \
+    && rm -rf $POETRY_CACHE_DIR \
+    && pip cache purge \
+    && apt-get purge -y --auto-remove build-essential
 
 ####################################################################################################
-# udf: used for running the udf vertices
+# runtime: used for running the udf vertices
 ####################################################################################################
-FROM builder AS udf
+FROM python:${PYTHON_VERSION}-slim-bookworm AS runtime
 
-ARG INSTALL_EXTRAS
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y dumb-init \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* \
+    && apt-get purge -y --auto-remove
 
-WORKDIR $PYSETUP_PATH
-COPY ./pyproject.toml ./poetry.lock ./
 
-# TODO install cpu/gpu based on args/arch
-RUN poetry install --without dev --no-cache --no-root --extras "${INSTALL_EXTRAS}" && \
-    poetry run pip install --no-cache "torch>=2.0,<3.0" --index-url https://download.pytorch.org/whl/cpu && \
-    poetry run pip install --no-cache "pytorch-lightning>=2.0<3.0" && \
-    rm -rf ~/.cache/pypoetry/
+ENV VIRTUAL_ENV=/app/.venv
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 COPY . /app
 WORKDIR /app
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
 EXPOSE 5000
diff --git a/tests/udfs/resources/data/stream.json b/tests/udfs/resources/data/stream.json
@@ -1 +1,195 @@
-{"uuid":"dd7dfb43-532b-49a3-906e-f78f82ad9c4b","config_id":"druid-config","data":[{"degraded":14,"degraded_rate":0.003626943005181347,"error_rate":0.009067357512953367,"failed":21,"failed_rate":0.005440414507772021,"success":3825,"timestamp":1691622660000},{"degraded":10,"degraded_rate":0.002484472049689441,"error_rate":0.009937888198757764,"failed":30,"failed_rate":0.007453416149068323,"success":3985,"timestamp":1691622720000},{"degraded":9,"degraded_rate":0.0025906735751295338,"error_rate":0.005469199769717904,"failed":10,"failed_rate":0.0028785261945883708,"success":3455,"timestamp":1691622780000},{"degraded":13,"degraded_rate":0.003798947983635301,"error_rate":0.007597895967270602,"failed":13,"failed_rate":0.003798947983635301,"success":3396,"timestamp":1691622840000},{"degraded":11,"degraded_rate":0.0030328094844223876,"error_rate":0.005789909015715467,"failed":10,"failed_rate":0.0027570995312930797,"success":3606,"timestamp":1691622900000},{"degraded":17,"degraded_rate":0.004964953271028037,"error_rate":0.009345794392523364,"failed":15,"failed_rate":0.004380841121495327,"success":3392,"timestamp":1691622960000},{"degraded":10,"degraded_rate":0.002774694783573807,"error_rate":0.006104328523862375,"failed":12,"failed_rate":0.003329633740288568,"success":3582,"timestamp":1691623020000},{"degraded":12,"degraded_rate":0.003770028275212064,"error_rate":0.00942507068803016,"failed":18,"failed_rate":0.005655042412818096,"success":3153,"timestamp":1691623080000},{"degraded":14,"degraded_rate":0.003469640644361834,"error_rate":0.006195786864931847,"failed":11,"failed_rate":0.0027261462205700124,"success":4010,"timestamp":1691623140000},{"degraded":4,"degraded_rate":0.0010887316276537834,"error_rate":0.005715841045182362,"failed":17,"failed_rate":0.004627109417528579,"success":3653,"timestamp":1691623200000},{"degraded":13,"degraded_rate":0.0035278154681139757,"error_rate":0.009497964721845319,"failed":22,"failed_rate":0.005970149253731343,"success":3650,"timestamp":1691623260000},{"degraded":7,"degraded_rate":0.0019358407079646017,"error_rate":0.00663716814159292,"failed":17,"failed_rate":0.004701327433628319,"success":3592,"timestamp":1691623320000},{"degraded":18,"degraded_rate":0.004288777698355968,"error_rate":0.00976888253514415,"failed":23,"failed_rate":0.005480104836788182,"success":4156,"timestamp":1691623380000},{"degraded":15,"degraded_rate":0.00487012987012987,"error_rate":0.00974025974025974,"failed":15,"failed_rate":0.00487012987012987,"success":3050,"timestamp":1691623440000},{"degraded":9,"degraded_rate":0.0024563318777292577,"error_rate":0.006823144104803494,"failed":16,"failed_rate":0.004366812227074236,"success":3639,"timestamp":1691623500000},{"degraded":10,"degraded_rate":0.0031959092361776927,"error_rate":0.006391818472355385,"failed":10,"failed_rate":0.0031959092361776927,"success":3109,"timestamp":1691623560000},{"degraded":12,"degraded_rate":0.003827751196172249,"error_rate":0.004784688995215311,"failed":3,"failed_rate":0.0009569377990430622,"success":3120,"timestamp":1691623620000},{"degraded":21,"degraded_rate":0.007123473541383989,"error_rate":0.009158751696065129,"failed":6,"failed_rate":0.0020352781546811396,"success":2921,"timestamp":1691623680000},{"degraded":7,"degraded_rate":0.002027222704894295,"error_rate":0.0034752389226759338,"failed":5,"failed_rate":0.0014480162177816392,"success":3441,"timestamp":1691623740000},{"degraded":8,"degraded_rate":0.0022179096201829776,"error_rate":0.0049902966454117,"failed":10,"failed_rate":0.0027723870252287217,"success":3589,"timestamp":1691623800000}],"start_time":1691622660000,"end_time":1691623860000,"metadata":{"tags":{"asset_alias":"data","asset_id":"123456789","env":"prd"}}}
+{
+  "uuid": "dd7dfb43-532b-49a3-906e-f78f82ad9c4b",
+  "config_id": "druid-config",
+  "data": [
+    {
+      "degraded": 14,
+      "degraded_rate": 0.003626943005181347,
+      "error_rate": 0.009067357512953367,
+      "failed": 21,
+      "failed_rate": 0.005440414507772021,
+      "success": 3825,
+      "timestamp": 1691622660000
+    },
+    {
+      "degraded": 10,
+      "degraded_rate": 0.002484472049689441,
+      "error_rate": 0.009937888198757764,
+      "failed": 30,
+      "failed_rate": 0.007453416149068323,
+      "success": 3985,
+      "timestamp": 1691622720000
+    },
+    {
+      "degraded": 9,
+      "degraded_rate": 0.0025906735751295338,
+      "error_rate": 0.005469199769717904,
+      "failed": 10,
+      "failed_rate": 0.0028785261945883708,
+      "success": 3455,
+      "timestamp": 1691622780000
+    },
+    {
+      "degraded": 13,
+      "degraded_rate": 0.003798947983635301,
+      "error_rate": 0.007597895967270602,
+      "failed": 13,
+      "failed_rate": 0.003798947983635301,
+      "success": 3396,
+      "timestamp": 1691622840000
+    },
+    {
+      "degraded": 11,
+      "degraded_rate": 0.0030328094844223876,
+      "error_rate": 0.005789909015715467,
+      "failed": 10,
+      "failed_rate": 0.0027570995312930797,
+      "success": 3606,
+      "timestamp": 1691622900000
+    },
+    {
+      "degraded": 17,
+      "degraded_rate": 0.004964953271028037,
+      "error_rate": 0.009345794392523364,
+      "failed": 15,
+      "failed_rate": 0.004380841121495327,
+      "success": 3392,
+      "timestamp": 1691622960000
+    },
+    {
+      "degraded": 10,
+      "degraded_rate": 0.002774694783573807,
+      "error_rate": 0.006104328523862375,
+      "failed": 12,
+      "failed_rate": 0.003329633740288568,
+      "success": 3582,
+      "timestamp": 1691623020000
+    },
+    {
+      "degraded": 12,
+      "degraded_rate": 0.003770028275212064,
+      "error_rate": 0.00942507068803016,
+      "failed": 18,
+      "failed_rate": 0.005655042412818096,
+      "success": 3153,
+      "timestamp": 1691623080000
+    },
+    {
+      "degraded": 14,
+      "degraded_rate": 0.003469640644361834,
+      "error_rate": 0.006195786864931847,
+      "failed": 11,
+      "failed_rate": 0.0027261462205700124,
+      "success": 4010,
+      "timestamp": 1691623140000
+    },
+    {
+      "degraded": 4,
+      "degraded_rate": 0.0010887316276537834,
+      "error_rate": 0.005715841045182362,
+      "failed": 17,
+      "failed_rate": 0.004627109417528579,
+      "success": 3653,
+      "timestamp": 1691623200000
+    },
+    {
+      "degraded": 13,
+      "degraded_rate": 0.0035278154681139757,
+      "error_rate": 0.009497964721845319,
+      "failed": 22,
+      "failed_rate": 0.005970149253731343,
+      "success": 3650,
+      "timestamp": 1691623260000
+    },
+    {
+      "degraded": 7,
+      "degraded_rate": 0.0019358407079646017,
+      "error_rate": 0.00663716814159292,
+      "failed": 17,
+      "failed_rate": 0.004701327433628319,
+      "success": 3592,
+      "timestamp": 1691623320000
+    },
+    {
+      "degraded": 18,
+      "degraded_rate": 0.004288777698355968,
+      "error_rate": 0.00976888253514415,
+      "failed": 23,
+      "failed_rate": 0.005480104836788182,
+      "success": 4156,
+      "timestamp": 1691623380000
+    },
+    {
+      "degraded": 15,
+      "degraded_rate": 0.00487012987012987,
+      "error_rate": 0.00974025974025974,
+      "failed": 15,
+      "failed_rate": 0.00487012987012987,
+      "success": 3050,
+      "timestamp": 1691623440000
+    },
+    {
+      "degraded": 9,
+      "degraded_rate": 0.0024563318777292577,
+      "error_rate": 0.006823144104803494,
+      "failed": 16,
+      "failed_rate": 0.004366812227074236,
+      "success": 3639,
+      "timestamp": 1691623500000
+    },
+    {
+      "degraded": 10,
+      "degraded_rate": 0.0031959092361776927,
+      "error_rate": 0.006391818472355385,
+      "failed": 10,
+      "failed_rate": 0.0031959092361776927,
+      "success": 3109,
+      "timestamp": 1691623560000
+    },
+    {
+      "degraded": 12,
+      "degraded_rate": 0.003827751196172249,
+      "error_rate": 0.004784688995215311,
+      "failed": 3,
+      "failed_rate": 0.0009569377990430622,
+      "success": 3120,
+      "timestamp": 1691623620000
+    },
+    {
+      "degraded": 21,
+      "degraded_rate": 0.007123473541383989,
+      "error_rate": 0.009158751696065129,
+      "failed": 6,
+      "failed_rate": 0.0020352781546811396,
+      "success": 2921,
+      "timestamp": 1691623680000
+    },
+    {
+      "degraded": 7,
+      "degraded_rate": 0.002027222704894295,
+      "error_rate": 0.0034752389226759338,
+      "failed": 5,
+      "failed_rate": 0.0014480162177816392,
+      "success": 3441,
+      "timestamp": 1691623740000
+    },
+    {
+      "degraded": 8,
+      "degraded_rate": 0.0022179096201829776,
+      "error_rate": 0.0049902966454117,
+      "failed": 10,
+      "failed_rate": 0.0027723870252287217,
+      "success": 3589,
+      "timestamp": 1691623800000
+    }
+  ],
+  "start_time": 1691622660000,
+  "end_time": 1691623860000,
+  "metadata": {
+    "tags": {
+      "asset_alias": "data",
+      "asset_id": "123456789",
+      "env": "prd"
+    }
+  }
+}