Development image to test PopPunk branches

.github/workflows/build_and_push.yml

@@ -0,0 +1,25 @@
+name: build and push docker images
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+env:
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to GHCR (GitHub Packages)
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build docker image
+        run: ./docker/build --with-dev # TODO: remove --with_dev before merging
+      - name: Push SHA docker image
+        run: ./docker/push --with-dev  # TODO: remove --with_dev before merging

README.md

@@ -101,3 +101,19 @@ Testing can be done in a second terminal (make sure to activate 'beebop_py') by
 ```
 TESTING=True poetry run pytest
 ```
+
+### Use/Deploy specific version of PopPUNK
+
+To use a specific version, commit or branch of PopPUNK in a beebop_py deployment, you can update `POPPUNK_VERSION` in `common`.
+
+The new dev images built with `/docker/build --with-dev` will have a *-dev* postfix.
+
+### Local Development
+
+You can build the image with `/docker/build --with-dev`, this new image can now be used by Beebop.
+
+### Deployment
+
+A pull request can be created so GHA pushes the images to the docker hub. Add `--with-dev` to the build & push commands `pipeline.yaml`.
+**Ensure to remove the `--with-dev` flag before merging the PR.**
+Then on the `beebop-deploy` the api image can be updated with the new dev image.

docker/Dockerfile.dev

@@ -0,0 +1,61 @@
+FROM continuumio/miniconda3
+
+# ensure conda does not install packages for the wrong architecture
+ENV CONDA_OVERRIDE_ARCHSPEC=skylake
+
+ARG POPPUNK_VERSION="v2.6.7"
+
+# Make RUN commands use the new environment:
+SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"]
+
+# Set up conda environment
+RUN conda install python=3.10
+RUN conda config --append channels conda-forge && \
+    conda config --append channels bioconda
+
+# Install mamba: which is a faster package manager than conda
+RUN conda install -c conda-forge mamba
+RUN conda config --set channel_priority flexible
+
+#Install PopPUNK conda dependencies
+RUN mamba install -y -c conda-forge -y graph-tool mandrake 
+RUN mamba install -y \
+    # Core data packages
+    pandas \
+    requests \
+    networkx \
+    scikit-learn \
+    # Bioinformatics tools
+    pp-sketchlib \
+    biopython \
+    treeswift \
+    rapidnj \
+    # Analysis tools
+    hdbscan \
+    # Progress tracking
+    tqdm \
+    && mamba clean -afy
+
+# System dependencies 
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    cmake \
+    libeigen3-dev \
+    libhdf5-dev \
+    libopenblas-dev
+
+
+RUN pip install git+https://github.com/bacpop/PopPUNK@${POPPUNK_VERSION}#egg=PopPUNK 
+
+# Poetry setup
+RUN pip install poetry
+COPY *.toml *.lock /
+RUN poetry config virtualenvs.create false && \
+    poetry install
+
+COPY . /beebop
+WORKDIR /beebop
+EXPOSE 5000
+
+CMD ["conda", "run" ,"--no-capture-output", "-n", "base", "poetry", "run", "waitress-serve", "--port=5000", "beebop.app:app"]

docker/build

@@ -4,11 +4,21 @@ set -ex
 HERE=$(dirname $0)
 . $HERE/common
 
+# Build and push the production image
 docker build --pull \
        --tag $TAG_SHA \
-       -f docker/Dockerfile \
+       -f docker/Dockerfile.prod \
        $PACKAGE_ROOT
-
 # We always push the SHA tagged versions, for debugging if the tests
 # after this step fail
 docker push $TAG_SHA
+
+# Build and push dev image only if --with-dev flag is set
+if [ "$1" == "--with-dev" ]; then
+    docker build --pull \
+           --build-arg POPPUNK_VERSION=$POPPUNK_VERSION \
+           --tag $TAG_DEV_SHA \
+           -f docker/Dockerfile.dev \
+           $PACKAGE_ROOT
+    docker push $TAG_DEV_SHA
+fi

docker/common

@@ -1,25 +1,24 @@
 #!/usr/bin/env bash
 set -e
-
+REGISTRY=ghcr.io
 PACKAGE_ROOT=$(realpath $HERE/..)
 PACKAGE_NAME=beebop-py
-PACKAGE_ORG=mrcide
+PACKAGE_ORG=bacpop
+PACKAGE_DEV=dev
 
-# Buildkite doesn't check out a full history from the remote (just the
-# single commit) so you end up with a detached head and git rev-parse
-# doesn't work
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_SHA=${BUILDKITE_COMMIT:0:7}
+GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+if [[ -v "BRANCH_NAME" ]]; then
+    GIT_BRANCH=${BRANCH_NAME}
 else
-    GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+    GIT_BRANCH=$(git symbolic-ref --short HEAD)
 fi
 
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_BRANCH=$BUILDKITE_BRANCH
-else
-    GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD)
-fi
+# production image
+TAG_SHA="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
+TAG_BRANCH="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
+TAG_LATEST="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
 
-TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
-TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
-TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
+# development image
+TAG_DEV_SHA="${REGISTRY}/${TAG_SHA}-${PACKAGE_DEV}"
+TAG_DEV_BRANCH="${REGISTRY}/${TAG_BRANCH}-${PACKAGE_DEV}"
+POPPUNK_VERSION=v2.6.7 # can be version, branch or commit

docker/push

@@ -5,11 +5,16 @@ HERE=$(dirname $0)
 . $HERE/common
 
 # In case we switch agents between steps
-[ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA
 
 docker tag $TAG_SHA $TAG_BRANCH
 docker push $TAG_BRANCH
 
+if [ "$1" == "--with-dev" ]; then
+    [ ! -z $(docker images -q $TAG_DEV_SHA) ] || docker pull $TAG_DEV_SHA
+    docker tag $TAG_DEV_SHA $TAG_DEV_BRANCH
+    docker push $TAG_DEV_BRANCH
+fi
+
 if [ $GIT_BRANCH == "main" ]; then
    docker tag $TAG_SHA $TAG_LATEST
    docker push $TAG_LATEST