Development image to test PopPunk branches

README.md

@@ -101,3 +101,19 @@ Testing can be done in a second terminal (make sure to activate 'beebop_py') by
 ```
 TESTING=True poetry run pytest
 ```
+
+### Use/Deploy specific version of PopPUNK
+
+To use deploy a specific version/commit/branch of PopPUNK, you can update `RUN pip install git+https://github.com/bacpop/[email protected]#egg=PopPUNK `
+in `DockerFile.dev` with the desired version/commit/branch.
+
+The new images built with `/docker/build` will have a *-dev* postfix.
+
+### Local Development
+
+You can build the image with `/docker/build`, this new image can now be used by Beebop.
+
+### Deployment
+
+A pull request can be created so buildkite pushes the images to the docker hub.
+Then on the `beebop-deploy` the api image can be updated with the new dev image.

docker/Dockerfile.dev

@@ -0,0 +1,56 @@
+FROM continuumio/miniconda3
+
+# Make RUN commands use the new environment:
+SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"]
+
+# Set up conda environment
+RUN conda install python=3.10
+RUN conda config --append channels conda-forge && \
+    conda config --append channels bioconda
+
+# Install mamba: which is a faster package manager than conda
+RUN conda install -c conda-forge mamba
+RUN conda config --set channel_priority flexible
+
+# Install PopPUNK conda dependencies
+RUN mamba install -y -c conda-forge -y graph-tool mandrake 
+RUN mamba install -y \
+    # Core data packages
+    pandas \
+    requests \
+    networkx \
+    scikit-learn \
+    # Bioinformatics tools
+    pp-sketchlib \
+    biopython \
+    treeswift \
+    rapidnj \
+    # Analysis tools
+    hdbscan \
+    # Progress tracking
+    tqdm \
+    && mamba clean -afy
+
+# System dependencies 
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    cmake \
+    libeigen3-dev \
+    libhdf5-dev \
+    libopenblas-dev
+
+# Update PopPUNK to desired version
+RUN pip install git+https://github.com/bacpop/[email protected]#egg=PopPUNK 
+
+# Poetry setup
+RUN pip install poetry
+COPY *.toml *.lock /
+RUN poetry config virtualenvs.create false && \
+    poetry install
+
+COPY . /beebop
+WORKDIR /beebop
+EXPOSE 5000
+
+CMD ["bash", "-c", "mamba run --no-capture-output -n base poetry run waitress-serve --port=5000 'beebop.app:app'"]

docker/build

@@ -4,11 +4,19 @@ set -ex
 HERE=$(dirname $0)
 . $HERE/common
 
+# Build the production image
 docker build --pull \
        --tag $TAG_SHA \
-       -f docker/Dockerfile \
+       -f docker/Dockerfile.prod \
+       $PACKAGE_ROOT
+
+# Build the development image
+docker build --pull \
+       --tag $TAG_DEV_SHA \
+       -f docker/Dockerfile.dev \
        $PACKAGE_ROOT
 
 # We always push the SHA tagged versions, for debugging if the tests
 # after this step fail
 docker push $TAG_SHA
+docker push $TAG_DEV_SHA

docker/common

@@ -4,6 +4,7 @@ set -e
 PACKAGE_ROOT=$(realpath $HERE/..)
 PACKAGE_NAME=beebop-py
 PACKAGE_ORG=mrcide
+PACKAGE_DEV=dev
 
 # Buildkite doesn't check out a full history from the remote (just the
 # single commit) so you end up with a detached head and git rev-parse
@@ -20,6 +21,11 @@ else
     GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD)
 fi
 
+# production images
 TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
 TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
-TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
+TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
+
+# development images
+TAG_DEV_SHA="${TAG_SHA}-${PACKAGE_DEV}"
+TAG_DEV_BRANCH="${TAG_BRANCH}-${PACKAGE_DEV}"

docker/push

@@ -7,8 +7,12 @@ HERE=$(dirname $0)
 # In case we switch agents between steps
 [ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA
 
+# production image
 docker tag $TAG_SHA $TAG_BRANCH
 docker push $TAG_BRANCH
+# development image
+docker tag $TAG_DEV_SHA $TAG_DEV_BRANCH
+docker push $TAG_DEV_BRANCH
 
 if [ $GIT_BRANCH == "main" ]; then
    docker tag $TAG_SHA $TAG_LATEST