Merge pull request #28 from nasa-fornax/gitlab-migration

Migration of fornax-images from gitlab

.github/workflows/check-build-code.yml

@@ -0,0 +1,25 @@
+name: Run tests for the build scripts
+
+on:
+  push:
+    branches-ignore: ['releases/**']
+    paths: ['scripts/**']
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        id: setup_python
+        with:
+          python-version: '3.12'
+
+      - name: Run builder tests
+        id: test
+        run: |
+           python3 tests/test_build_code.py

.github/workflows/image-build.yml

@@ -0,0 +1,81 @@
+name: Build and push images
+
+on:
+  push:
+    branches-ignore: ['releases/**']
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+
+  build-and-push-images:
+
+    runs-on: ubuntu-latest
+
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in
+    # this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    outputs:
+      dir_changes: ${{ steps.changed-images.outputs.changed_dirs }}
+
+    steps:
+      - name: Debug Event Payload
+        env:
+          EVENT_PAYLOAD: ${{ toJson(github.event) }}
+        run: echo "$EVENT_PAYLOAD"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Git branch name
+        id: git-branch-name
+        run: echo "${{ github.head_ref || github.ref_name }}"
+
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        id: setup_python
+        with:
+          python-version: '3.12'
+
+      - name: Changed Images
+        id: changed-images
+        run: |
+          cat <<EOF > tmp_github.json
+          ${{ toJson(github) }}
+          EOF
+          changed_dirs=`python scripts/changed_images.py tmp_github.json --debug`
+          echo "changed_dirs=$changed_dirs" >> $GITHUB_OUTPUT
+
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push images
+        id: push
+        env:
+          DIR_CHANGES: ${{ steps.changed-images.outputs.changed_dirs }}
+        run: |
+          echo "DIR_CHANGES: $DIR_CHANGES"
+          python scripts/build.py --debug \
+              --registry "$REGISTRY" --repository "${{ github.repository }}" \
+              --tag "${{ github.head_ref || github.ref_name }}" \
+              --push $DIR_CHANGES
+  
+  test-images:
+    needs: build-and-push-images
+    uses: ./.github/workflows/run-tests.yml
+    secrets: inherit
+    with:
+      # pass the images that changed for testing
+      images: ${{ needs.build-and-push-images.outputs.dir_changes }}

.github/workflows/release.yml

@@ -0,0 +1,53 @@
+name: Tag released images
+
+on:
+  release:
+    types: [published]
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+
+  build-and-push-images:
+    runs-on: ubuntu-latest
+
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in
+    # this job.
+    permissions:
+
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+
+      - name: Debug Event Payload
+        env:
+          EVENT_PAYLOAD: ${{ toJson(github.event) }}
+        run: echo "$EVENT_PAYLOAD"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        id: setup_python
+        with:
+          python-version: '3.12'
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Tag main images with release tags
+        id: push
+        run: |
+          python scripts/build.py --debug --no-build \
+              --registry "$REGISTRY" --repository "${{ github.repository }}" \
+              --tag "${{ github.event.release.target_commitish }}" \
+              --release "${{ github.event.release.tag_name }}" stable

.github/workflows/run-tests.yml

@@ -0,0 +1,63 @@
+name: image-tests
+on:
+  # call from the image build pipeline
+  workflow_call:
+    inputs:
+      images:
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      images:
+        description: 'Images of the form: ["image-1","image-2"]'
+        required: true
+        type: string
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  test-images:
+    runs-on: ubuntu-latest
+    if: inputs.images != '[]'
+    strategy:
+      matrix:
+        image: ${{ fromJson(inputs.images) }}
+
+    steps:
+      - name: Configure tests
+        id: configure
+        run: |
+          echo "uid=$(id -u)" >> $GITHUB_OUTPUT
+          echo "gid=$(id -g)" >> $GITHUB_OUTPUT
+          echo "Images passed for testing: ${{ inputs.images }}"
+  
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Pull Image: ${{ matrix.image }}"
+        run: |
+          tag=${{ github.head_ref || github.ref_name }}
+          image=${{ matrix.image }}
+          docker pull ghcr.io/${{ github.repository }}/$image:$tag
+          docker tag ghcr.io/${{ github.repository }}/$image:$tag $image
+
+      - name: "Run tests inside the image: ${{ matrix.image }}"
+        run: |
+          image=${{ matrix.image }}
+          docker run -i --rm -v .:/opt/workspace/repo-code \
+            $image \
+            bash -c "pytest -s -v /opt/workspace/repo-code/tests/test_${image}.py -o cache_dir=/tmp/"
+    
+      - name: "Run jupyter-lab: ${{ matrix.image }}"
+        run: |
+          image=${{ matrix.image }}
+          cmd="docker run -i --rm $image"
+          timeout 10 $cmd || [[ $? -eq 124 ]] && { echo "Timed out as expeced."; } || exit $?

README.md

@@ -1,8 +1,50 @@
 # fornax-images
-Customised Jupyterhub images for the Fornax Platform deployments
+This repo contains the Docker images for the Fornax Platform deployments.
+It produces reproducible computing environments. Some of the parts are
+adapted from the [Pangeo](https://github.com/pangeo-data/pangeo-docker-images) project.
 
-Have separate subdirectories for the different images, and please list them below for documentation purposes.
+Reproducibility is achived by keeping track of the software environments using conda yaml files.
+The following is a general description of the images:
 
+- Each image is in its own directory (e.g. `base_image` and `astro-defaul`).
+
+- `base_image` is a base image that contains basic JupyterHub and Lab setup, and many astronomy packages.
+Other images should use it as a starting point (i.e. using `FROM ...`).
+
+- Jupyterlab is installed in a conda environment called `notebook`, and it is the
+default environment when running the images. It is also the environment that contains `dask`.
+
+- The `scripts/build.py` script should be used when building the image locally. It takes as parameter
+the name of the folder that contains Dockerfile, which is also the name of the image.
+For example: `python scripts/build.py base_image` builds the base image, and
+`python scripts/build.py astro-default` builds the default image, etc.
+
+- The Dockerfile of each image (other than `base_image`) should start from the base image.
+
+- Starting from `base_image` will trigger the `ONBUILD` sections defined in the
+`base_image/Dockerfile`, which include:
+  - If `apt.txt` exits, it will be parsed for the list of the system software to be installed with `apt-get`.
+  - If `build-*` files exist, the scripts are run during the build.
+  - If `conda-{env}.yml` exists, it defines a conda environment called `{env}`, which typically what gets modified by hand.
+  - Additionally, if `conda-{env}-lock.yml` exists, it locks
+the versions of the installed libraries. To create this `-lock` file, or updated it, pass `--update-lock` to the
+build script `scripts/build.py`. This will first create or update the conda environment from the `conda-{env}.yml` file, then generate a new `conda-{env}-lock.yml` from the installed packages.
+  - If `introduction.md` file exists, it is copied to `/opt/scritps`, then copied to the user's `~/notebooks` (by the pre-notebook script) and serve as a landing page (through `JUPYTERHUB_DEFAULT_URL` defined in the jupyterhub depolyment code).
+
+The recommonded workflow is therefore like this:
+
+- Define the libraries requirement in `conda-{env}.yml`.
+
+- Build the image with `python scripts/build.py {image-name} --update-lock`.
+
+- This will generate `conda-{env}-lock.yml`, which should be kept under
+version control. The next time the image is built with `python scripts/build.py {image-name}`, the lock
+file will be used inside the Dockerfile to reproduce the exact build.
+
+# The images
+- `base_image`: is the base image that all other images should start from. It contains jupyter and the basic tools needed for deployment in the fornax project.
+
+- `astro-default`: Main Astro image that was used for the demo notebooks. It contains tractor and other general useful tools.
+
+- `heasoft`: high energy image containing heasoft.
 
-#### fornax_forced_photometry
- - Basic image with tractor installed

astro-default/Dockerfile

@@ -0,0 +1,22 @@
+# ONBUILD instructions in base-image/Dockerfile are used to
+# perform certain actions based on the presence of specific
+# files (such as conda-linux-64.lock, start) in this repo.
+# Refer to the base-image/Dockerfile for documentation.
+ARG BASE_TAG=latest
+ARG REPOSITORY=nasa-fornax/fornax-images
+ARG REGISTRY=ghcr.io
+
+FROM ${REGISTRY}/${REPOSITORY}/base_image:${BASE_TAG}
+
+
+LABEL org.opencontainers.image.source=https://github.com/nasa-fornax/fornax-images
+LABEL org.opencontainers.image.description "Fornax Main Astronomy Image"
+LABEL maintainer="Fornax Project"
+
+# add notebook updater script to run when jupyter starts
+USER root
+COPY --chmod=0755 update-notebooks.sh /usr/local/bin/before-notebook.d
+USER $NB_USER
+
+# For firefly
+ENV FIREFLY_URL=https://irsa.ipac.caltech.edu/irsaviewer

astro-default/build-tractor.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e 
+set -o pipefail
+
+pythonenv=notebook
+astrometry_commit=e868ccd
+tractor_commit=8059ae0
+
+# Install astrometry.net and tractor
+cd /tmp
+git clone https://github.com/dstndstn/astrometry.net.git
+cd astrometry.net
+git config --global --add safe.directory $PWD
+git checkout $astrometry_commit
+conda run -n $pythonenv make
+conda run -n $pythonenv make py
+conda run -n $pythonenv make extra
+conda run -n $pythonenv make install INSTALL_DIR=${CONDA_DIR}/envs/${pythonenv}
+mv ${CONDA_DIR}/envs/$pythonenv/lib/python/astrometry \
+   ${CONDA_DIR}/envs/$pythonenv/lib/python3.??/
+
+cd /tmp
+git clone https://github.com/dstndstn/tractor.git
+cd tractor
+git checkout $tractor_commit
+conda run -n $pythonenv python setup.py build_ext --inplace --with-cython
+conda run -n $pythonenv pip install --no-cache-dir . --target ${CONDA_DIR}/envs/$pythonenv/lib/python3.??/
+cd $HOME
+rm -rf /tmp/astrometry.net /tmp/tractor