diff --git a/.github/workflows/check-build-code.yml b/.github/workflows/check-build-code.yml new file mode 100644 index 0000000..1836910 --- /dev/null +++ b/.github/workflows/check-build-code.yml @@ -0,0 +1,25 @@ +name: Run tests for the build scripts + +on: + push: + branches-ignore: ['releases/**'] + paths: ['scripts/**'] + +jobs: + run-tests: + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + id: setup_python + with: + python-version: '3.12' + + - name: Run builder tests + id: test + run: | + python3 tests/test_build_code.py diff --git a/.github/workflows/image-build.yml b/.github/workflows/image-build.yml new file mode 100644 index 0000000..46c8519 --- /dev/null +++ b/.github/workflows/image-build.yml @@ -0,0 +1,81 @@ +name: Build and push images + +on: + push: + branches-ignore: ['releases/**'] + +env: + REGISTRY: ghcr.io + +jobs: + + build-and-push-images: + + runs-on: ubuntu-latest + + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in + # this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + + outputs: + dir_changes: ${{ steps.changed-images.outputs.changed_dirs }} + + steps: + - name: Debug Event Payload + env: + EVENT_PAYLOAD: ${{ toJson(github.event) }} + run: echo "$EVENT_PAYLOAD" + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Git branch name + id: git-branch-name + run: echo "${{ github.head_ref || github.ref_name }}" + + + - name: Setup Python + uses: actions/setup-python@v5 + id: setup_python + with: + python-version: '3.12' + + - name: Changed Images + id: changed-images + run: | + cat < tmp_github.json + ${{ toJson(github) }} + EOF + changed_dirs=`python scripts/changed_images.py tmp_github.json --debug` + echo "changed_dirs=$changed_dirs" >> $GITHUB_OUTPUT + + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push images + id: push + env: + DIR_CHANGES: ${{ steps.changed-images.outputs.changed_dirs }} + run: | + echo "DIR_CHANGES: $DIR_CHANGES" + python scripts/build.py --debug \ + --registry "$REGISTRY" --repository "${{ github.repository }}" \ + --tag "${{ github.head_ref || github.ref_name }}" \ + --push $DIR_CHANGES + + test-images: + needs: build-and-push-images + uses: ./.github/workflows/run-tests.yml + secrets: inherit + with: + # pass the images that changed for testing + images: ${{ needs.build-and-push-images.outputs.dir_changes }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e8f521c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,53 @@ +name: Tag released images + +on: + release: + types: [published] + +env: + REGISTRY: ghcr.io + +jobs: + + build-and-push-images: + runs-on: ubuntu-latest + + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in + # this job. + permissions: + + contents: read + packages: write + attestations: write + id-token: write + + steps: + + - name: Debug Event Payload + env: + EVENT_PAYLOAD: ${{ toJson(github.event) }} + run: echo "$EVENT_PAYLOAD" + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + id: setup_python + with: + python-version: '3.12' + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tag main images with release tags + id: push + run: | + python scripts/build.py --debug --no-build \ + --registry "$REGISTRY" --repository "${{ github.repository }}" \ + --tag "${{ github.event.release.target_commitish }}" \ + --release "${{ github.event.release.tag_name }}" stable diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..3025fb8 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,63 @@ +name: image-tests +on: + # call from the image build pipeline + workflow_call: + inputs: + images: + required: true + type: string + workflow_dispatch: + inputs: + images: + description: 'Images of the form: ["image-1","image-2"]' + required: true + type: string + +env: + REGISTRY: ghcr.io + +jobs: + test-images: + runs-on: ubuntu-latest + if: inputs.images != '[]' + strategy: + matrix: + image: ${{ fromJson(inputs.images) }} + + steps: + - name: Configure tests + id: configure + run: | + echo "uid=$(id -u)" >> $GITHUB_OUTPUT + echo "gid=$(id -g)" >> $GITHUB_OUTPUT + echo "Images passed for testing: ${{ inputs.images }}" + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: "Pull Image: ${{ matrix.image }}" + run: | + tag=${{ github.head_ref || github.ref_name }} + image=${{ matrix.image }} + docker pull ghcr.io/${{ github.repository }}/$image:$tag + docker tag ghcr.io/${{ github.repository }}/$image:$tag $image + + - name: "Run tests inside the image: ${{ matrix.image }}" + run: | + image=${{ matrix.image }} + docker run -i --rm -v .:/opt/workspace/repo-code \ + $image \ + bash -c "pytest -s -v /opt/workspace/repo-code/tests/test_${image}.py -o cache_dir=/tmp/" + + - name: "Run jupyter-lab: ${{ matrix.image }}" + run: | + image=${{ matrix.image }} + cmd="docker run -i --rm $image" + timeout 10 $cmd || [[ $? -eq 124 ]] && { echo "Timed out as expeced."; } || exit $? diff --git a/README.md b/README.md index 04491c1..53ce028 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,50 @@ # fornax-images -Customised Jupyterhub images for the Fornax Platform deployments +This repo contains the Docker images for the Fornax Platform deployments. +It produces reproducible computing environments. Some of the parts are +adapted from the [Pangeo](https://github.com/pangeo-data/pangeo-docker-images) project. -Have separate subdirectories for the different images, and please list them below for documentation purposes. +Reproducibility is achived by keeping track of the software environments using conda yaml files. +The following is a general description of the images: +- Each image is in its own directory (e.g. `base_image` and `astro-defaul`). + +- `base_image` is a base image that contains basic JupyterHub and Lab setup, and many astronomy packages. +Other images should use it as a starting point (i.e. using `FROM ...`). + +- Jupyterlab is installed in a conda environment called `notebook`, and it is the +default environment when running the images. It is also the environment that contains `dask`. + +- The `scripts/build.py` script should be used when building the image locally. It takes as parameter +the name of the folder that contains Dockerfile, which is also the name of the image. +For example: `python scripts/build.py base_image` builds the base image, and +`python scripts/build.py astro-default` builds the default image, etc. + +- The Dockerfile of each image (other than `base_image`) should start from the base image. + +- Starting from `base_image` will trigger the `ONBUILD` sections defined in the +`base_image/Dockerfile`, which include: + - If `apt.txt` exits, it will be parsed for the list of the system software to be installed with `apt-get`. + - If `build-*` files exist, the scripts are run during the build. + - If `conda-{env}.yml` exists, it defines a conda environment called `{env}`, which typically what gets modified by hand. + - Additionally, if `conda-{env}-lock.yml` exists, it locks +the versions of the installed libraries. To create this `-lock` file, or updated it, pass `--update-lock` to the +build script `scripts/build.py`. This will first create or update the conda environment from the `conda-{env}.yml` file, then generate a new `conda-{env}-lock.yml` from the installed packages. + - If `introduction.md` file exists, it is copied to `/opt/scritps`, then copied to the user's `~/notebooks` (by the pre-notebook script) and serve as a landing page (through `JUPYTERHUB_DEFAULT_URL` defined in the jupyterhub depolyment code). + +The recommonded workflow is therefore like this: + +- Define the libraries requirement in `conda-{env}.yml`. + +- Build the image with `python scripts/build.py {image-name} --update-lock`. + +- This will generate `conda-{env}-lock.yml`, which should be kept under +version control. The next time the image is built with `python scripts/build.py {image-name}`, the lock +file will be used inside the Dockerfile to reproduce the exact build. + +# The images +- `base_image`: is the base image that all other images should start from. It contains jupyter and the basic tools needed for deployment in the fornax project. + +- `astro-default`: Main Astro image that was used for the demo notebooks. It contains tractor and other general useful tools. + +- `heasoft`: high energy image containing heasoft. -#### fornax_forced_photometry - - Basic image with tractor installed diff --git a/astro-default/Dockerfile b/astro-default/Dockerfile new file mode 100644 index 0000000..44b2725 --- /dev/null +++ b/astro-default/Dockerfile @@ -0,0 +1,22 @@ +# ONBUILD instructions in base-image/Dockerfile are used to +# perform certain actions based on the presence of specific +# files (such as conda-linux-64.lock, start) in this repo. +# Refer to the base-image/Dockerfile for documentation. +ARG BASE_TAG=latest +ARG REPOSITORY=nasa-fornax/fornax-images +ARG REGISTRY=ghcr.io + +FROM ${REGISTRY}/${REPOSITORY}/base_image:${BASE_TAG} + + +LABEL org.opencontainers.image.source=https://github.com/nasa-fornax/fornax-images +LABEL org.opencontainers.image.description "Fornax Main Astronomy Image" +LABEL maintainer="Fornax Project" + +# add notebook updater script to run when jupyter starts +USER root +COPY --chmod=0755 update-notebooks.sh /usr/local/bin/before-notebook.d +USER $NB_USER + +# For firefly +ENV FIREFLY_URL=https://irsa.ipac.caltech.edu/irsaviewer diff --git a/astro-default/build-tractor.sh b/astro-default/build-tractor.sh new file mode 100644 index 0000000..2b9be26 --- /dev/null +++ b/astro-default/build-tractor.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +set -o pipefail + +pythonenv=notebook +astrometry_commit=e868ccd +tractor_commit=8059ae0 + +# Install astrometry.net and tractor +cd /tmp +git clone https://github.com/dstndstn/astrometry.net.git +cd astrometry.net +git config --global --add safe.directory $PWD +git checkout $astrometry_commit +conda run -n $pythonenv make +conda run -n $pythonenv make py +conda run -n $pythonenv make extra +conda run -n $pythonenv make install INSTALL_DIR=${CONDA_DIR}/envs/${pythonenv} +mv ${CONDA_DIR}/envs/$pythonenv/lib/python/astrometry \ + ${CONDA_DIR}/envs/$pythonenv/lib/python3.??/ + +cd /tmp +git clone https://github.com/dstndstn/tractor.git +cd tractor +git checkout $tractor_commit +conda run -n $pythonenv python setup.py build_ext --inplace --with-cython +conda run -n $pythonenv pip install --no-cache-dir . --target ${CONDA_DIR}/envs/$pythonenv/lib/python3.??/ +cd $HOME +rm -rf /tmp/astrometry.net /tmp/tractor \ No newline at end of file diff --git a/astro-default/conda-notebook-lock.yml b/astro-default/conda-notebook-lock.yml new file mode 100644 index 0000000..0436009 --- /dev/null +++ b/astro-default/conda-notebook-lock.yml @@ -0,0 +1,529 @@ +name: notebook +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - acstools=3.7.2=pyhd8ed1ab_0 + - affine=2.4.0=pyhd8ed1ab_0 + - aiobotocore=2.15.2=pyhd8ed1ab_0 + - aiohappyeyeballs=2.4.3=pyhd8ed1ab_0 + - aiohttp=3.11.7=py311h2dc5d0c_0 + - aioitertools=0.12.0=pyhd8ed1ab_0 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - alembic=1.14.0=pyhd8ed1ab_0 + - alsa-lib=1.2.13=hb9d3cd8_0 + - annotated-types=0.7.0=pyhd8ed1ab_0 + - anyio=4.6.2.post1=pyhd8ed1ab_0 + - aom=3.9.1=hac33072_0 + - argon2-cffi=23.1.0=pyhd8ed1ab_0 + - argon2-cffi-bindings=21.2.0=py311h9ecbd09_5 + - arrow=1.3.0=pyhd8ed1ab_0 + - asciitree=0.3.3=py_2 + - asdf=4.0.0=pyhd8ed1ab_0 + - asdf-astropy=0.7.0=pyhd8ed1ab_0 + - asdf-coordinates-schemas=0.3.0=pyhd8ed1ab_0 + - asdf-standard=1.1.1=pyhd8ed1ab_0 + - asdf-transform-schemas=0.5.0=pyhd8ed1ab_0 + - asdf-wcs-schemas=0.4.0=pyhd8ed1ab_0 + - astropy=6.1.6=py311h9f3472d_0 + - astropy-healpix=1.0.3=py311h9f3472d_2 + - astropy-iers-data=0.2024.11.18.0.35.2=pyhd8ed1ab_0 + - astroscrappy=1.2.0=py311h9ecbd09_1 + - asttokens=2.4.1=pyhd8ed1ab_0 + - async-lru=2.0.4=pyhd8ed1ab_0 + - async_generator=1.10=pyhd8ed1ab_1 + - attrs=24.2.0=pyh71513ae_0 + - autograd=1.7.0=pyhd8ed1ab_0 + - aws-c-auth=0.8.0=hb88c0a9_10 + - aws-c-cal=0.8.0=hecf86a2_2 + - aws-c-common=0.10.3=hb9d3cd8_0 + - aws-c-compression=0.3.0=hf42f96a_2 + - aws-c-event-stream=0.5.0=h1ffe551_7 + - aws-c-http=0.9.1=hab05fe4_2 + - aws-c-io=0.15.2=hdeadb07_2 + - aws-c-mqtt=0.11.0=h7bd072d_8 + - aws-c-s3=0.7.1=h3a84f74_3 + - aws-c-sdkutils=0.2.1=hf42f96a_1 + - aws-checksums=0.2.2=hf42f96a_1 + - aws-crt-cpp=0.29.5=h21d7256_0 + - aws-sdk-cpp=1.11.449=hdaa582e_3 + - azure-core-cpp=1.14.0=h5cfcd09_0 + - azure-identity-cpp=1.10.0=h113e628_0 + - azure-storage-blobs-cpp=12.13.0=h3cf044e_1 + - azure-storage-common-cpp=12.8.0=h736e048_1 + - azure-storage-files-datalake-cpp=12.12.0=ha633028_1 + - babel=2.16.0=pyhd8ed1ab_0 + - backports=1.0=pyhd8ed1ab_4 + - backports.tarfile=1.2.0=pyhd8ed1ab_0 + - beautifulsoup4=4.12.3=pyha770c72_0 + - bleach=6.2.0=pyhd8ed1ab_0 + - blinker=1.9.0=pyhff2d567_0 + - blosc=1.21.6=hef167b5_0 + - bokeh=3.6.1=pyhd8ed1ab_0 + - boto3=1.35.36=pyhd8ed1ab_0 + - botocore=1.35.36=pyge310_1234567_0 + - bottleneck=1.4.2=py311h9f3472d_0 + - brotli=1.1.0=hb9d3cd8_2 + - brotli-bin=1.1.0=hb9d3cd8_2 + - brotli-python=1.1.0=py311hfdbb021_2 + - brunsli=0.1=h9c3ff4c_0 + - bzip2=1.0.8=h4bc722e_7 + - c-ares=1.34.3=heb4867d_0 + - c-blosc2=2.15.1=hc57e6cf_0 + - ca-certificates=2024.8.30=hbcca054_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cairo=1.18.0=hebfffa5_3 + - ccdproc=2.4.2=pyhd8ed1ab_0 + - cdshealpix=0.7.0=py311h9ecbd09_1 + - ceres-solver=2.2.0=ha77e7a2_4 + - certifi=2024.8.30=pyhd8ed1ab_0 + - certipy=0.2.1=pyhd8ed1ab_0 + - cffi=1.17.1=py311hf29c0ef_0 + - cfitsio=4.4.1=ha728647_2 + - charls=2.4.2=h59595ed_0 + - charset-normalizer=3.4.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - click-plugins=1.1.1=py_0 + - cligj=0.7.2=pyhd8ed1ab_1 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.2.2=pyhd8ed1ab_0 + - configurable-http-proxy=4.6.2=he2f69ee_0 + - contourpy=1.3.1=py311hd18a35c_0 + - cryptography=43.0.3=py311hafd3f86_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cyrus-sasl=2.1.27=h54b06d7_7 + - cython=3.0.11=py311h55d416d_3 + - cytoolz=1.0.0=py311h9ecbd09_1 + - dask=2024.11.2=pyhff2d567_1 + - dask-core=2024.11.2=pyhff2d567_1 + - dask-expr=1.1.19=pyhd8ed1ab_0 + - dask-labextension=7.0.0=pyhd8ed1ab_0 + - dav1d=1.2.1=hd590300_0 + - dbus=1.13.6=h5008d03_3 + - debugpy=1.8.9=py311hfdbb021_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - deprecated=1.2.15=pyhff2d567_0 + - distributed=2024.11.2=pyhff2d567_1 + - double-conversion=3.3.0=h59595ed_0 + - eigen=3.4.0=h00ab1b0_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - exceptiongroup=1.2.2=pyhd8ed1ab_0 + - executing=2.1.0=pyhd8ed1ab_0 + - expat=2.6.4=h5888daf_0 + - fasteners=0.17.3=pyhd8ed1ab_0 + - fbpca=1.0=py_0 + - firefly-client=3.1.0=pyhd8ed1ab_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_3 + - fontconfig=2.15.0=h7e30c49_1 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.55.0=py311h2dc5d0c_0 + - fqdn=1.5.1=pyhd8ed1ab_0 + - freetype=2.12.1=h267a509_2 + - freexl=2.0.0=h743c826_0 + - frozenlist=1.5.0=py311h9ecbd09_0 + - fsspec=2024.10.0=pyhff2d567_0 + - future=1.0.0=pyhd8ed1ab_0 + - geos=3.13.0=h5888daf_0 + - geotiff=1.7.3=h77b800c_3 + - gflags=2.2.2=h5888daf_1005 + - ghostscript=10.04.0=h5888daf_0 + - giflib=5.2.2=hd590300_0 + - glog=0.7.1=hbabe93e_0 + - gmp=6.3.0=hac33072_2 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.1.1=py311hfdbb021_0 + - gwcs=0.21.0=pyhd8ed1ab_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - harfbuzz=9.0.0=hda332d3_1 + - healpy=1.18.0=py311hc30af9e_0 + - hipscat=0.3.5=pyhd8ed1ab_0 + - hpack=4.0.0=pyh9f0ad1d_0 + - hpgeom=1.4.0=py311h9f3472d_0 + - html5lib=1.1=pyhd8ed1ab_1 + - httpcore=1.0.7=pyh29332c3_1 + - httpx=0.27.2=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=75.1=he02047a_0 + - idna=3.10=pyhd8ed1ab_0 + - imagecodecs=2024.9.22=py311h7d28041_0 + - imageio=2.36.0=pyh12aca89_1 + - importlib-metadata=8.5.0=pyha770c72_0 + - importlib_metadata=8.5.0=hd8ed1ab_0 + - importlib_resources=6.4.5=pyhd8ed1ab_0 + - iniconfig=2.0.0=pyhd8ed1ab_1 + - ipykernel=6.29.5=pyh3099207_0 + - ipython=8.29.0=pyh707e725_0 + - ipywidgets=8.1.5=pyhd8ed1ab_0 + - isoduration=20.11.0=pyhd8ed1ab_0 + - jaraco.classes=3.4.0=pyhd8ed1ab_1 + - jaraco.context=5.3.0=pyhd8ed1ab_1 + - jaraco.functools=4.0.0=pyhd8ed1ab_0 + - jedi=0.19.2=pyhff2d567_0 + - jeepney=0.8.0=pyhd8ed1ab_0 + - jinja2=3.1.4=pyhd8ed1ab_0 + - jmespath=1.0.1=pyhd8ed1ab_0 + - joblib=1.4.2=pyhd8ed1ab_0 + - json-c=0.18=h6688a6e_0 + - json5=0.9.28=pyhff2d567_0 + - jsonpointer=3.0.0=py311h38be061_1 + - jsonschema=4.23.0=pyhd8ed1ab_0 + - jsonschema-specifications=2024.10.1=pyhd8ed1ab_0 + - jsonschema-with-format-nongpl=4.23.0=hd8ed1ab_0 + - jupyter-lsp=2.2.5=pyhd8ed1ab_0 + - jupyter-server-proxy=4.4.0=pyhd8ed1ab_0 + - jupyter-vscode-proxy=0.6=pyhd8ed1ab_0 + - jupyter_client=8.6.3=pyhd8ed1ab_0 + - jupyter_core=5.7.2=pyh31011fe_1 + - jupyter_events=0.10.0=pyhd8ed1ab_0 + - jupyter_server=2.14.2=pyhd8ed1ab_0 + - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0 + - jupyterhub=5.1.0=pyh31011fe_0 + - jupyterhub-base=5.1.0=pyh31011fe_0 + - jupyterlab=4.2.4=pyhd8ed1ab_0 + - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1 + - jupyterlab_server=2.27.3=pyhd8ed1ab_0 + - jupyterlab_widgets=3.0.13=pyhd8ed1ab_0 + - jupytext=1.16.4=pyh80e38bb_0 + - jxrlib=1.1=hd590300_3 + - keyring=25.5.0=pyha804496_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.7=py311hd18a35c_0 + - krb5=1.21.3=h659f571_0 + - lazy-loader=0.4=pyhd8ed1ab_1 + - lazy_loader=0.4=pyhd8ed1ab_1 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.43=h712a8e2_2 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240722.0=cxx17_h5888daf_1 + - libaec=1.1.3=h59595ed_0 + - libarchive=3.7.7=hadbb8c3_0 + - libarrow=18.0.0=h94eee4b_8_cpu + - libarrow-acero=18.0.0=h5888daf_8_cpu + - libarrow-dataset=18.0.0=h5888daf_8_cpu + - libarrow-substrait=18.0.0=h5c8f2c3_8_cpu + - libavif16=1.1.1=h1909e37_2 + - libblas=3.9.0=25_linux64_openblas + - libbrotlicommon=1.1.0=hb9d3cd8_2 + - libbrotlidec=1.1.0=hb9d3cd8_2 + - libbrotlienc=1.1.0=hb9d3cd8_2 + - libcblas=3.9.0=25_linux64_openblas + - libclang-cpp19.1=19.1.4=default_hb5137d0_0 + - libclang13=19.1.4=default_h9c6a7e4_0 + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcups=2.3.3=h4637d8d_4 + - libcurl=8.10.1=hbbe4b11_0 + - libde265=1.0.15=h00ab1b0_0 + - libdeflate=1.22=hb9d3cd8_0 + - libdrm=2.4.123=hb9d3cd8_0 + - libedit=3.1.20191231=he28a2e2_2 + - libegl=1.7.0=ha4b6fd6_2 + - libev=4.33=hd590300_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.4=h5888daf_0 + - libffi=3.4.2=h7f98852_5 + - libgcc=14.2.0=h77fa898_1 + - libgcc-ng=14.2.0=h69a702a_1 + - libgcrypt=1.11.0=h4ab18f5_1 + - libgdal-core=3.10.0=hef9eae6_1 + - libgfortran=14.2.0=h69a702a_1 + - libgfortran-ng=14.2.0=h69a702a_1 + - libgfortran5=14.2.0=hd5240d6_1 + - libgl=1.7.0=ha4b6fd6_2 + - libglib=2.82.2=h2ff4ddf_0 + - libglvnd=1.7.0=ha4b6fd6_2 + - libglx=1.7.0=ha4b6fd6_2 + - libgomp=14.2.0=h77fa898_1 + - libgoogle-cloud=2.31.0=h804f50b_0 + - libgoogle-cloud-storage=2.31.0=h0121fbd_0 + - libgpg-error=1.51=hbd13f7d_1 + - libgrpc=1.67.1=hc2c308b_0 + - libheif=1.18.2=gpl_hffcb242_100 + - libhwloc=2.11.2=default_h0d58e46_1001 + - libhwy=1.1.0=h00ab1b0_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - libjxl=0.11.1=hdb8da77_0 + - libkml=1.3.0=hf539b9f_1021 + - liblapack=3.9.0=25_linux64_openblas + - libllvm14=14.0.6=hcd5def8_4 + - libllvm19=19.1.4=ha7bfdaf_0 + - libnghttp2=1.64.0=h161d5f1_0 + - libnsl=2.0.1=hd590300_0 + - libntlm=1.4=h7f98852_1002 + - libopenblas=0.3.28=pthreads_h94d23a6_1 + - libopengl=1.7.0=ha4b6fd6_2 + - libparquet=18.0.0=h6bd9018_8_cpu + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.44=hadc24fc_0 + - libpq=17.2=h04577a9_0 + - libprotobuf=5.28.2=h5b01275_0 + - libre2-11=2024.07.02=hbbce691_1 + - librttopo=1.1.0=h97f6797_17 + - libsecret=0.18.8=h329b89f_2 + - libsodium=1.0.20=h4ab18f5_0 + - libspatialite=5.1.0=h1b4f908_11 + - libsqlite=3.47.0=hadc24fc_1 + - libssh2=1.11.0=h0841786_0 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libstdcxx-ng=14.2.0=h4852527_1 + - libthrift=0.21.0=h0e7cc3e_0 + - libtiff=4.7.0=he137b08_1 + - libutf8proc=2.8.0=h166bdaf_0 + - libuuid=2.38.1=h0b41bf4_0 + - libuv=1.49.2=hb9d3cd8_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.17.0=h8a09558_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.7.0=h2c5496b_1 + - libxml2=2.13.5=hb346dea_0 + - libxslt=1.1.39=h76b75d6_0 + - libzlib=1.3.1=hb9d3cd8_2 + - libzopfli=1.0.3=h9c3ff4c_0 + - lightkurve=2.5.0=pyhd8ed1ab_0 + - llvmlite=0.43.0=py311h9c9ff8c_1 + - locket=1.0.0=pyhd8ed1ab_0 + - lsdb=0.2.6=pyhd8ed1ab_0 + - lsst-sphgeom=27.2024.4400=py311h7db5c69_0 + - lz4=4.3.3=py311h2cbdf9a_1 + - lz4-c=1.9.4=hcb278e6_0 + - lzo=2.10=hd590300_1001 + - mako=1.3.6=pyhff2d567_0 + - markdown-it-py=3.0.0=pyhd8ed1ab_0 + - markupsafe=3.0.2=py311h2dc5d0c_0 + - matplotlib=3.9.2=py311h38be061_2 + - matplotlib-base=3.9.2=py311h2b939e6_2 + - matplotlib-inline=0.1.7=pyhd8ed1ab_0 + - mdit-py-plugins=0.4.2=pyhd8ed1ab_0 + - mdurl=0.1.2=pyhd8ed1ab_0 + - memoization=0.4.0=pyhd8ed1ab_1 + - metis=5.1.0=hd0bcaf9_1007 + - minizip=4.0.7=h401b404_0 + - mistune=3.0.2=pyhd8ed1ab_0 + - mocpy=0.17.0=py311h9ecbd09_1 + - more-itertools=10.5.0=pyhd8ed1ab_0 + - mpfr=4.2.1=h90cbb55_3 + - mpl_animators=1.2.0=pyhd8ed1ab_0 + - mpld3=0.5.10=pyhd8ed1ab_0 + - msgpack-python=1.1.0=py311hd18a35c_0 + - multidict=6.1.0=py311h2dc5d0c_1 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=9.0.1=h266115a_2 + - mysql-libs=9.0.1=he0572af_2 + - nbclient=0.10.0=pyhd8ed1ab_0 + - nbconvert-core=7.16.4=pyhd8ed1ab_1 + - nbformat=5.10.4=pyhd8ed1ab_0 + - nbgitpuller=1.2.1=pyhd8ed1ab_0 + - ncurses=6.5=he02047a_1 + - ndcube=2.2.4=pyhd8ed1ab_0 + - nest-asyncio=1.6.0=pyhd8ed1ab_0 + - netpbm=10.73.43=pl5321h6ae6222_4 + - networkx=3.4.2=pyh267e887_2 + - nodejs=18.20.4=hc55a1b2_1 + - notebook=7.2.1=pyhd8ed1ab_0 + - notebook-shim=0.2.4=pyhd8ed1ab_0 + - numba=0.60.0=py311h4bc866e_0 + - numcodecs=0.14.1=py311h7db5c69_0 + - numpy=1.26.4=py311h64a7726_0 + - oauthlib=3.2.2=pyhd8ed1ab_0 + - oktopus=0.1.2=py_0 + - openjpeg=2.5.2=h488ebb8_0 + - openldap=2.6.8=hedd0468_0 + - openssl=3.4.0=hb9d3cd8_0 + - openvscode-server=1.92.1=hb09f993_0 + - orc=2.0.3=he039a57_0 + - overrides=7.7.0=pyhd8ed1ab_0 + - packaging=24.2=pyhff2d567_1 + - pamela=1.2.0=pyhff2d567_0 + - pandas=2.2.3=py311h7db5c69_1 + - pandocfilters=1.5.0=pyhd8ed1ab_0 + - parso=0.8.4=pyhd8ed1ab_0 + - partd=1.4.2=pyhd8ed1ab_0 + - patsy=1.0.1=pyhff2d567_0 + - pcre2=10.44=hba22ea6_2 + - perl=5.32.1=7_hd590300_perl5 + - pexpect=4.9.0=pyhd8ed1ab_0 + - pgplot=5.2.2=hbeaba86_1009 + - photutils=2.0.2=py311h9f3472d_1 + - pickleshare=0.7.5=py_1003 + - pillow=11.0.0=py311h49e9ac3_0 + - pip=24.3.1=pyh8b19718_0 + - pixman=0.43.2=h59595ed_0 + - pkg-config=0.29.2=h4bc722e_1009 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.3.6=pyhd8ed1ab_0 + - pluggy=1.5.0=pyhd8ed1ab_1 + - proj=9.5.0=h12925eb_0 + - prometheus_client=0.21.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.48=pyha770c72_0 + - propcache=0.2.0=py311h9ecbd09_2 + - pthread-stubs=0.4=hb9d3cd8_1002 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.3=pyhd8ed1ab_0 + - pyarrow=18.0.0=py311h38be061_1 + - pyarrow-core=18.0.0=py311h4854187_1_cpu + - pycparser=2.22=pyhd8ed1ab_0 + - pycurl=7.45.3=py311h0ad5ee3_3 + - pydantic=2.10.0=pyh10f6f8f_0 + - pydantic-core=2.27.0=py311h9e33e62_0 + - pyerfa=2.0.1.5=py311h9f3472d_0 + - pygments=2.18.0=pyhd8ed1ab_0 + - pyjwt=2.10.0=pyhff2d567_0 + - pynndescent=0.5.13=pyhff2d567_0 + - pyparsing=3.2.0=pyhd8ed1ab_1 + - pyside6=6.8.0.2=py311h9053184_0 + - pysocks=1.7.1=pyha2e5f31_6 + - pytest=8.3.4=pyhd8ed1ab_1 + - python=3.11.0=he550d4f_1_cpython + - python-dateutil=2.9.0.post0=pyhff2d567_0 + - python-fastjsonschema=2.20.0=pyhd8ed1ab_0 + - python-json-logger=2.0.7=pyhd8ed1ab_0 + - python-tzdata=2024.2=pyhd8ed1ab_0 + - python_abi=3.11=5_cp311 + - pytz=2024.1=pyhd8ed1ab_0 + - pyvo=1.6=pyhd8ed1ab_0 + - pywavelets=1.7.0=py311h9f3472d_2 + - pyyaml=6.0.2=py311h9ecbd09_1 + - pyzmq=26.2.0=py311h7deb3e3_3 + - qhull=2020.2=h434a139_5 + - qt6-main=6.8.0=h6e8976b_0 + - rasterio=1.4.2=py311h5394301_1 + - rav1e=0.6.6=he8a937b_2 + - re2=2024.07.02=h77b4e00_1 + - readline=8.2=h8228510_1 + - referencing=0.35.1=pyhd8ed1ab_0 + - regions=0.10=py311h9f3472d_0 + - reproject=0.14.1=py311h9f3472d_0 + - requests=2.32.3=pyhd8ed1ab_0 + - rfc3339-validator=0.1.4=pyhd8ed1ab_0 + - rfc3986-validator=0.1.1=pyh9f0ad1d_0 + - ripgrep=14.1.1=h8fae777_0 + - rpds-py=0.21.0=py311h9e33e62_0 + - s2n=1.5.9=h0fd0ee4_0 + - s3fs=2024.10.0=pyhd8ed1ab_0 + - s3transfer=0.10.4=pyhd8ed1ab_0 + - scikit-image=0.24.0=py311h7db5c69_3 + - scikit-learn=1.5.2=py311h57cc02b_1 + - scipy=1.14.1=py311he9a78e4_1 + - seaborn=0.13.2=hd8ed1ab_2 + - seaborn-base=0.13.2=pyhd8ed1ab_2 + - secretstorage=3.3.3=py311h38be061_3 + - semantic_version=2.10.0=pyhd8ed1ab_0 + - send2trash=1.8.3=pyh0d859eb_0 + - setuptools=75.6.0=pyhff2d567_1 + - shapely=2.0.6=py311h2fdb869_2 + - simpervisor=1.0.0=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.2.1=ha2e4443_0 + - sniffio=1.3.1=pyhd8ed1ab_0 + - snuggs=1.4.7=pyhd8ed1ab_1 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - soupsieve=2.5=pyhd8ed1ab_1 + - specutils=1.19.0=pyhd8ed1ab_0 + - sqlalchemy=2.0.36=py311h9ecbd09_0 + - sqlite=3.47.0=h9eae976_1 + - stack_data=0.6.2=pyhd8ed1ab_0 + - statsmodels=0.14.4=py311h9f3472d_0 + - suitesparse=7.8.3=hb42a789_1 + - svt-av1=2.3.0=h5888daf_0 + - swig=4.3.0=heed6a68_0 + - tbb=2022.0.0=hceb3a55_0 + - tblib=3.0.0=pyhd8ed1ab_0 + - terminado=0.18.1=pyh0d859eb_0 + - threadpoolctl=3.5.0=pyhc1e730c_0 + - tifffile=2024.9.20=pyhd8ed1ab_0 + - tinycss2=1.4.0=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tomli=2.1.0=pyhff2d567_0 + - toolz=1.0.0=pyhd8ed1ab_0 + - tornado=6.4.1=py311h9ecbd09_1 + - tqdm=4.67.0=pyhd8ed1ab_0 + - traitlets=5.14.3=pyhd8ed1ab_0 + - types-python-dateutil=2.9.0.20241003=pyhff2d567_0 + - typing-extensions=4.12.2=hd8ed1ab_0 + - typing_extensions=4.12.2=pyha770c72_0 + - typing_utils=0.1.0=pyhd8ed1ab_0 + - tzdata=2024b=hc8b5060_0 + - umap-learn=0.5.7=py311h38be061_0 + - uncertainties=3.2.2=pyhd8ed1ab_1 + - unicodedata2=15.1.0=py311h9ecbd09_1 + - uri-template=1.3.0=pyhd8ed1ab_0 + - uriparser=0.9.8=hac33072_0 + - urllib3=2.2.3=pyhd8ed1ab_0 + - wayland=1.23.1=h3e06ad9_0 + - wcslib=8.2.2=ha708777_2 + - wcwidth=0.2.13=pyhd8ed1ab_0 + - webcolors=24.8.0=pyhd8ed1ab_0 + - webencodings=0.5.1=pyhd8ed1ab_2 + - websocket-client=1.8.0=pyhd8ed1ab_0 + - wheel=0.45.0=pyhd8ed1ab_0 + - widgetsnbextension=4.0.13=pyhd8ed1ab_0 + - wrapt=1.17.0=py311h9ecbd09_0 + - x265=3.5=h924138e_3 + - xcb-util=0.4.1=hb711507_2 + - xcb-util-cursor=0.1.5=hb9d3cd8_0 + - xcb-util-image=0.4.0=hb711507_2 + - xcb-util-keysyms=0.4.1=hb711507_0 + - xcb-util-renderutil=0.3.10=hb711507_0 + - xcb-util-wm=0.4.2=hb711507_0 + - xerces-c=3.2.5=h988505b_2 + - xkeyboard-config=2.43=hb9d3cd8_0 + - xorg-libice=1.1.1=hb9d3cd8_1 + - xorg-libsm=1.2.4=he73a12e_1 + - xorg-libx11=1.8.10=h4f16b4b_0 + - xorg-libxau=1.0.11=hb9d3cd8_1 + - xorg-libxcomposite=0.4.6=hb9d3cd8_2 + - xorg-libxcursor=1.2.3=hb9d3cd8_0 + - xorg-libxdamage=1.1.6=hb9d3cd8_0 + - xorg-libxdmcp=1.1.5=hb9d3cd8_0 + - xorg-libxext=1.3.6=hb9d3cd8_0 + - xorg-libxfixes=6.0.1=hb9d3cd8_0 + - xorg-libxi=1.8.2=hb9d3cd8_0 + - xorg-libxrandr=1.5.4=hb9d3cd8_0 + - xorg-libxrender=0.9.11=hb9d3cd8_1 + - xorg-libxtst=1.2.5=hb9d3cd8_3 + - xorg-libxxf86vm=1.1.5=hb9d3cd8_4 + - xorg-xorgproto=2024.1=hb9d3cd8_1 + - xyzservices=2024.9.0=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yarl=1.18.0=py311h9ecbd09_0 + - zarr=2.18.3=pyhd8ed1ab_0 + - zeromq=4.3.5=h3b0a872_7 + - zfp=1.0.1=h5888daf_2 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.21.0=pyhd8ed1ab_0 + - zlib=1.3.1=hb9d3cd8_2 + - zlib-ng=2.2.2=h5888daf_0 + - zstandard=0.23.0=py311hbc35293_1 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - alerce==1.2.0 + - anywidget==0.9.13 + - astroquery==0.4.8.dev9474 + - filelock==3.16.1 + - gdown==5.2.0 + - gitdb==4.0.11 + - gitpython==3.1.43 + - ipyaladin==0.5.2 + - jupyter-cpu-alive==0.1.2 + - jupyter-firefly-extensions==4.3.0 + - jupyter-resource-usage==1.1.0 + - jupyter-server-mathjax==0.2.6 + - jupyterlab-execute-time==3.2.0 + - jupyterlab-git==0.50.2 + - jupyterlab-myst==2.4.2 + - nbdime==4.0.2 + - psutil==5.9.8 + - psygnal==0.11.1 + - smmap==5.0.1 + - specreduce==1.4.1 +prefix: /opt/conda/envs/notebook diff --git a/astro-default/conda-notebook.yml b/astro-default/conda-notebook.yml new file mode 100644 index 0000000..57a5912 --- /dev/null +++ b/astro-default/conda-notebook.yml @@ -0,0 +1,51 @@ +name: notebook +channels: + - conda-forge + - nodefaults +dependencies: + - numpy + - pandas + - scipy + - pyarrow + - swig + - ipykernel + - cairo + - cython + - ceres-solver + - pkg-config + - setuptools + - wcslib + - libjpeg-turbo + - netpbm + - acstools + - pyvo + - boto3 + - firefly-client + - hpgeom + - lightkurve + - matplotlib + - seaborn + - mpld3 + - reproject + - s3fs + - scikit-learn + - scikit-image + - statsmodels + - astropy-healpix + - ccdproc + - photutils + - regions + - reproject + - specutils + - umap-learn + - statsmodels + - lsdb + - pip + - pip: + - specreduce + - astroquery>=0.4.8.dev0 + - alerce + - gdown + - ipyaladin + - jupyter_firefly_extensions +prefix: /opt/conda/envs/notebook \ No newline at end of file diff --git a/astro-default/introduction.md b/astro-default/introduction.md new file mode 100644 index 0000000..b69ba84 --- /dev/null +++ b/astro-default/introduction.md @@ -0,0 +1,55 @@ +# +Fornax Science Console + +# Welcome to the Fornax Science Console! +--- + +# Documentation +The general documentation for using the Fornax Science Console are available in the +[documentation repository](fornax-documentation/README.md). These can also be browsed +in the [documentation page on github](https://nasa-fornax.github.io/fornax-demo-notebooks/#user-documentation). + +# Notebooks +The `notebooks` folder in the home directory contains notebooks that are actively being +developed. They currently include: +- [fornax-demo-notebooks](fornax-demo-notebooks/README.md): These are the main notbeooks developed + by the Fornax team. The rendered version is availale on the + [documentation page](https://nasa-fornax.github.io/fornax-demo-notebooks). +- [IVOA_2024_demo](IVOA_2024_demo/README.md): Notebooks developed in collaboration with the LINCC team, + demonstrating the use of Hispcat and LSDB for large catalog cross matching. +- Others will be added. + +The content of the `notebooks` folder in the home directory will be updated automatically +at the start of every new session. To disable these updates, add an empty file called +`.no-notebook-update.txt` in your home directory. + +--- +# Latest Changes +#### 11/26/2024 +- The primary conda environment is changed to `notebook`. It is the environment +where the notebooks should be run. With this change, the dask extension should +work naturally. +- Added the openvscode extension. +- Updates to prevent sessions with CPU activity from being stopped. The policy now is: + - If there is CPU activity, the notebook will not be stopped, even if the browser + is closed. + - If there is no activity (e.g. the notebook or browser tab is closed), + the session terminates after 15 min. +- The notebooks are updated automatically using `nbgitpuller` and they are +stored in the user's home directory. The update policy for `nbgitpuller`can be found +[here](https://nbgitpuller.readthedocs.io/en/latest/topic/automatic-merging.html#topic-automatic-merging). +The summary is: + - 1. A file that is changed in the remote repo but not in the local clone will be updated. + - 2. If different lines are changed by both the remote and local clone, the remote + changes will be merged similar to case 1. + - 3. If the same lines are changed by both the remote and local clone, the local + changes are kept and the remote changes are discarded. + - 4. If a file is deleted locally but still present in the remote repo, it will be restored. + - 5. If a new file is added in the locall clone, and the remote repo has a new file with + the same name, the local copy will be renamed by adding `_`, and the remote copy + will be used. +If the user has a file (it can be empty) called `.no-notebook-update.txt` in their home +directory, then `nbgitpuller` will not be used and the notebook folder in the home +directory will **not** be updated. +- Switched to using conda yaml files to keep track of the installed software. \ No newline at end of file diff --git a/astro-default/update-notebooks.sh b/astro-default/update-notebooks.sh new file mode 100644 index 0000000..e5512e9 --- /dev/null +++ b/astro-default/update-notebooks.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set +e +timeout=10 + +notebook_repos=( + # main demo notebooks + https://github.com/nasa-fornax/fornax-demo-notebooks.git + # Documentation + https://github.com/nasa-fornax/fornax-documentation.git + # lsdb notbeooks + https://github.com/lincc-frameworks/IVOA_2024_demo.git +) + +if test -z $NOTEBOOK_DIR; then export NOTEBOOK_DIR=$HOME/notebooks; fi +mkdir -p $NOTEBOOK_DIR +cd $NOTEBOOK_DIR +# copy notebooks +echo "Cloning the notebooks to $notebook_dir ..." +for repo in ${notebook_repos[@]}; do + name=`echo $repo | sed 's#.*/\([^/]*\)\.git#\1#'` + if ! test -f $NOUPDATE; then + timeout $timeout python -m nbgitpuller.pull $repo main $name + fi +done +cd $HOME diff --git a/base_image/Dockerfile b/base_image/Dockerfile new file mode 100644 index 0000000..99f2016 --- /dev/null +++ b/base_image/Dockerfile @@ -0,0 +1,144 @@ +ARG BASE_TAG=2024-08-12 + +FROM quay.io/jupyter/base-notebook:${BASE_TAG} as base +FROM quay.io/jupyter/docker-stacks-foundation:${BASE_TAG} + + +LABEL org.opencontainers.image.source=https://github.com/nasa-fornax/fornax-images +LABEL org.opencontainers.image.description "Fornax Base Astronomy Image" +LABEL org.opencontainers.image.authors "Fornax Project" + + +# Bring some commands from jupyter/base-notebook +# Skip the install of jupyterhub/lab etc +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV JUPYTER_PORT=8888 +EXPOSE $JUPYTER_PORT +CMD ["start-notebook.py"] +# Copy files from the image directly +COPY --from=base /usr/local/bin/start-* /usr/local/bin/ +COPY --from=base /etc/jupyter/*py /etc/jupyter/ +USER root +HEALTHCHECK --interval=3s --timeout=1s --start-period=3s --retries=3 \ + CMD /etc/jupyter/docker_healthcheck.py || exit 1 +RUN fix-permissions /etc/jupyter/ /home/${NB_USER} /opt/ +USER $NB_USER +WORKDIR $HOME +# End of imports from jupyter/base-notebook # +# ----------------------------------------- # + + +ENV CONDA_ENV=notebook + +# Ask dask to read config from ${CONDA_DIR}/etc rather than +# the default of /etc, since the non-root jovyan user can write +# to ${CONDA_DIR}/etc but not to /etc +ENV DASK_ROOT_CONFIG=${CONDA_DIR}/etc + +# COPY the current content to $HOME/build +RUN mkdir -p $HOME/build/ +COPY --chown=$NB_UID:$NB_GID apt* conda*yml build-* $HOME/build/ +COPY --chown=$NB_UID:$NB_GID overrides.json $HOME/build/ + +USER root + +# Make /opt/ user writeable so it can be used by build-* scripts +RUN fix-permissions /opt/ + +# Install OS packages and then clean up +COPY --chown=$NB_UID:$NB_GID scripts/*.sh /opt/scripts/ +# Read apt.txt line by line, and execute apt-get install for each line +RUN cd build && bash /opt/scripts/apt-install.sh + +USER $NB_USER +# setup conda environments +RUN cd build && bash /opt/scripts/conda-env-install.sh \ + # Change dispaly name of the default kernel + && mamba run -n $CONDA_ENV python -m ipykernel install --sys-prefix --display-name "$CONDA_ENV" + +# Any other build-* scripts # +RUN cd $HOME/build \ + ; for script in `ls build-*`; do \ + echo "Found script ${script} ..." \ + && chmod +x $script \ + && ./$script \ + ; done +# --------------------------- # + +# cache location: use /tmp/ so we don't fill up $HOME +ENV CACHE_DIR=/tmp/cache +RUN mkdir -p $CACHE_DIR +ENV XDG_CACHE_HOME=$CACHE_DIR +ENV XDG_CACHE_DIR=$CACHE_DIR +ENV XDG_CACHE_DIR=$CACHE_DIR + +# Set default config for pip and conda +# change default pip cache from ~/.cache/pip +ENV PIP_CACHE_DIR=$CACHE_DIR/pip +RUN mkdir -p $PIP_CACHE_DIR + +COPY --chown=$NB_UID:$NB_GID condarc ${CONDA_DIR}/.condarc +# ------------------------------------# + +# add a script for to run before the jupyter session +USER root +COPY --chmod=0755 pre-notebook.sh /usr/local/bin/before-notebook.d/20-pre-notebook.sh +USER $NB_USER + +# Make $CONDA_ENV default; do it at the global level +# because ~/.bashrc is not loaded when user space is mounted +# Also, add it to before-notebook.d main script +USER root +ENV PATH=$CONDA_DIR/envs/$CONDA_ENV/bin:$PATH +RUN cat $HOME/.bashrc >> /etc/bash.bashrc \ + && printf "\nconda activate \$CONDA_ENV\n" >> /etc/bash.bashrc \ + && printf "\nconda activate \$CONDA_ENV\n" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh \ + && printf "" > $HOME/.bashrc +USER $NB_USER + +# For vscode +ENV CODE_EXECUTABLE=openvscode-server + + +# For outside mount when using outside fornax +RUN mkdir -p /opt/workspace +VOLUME /opt/workspace + +# reset user and location +RUN rm -r $HOME/build $HOME/work /tmp/* +WORKDIR ${HOME} + +# Useful environment variables +ENV NOTEBOOK_DIR=${HOME}/notebooks \ + NOUPDATE=${HOME}/.no-notebook-update.txt + +# Install OS packages and then clean up +ONBUILD RUN mkdir -p $HOME/build +ONBUILD COPY --chown=$NB_UID:$NB_GID apt* conda*yml build-* $HOME/build/ +ONBUILD USER root +ONBUILD RUN mkdir -p build && cd build && bash /opt/scripts/apt-install.sh +ONBUILD USER $NB_USER +# ------------------------------------ # + + +# setup conda environments +ONBUILD RUN cd build && bash /opt/scripts/conda-env-install.sh +# and ensure the correct display name +ONBUILD RUN mamba run -n $CONDA_ENV python -m ipykernel install --sys-prefix --display-name "$CONDA_ENV" +# ----------------------- # + +# Any other build-* scripts # +ONBUILD RUN cd build \ + ; for script in `ls build-*`; do \ + echo "Found script ${script} ..." \ + && chmod +x $script \ + && ./$script \ + ; done +# --------------------------- # + +# landing page +ONBUILD COPY --chown=$NB_UID:$NB_GID --chmod=644 introduction.md* /opt/scripts/ + +ONBUILD RUN rm -r $HOME/build +ONBUILD USER ${NB_USER} +ONBUILD WORKDIR ${HOME} \ No newline at end of file diff --git a/base_image/apt.txt b/base_image/apt.txt new file mode 100644 index 0000000..85ae466 --- /dev/null +++ b/base_image/apt.txt @@ -0,0 +1,14 @@ +fonts-liberation +pandoc +vim +nano +emacs-nox +fuse +bzip2 +git +curl +zip +build-essential +gcc +make +gfortran \ No newline at end of file diff --git a/base_image/build-config.sh b/base_image/build-config.sh new file mode 100644 index 0000000..25a5266 --- /dev/null +++ b/base_image/build-config.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +env=notebook + +# jupyter overrides +mkdir -p $CONDA_DIR/envs/$env/share/jupyter/lab/settings/ +mv ~/build/overrides.json $CONDA_DIR/envs/$env/share/jupyter/lab/settings/ + + +# disable the annoucement extension +mamba run -n $env jupyter labextension disable "@jupyterlab/apputils-extension:announcements" diff --git a/base_image/conda-notebook-lock.yml b/base_image/conda-notebook-lock.yml new file mode 100644 index 0000000..2936c4a --- /dev/null +++ b/base_image/conda-notebook-lock.yml @@ -0,0 +1,397 @@ +name: notebook +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - aiohappyeyeballs=2.4.3=pyhd8ed1ab_0 + - aiohttp=3.11.7=py311h2dc5d0c_0 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - alembic=1.14.0=pyhd8ed1ab_0 + - alsa-lib=1.2.13=hb9d3cd8_0 + - annotated-types=0.7.0=pyhd8ed1ab_0 + - anyio=4.6.2.post1=pyhd8ed1ab_0 + - argon2-cffi=23.1.0=pyhd8ed1ab_0 + - argon2-cffi-bindings=21.2.0=py311h9ecbd09_5 + - arrow=1.3.0=pyhd8ed1ab_0 + - astropy=6.1.6=py311h9f3472d_0 + - astropy-iers-data=0.2024.11.18.0.35.2=pyhd8ed1ab_0 + - astroquery=0.4.7=pyhd8ed1ab_0 + - asttokens=2.4.1=pyhd8ed1ab_0 + - async-lru=2.0.4=pyhd8ed1ab_0 + - async_generator=1.10=pyhd8ed1ab_1 + - attrs=24.2.0=pyh71513ae_0 + - aws-c-auth=0.8.0=hb88c0a9_10 + - aws-c-cal=0.8.0=hecf86a2_2 + - aws-c-common=0.10.3=hb9d3cd8_0 + - aws-c-compression=0.3.0=hf42f96a_2 + - aws-c-event-stream=0.5.0=h1ffe551_7 + - aws-c-http=0.9.1=hab05fe4_2 + - aws-c-io=0.15.2=hdeadb07_2 + - aws-c-mqtt=0.11.0=h7bd072d_8 + - aws-c-s3=0.7.1=h3a84f74_3 + - aws-c-sdkutils=0.2.1=hf42f96a_1 + - aws-checksums=0.2.2=hf42f96a_1 + - aws-crt-cpp=0.29.5=h21d7256_0 + - aws-sdk-cpp=1.11.449=hdaa582e_3 + - azure-core-cpp=1.14.0=h5cfcd09_0 + - azure-identity-cpp=1.10.0=h113e628_0 + - azure-storage-blobs-cpp=12.13.0=h3cf044e_1 + - azure-storage-common-cpp=12.8.0=h736e048_1 + - azure-storage-files-datalake-cpp=12.12.0=ha633028_1 + - babel=2.16.0=pyhd8ed1ab_0 + - backports=1.0=pyhd8ed1ab_4 + - backports.tarfile=1.2.0=pyhd8ed1ab_0 + - beautifulsoup4=4.12.3=pyha770c72_0 + - bleach=6.2.0=pyhd8ed1ab_0 + - blinker=1.9.0=pyhff2d567_0 + - bokeh=3.6.1=pyhd8ed1ab_0 + - brotli=1.1.0=hb9d3cd8_2 + - brotli-bin=1.1.0=hb9d3cd8_2 + - brotli-python=1.1.0=py311hfdbb021_2 + - bzip2=1.0.8=h4bc722e_7 + - c-ares=1.34.3=heb4867d_0 + - ca-certificates=2024.8.30=hbcca054_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cairo=1.18.0=hebfffa5_3 + - certifi=2024.8.30=pyhd8ed1ab_0 + - certipy=0.2.1=pyhd8ed1ab_0 + - cffi=1.17.1=py311hf29c0ef_0 + - charset-normalizer=3.4.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.2.2=pyhd8ed1ab_0 + - configurable-http-proxy=4.6.2=he2f69ee_0 + - contourpy=1.3.1=py311hd18a35c_0 + - cryptography=43.0.3=py311hafd3f86_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cyrus-sasl=2.1.27=h54b06d7_7 + - cytoolz=1.0.0=py311h9ecbd09_1 + - dask=2024.11.2=pyhff2d567_1 + - dask-core=2024.11.2=pyhff2d567_1 + - dask-expr=1.1.19=pyhd8ed1ab_0 + - dask-labextension=7.0.0=pyhd8ed1ab_0 + - dbus=1.13.6=h5008d03_3 + - debugpy=1.8.9=py311hfdbb021_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - distributed=2024.11.2=pyhff2d567_1 + - double-conversion=3.3.0=h59595ed_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - exceptiongroup=1.2.2=pyhd8ed1ab_0 + - executing=2.1.0=pyhd8ed1ab_0 + - expat=2.6.4=h5888daf_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_3 + - fontconfig=2.15.0=h7e30c49_1 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.55.0=py311h2dc5d0c_0 + - fqdn=1.5.1=pyhd8ed1ab_0 + - freetype=2.12.1=h267a509_2 + - frozenlist=1.5.0=py311h9ecbd09_0 + - fsspec=2024.10.0=pyhff2d567_0 + - gflags=2.2.2=h5888daf_1005 + - glog=0.7.1=hbabe93e_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.1.1=py311hfdbb021_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - harfbuzz=9.0.0=hda332d3_1 + - hpack=4.0.0=pyh9f0ad1d_0 + - html5lib=1.1=pyhd8ed1ab_1 + - httpcore=1.0.7=pyh29332c3_1 + - httpx=0.27.2=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=75.1=he02047a_0 + - idna=3.10=pyhd8ed1ab_0 + - importlib-metadata=8.5.0=pyha770c72_0 + - importlib_metadata=8.5.0=hd8ed1ab_0 + - importlib_resources=6.4.5=pyhd8ed1ab_0 + - iniconfig=2.0.0=pyhd8ed1ab_1 + - ipykernel=6.29.5=pyh3099207_0 + - ipython=8.29.0=pyh707e725_0 + - ipywidgets=8.1.5=pyhd8ed1ab_0 + - isoduration=20.11.0=pyhd8ed1ab_0 + - jaraco.classes=3.4.0=pyhd8ed1ab_1 + - jaraco.context=5.3.0=pyhd8ed1ab_1 + - jaraco.functools=4.0.0=pyhd8ed1ab_0 + - jedi=0.19.2=pyhff2d567_0 + - jeepney=0.8.0=pyhd8ed1ab_0 + - jinja2=3.1.4=pyhd8ed1ab_0 + - json5=0.9.28=pyhff2d567_0 + - jsonpointer=3.0.0=py311h38be061_1 + - jsonschema=4.23.0=pyhd8ed1ab_0 + - jsonschema-specifications=2024.10.1=pyhd8ed1ab_0 + - jsonschema-with-format-nongpl=4.23.0=hd8ed1ab_0 + - jupyter-lsp=2.2.5=pyhd8ed1ab_0 + - jupyter-server-proxy=4.4.0=pyhd8ed1ab_0 + - jupyter-vscode-proxy=0.6=pyhd8ed1ab_0 + - jupyter_client=8.6.3=pyhd8ed1ab_0 + - jupyter_core=5.7.2=pyh31011fe_1 + - jupyter_events=0.10.0=pyhd8ed1ab_0 + - jupyter_server=2.14.2=pyhd8ed1ab_0 + - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0 + - jupyterhub=5.1.0=pyh31011fe_0 + - jupyterhub-base=5.1.0=pyh31011fe_0 + - jupyterlab=4.2.4=pyhd8ed1ab_0 + - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1 + - jupyterlab_server=2.27.3=pyhd8ed1ab_0 + - jupyterlab_widgets=3.0.13=pyhd8ed1ab_0 + - jupytext=1.16.4=pyh80e38bb_0 + - keyring=25.5.0=pyha804496_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.7=py311hd18a35c_0 + - krb5=1.21.3=h659f571_0 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.43=h712a8e2_2 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240722.0=cxx17_h5888daf_1 + - libarrow=18.0.0=h94eee4b_8_cpu + - libarrow-acero=18.0.0=h5888daf_8_cpu + - libarrow-dataset=18.0.0=h5888daf_8_cpu + - libarrow-substrait=18.0.0=h5c8f2c3_8_cpu + - libblas=3.9.0=25_linux64_openblas + - libbrotlicommon=1.1.0=hb9d3cd8_2 + - libbrotlidec=1.1.0=hb9d3cd8_2 + - libbrotlienc=1.1.0=hb9d3cd8_2 + - libcblas=3.9.0=25_linux64_openblas + - libclang-cpp19.1=19.1.4=default_hb5137d0_0 + - libclang13=19.1.4=default_h9c6a7e4_0 + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcups=2.3.3=h4637d8d_4 + - libcurl=8.10.1=hbbe4b11_0 + - libdeflate=1.22=hb9d3cd8_0 + - libdrm=2.4.123=hb9d3cd8_0 + - libedit=3.1.20191231=he28a2e2_2 + - libegl=1.7.0=ha4b6fd6_2 + - libev=4.33=hd590300_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.4=h5888daf_0 + - libffi=3.4.2=h7f98852_5 + - libgcc=14.2.0=h77fa898_1 + - libgcc-ng=14.2.0=h69a702a_1 + - libgcrypt=1.11.0=h4ab18f5_1 + - libgfortran=14.2.0=h69a702a_1 + - libgfortran5=14.2.0=hd5240d6_1 + - libgl=1.7.0=ha4b6fd6_2 + - libglib=2.82.2=h2ff4ddf_0 + - libglvnd=1.7.0=ha4b6fd6_2 + - libglx=1.7.0=ha4b6fd6_2 + - libgomp=14.2.0=h77fa898_1 + - libgoogle-cloud=2.31.0=h804f50b_0 + - libgoogle-cloud-storage=2.31.0=h0121fbd_0 + - libgpg-error=1.51=hbd13f7d_1 + - libgrpc=1.67.1=hc2c308b_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - liblapack=3.9.0=25_linux64_openblas + - libllvm19=19.1.4=ha7bfdaf_0 + - libnghttp2=1.64.0=h161d5f1_0 + - libnsl=2.0.1=hd590300_0 + - libntlm=1.4=h7f98852_1002 + - libopenblas=0.3.28=pthreads_h94d23a6_1 + - libopengl=1.7.0=ha4b6fd6_2 + - libparquet=18.0.0=h6bd9018_8_cpu + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.44=hadc24fc_0 + - libpq=17.2=h04577a9_0 + - libprotobuf=5.28.2=h5b01275_0 + - libre2-11=2024.07.02=hbbce691_1 + - libsecret=0.18.8=h329b89f_2 + - libsodium=1.0.20=h4ab18f5_0 + - libsqlite=3.47.0=hadc24fc_1 + - libssh2=1.11.0=h0841786_0 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libstdcxx-ng=14.2.0=h4852527_1 + - libthrift=0.21.0=h0e7cc3e_0 + - libtiff=4.7.0=he137b08_1 + - libutf8proc=2.8.0=h166bdaf_0 + - libuuid=2.38.1=h0b41bf4_0 + - libuv=1.49.2=hb9d3cd8_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.17.0=h8a09558_0 + - libxkbcommon=1.7.0=h2c5496b_1 + - libxml2=2.13.5=hb346dea_0 + - libxslt=1.1.39=h76b75d6_0 + - libzlib=1.3.1=hb9d3cd8_2 + - locket=1.0.0=pyhd8ed1ab_0 + - lz4=4.3.3=py311h2cbdf9a_1 + - lz4-c=1.9.4=hcb278e6_0 + - mako=1.3.6=pyhff2d567_0 + - markdown-it-py=3.0.0=pyhd8ed1ab_0 + - markupsafe=3.0.2=py311h2dc5d0c_0 + - matplotlib=3.9.2=py311h38be061_2 + - matplotlib-base=3.9.2=py311h2b939e6_2 + - matplotlib-inline=0.1.7=pyhd8ed1ab_0 + - mdit-py-plugins=0.4.2=pyhd8ed1ab_0 + - mdurl=0.1.2=pyhd8ed1ab_0 + - mistune=3.0.2=pyhd8ed1ab_0 + - more-itertools=10.5.0=pyhd8ed1ab_0 + - msgpack-python=1.1.0=py311hd18a35c_0 + - multidict=6.1.0=py311h2dc5d0c_1 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=9.0.1=h266115a_2 + - mysql-libs=9.0.1=he0572af_2 + - nbclient=0.10.0=pyhd8ed1ab_0 + - nbconvert-core=7.16.4=pyhd8ed1ab_1 + - nbformat=5.10.4=pyhd8ed1ab_0 + - nbgitpuller=1.2.1=pyhd8ed1ab_0 + - ncurses=6.5=he02047a_1 + - nest-asyncio=1.6.0=pyhd8ed1ab_0 + - nodejs=18.20.4=hc55a1b2_1 + - notebook=7.2.1=pyhd8ed1ab_0 + - notebook-shim=0.2.4=pyhd8ed1ab_0 + - numpy=1.26.4=py311h64a7726_0 + - oauthlib=3.2.2=pyhd8ed1ab_0 + - openjpeg=2.5.2=h488ebb8_0 + - openldap=2.6.8=hedd0468_0 + - openssl=3.4.0=hb9d3cd8_0 + - openvscode-server=1.92.1=hb09f993_0 + - orc=2.0.3=he039a57_0 + - overrides=7.7.0=pyhd8ed1ab_0 + - packaging=24.2=pyhff2d567_1 + - pamela=1.2.0=pyhff2d567_0 + - pandas=2.2.3=py311h7db5c69_1 + - pandocfilters=1.5.0=pyhd8ed1ab_0 + - parso=0.8.4=pyhd8ed1ab_0 + - partd=1.4.2=pyhd8ed1ab_0 + - patsy=1.0.1=pyhff2d567_0 + - pcre2=10.44=hba22ea6_2 + - pexpect=4.9.0=pyhd8ed1ab_0 + - pickleshare=0.7.5=py_1003 + - pillow=11.0.0=py311h49e9ac3_0 + - pip=24.3.1=pyh8b19718_0 + - pixman=0.43.2=h59595ed_0 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.3.6=pyhd8ed1ab_0 + - pluggy=1.5.0=pyhd8ed1ab_1 + - prometheus_client=0.21.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.48=pyha770c72_0 + - propcache=0.2.0=py311h9ecbd09_2 + - pthread-stubs=0.4=hb9d3cd8_1002 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.3=pyhd8ed1ab_0 + - pyarrow=18.0.0=py311h38be061_1 + - pyarrow-core=18.0.0=py311h4854187_1_cpu + - pycparser=2.22=pyhd8ed1ab_0 + - pycurl=7.45.3=py311h0ad5ee3_3 + - pydantic=2.10.0=pyh10f6f8f_0 + - pydantic-core=2.27.0=py311h9e33e62_0 + - pyerfa=2.0.1.5=py311h9f3472d_0 + - pygments=2.18.0=pyhd8ed1ab_0 + - pyjwt=2.10.0=pyhff2d567_0 + - pyparsing=3.2.0=pyhd8ed1ab_1 + - pyside6=6.8.0.2=py311h9053184_0 + - pysocks=1.7.1=pyha2e5f31_6 + - pytest=8.3.4=pyhd8ed1ab_1 + - python=3.11.0=he550d4f_1_cpython + - python-dateutil=2.9.0.post0=pyhff2d567_0 + - python-fastjsonschema=2.20.0=pyhd8ed1ab_0 + - python-json-logger=2.0.7=pyhd8ed1ab_0 + - python-tzdata=2024.2=pyhd8ed1ab_0 + - python_abi=3.11=5_cp311 + - pytz=2024.1=pyhd8ed1ab_0 + - pyvo=1.6=pyhd8ed1ab_0 + - pyyaml=6.0.2=py311h9ecbd09_1 + - pyzmq=26.2.0=py311h7deb3e3_3 + - qhull=2020.2=h434a139_5 + - qt6-main=6.8.0=h6e8976b_0 + - re2=2024.07.02=h77b4e00_1 + - readline=8.2=h8228510_1 + - referencing=0.35.1=pyhd8ed1ab_0 + - requests=2.32.3=pyhd8ed1ab_0 + - rfc3339-validator=0.1.4=pyhd8ed1ab_0 + - rfc3986-validator=0.1.1=pyh9f0ad1d_0 + - ripgrep=14.1.1=h8fae777_0 + - rpds-py=0.21.0=py311h9e33e62_0 + - s2n=1.5.9=h0fd0ee4_0 + - scipy=1.14.1=py311he9a78e4_1 + - seaborn=0.13.2=hd8ed1ab_2 + - seaborn-base=0.13.2=pyhd8ed1ab_2 + - secretstorage=3.3.3=py311h38be061_3 + - send2trash=1.8.3=pyh0d859eb_0 + - setuptools=75.6.0=pyhff2d567_0 + - simpervisor=1.0.0=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.2.1=ha2e4443_0 + - sniffio=1.3.1=pyhd8ed1ab_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - soupsieve=2.5=pyhd8ed1ab_1 + - sqlalchemy=2.0.36=py311h9ecbd09_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - statsmodels=0.14.4=py311h9f3472d_0 + - tblib=3.0.0=pyhd8ed1ab_0 + - terminado=0.18.1=pyh0d859eb_0 + - tinycss2=1.4.0=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tomli=2.1.0=pyhff2d567_0 + - toolz=1.0.0=pyhd8ed1ab_0 + - tornado=6.4.1=py311h9ecbd09_1 + - tqdm=4.67.0=pyhd8ed1ab_0 + - traitlets=5.14.3=pyhd8ed1ab_0 + - types-python-dateutil=2.9.0.20241003=pyhff2d567_0 + - typing-extensions=4.12.2=hd8ed1ab_0 + - typing_extensions=4.12.2=pyha770c72_0 + - typing_utils=0.1.0=pyhd8ed1ab_0 + - tzdata=2024b=hc8b5060_0 + - unicodedata2=15.1.0=py311h9ecbd09_1 + - uri-template=1.3.0=pyhd8ed1ab_0 + - urllib3=2.2.3=pyhd8ed1ab_0 + - wayland=1.23.1=h3e06ad9_0 + - wcwidth=0.2.13=pyhd8ed1ab_0 + - webcolors=24.8.0=pyhd8ed1ab_0 + - webencodings=0.5.1=pyhd8ed1ab_2 + - websocket-client=1.8.0=pyhd8ed1ab_0 + - wheel=0.45.0=pyhd8ed1ab_0 + - widgetsnbextension=4.0.13=pyhd8ed1ab_0 + - xcb-util=0.4.1=hb711507_2 + - xcb-util-cursor=0.1.5=hb9d3cd8_0 + - xcb-util-image=0.4.0=hb711507_2 + - xcb-util-keysyms=0.4.1=hb711507_0 + - xcb-util-renderutil=0.3.10=hb711507_0 + - xcb-util-wm=0.4.2=hb711507_0 + - xkeyboard-config=2.43=hb9d3cd8_0 + - xorg-libice=1.1.1=hb9d3cd8_1 + - xorg-libsm=1.2.4=he73a12e_1 + - xorg-libx11=1.8.10=h4f16b4b_0 + - xorg-libxau=1.0.11=hb9d3cd8_1 + - xorg-libxcomposite=0.4.6=hb9d3cd8_2 + - xorg-libxcursor=1.2.3=hb9d3cd8_0 + - xorg-libxdamage=1.1.6=hb9d3cd8_0 + - xorg-libxdmcp=1.1.5=hb9d3cd8_0 + - xorg-libxext=1.3.6=hb9d3cd8_0 + - xorg-libxfixes=6.0.1=hb9d3cd8_0 + - xorg-libxi=1.8.2=hb9d3cd8_0 + - xorg-libxrandr=1.5.4=hb9d3cd8_0 + - xorg-libxrender=0.9.11=hb9d3cd8_1 + - xorg-libxtst=1.2.5=hb9d3cd8_3 + - xorg-libxxf86vm=1.1.5=hb9d3cd8_4 + - xorg-xorgproto=2024.1=hb9d3cd8_1 + - xyzservices=2024.9.0=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yarl=1.18.0=py311h9ecbd09_0 + - zeromq=4.3.5=h3b0a872_7 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.21.0=pyhd8ed1ab_0 + - zlib=1.3.1=hb9d3cd8_2 + - zstandard=0.23.0=py311hbc35293_1 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - gitdb==4.0.11 + - gitpython==3.1.43 + - jupyter-cpu-alive==0.1.2 + - jupyter-resource-usage==1.1.0 + - jupyter-server-mathjax==0.2.6 + - jupyterlab-execute-time==3.2.0 + - jupyterlab-git==0.50.2 + - jupyterlab-myst==2.4.2 + - nbdime==4.0.2 + - psutil==5.9.8 + - smmap==5.0.1 +prefix: /opt/conda/envs/notebook diff --git a/base_image/conda-notebook.yml b/base_image/conda-notebook.yml new file mode 100644 index 0000000..33f1b1b --- /dev/null +++ b/base_image/conda-notebook.yml @@ -0,0 +1,33 @@ +channels: + - conda-forge + - nodefaults +dependencies: + - python==3.11 + - jupyterlab==4.2.4 + - jupyterhub==5.1.0 + - notebook==7.2.1 + - ipykernel + - ipywidgets + - jupytext + - nbgitpuller + - dask + - dask-labextension + - numpy<2 + - scipy + - pandas + - astropy + - pyvo + - astroquery + - tqdm + - matplotlib + - seaborn + - openvscode-server + - jupyter-vscode-proxy + - pytest + - pip + - pip: + - jupyterlab_execute_time + - jupyter-resource-usage + - jupyterlab-git + - jupyterlab-myst + - jupyter_cpu_alive diff --git a/base_image/condarc b/base_image/condarc new file mode 100644 index 0000000..1cff18c --- /dev/null +++ b/base_image/condarc @@ -0,0 +1,8 @@ +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge +envs_dirs: + - /opt/conda/envs +pkgs_dirs: + - /tmp/cache/conda \ No newline at end of file diff --git a/base_image/overrides.json b/base_image/overrides.json new file mode 100644 index 0000000..43d621c --- /dev/null +++ b/base_image/overrides.json @@ -0,0 +1,13 @@ +{ + "@jupyterlab/docmanager-extension:plugin": { + "defaultViewers": { + "markdown": "Jupytext Notebook" + } + }, + "dask-labextension:plugin": { + "autoStartClient": true + }, + "@jupyterlab/notebook-extension": { + "scrollHeadingToTop": false + } +} \ No newline at end of file diff --git a/base_image/pre-notebook.sh b/base_image/pre-notebook.sh new file mode 100644 index 0000000..2f69c1d --- /dev/null +++ b/base_image/pre-notebook.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# if something fails, keep going, we don't want to the stop the server from loading +set +e + +# include any code that needs to be run before the start of the jupyter session + +# remove the old nb_conda_kernels defined in the user directory +# so they don't show up in the Jupyterlab launcher +rm -rf ~/.local/share/jupyter/kernels/conda-base-py &>/dev/null +rm -rf ~/.local/share/jupyter/kernels/conda-env-science_demo-py &>/dev/null + +# remove old condarc in the user's home; continue gracefully +find ~/.condarc -type f ! -newermt "2024-11-26" -delete &>/dev/null || true + +# cleanup cache accmulated before the new cache location at /tmp/cache +cd $HOME +for dir in users_conda_envs .astropy/cache .cache; do + rm $dir &>/dev/null +done + +# remove any old pip cache +pip cache purge &>/dev/null + +# clean any conda cache +mamba clean -yaf &>/dev/null + +# Make sure we have a landing page +if test -z $NOTEBOOK_DIR; then export NOTEBOOK_DIR=$HOME/notebooks; fi +mkdir -p $NOTEBOOK_DIR +if test -f /opt/scripts/introduction.md && ! test -f $HOME/.no-notebook-update.txt; then + mv /opt/scripts/introduction.md $NOTEBOOK_DIR +fi +touch $NOTEBOOK_DIR/introduction.md diff --git a/base_image/scripts/apt-install.sh b/base_image/scripts/apt-install.sh new file mode 100644 index 0000000..f0dcf68 --- /dev/null +++ b/base_image/scripts/apt-install.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# This will be run as root inside Dockerfile + +if test -f "apt.txt" ; then + echo "Found apt.txt; using it ..." + apt-get update --fix-missing > /dev/null + xargs -a apt.txt apt-get install -y + apt-get clean + apt-get -y autoremove + rm -rf /var/lib/apt/lists/* +fi \ No newline at end of file diff --git a/base_image/scripts/conda-env-install.sh b/base_image/scripts/conda-env-install.sh new file mode 100644 index 0000000..2556ab4 --- /dev/null +++ b/base_image/scripts/conda-env-install.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e +set -o pipefail + +# handle conda environment and lock files +# look for conda-{env}-lock.yml and conda-{env}.yml files + +for envfile in `ls conda-*.yml | grep -v lock`; do + env=`echo $envfile | sed -n 's/conda-\(.*\)\.yml/\1/p'` + if test -f conda-${env}-lock.yml; then + ENVFILE=conda-${env}-lock.yml + echo "Found $ENVFILE, using it ..." + mamba env update -n ${env} -f $ENVFILE + elif test -f conda-${env}.yml; then + ENVFILE=conda-${env}.yml + echo "Found $ENVFILE, using it ..." + mamba env update -n ${env} -f $ENVFILE + elif [[ "$env"=="$CONDA_ENV" ]]; then + echo "Defaulting to basic env ..." + mamba create --name $env python=3.11 jupyterlab + fi + if [ "$env" != "$CONDA_ENV" ]; then + # add the environement as a jupyter kernel + # CONDA_ENV is defined in the dockerfile + mamba install -n $env ipykernel + mamba run -n $env python -m ipykernel install --name $env --prefix $CONDA_DIR/envs/$CONDA_ENV + fi +done + +# clean +mamba clean -yaf +pip cache purge +find ${CONDA_DIR} -follow -type f -name '*.a' -name '*.pyc' -delete +find ${CONDA_DIR} -follow -type f -name '*.js.map' -delete diff --git a/fornax_forced_photometry/README.md b/fornax_forced_photometry/README.md deleted file mode 100644 index 7cbc55b..0000000 --- a/fornax_forced_photometry/README.md +++ /dev/null @@ -1 +0,0 @@ -Packages from the astropy ecosystem + tractor and its deepndencies diff --git a/heasoft/Dockerfile b/heasoft/Dockerfile new file mode 100644 index 0000000..0118903 --- /dev/null +++ b/heasoft/Dockerfile @@ -0,0 +1,23 @@ +# ONBUILD instructions in base-image/Dockerfile are used to +# perform certain actions based on the presence of specific +# files (such as conda-linux-64.lock, start) in this repo. +# Refer to the base-image/Dockerfile for documentation. +ARG BASE_TAG=latest +ARG REPOSITORY=nasa-fornax/fornax-images +ARG REGISTRY=ghcr.io + +FROM ${REGISTRY}/${REPOSITORY}/base_image:${BASE_TAG} + + +LABEL org.opencontainers.image.source=https://github.com/nasa-fornax/fornax-images +LABEL org.opencontainers.image.description "Fornax High Energy Astronomy Image" +LABEL maintainer="Fornax Project" + +# set default conda env +ENV CONDA_ENV=notebook + + +LABEL org.opencontainers.image.ref.name="Fornax High Energy Astrophysics" +LABEL gov.nasa.smce.fornax.jupyterhub.image="${IMAGE_TAG}" +LABEL gov.nasa.smce.fornax.jupyterhub.base_image="${BASE_IMAGE_TAG}" +LABEL gov.nasa.smce.fornax.jupyterhub.repository="${REPOSITORY}" diff --git a/heasoft/apt.txt b/heasoft/apt.txt new file mode 100644 index 0000000..baa2a47 --- /dev/null +++ b/heasoft/apt.txt @@ -0,0 +1,13 @@ +g++ +libncurses-dev +libreadline-dev +libgsl-dev +file +make +ncurses-dev +perl-modules +tcsh +wget +xorg-dev +libcurl4 +libssl3 \ No newline at end of file diff --git a/heasoft/build-heasoft.sh b/heasoft/build-heasoft.sh new file mode 100644 index 0000000..5c93225 --- /dev/null +++ b/heasoft/build-heasoft.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +set -e +set -o pipefail + +pythonenv=notebook +heasoft_version=6.34 +install_dir=/opt/heasoft +caldb_dir=/opt/caldb + +# option to compile only part of heasoft when testing +# set to "yes" to activate +fast="no" + +export PYTHON=${CONDA_DIR}/envs/${pythonenv}/bin/python +heasoft_tarfile_suffix=src_no_xspec_modeldata +echo " --- Downloading heasoft ---" +wget -q https://heasarc.gsfc.nasa.gov/FTP/software/lheasoft/lheasoft${heasoft_version}/heasoft-${heasoft_version}${heasoft_tarfile_suffix}.tar.gz +tar xzf heasoft-${heasoft_version}${heasoft_tarfile_suffix}.tar.gz > tar.log.txt +rm -f heasoft-${heasoft_version}${heasoft_tarfile_suffix}.tar.gz +cd heasoft-${heasoft_version} + +if [ $fast == "yes" ]; then + rm -r demo integral nicer suzaku Xspec calet heagen heasim hitomixrism ixpe maxi nustar swift + rm -r attitude ftools heasptools heatools tcltk +fi + +# ----------------------- # +# Handle large refdata that we don't want in the image +# we add them later +refdata=( + "ftools/xstar/data/atdb.fits" + "heasim/skyback/torus1006.fits" +) +# remove refdata from the image +for file in "${refdata[@]}"; do + rm -f $file +done +# ----------------------- # + +## Configure, make, and install ... +echo " --- Configure heasoft ---" +cd BUILD_DIR/ +# write stdout to a file (can be long), and leave stderr on screen +./configure --prefix=$install_dir --enable-collapse > config.log.txt + +echo " --- Build heasoft ---" +make > build.log.txt +make install > install.log.txt +make clean > clean.log.txt +gzip -9 *.log.txt && mv *.log.txt.gz $install_dir +cd .. +if [ ! $fast == "yes" ]; then + cp -p Xspec/BUILD_DIR/hmakerc $install_dir/x86_64*/bin/ + cp -p Xspec/BUILD_DIR/Makefile-std $install_dir/x86_64*/bin/ +fi +mv Release_Notes* $install_dir +cd .. && rm -rf heasoft-${heasoft_version} + +# Tweak Xspec settings for a no-X11 environment +if [ ! $fast == "yes" ]; then + printf "setplot splashpage off\ncpd /GIF\n" >> $install_dir/spectral/scripts/global_customize.tcl +fi + +# enable remote CALDB for now. +export CALDBCONFIG=$caldb_dir/caldb.config +export CALDBALIAS=$caldb_dir/alias_config.fits +export CALDB=/home/jovyan/efs/caldb +mkdir -p $caldb_dir +cd $caldb_dir +wget -q https://heasarc.gsfc.nasa.gov/FTP/caldb/software/tools/caldb.config +wget -q https://heasarc.gsfc.nasa.gov/FTP/caldb/software/tools/alias_config.fits + +# setup scripts so it runs when (heasoft) is activated +HEADAS=`ls -d $install_dir/x86_64*` +CALDB=https://heasarc.gsfc.nasa.gov/FTP/caldb + +# bash init script +cat < activate_heasoft.sh +export HEADAS=$HEADAS +export CALDB=$CALDB +source \$HEADAS/headas-init.sh +if [ -z \$CALDB ] ; then + echo "** No CALDB data. **" +elif [[ \$CALDB == http* ]]; then + echo "** Using Remote CALDB **" +elif [ -d \$CALDB ]; then + echo "** Using CALDB in \$CALDB **" + source \$CALDB/software/tools/caldbinit.sh +else + echo "** No CALDB data. **" +fi +EOF + +_activatedir=$CONDA_DIR/envs/${pythonenv}/etc/conda/activate.d/ +mkdir -p $_activatedir +mv activate_heasoft.sh $_activatedir + + +# ----------------------- # +# write a script to download the refdata +cat < $HEADAS/bin/download-refdata.sh +#!/usr/bin/env bash + +if test -z \$HEADAS; then + echo "\$HEADAS is not defined. Make sure heasoft is installed" +else + cd \$HEADAS/refdata + for file in ${refdata[@]}; do + if ! test -f \$(basename \$file); then + echo "downloading \$file ..." + wget -q https://heasarc.gsfc.nasa.gov/FTP/software/lheasoft/lheasoft${heasoft_version}/heasoft-${heasoft_version}/\$file + fi + done +fi +EOF +chmod +x $HEADAS/bin/download-refdata.sh \ No newline at end of file diff --git a/heasoft/conda-notebook-lock.yml b/heasoft/conda-notebook-lock.yml new file mode 100644 index 0000000..3801c15 --- /dev/null +++ b/heasoft/conda-notebook-lock.yml @@ -0,0 +1,395 @@ +name: notebook +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - aiohappyeyeballs=2.4.3=pyhd8ed1ab_0 + - aiohttp=3.11.7=py311h2dc5d0c_0 + - aiosignal=1.3.1=pyhd8ed1ab_0 + - alembic=1.14.0=pyhd8ed1ab_0 + - alsa-lib=1.2.13=hb9d3cd8_0 + - annotated-types=0.7.0=pyhd8ed1ab_0 + - anyio=4.6.2.post1=pyhd8ed1ab_0 + - argon2-cffi=23.1.0=pyhd8ed1ab_0 + - argon2-cffi-bindings=21.2.0=py311h9ecbd09_5 + - arrow=1.3.0=pyhd8ed1ab_0 + - astropy=6.1.6=py311h9f3472d_0 + - astropy-iers-data=0.2024.11.18.0.35.2=pyhd8ed1ab_0 + - astroquery=0.4.7=pyhd8ed1ab_0 + - asttokens=2.4.1=pyhd8ed1ab_0 + - async-lru=2.0.4=pyhd8ed1ab_0 + - async_generator=1.10=pyhd8ed1ab_1 + - attrs=24.2.0=pyh71513ae_0 + - aws-c-auth=0.8.0=hb88c0a9_10 + - aws-c-cal=0.8.0=hecf86a2_2 + - aws-c-common=0.10.3=hb9d3cd8_0 + - aws-c-compression=0.3.0=hf42f96a_2 + - aws-c-event-stream=0.5.0=h1ffe551_7 + - aws-c-http=0.9.1=hab05fe4_2 + - aws-c-io=0.15.2=hdeadb07_2 + - aws-c-mqtt=0.11.0=h7bd072d_8 + - aws-c-s3=0.7.1=h3a84f74_3 + - aws-c-sdkutils=0.2.1=hf42f96a_1 + - aws-checksums=0.2.2=hf42f96a_1 + - aws-crt-cpp=0.29.5=h21d7256_0 + - aws-sdk-cpp=1.11.449=hdaa582e_3 + - azure-core-cpp=1.14.0=h5cfcd09_0 + - azure-identity-cpp=1.10.0=h113e628_0 + - azure-storage-blobs-cpp=12.13.0=h3cf044e_1 + - azure-storage-common-cpp=12.8.0=h736e048_1 + - azure-storage-files-datalake-cpp=12.12.0=ha633028_1 + - babel=2.16.0=pyhd8ed1ab_0 + - backports=1.0=pyhd8ed1ab_4 + - backports.tarfile=1.2.0=pyhd8ed1ab_0 + - beautifulsoup4=4.12.3=pyha770c72_0 + - bleach=6.2.0=pyhd8ed1ab_0 + - blinker=1.9.0=pyhff2d567_0 + - bokeh=3.6.1=pyhd8ed1ab_0 + - brotli=1.1.0=hb9d3cd8_2 + - brotli-bin=1.1.0=hb9d3cd8_2 + - brotli-python=1.1.0=py311hfdbb021_2 + - bzip2=1.0.8=h4bc722e_7 + - c-ares=1.34.3=heb4867d_0 + - ca-certificates=2024.8.30=hbcca054_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cairo=1.18.0=hebfffa5_3 + - certifi=2024.8.30=pyhd8ed1ab_0 + - certipy=0.2.1=pyhd8ed1ab_0 + - cffi=1.17.1=py311hf29c0ef_0 + - charset-normalizer=3.4.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - colorama=0.4.6=pyhd8ed1ab_0 + - comm=0.2.2=pyhd8ed1ab_0 + - configurable-http-proxy=4.6.2=he2f69ee_0 + - contourpy=1.3.1=py311hd18a35c_0 + - cryptography=43.0.3=py311hafd3f86_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - cyrus-sasl=2.1.27=h54b06d7_7 + - cytoolz=1.0.0=py311h9ecbd09_1 + - dask=2024.11.2=pyhff2d567_1 + - dask-core=2024.11.2=pyhff2d567_1 + - dask-expr=1.1.19=pyhd8ed1ab_0 + - dask-labextension=7.0.0=pyhd8ed1ab_0 + - dbus=1.13.6=h5008d03_3 + - debugpy=1.8.9=py311hfdbb021_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - defusedxml=0.7.1=pyhd8ed1ab_0 + - distributed=2024.11.2=pyhff2d567_1 + - double-conversion=3.3.0=h59595ed_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - exceptiongroup=1.2.2=pyhd8ed1ab_0 + - executing=2.1.0=pyhd8ed1ab_0 + - expat=2.6.4=h5888daf_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_3 + - fontconfig=2.15.0=h7e30c49_1 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.55.0=py311h2dc5d0c_0 + - fqdn=1.5.1=pyhd8ed1ab_0 + - freetype=2.12.1=h267a509_2 + - frozenlist=1.5.0=py311h9ecbd09_0 + - fsspec=2024.10.0=pyhff2d567_0 + - gflags=2.2.2=h5888daf_1005 + - glog=0.7.1=hbabe93e_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.1.1=py311hfdbb021_0 + - h11=0.14.0=pyhd8ed1ab_0 + - h2=4.1.0=pyhd8ed1ab_0 + - harfbuzz=9.0.0=hda332d3_1 + - hpack=4.0.0=pyh9f0ad1d_0 + - html5lib=1.1=pyhd8ed1ab_1 + - httpcore=1.0.7=pyh29332c3_1 + - httpx=0.27.2=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=75.1=he02047a_0 + - idna=3.10=pyhd8ed1ab_0 + - importlib-metadata=8.5.0=pyha770c72_0 + - importlib_metadata=8.5.0=hd8ed1ab_0 + - importlib_resources=6.4.5=pyhd8ed1ab_0 + - ipykernel=6.29.5=pyh3099207_0 + - ipython=8.29.0=pyh707e725_0 + - ipywidgets=8.1.5=pyhd8ed1ab_0 + - isoduration=20.11.0=pyhd8ed1ab_0 + - jaraco.classes=3.4.0=pyhd8ed1ab_1 + - jaraco.context=5.3.0=pyhd8ed1ab_1 + - jaraco.functools=4.0.0=pyhd8ed1ab_0 + - jedi=0.19.2=pyhff2d567_0 + - jeepney=0.8.0=pyhd8ed1ab_0 + - jinja2=3.1.4=pyhd8ed1ab_0 + - json5=0.9.28=pyhff2d567_0 + - jsonpointer=3.0.0=py311h38be061_1 + - jsonschema=4.23.0=pyhd8ed1ab_0 + - jsonschema-specifications=2024.10.1=pyhd8ed1ab_0 + - jsonschema-with-format-nongpl=4.23.0=hd8ed1ab_0 + - jupyter-lsp=2.2.5=pyhd8ed1ab_0 + - jupyter-server-proxy=4.4.0=pyhd8ed1ab_0 + - jupyter-vscode-proxy=0.6=pyhd8ed1ab_0 + - jupyter_client=8.6.3=pyhd8ed1ab_0 + - jupyter_core=5.7.2=pyh31011fe_1 + - jupyter_events=0.10.0=pyhd8ed1ab_0 + - jupyter_server=2.14.2=pyhd8ed1ab_0 + - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0 + - jupyterhub=5.1.0=pyh31011fe_0 + - jupyterhub-base=5.1.0=pyh31011fe_0 + - jupyterlab=4.2.4=pyhd8ed1ab_0 + - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1 + - jupyterlab_server=2.27.3=pyhd8ed1ab_0 + - jupyterlab_widgets=3.0.13=pyhd8ed1ab_0 + - jupytext=1.16.4=pyh80e38bb_0 + - keyring=25.5.0=pyha804496_0 + - keyutils=1.6.1=h166bdaf_0 + - kiwisolver=1.4.7=py311hd18a35c_0 + - krb5=1.21.3=h659f571_0 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.43=h712a8e2_2 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240722.0=cxx17_h5888daf_1 + - libarrow=18.0.0=h94eee4b_8_cpu + - libarrow-acero=18.0.0=h5888daf_8_cpu + - libarrow-dataset=18.0.0=h5888daf_8_cpu + - libarrow-substrait=18.0.0=h5c8f2c3_8_cpu + - libblas=3.9.0=25_linux64_openblas + - libbrotlicommon=1.1.0=hb9d3cd8_2 + - libbrotlidec=1.1.0=hb9d3cd8_2 + - libbrotlienc=1.1.0=hb9d3cd8_2 + - libcblas=3.9.0=25_linux64_openblas + - libclang-cpp19.1=19.1.4=default_hb5137d0_0 + - libclang13=19.1.4=default_h9c6a7e4_0 + - libcrc32c=1.1.2=h9c3ff4c_0 + - libcups=2.3.3=h4637d8d_4 + - libcurl=8.10.1=hbbe4b11_0 + - libdeflate=1.22=hb9d3cd8_0 + - libdrm=2.4.123=hb9d3cd8_0 + - libedit=3.1.20191231=he28a2e2_2 + - libegl=1.7.0=ha4b6fd6_2 + - libev=4.33=hd590300_2 + - libevent=2.1.12=hf998b51_1 + - libexpat=2.6.4=h5888daf_0 + - libffi=3.4.2=h7f98852_5 + - libgcc=14.2.0=h77fa898_1 + - libgcc-ng=14.2.0=h69a702a_1 + - libgcrypt=1.11.0=h4ab18f5_1 + - libgfortran=14.2.0=h69a702a_1 + - libgfortran5=14.2.0=hd5240d6_1 + - libgl=1.7.0=ha4b6fd6_2 + - libglib=2.82.2=h2ff4ddf_0 + - libglvnd=1.7.0=ha4b6fd6_2 + - libglx=1.7.0=ha4b6fd6_2 + - libgomp=14.2.0=h77fa898_1 + - libgoogle-cloud=2.31.0=h804f50b_0 + - libgoogle-cloud-storage=2.31.0=h0121fbd_0 + - libgpg-error=1.51=hbd13f7d_1 + - libgrpc=1.67.1=hc2c308b_0 + - libiconv=1.17=hd590300_2 + - libjpeg-turbo=3.0.0=hd590300_1 + - liblapack=3.9.0=25_linux64_openblas + - libllvm19=19.1.4=ha7bfdaf_0 + - libnghttp2=1.64.0=h161d5f1_0 + - libnsl=2.0.1=hd590300_0 + - libntlm=1.4=h7f98852_1002 + - libopenblas=0.3.28=pthreads_h94d23a6_1 + - libopengl=1.7.0=ha4b6fd6_2 + - libparquet=18.0.0=h6bd9018_8_cpu + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.44=hadc24fc_0 + - libpq=17.2=h04577a9_0 + - libprotobuf=5.28.2=h5b01275_0 + - libre2-11=2024.07.02=hbbce691_1 + - libsecret=0.18.8=h329b89f_2 + - libsodium=1.0.20=h4ab18f5_0 + - libsqlite=3.47.0=hadc24fc_1 + - libssh2=1.11.0=h0841786_0 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libstdcxx-ng=14.2.0=h4852527_1 + - libthrift=0.21.0=h0e7cc3e_0 + - libtiff=4.7.0=he137b08_1 + - libutf8proc=2.8.0=h166bdaf_0 + - libuuid=2.38.1=h0b41bf4_0 + - libuv=1.49.2=hb9d3cd8_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.17.0=h8a09558_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.7.0=h2c5496b_1 + - libxml2=2.13.5=hb346dea_0 + - libxslt=1.1.39=h76b75d6_0 + - libzlib=1.3.1=hb9d3cd8_2 + - locket=1.0.0=pyhd8ed1ab_0 + - lz4=4.3.3=py311h2cbdf9a_1 + - lz4-c=1.9.4=hcb278e6_0 + - mako=1.3.6=pyhff2d567_0 + - markdown-it-py=3.0.0=pyhd8ed1ab_0 + - markupsafe=3.0.2=py311h2dc5d0c_0 + - matplotlib=3.9.2=py311h38be061_2 + - matplotlib-base=3.9.2=py311h2b939e6_2 + - matplotlib-inline=0.1.7=pyhd8ed1ab_0 + - mdit-py-plugins=0.4.2=pyhd8ed1ab_0 + - mdurl=0.1.2=pyhd8ed1ab_0 + - mistune=3.0.2=pyhd8ed1ab_0 + - more-itertools=10.5.0=pyhd8ed1ab_0 + - msgpack-python=1.1.0=py311hd18a35c_0 + - multidict=6.1.0=py311h2dc5d0c_1 + - munkres=1.1.4=pyh9f0ad1d_0 + - mysql-common=9.0.1=h266115a_2 + - mysql-libs=9.0.1=he0572af_2 + - nbclient=0.10.0=pyhd8ed1ab_0 + - nbconvert-core=7.16.4=pyhd8ed1ab_1 + - nbformat=5.10.4=pyhd8ed1ab_0 + - nbgitpuller=1.2.1=pyhd8ed1ab_0 + - ncurses=6.5=he02047a_1 + - nest-asyncio=1.6.0=pyhd8ed1ab_0 + - nodejs=18.20.4=hc55a1b2_1 + - notebook=7.2.1=pyhd8ed1ab_0 + - notebook-shim=0.2.4=pyhd8ed1ab_0 + - numpy=2.1.3=py311h71ddf71_0 + - oauthlib=3.2.2=pyhd8ed1ab_0 + - openjpeg=2.5.2=h488ebb8_0 + - openldap=2.6.8=hedd0468_0 + - openssl=3.4.0=hb9d3cd8_0 + - openvscode-server=1.92.1=hb09f993_0 + - orc=2.0.3=he039a57_0 + - overrides=7.7.0=pyhd8ed1ab_0 + - packaging=24.2=pyhff2d567_1 + - pamela=1.2.0=pyhff2d567_0 + - pandas=2.2.3=py311h7db5c69_1 + - pandocfilters=1.5.0=pyhd8ed1ab_0 + - parso=0.8.4=pyhd8ed1ab_0 + - partd=1.4.2=pyhd8ed1ab_0 + - patsy=1.0.1=pyhff2d567_0 + - pcre2=10.44=hba22ea6_2 + - pexpect=4.9.0=pyhd8ed1ab_0 + - pickleshare=0.7.5=py_1003 + - pillow=11.0.0=py311h49e9ac3_0 + - pip=24.3.1=pyh8b19718_0 + - pixman=0.43.2=h59595ed_0 + - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1 + - platformdirs=4.3.6=pyhd8ed1ab_0 + - prometheus_client=0.21.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.48=pyha770c72_0 + - propcache=0.2.0=py311h9ecbd09_2 + - pthread-stubs=0.4=hb9d3cd8_1002 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.3=pyhd8ed1ab_0 + - pyarrow=18.0.0=py311h38be061_1 + - pyarrow-core=18.0.0=py311h4854187_1_cpu + - pycparser=2.22=pyhd8ed1ab_0 + - pycurl=7.45.3=py311h0ad5ee3_3 + - pydantic=2.10.0=pyh10f6f8f_0 + - pydantic-core=2.27.0=py311h9e33e62_0 + - pyerfa=2.0.1.5=py311h9f3472d_0 + - pygments=2.18.0=pyhd8ed1ab_0 + - pyjwt=2.10.0=pyhff2d567_0 + - pyparsing=3.2.0=pyhd8ed1ab_1 + - pyside6=6.8.0.2=py311h9053184_0 + - pysocks=1.7.1=pyha2e5f31_6 + - python=3.11.10=hc5c86c4_3_cpython + - python-dateutil=2.9.0.post0=pyhff2d567_0 + - python-fastjsonschema=2.20.0=pyhd8ed1ab_0 + - python-json-logger=2.0.7=pyhd8ed1ab_0 + - python-tzdata=2024.2=pyhd8ed1ab_0 + - python_abi=3.11=5_cp311 + - pytz=2024.1=pyhd8ed1ab_0 + - pyvo=1.6=pyhd8ed1ab_0 + - pyyaml=6.0.2=py311h9ecbd09_1 + - pyzmq=26.2.0=py311h7deb3e3_3 + - qhull=2020.2=h434a139_5 + - qt6-main=6.8.0=h6e8976b_0 + - re2=2024.07.02=h77b4e00_1 + - readline=8.2=h8228510_1 + - referencing=0.35.1=pyhd8ed1ab_0 + - requests=2.32.3=pyhd8ed1ab_0 + - rfc3339-validator=0.1.4=pyhd8ed1ab_0 + - rfc3986-validator=0.1.1=pyh9f0ad1d_0 + - ripgrep=14.1.1=h8fae777_0 + - rpds-py=0.21.0=py311h9e33e62_0 + - s2n=1.5.9=h0fd0ee4_0 + - scipy=1.14.1=py311he9a78e4_1 + - seaborn=0.13.2=hd8ed1ab_2 + - seaborn-base=0.13.2=pyhd8ed1ab_2 + - secretstorage=3.3.3=py311h38be061_3 + - send2trash=1.8.3=pyh0d859eb_0 + - setuptools=75.6.0=pyhff2d567_0 + - simpervisor=1.0.0=pyhd8ed1ab_0 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.2.1=ha2e4443_0 + - sniffio=1.3.1=pyhd8ed1ab_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - soupsieve=2.5=pyhd8ed1ab_1 + - sqlalchemy=2.0.36=py311h9ecbd09_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - statsmodels=0.14.4=py311h9f3472d_0 + - tblib=3.0.0=pyhd8ed1ab_0 + - terminado=0.18.1=pyh0d859eb_0 + - tinycss2=1.4.0=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tomli=2.1.0=pyhff2d567_0 + - toolz=1.0.0=pyhd8ed1ab_0 + - tornado=6.4.1=py311h9ecbd09_1 + - tqdm=4.67.0=pyhd8ed1ab_0 + - traitlets=5.14.3=pyhd8ed1ab_0 + - types-python-dateutil=2.9.0.20241003=pyhff2d567_0 + - typing-extensions=4.12.2=hd8ed1ab_0 + - typing_extensions=4.12.2=pyha770c72_0 + - typing_utils=0.1.0=pyhd8ed1ab_0 + - tzdata=2024b=hc8b5060_0 + - unicodedata2=15.1.0=py311h9ecbd09_1 + - uri-template=1.3.0=pyhd8ed1ab_0 + - urllib3=2.2.3=pyhd8ed1ab_0 + - wayland=1.23.1=h3e06ad9_0 + - wcwidth=0.2.13=pyhd8ed1ab_0 + - webcolors=24.8.0=pyhd8ed1ab_0 + - webencodings=0.5.1=pyhd8ed1ab_2 + - websocket-client=1.8.0=pyhd8ed1ab_0 + - wheel=0.45.0=pyhd8ed1ab_0 + - widgetsnbextension=4.0.13=pyhd8ed1ab_0 + - xcb-util=0.4.1=hb711507_2 + - xcb-util-cursor=0.1.5=hb9d3cd8_0 + - xcb-util-image=0.4.0=hb711507_2 + - xcb-util-keysyms=0.4.1=hb711507_0 + - xcb-util-renderutil=0.3.10=hb711507_0 + - xcb-util-wm=0.4.2=hb711507_0 + - xkeyboard-config=2.43=hb9d3cd8_0 + - xorg-libice=1.1.1=hb9d3cd8_1 + - xorg-libsm=1.2.4=he73a12e_1 + - xorg-libx11=1.8.10=h4f16b4b_0 + - xorg-libxau=1.0.11=hb9d3cd8_1 + - xorg-libxcomposite=0.4.6=hb9d3cd8_2 + - xorg-libxcursor=1.2.3=hb9d3cd8_0 + - xorg-libxdamage=1.1.6=hb9d3cd8_0 + - xorg-libxdmcp=1.1.5=hb9d3cd8_0 + - xorg-libxext=1.3.6=hb9d3cd8_0 + - xorg-libxfixes=6.0.1=hb9d3cd8_0 + - xorg-libxi=1.8.2=hb9d3cd8_0 + - xorg-libxrandr=1.5.4=hb9d3cd8_0 + - xorg-libxrender=0.9.11=hb9d3cd8_1 + - xorg-libxtst=1.2.5=hb9d3cd8_3 + - xorg-libxxf86vm=1.1.5=hb9d3cd8_4 + - xorg-xorgproto=2024.1=hb9d3cd8_1 + - xyzservices=2024.9.0=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - yarl=1.18.0=py311h9ecbd09_0 + - zeromq=4.3.5=h3b0a872_7 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.21.0=pyhd8ed1ab_0 + - zlib=1.3.1=hb9d3cd8_2 + - zstandard=0.23.0=py311hbc35293_1 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - gitdb==4.0.11 + - gitpython==3.1.43 + - jupyter-cpu-alive==0.1.2 + - jupyter-resource-usage==1.1.0 + - jupyter-server-mathjax==0.2.6 + - jupyterlab-execute-time==3.2.0 + - jupyterlab-git==0.50.2 + - jupyterlab-myst==2.4.2 + - nbdime==4.0.2 + - psutil==5.9.8 + - smmap==5.0.1 +prefix: /opt/conda/envs/notebook diff --git a/heasoft/conda-notebook.yml b/heasoft/conda-notebook.yml new file mode 100644 index 0000000..f2ff1e1 --- /dev/null +++ b/heasoft/conda-notebook.yml @@ -0,0 +1,15 @@ +name: notebook +channels: + - conda-forge + - nodefaults +dependencies: + - python=3.11 + - numpy + - scipy>1.6 + - astropy>=4.0 + - matplotlib + - ipykernel + - pip + - pip: + - astroquery +prefix: /opt/conda/envs/notebook \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..4f8d55e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,65 @@ +# Fornax Images Repo (beta) + +This repo houses Dockerfiles for building the Fornax Science Enivronment images. +Each image is hosted in its directory. base_image is the base image that all +others start from. + +It also includes three GitHub workflows: + +- `image-build.yml`: build images whose code has changed in a push commit. The images + will be tagged with branch name (e.g. `base_image:fix-issue-1` or `astro-default:fix-issue-34`). + The images will be built and pushed to the container registry of the repo. + + This happens for every GitHub branch, not just "main". + + NB: when `base_image` changes, its dependencies will **not** be rebuilt against the new + `base_image` unless files change in that image itself. + +- `release.yml`: runs on a release, and it tags the image from which the release is + coming from (typically main) with a release tag and symbolic tag named 'stable'. + +- A workflow that runs the tests of the building scripts and tagging machinery when + anything in "scripts" changes. + +For each image built, it is pushed to the GitHub container registry associated +with this repostory. + +See the "Packages" link on the right hand side of the main repository page for +a list of images in the container registry. + +See the "Actions" tab of the repository to see the results of each workflow. + +NB: The code in this directory has only been tested with Python3.11 and better. + +# Notable Changes After Moving to GH + +- We use GitHub Container Registry instead of Amazon's. + +- Instead of image names like `fornax_images:base-image-XYZ`, and + `fornax_images:heasoft-XYZ`, we produce images like `base_image:XYZ` and + `astro-default:XYZ` as it is easy enough to do when we use the GitHub container + registry, and it's more "normal". + +- All of the logic to build and push (or not build or push) exists within + `scripts/build.py`, which is executed by the GitHub workflow actions + defined within `.github/workflows/image-build.yml`. + +- The event that fires off the worfklow that produces the images is currently a + a push or pull request event. It happens on every push for every branch. + +- The event that creates "imagename:v0.1.1" tags is a GitHub release event. It + tags all "main" images as released using the release tag name supplied plus a + symbolic "stable" tag. + +- `src/build.py` can be run standalone from any machine. It can only + push images if it is logged in to a GitHub account with an API token that + permitted to create "packages" using `docker login` . + +- The limitations of a free GitHub account with respect to action runners + appears to be these: + + - Only one runner may be available at any time. + + - The active runner is executed on a run-of-the-mill machine. + + - The runner cannot be self-hosted. diff --git a/scripts/build.py b/scripts/build.py new file mode 100644 index 0000000..56b3d22 --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,430 @@ +import logging +import subprocess +import sys +import os +import glob +import re +import argparse +import json + +IMAGE_ORDER = ( + 'base_image', + 'astro-default', + #'heasoft' +) + +class TaskRunner: + """Class for managing logging and system calls""" + + def __init__(self, logger, dryrun=False): + """Create a new TaskRunner + + Parameters: + ----------- + logger: logging.Logger + Logging object + dryrun: bool + If True, print the commands without running them + + """ + self.logger = logger + self.dryrun = dryrun + + def out(self, msg, severity=logging.INFO): + """Log progress message + + Parameters + ---------- + msg: str + Message to log + severity: int + Logging level: logging.INFO, DEBUG, ERROR etc. + + """ + self.logger.log(severity, msg) + sys.stdout.flush() + + def run(self, command, timeout, **runargs): + """Run system command {command} with a timeout + + Parameters: + ----------- + command: str + Command to pass to subprocess.run() + timout: int + Timeout in sec + **runargs: + to be passed to subprocess.run + + Returns an instance of subprocess.CompletedProcess + + """ + self.out(f"Running ::\n{command}") + result = None + if not self.dryrun: + result = subprocess.run( + command, + shell=True, + check=True, + text=True, + timeout=timeout, + **runargs, + ) + return result + + +class Builder(TaskRunner): + """Class for managing the docker build commands""" + + def __init__(self, repository, logger, registry='ghcr.io', dryrun=False): + """Create a new Builder + + Parameters: + ----------- + repository: str + Repository name. e.g. nasa-fornax/fornax-images + logger: logging.Logger + Logging object + registry: str + Name of the docker registry. Default: ghcr.io + dryrun: bool + If True, print the commands without running them + + """ + super().__init__(logger, dryrun) + self.repository = repository + self.registry = registry + + def check_tag(self, tag): + """Check the tag""" + if not isinstance(tag, str) or ':' in tag: + raise ValueError(f'tag: {tag} is not a str without :') + + def get_full_tag(self, image, tag): + """Return full tag that includes the registry and repository + + Parameters: + ----------- + image: str + Image name (e.g. astro-default or heasoft) + tag: str + The image tag. + + """ + self.check_tag(tag) + full_tag = f'{self.registry}/{self.repository}/{image}:{tag}' + return full_tag + + def build(self, image, tag, build_args=None, extra_args=None): + """Build an image by calling 'docker build ..' + + Parameters: + ----------- + repo: str + repository name + image: str + The name of the image to be built (e.g. astro-default or heasoft) + tag: str + The image tag. + build_args: list + A list of str arguments to be passed directly to 'docker build'. e.g. + 'SOME_ENV=myvalue OTHER_ENV=value' + extra_args: str + Extra command line arguments to be passed to 'docker build' + e.g. '--no-cache --network=host' + + """ + cmd_args = [] + + # build_args is a list + build_args = build_args or [] + if not isinstance(build_args, list): + raise ValueError(f'build_args is of type {type(build_args)}. Expected a list.') + if ':' in tag: + tag = tag.split(':')[-1] + + # Ensure we have: name=value + build_args = [arg.strip() for arg in build_args] + + # add some defaults to build_args + # For base_image, the tags are external and should be updated in the Dockerfile + if image != 'base_image': + mapping = { + 'REPOSITORY': self.repository, + 'REGISTRY': self.registry, + 'BASE_TAG': tag, + } + for key,val in mapping.items(): + if not any([arg.startswith(f'{key}=') for arg in build_args]): + build_args.append(f'{key}={val}') + + # loop through the build_args and add them to cmd_args + for arg in build_args: + if not arg.count("=") == 1: + raise ValueError( + f"build_args should be of the form 'name=value'. " + f"Got '{arg}'." + ) + name, val = arg.split("=", 1) + cmd_args.append(f"--build-arg {name}={val}") + + # now add any other line parameters + if extra_args: + cmd_args.append(extra_args) + + cmd_args = " ".join(cmd_args) + full_tag = self.get_full_tag(image, tag) + build_cmd = f"docker build {cmd_args} --tag {full_tag} {image}" + self.out(f"Building {image} ...") + result = self.run(build_cmd, timeout=10000) + + def push(self, image, tag): + """Push the image with 'docker push' + + Parameters: + ----------- + image: str + The name of Image to be pushed (e.g. astro-default or heasoft) + tag: str + The image tag. + + """ + self.check_tag(tag) + full_tag = self.get_full_tag(image, tag) + push_command = f'docker push {full_tag}' + self.out(f"Pushing {full_tag} ...") + result = self.run(push_command, timeout=1000) + + def release(self, source_tag, release_tags, images=None): + """Make an image release by tagging the image with release_tags + + Parameters: + ----------- + source_tag: str + The tag name for the image (no repo name) + release_tags: list + A list of target tag names for the release (no repo name) + images: list or None + The list of images to tag for release. By default, all images + + """ + # check the passed tags + self.check_tag(source_tag) + if not isinstance(release_tags, list): + raise ValueError(f'release_tags: {release_tags} is not a list') + for release_tag in release_tags: + self.check_tag(release_tag) + + if images is not None and not isinstance(images, list): + raise ValueError(f'Expected images to be a list; got {images}') + + # get a list of images to release + to_release = images if images is not None else list(IMAGE_ORDER) + for image in to_release: + if image not in IMAGE_ORDER: + raise ValueError(f'Requested image to release {image} is unknown') + + # Loop through the images + for image in to_release: + full_source_tag = self.get_full_tag(image, source_tag) + + # pull + command = f'docker pull {full_source_tag}' + self.out(f"Pulling {full_source_tag} ...") + self.run(command, timeout=1000) + + # loog through release tags + for release_tag in release_tags: + full_release_tag = self.get_full_tag(image, release_tag) + + # tag + command = f'docker tag {full_source_tag} {full_release_tag}' + self.out(f"Tagging {full_source_tag} with {full_release_tag} ...") + self.run(command, timeout=1000) + + # push + command = f'docker push {full_release_tag}' + self.out(f"Pushing {full_release_tag} ...") + self.run(command, timeout=1000) + + def remove_lockfiles(self, image): + """Remove conda lock files from an image + + Parameters + ---------- + image: str + Image name (e.g. astro-default or heasoft) + """ + self.out(f"Removing the lock files for {image}") + lockfiles = glob.glob(f"{image}/conda-*lock.yml") + for lockfile in lockfiles: + self.out(f"Removing {lockfile}") + cmd = f'rm -f {lockfile}' + result = self.run(cmd, timeout=100) + + def update_lockfiles(self, image, tag, extra_args=None): + """Update the conda lock files in {image} using image {tag} + + Parameters + ---------- + image: str + The name of the image to be updated (e.g. astro-default or heasoft) + tag: str + The image tag. + extra_args: str + Extra command line arguments to be passed to 'docker run' + e.g. '--network=host' + + """ + self.check_tag(tag) + + extra_args = extra_args or '' + if not isinstance(extra_args, str): + raise ValueError(f'Expected str for extra_args; got: {extra_args}') + + self.out(f'Updating the lock files for {image}') + envfiles = [env for env in glob.glob(f'{image}/conda-*.yml') + if 'lock' not in env] + for env in envfiles: + match = re.match(rf"{image}/conda-(.*).yml", env) + env_name = match[1] if match else 'base' + cmd = (f'docker run --entrypoint="" --rm {extra_args} {tag} ' + f'mamba env export -n {env_name}') + result = self.run(cmd, 500, capture_output=True) + if result is not None: # dryrun=True + # capture lines after: 'name:' + lines = [] + include = False + for line in result.stdout.split('\n'): + if "name:" in line: + include = True + if include: + lines.append(line) + with open(f"{image}/conda-{env_name}-lock.yml", "w") as fp: + fp.write("\n".join(lines)) + +if __name__ == '__main__': + + ap = argparse.ArgumentParser() + + ap.add_argument('images', nargs='*', + help="Image names to build separated by spaces e.g. 'base_image astro-default'") + + ap.add_argument('--tag', + help="Container registry tag name (e.g. 'mybranch'). Default is current git branch") + + ap.add_argument('--registry', + help='Container registry name (e.g. ghcr.io)', + default='ghcr.io') + + ap.add_argument('--repository', + help="GH repository name (e.g. 'nasa-fornax/fornax-images')", + default='nasa-fornax/fornax-images') + + ap.add_argument('--push', action='store_true', + help='After building, push to container registry', + default=False) + + ap.add_argument('--release', nargs='*', + help='Release using the given tag') + + ap.add_argument('--no-build', action='store_true', + help="Do not run 'docker build' command", + default=False) + + ap.add_argument('--update-lock', action='store_true', + help='Update conda lock files.', + default=False) + + ap.add_argument('--dryrun', action='store_true', + help='prepare but do not run commands', + default=False) + + ap.add_argument('--build-args', nargs='*', + help="Extra --build-arg arguments passed to docker build e.g. 'a=b c=d'") + + ap.add_argument('--extra-pars', + help="Arguments to be passed directly to `docker build` or `docker run`", + default=None) + + ap.add_argument('--debug', action='store_true', + help='Print debug messages', + default=False) + + args = ap.parse_args() + + # get parameters + dryrun = args.dryrun + debug = args.debug + images = args.images + registry = args.registry + repository = args.repository + tag = args.tag + push = args.push + release = args.release + update_lock = args.update_lock + no_build = args.no_build + build_args = args.build_args + extra_pars = args.extra_pars + + # in case images is of the form: '["dir_1", "dir_2"]' + if len(images) == 1 and '[' in images[0]: + images = json.loads(images[0]) + + os.environ["DOCKER_BUILDKIT"] = "1" + logging.basicConfig( + format="%(asctime)s|%(levelname)5s| %(message)s", + datefmt="%Y-%m-%d|%H:%M:%S", + ) + + logger = logging.getLogger('::Builder::') + logger.setLevel(level=logging.DEBUG if debug else logging.INFO) + builder = Builder(repository, logger, dryrun=dryrun, registry=registry) + + # get current branch name as default tag + if (len(images) or release is not None) and tag is None: + out = builder.run('git branch --show-current', timeout=100, capture_output=True) + if out is not None: + tag = out.stdout.strip() + # if out is None, we are in --dryrun mode, add a dummy tag + tag = 'no-tag' if out is None else out.stdout.strip() + + # some logging: + builder.out('Builder initialized ..', logging.DEBUG) + builder.out('+++ INPUT +++', logging.DEBUG) + builder.out(f'images: {images}', logging.DEBUG) + builder.out(f'registry: {registry}', logging.DEBUG) + builder.out(f'repository: {repository}', logging.DEBUG) + builder.out(f'tag: {tag}', logging.DEBUG) + builder.out(f'push: {push}', logging.DEBUG) + builder.out(f'release: {release}', logging.DEBUG) + builder.out(f'update_lock: {update_lock}', logging.DEBUG) + builder.out(f'no_build: {no_build}', logging.DEBUG) + builder.out(f'build_args: {build_args}', logging.DEBUG) + builder.out(f'extra_pars: {extra_pars}', logging.DEBUG) + builder.out('+++++++++++++', logging.DEBUG) + + # get a sorted list of images to build + to_build = [] + for image in IMAGE_ORDER: + if image in images: + to_build.append(image) + + builder.out(f'Images to build: {to_build}', logging.DEBUG) + for image in to_build: + builder.out(f'Working on: {image}', logging.DEBUG) + + if update_lock: + builder.remove_lockfiles(image) + + if not no_build: + builder.build(image, tag, build_args, extra_pars) + + if update_lock: + builder.update_lockfiles(image, tag) + + if push: + builder.push(image, tag) + + if release is not None: + builder.release(tag, release) diff --git a/scripts/changed_images.py b/scripts/changed_images.py new file mode 100644 index 0000000..0b9e5d2 --- /dev/null +++ b/scripts/changed_images.py @@ -0,0 +1,100 @@ +import argparse +import json +import os +import sys +import logging + +sys.path.insert(0, f'{os.path.dirname(__file__)}') +from build import TaskRunner + +def find_changed_images(github_data:dict, runner:TaskRunner): + """Find changed images + + Returns a list of image names that changed after the git event + """ + + if github_data['event_name'] == 'pull_request': + base_ref = github_data['event']['base_ref'] + + cmd = f'git fetch origin {base_ref}' + out = runner.run(cmd, 500, capture_output=True) + + cmd = f'git --no-pager diff --name-only HEAD origin/${base_ref} | xargs -n1 dirname | sort -u' + final_out = runner.run(cmd, 500, capture_output=True) + + elif github_data['event_name'] == 'push': + before = github_data['event']['before'] + after = github_data['event']['after'] + + cmd = f'git fetch origin {before}' + out = runner.run(cmd, 500, capture_output=True) + + cmd = f'git --no-pager diff-tree --name-only -r {before}..{after} | xargs -n1 dirname | sort -u' + final_out = runner.run(cmd, 500, capture_output=True) + + else: + cmd = 'git ls-files | xargs -n1 dirname | sort -u' + final_out = runner.run(cmd, 500, capture_output=True) + + changed_images = [] + if not runner.dryrun: + changed_images = final_out.stdout.strip() + + # from str to list + changed_images = changed_images.split() + + # keep a list of images, i.e. folders that contain Dockerfile + changed_images = [cdir for cdir in changed_images if os.path.exists(f'{cdir}/Dockerfile')] + + return changed_images + + +if __name__ == '__main__': + + ap = argparse.ArgumentParser() + + ap.add_argument('jsonfile', + help="File name of the json file that contains Github action context") + + ap.add_argument('--dryrun', action='store_true', + help='prepare but do not run commands', + default=False) + + ap.add_argument('--debug', action='store_true', + help='Print debug messages', + default=False) + + args = ap.parse_args() + # get parameters + dryrun = args.dryrun + debug = args.debug + jsonfile = args.jsonfile + + with open(args.jsonfile, "r") as file: + data = json.load(file) + + logging.basicConfig( + format="%(asctime)s|%(levelname)5s| %(message)s", + datefmt="%Y-%m-%d|%H:%M:%S", + ) + logger = logging.getLogger('::Changed-dirs::') + logger.setLevel(level=logging.DEBUG if debug else logging.INFO) + runner = TaskRunner(logger, dryrun) + + # some logging: + runner.out('+++ INPUT +++', logging.DEBUG) + runner.out(f'jsonfile: {jsonfile}', logging.DEBUG) + runner.out(f'debug: {debug}', logging.DEBUG) + runner.out(f'dryrun: {dryrun}', logging.DEBUG) + runner.out(f'event_name: {data["event_name"]}', logging.DEBUG) + runner.out('+++++++++++++', logging.DEBUG) + + changed_images = find_changed_images(data, runner) + + # print the result in json text so it is picked up in the CI script + res = json.dumps(changed_images) + runner.out('+++ OUTPUT +++', logging.DEBUG) + runner.out(res) + runner.out('++++++++++++++', logging.DEBUG) + # clean print so it is picked up by the CI + print(res) diff --git a/tests/test_astro-default.py b/tests/test_astro-default.py new file mode 100644 index 0000000..463dcd0 --- /dev/null +++ b/tests/test_astro-default.py @@ -0,0 +1,80 @@ +import unittest +import subprocess +import sys +import os +from packaging import version +import contextlib +import pytest + +sys.path.insert(0, os.getcwd()) +from test_base_image import CommonTests + +notebook_dir = os.environ.get('NOTEBOOK_DIR', '/home/jovyan/notebooks') + +notebooks = { + 'multiband_photometry': { + 'file': 'fornax-demo-notebooks/forced_photometry/multiband_photometry.md', + 'req': 'requirements_multiband_photometry.txt' + }, + 'light_curve_classifier': { + 'file': 'fornax-demo-notebooks/light_curves/light_curve_classifier.md', + 'req': 'requirements_light_curve_classifier.txt' + }, + 'light_curve_generator': { + 'file': 'fornax-demo-notebooks/light_curves/light_curve_generator.md', + 'req': 'requirements_light_curve_generator.txt' + }, + 'scale_up': { + 'file': 'fornax-demo-notebooks/light_curves/scale_up.md', + 'req': 'requirements_scale_up.txt' + }, + 'ML_AGNzoo': { + 'file': 'fornax-demo-notebooks/light_curves/ML_AGNzoo.md', + 'req': 'requirements_ML_AGNzoo.txt' + } +} + +@contextlib.contextmanager +def change_dir(destination): + """A context manager to change the current working directory.""" + try: + current_dir = os.getcwd() + os.chdir(destination) + yield + finally: + os.chdir(current_dir) + +class Test_astro_default(unittest.TestCase, CommonTests): + + def test_conda_env_file(self): + self._test_conda_env_file('astro-default') + + def test_check_packages(self): + import tractor + import astrometry + import lsdb + self.assertLess(version.parse(lsdb.__version__), version.parse('0.4')) + + def test_notebooks_folder(self): + self.assertTrue(os.path.exists(notebook_dir)) + self.assertTrue(os.path.exists(f'{notebook_dir}/fornax-documentation')) + self.assertTrue(os.path.exists(f'{notebook_dir}/fornax-demo-notebooks')) + +@pytest.mark.parametrize("notebook", list(notebooks.keys())) +def test_notebooks(notebook): + comman = CommonTests() + nb_file = notebooks[notebook]['file'] + nb_req = notebooks[notebook]['req'] + nb_path = os.path.dirname(nb_file) + nb_filename = os.path.basename(nb_file) + py_filename = nb_filename.replace('md', 'py') + assert os.path.exists(f'{notebook_dir}/{nb_file}') + with change_dir(f'{notebook_dir}/{nb_path}'): + comman.run_cmd(f'pip install -r {nb_req}') + comman.run_cmd(f'jupytext --to py {nb_filename}') + comman.run_cmd(f'python {py_filename}') + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_base_image.py b/tests/test_base_image.py new file mode 100644 index 0000000..3e2f83a --- /dev/null +++ b/tests/test_base_image.py @@ -0,0 +1,57 @@ +import unittest +import subprocess +import sys +import os + +class CommonTests: + + def run_cmd(self, command, **runargs): + """Run shell command""" + result = subprocess.run(command, shell=True, check=False, text=True, + capture_output=True, **runargs) + if result.returncode != 0: + sep = '\n' + ('+'*20) + '\n' + raise RuntimeError(f'*** ERROR running: {command}' + sep + result.stdout + sep + result.stderr) + return result + + def test_python_path(self): + version = f'{sys.version_info.major}.{sys.version_info.minor}' + self.assertTrue(sys.executable in + ['/opt/conda/envs/notebook/bin/python', f'/opt/conda/envs/notebook/bin/python{version}']) + + def test_python_path2(self): + path = self.run_cmd('which python') + self.assertEqual(path.stdout.strip(), f'/opt/conda/envs/notebook/bin/python') + + def test_conda_prefix(self): + self.assertTrue('CONDA_PREFIX' in os.environ) + self.assertEqual(os.environ['CONDA_PREFIX'], '/opt/conda/envs/notebook') + + def _test_conda_env_file(self, image): + result = self.run_cmd('mamba env export -n notebook') + lines = [] + include = False + for line in result.stdout.split('\n'): + if "name:" in line: + include = True + if include: + lines.append(line) + with open(f"tmp-notebook-lock.yml", "w") as fp: + fp.write("\n".join(lines)) + diff_cmd = f'diff tmp-notebook-lock.yml {os.path.dirname(__file__)}/../{image}/conda-notebook-lock.yml' + result = self.run_cmd(diff_cmd) + self.assertEqual(result.stdout, '') + self.assertEqual(result.stderr, '') + + +class Test_base_image(unittest.TestCase, CommonTests): + + def test_conda_env_file(self): + self._test_conda_env_file('base_image') + + def test_unqiue_base_image_test(self): + self.assertEqual(1, 1) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_build_code.py b/tests/test_build_code.py new file mode 100644 index 0000000..0a57067 --- /dev/null +++ b/tests/test_build_code.py @@ -0,0 +1,270 @@ +import unittest +from unittest.mock import patch +import logging +import sys +import os +import pathlib +import tempfile +import glob +import json +import subprocess +from io import StringIO + + +sys.path.insert(0, f'{os.path.dirname(__file__)}/../scripts/') +from build import TaskRunner, Builder +from changed_images import find_changed_images + +class TestTaskRunner(unittest.TestCase): + + def setUp(self): + logger = logging.getLogger() + self.builder_run = TaskRunner(logger, dryrun=False) + self.builder_dry = TaskRunner(logger, dryrun=True) + self.logger = logger + + def test_run(self): + out = self.builder_run.run('pwd', timeout=100, capture_output=True) + self.assertEqual(out.stdout.strip().lower(), os.getcwd().lower()) + + + def test_out(self): + msg = 'test logging ...' + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + self.builder_run.out(msg) + output = mock_out.getvalue().strip() + self.assertEqual(msg, output.split(':')[-1].strip()) + self.logger.handlers.clear() + + def test_dryrun(self): + out = self.builder_dry.run('ls', timeout=100, capture_output=True) + self.assertEqual(out, None) + + out = self.builder_run.run('ls', timeout=100, capture_output=True) + self.assertNotEqual(out, None) + + +class TestBuilder(unittest.TestCase): + + def setUp(self): + logger = logging.getLogger() + self.repo = 'some-repo' + self.registry = 'my-registry' + self.tag = 'some-tag' + self.image = 'some-image' + + self.builder_run = Builder(self.repo, logger, registry=self.registry, dryrun=False) + self.builder_dry = Builder(self.repo, logger, registry=self.registry, dryrun=True) + self.logger = logger + + def test_get_full_tag(self): + full_tag = self.builder_dry.get_full_tag(self.image, self.tag) + self.assertEqual(full_tag, f'{self.registry}/{self.repo}/{self.image}:{self.tag}') + + def test_check_tag(self): + with self.assertRaises(ValueError): + self.builder_dry.check_tag('repo:tag') + with self.assertRaises(ValueError): + self.builder_dry.check_tag(['repo']) + self.builder_dry.check_tag('tag') + self.logger.handlers.clear() + + def test_build__basic(self): + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + self.builder_dry.build(self.image, self.tag) + output = mock_out.getvalue().strip() + full_tag = self.builder_dry.get_full_tag(self.image, self.tag) + cmd = (f'docker build --build-arg REPOSITORY={self.repo} ' + f'--build-arg REGISTRY={self.registry} ' + f'--build-arg BASE_TAG={self.tag} --tag {full_tag} {self.image}') + self.assertEqual(cmd, output.split('::')[-1].strip()) + self.logger.handlers.clear() + + def test_build__build_args_is_list(self): + with self.assertRaises(ValueError): + self.builder_dry.build(self.image, self.tag, build_args='ENV=val') + self.logger.handlers.clear() + + def test_build__build_args(self): + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + self.builder_dry.build(self.image, self.tag, + build_args=['ENV=val', 'ENV2=val']) + output = mock_out.getvalue().strip() + full_tag = self.builder_dry.get_full_tag(self.image, self.tag) + cmd = (f'docker build --build-arg ENV=val --build-arg ENV2=val ' + f'--build-arg REPOSITORY={self.repo} ' + f'--build-arg REGISTRY={self.registry} ' + f'--build-arg BASE_TAG={self.tag} --tag {full_tag} {self.image}') + self.assertEqual(cmd, output.split('::')[-1].strip()) + self.logger.handlers.clear() + + def test_build__extra_args(self): + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + self.builder_dry.build(self.image, self.tag, + extra_args='--some-par') + output = mock_out.getvalue().strip() + full_tag = self.builder_dry.get_full_tag(self.image, self.tag) + cmd = (f'docker build --build-arg REPOSITORY={self.repo} ' + f'--build-arg REGISTRY={self.registry} ' + f'--build-arg BASE_TAG={self.tag} --some-par --tag {full_tag} {self.image}') + self.assertEqual(cmd, output.split('::')[-1].strip()) + self.logger.handlers.clear() + + def test_build__push_not_str(self): + with self.assertRaises(ValueError): + self.builder_dry.push(self.image, 123) + self.logger.handlers.clear() + + def test_build__push_wrong_format(self): + with self.assertRaises(ValueError): + self.builder_dry.push(self.image, f'{self.image}:{self.tag}') + self.logger.handlers.clear() + + def test_build__push(self): + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + self.builder_dry.push(self.image, self.tag) + output = mock_out.getvalue().strip() + cmd = (f'docker push {self.registry}/{self.repo}/{self.image}:{self.tag}') + self.assertEqual(cmd, output.split('::')[-1].strip()) + self.logger.handlers.clear() + + def test_build__release__tag_not_list(self): + with self.assertRaises(ValueError): + self.builder_dry.release('tag', 'out') + # the following should work + self.builder_dry.release('tag', ['out']) + + def test_build__release__wrong_tag(self): + # wrong release tag + with self.assertRaises(ValueError): + self.builder_dry.release('tag', ['tag:out']) + + # wrong source tag + with self.assertRaises(ValueError): + self.builder_dry.release('tag:in', ['tag']) + + def test_build__release__images_not_list(self): + with self.assertRaises(ValueError): + self.builder_dry.release('tag', ['out'], images='images') + + def test_build__release__images_unkown_image(self): + with self.assertRaises(ValueError): + self.builder_dry.release('tag', ['out'], images=['some_image']) + # the following should work + self.builder_dry.release('tag', ['out'], ['base_image']) + + def test_build__release(self): + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + self.builder_dry.release(f'{self.tag}', [f'{self.tag}-out'], images=['base_image']) + output = mock_out.getvalue().strip() + source_tag = self.builder_dry.get_full_tag('base_image', self.tag) + release_tag = self.builder_dry.get_full_tag('base_image', f'{self.tag}-out') + self.assertTrue(f'docker pull {source_tag}' in output) + self.assertTrue(f'docker tag {source_tag} {release_tag}' in output) + self.assertTrue(f'docker push {release_tag}' in output) + self.logger.handlers.clear() + + def test_remove_lockfiles(self): + with tempfile.TemporaryDirectory() as tmpdir: + files = ["conda-lock.yml", "conda-notebook-lock.yml", "conda-a.yml"] + for fn in files: + fn = os.path.join(tmpdir, fn) + pathlib.Path(fn).touch() + self.builder_run.remove_lockfiles(tmpdir) + self.assertEqual(glob.glob(f'{tmpdir}/conda-*lock.yml'), []) + self.assertEqual( + glob.glob(f"{tmpdir}/conda-*"), + [os.path.join(tmpdir, "conda-a.yml")], + ) + + def test_update_lockfiles(self): + # dummy builder to write to stdout + class DummyResult: + stdout = "woo\nyoo\nname:\nnextline" + ran = [] + def run(cmd, timeout, **kw): + ran.append((cmd, timeout)) + return DummyResult + + with tempfile.TemporaryDirectory() as tmpdir: + files = [ + "conda-notebook-lock.yml", + "conda-a.yml", + "conda-a-lock.yml", + ] + for fn in files: + fn = os.path.join(tmpdir, fn) + pathlib.Path(fn).touch() + _run = self.builder_dry.run + self.builder_dry.run = run + self.builder_dry.update_lockfiles(tmpdir, self.tag) + self.builder_dry.run = _run + with open(os.path.join(tmpdir, "conda-a-lock.yml"), "r") as f: + result = f.read() + nowfiles = os.listdir(tmpdir) + self.assertEqual(len(nowfiles), 3) + self.assertEqual(result, "name:\nnextline") + self.logger.handlers.clear() + +class TestChangedImages(unittest.TestCase): + + def setUp(self): + logger = logging.getLogger() + self.runner = TaskRunner(logger, dryrun=True) + self.logger = logger + + def test_pull_request(self): + pull_request_event = { + 'event_name': 'pull_request', + 'event': { + 'base_ref': '7905b4edab6' + } + } + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + res = find_changed_images(pull_request_event, self.runner) + output = mock_out.getvalue().strip() + base_ref = pull_request_event['event']['base_ref'] + self.assertTrue(f'git fetch origin {base_ref}' in output) + self.assertTrue(f'git --no-pager diff --name-only HEAD origin/${base_ref} | xargs -n1 dirname | sort -u' in output) + self.assertEqual(res, []) + self.logger.handlers.clear() + + def test_push(self): + push_event = { + 'event_name': 'push', + 'event': { + 'before': '299390bb5c8', + 'after': '2add5c8e038' + } + } + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + res = find_changed_images(push_event, self.runner) + output = mock_out.getvalue().strip() + before = push_event['event']['before'] + after = push_event['event']['after'] + self.assertTrue(f'git fetch origin {before}' in output) + self.assertTrue(f'git --no-pager diff-tree --name-only -r {before}..{after} | xargs -n1 dirname | sort -u' in output) + self.assertEqual(res, []) + self.logger.handlers.clear() + + def test_else_event(self): + else_event = {'event_name': 'other'} + with patch('sys.stderr', new=StringIO()) as mock_out: + logging.basicConfig(level=logging.DEBUG) + res = find_changed_images(else_event, self.runner) + output = mock_out.getvalue().strip() + self.assertTrue(f'git ls-files | xargs -n1 dirname | sort -u' in output) + self.assertEqual(res, []) + self.logger.handlers.clear() + + +if __name__ == "__main__": + unittest.main()