From 9b2647c90dfbbc3abde929257bf2e3bf043249c4 Mon Sep 17 00:00:00 2001 From: aradhakrishnanGFDL Date: Fri, 22 Sep 2023 13:18:27 -0400 Subject: [PATCH 1/7] docker container tests --- .github/workflows/docker-build-push-aws.yml | 53 ++++++++++++ .github/workflows/docker-run-test.yml | 18 ++++ Dockerfile | 48 +++++++++++ src/conda/micromamba_init.sh | 32 ++++++++ src/environment_manager.py | 24 ++---- tests/docker_test_set1.jsonc | 91 +++++++++++++++++++++ 6 files changed, 251 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/docker-build-push-aws.yml create mode 100644 .github/workflows/docker-run-test.yml create mode 100644 Dockerfile create mode 100755 src/conda/micromamba_init.sh create mode 100644 tests/docker_test_set1.jsonc diff --git a/.github/workflows/docker-build-push-aws.yml b/.github/workflows/docker-build-push-aws.yml new file mode 100644 index 000000000..3f400a384 --- /dev/null +++ b/.github/workflows/docker-build-push-aws.yml @@ -0,0 +1,53 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Build and push image to Amazon ECR. + +on: + pull_request: + branches: + - main + +env: + AWS_REGION: us-east-1 # set this to your preferred AWS region, e.g. us-west-1 + ECR_REPOSITORY: mdtf/mdtf-test # set this to your Amazon ECR repository name + ECS_SERVICE: custom-service # MY_ECS_SERVICE set this to your Amazon ECS service name + ECS_CLUSTER: mdtf-east1 # MY_ECS_CLUSTER set this to your Amazon ECS cluster name + ECS_TASK_DEFINITION: ecs-task-defn.json # set this to the path to your Amazon ECS task definition + # file, e.g. .aws/task-definition.json + CONTAINER_NAME: mdtf-test-a1r # set this to the name of the container in the + # containerDefinitions section of your task definition MY_CONTAINER_NAME + +jobs: + deploy: + name: push-to-public-ecr-cache-test + runs-on: ubuntu-latest + environment: dev + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@13d241b293754004c80624b5567555c4a39ffbe3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Build, tag, and push image to Amazon ECR + id: build-image + uses: aradhakrishnanGFDL/docker-build-with-cache-action@master #support-aws-ecr-public + env: + ECR_REGISTRY: public.ecr.aws + IMAGE_TAG: "latest" + ECS_IMAGE: $ECR_REGISTRY/ECR_REPOSITORY:$env.IMAGE_TAG + with: + registry: public.ecr.aws + username: "${{ secrets.AWS_ACCESS_KEY_ID }}" + password: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + image_name: c6h4h4s8/mdtf/mdtf-test + push_image_and_stages: true + image_tag: latest diff --git a/.github/workflows/docker-run-test.yml b/.github/workflows/docker-run-test.yml new file mode 100644 index 000000000..dfddcb24c --- /dev/null +++ b/.github/workflows/docker-run-test.yml @@ -0,0 +1,18 @@ +name: docker-run-minitest1 +on: + pull_request: + branches: [ main ] +jobs: + container-test-job: + runs-on: ubuntu-latest + container: + image: public.ecr.aws/c6h4h4s8/mdtf/mdtf-test:latest + env: + NODE_ENV: development + options: --cpus 1 + steps: + - name: Check PATH + run: (echo $PATH) || (echo "No path") + - name: Activate environment + shell: bash + run: (eval "$(micromamba shell hook -s bash -p ~/conda)"; micromamba activate; micromamba activate _MDTF_base;mdtf_framework.py --version) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..749a8a17a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,48 @@ +# Base OS +FROM ubuntu:latest +FROM mambaorg/micromamba:latest as micromamba + +#micromamba set up + +USER root + +RUN micromamba info + + +RUN micromamba shell hook --shell bash +#RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml + +## +# Container Metadata +LABEL maintainer="20195932+wrongkindofdoctor@users.noreply.github.com" +LABEL version="alpha-01" +LABEL description="This is a docker image for the MDTF-diagnostics package" +# Disable Prompt During Packages Installation +#ARG DEBIAN_FRONTEND=noninteractive + +# Copy the MDTF-diagnostics package contents from local machine to image +ENV CODE_ROOT=/proj/MDTF-diagnostics +COPY src ${CODE_ROOT}/src +COPY data ${CODE_ROOT}/data +#COPY diagnostics ${CODE_ROOT}/diagnostics +COPY mdtf_framework.py ${CODE_ROOT} +COPY shared ${CODE_ROOT}/shared +COPY sites ${CODE_ROOT}/sites +COPY tests ${CODE_ROOT}/tests +# Install conda environments +ENV CONDA_ROOT=/opt/conda/ +ENV CONDA_ENV_DIR=/opt/conda/envs + +#USER mambauser +RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml +RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml + + +ENV PATH="${PATH}:/proj/MDTF-diagnostics/" +#cRUN micromamba activate _MDTF_base +# Verify installation +#RUN /proj/MDTF-diagnostics/mdtf_framework.py --help +# Run mdtf on src/default_tests.jsonc +# CMD ["${CODE_ROOT}/mdtf", "-f","${CODE_ROOT}/src/default_tests.jsonc"] +ENTRYPOINT ["micromamba activate _MDTF_base"] +CMD ["/bin/bash"] diff --git a/src/conda/micromamba_init.sh b/src/conda/micromamba_init.sh new file mode 100755 index 000000000..8cdb91397 --- /dev/null +++ b/src/conda/micromamba_init.sh @@ -0,0 +1,32 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. + +# Note: PS1 and umask are already set in /etc/profile. You should not +# need this unless you want different defaults for root. +# PS1='${debian_chroot:+($debian_chroot)}\h:\w\$ ' +# umask 022 + +# You may uncomment the following lines if you want `ls' to be colorized: +# export LS_OPTIONS='--color=auto' +# eval "$(dircolors)" +# alias ls='ls $LS_OPTIONS' +# alias ll='ls $LS_OPTIONS -l' +# alias l='ls $LS_OPTIONS -lA' +# +# Some more alias to avoid making mistakes: +# alias rm='rm -i' +# alias cp='cp -i' +# alias mv='mv -i' +source /usr/local/bin/_activate_current_env.sh + +# >>> mamba initialize >>> +# !! Contents within this block are managed by 'mamba init' !! +export MAMBA_EXE='/usr/bin/micromamba'; +export MAMBA_ROOT_PREFIX='/root/micromamba'; +__mamba_setup="$("$MAMBA_EXE" shell hook --shell bash --root-prefix "$MAMBA_ROOT_PREFIX" 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__mamba_setup" +else + alias micromamba="$MAMBA_EXE" # Fallback on help from mamba activate +fi +unset __mamba_setup +# <<< mamba initialize <<< diff --git a/src/environment_manager.py b/src/environment_manager.py index 51b86f4c5..83c0c8e16 100644 --- a/src/environment_manager.py +++ b/src/environment_manager.py @@ -239,19 +239,11 @@ def __init__(self, log=_log): # conda_init for bash defines conda as a shell function; will get error # if we try to call the conda executable directly try: - conda_info = util.run_shell_command( - f"{self.conda_dir}/conda_init.sh {paths.get('conda_root','')}", - log=self.log - ) - for line in conda_info: - key, val = line.split('=') - if key == '_CONDA_EXE': - self.conda_exe = val - assert os.path.exists(self.conda_exe) - elif key == '_CONDA_ROOT': - self.conda_root = val + print('Looking for micromamba') + self.conda_exe = 'micromamba' + self.conda_root = '/opt/conda' except Exception as exc: - raise util.PodRuntimeError("Can't find conda.") from exc + raise util.PodRuntimeError("Can't find conda.") from exc # find where environments are installed if 'conda_env_root' in paths and paths.conda_env_root: @@ -332,10 +324,10 @@ def activate_env_commands(self, env_name): # if we try to call the conda executable directly conda_prefix = os.path.join(self.conda_env_root, env_name) return [ - f'source {self.conda_dir}/conda_init.sh {self.conda_root}', - f'conda activate {conda_prefix}' + f'source {self.conda_dir}/micromamba_init.sh', + f'micromamba activate {conda_prefix}' ] - + def deactivate_env_commands(self, env_name): return [] @@ -531,12 +523,14 @@ def setup(self): self.env_mgr.create_environment(env) def spawn_subprocess(self, p, env_vars_base): + print("SPAWN SUBPROCESS") run_cmds = p.validate_commands() + p.run_commands() if self.test_mode: run_cmds = ['echo "TEST MODE: call {}"'.format('; '.join(run_cmds))] commands = self.env_mgr.activate_env_commands(p.env) \ + run_cmds \ + self.env_mgr.deactivate_env_commands(p.env) + print(commands) if self.test_mode: for cmd in commands: print('\tTEST MODE: call {}'.format(cmd)) diff --git a/tests/docker_test_set1.jsonc b/tests/docker_test_set1.jsonc new file mode 100644 index 000000000..6076d3b2c --- /dev/null +++ b/tests/docker_test_set1.jsonc @@ -0,0 +1,91 @@ +// Configuration for MDTF-diagnostics driver script self-test. +// All text to the right of an unquoted "//" is a comment and ignored, as well +// as blank lines (JSONC quasi-standard.) +{ + "case_list" : [ + // The cases below correspond to the different sample model data sets. Note + // that the MDTF package does not currently support analyzing multiple + // models in a single invocation. Comment out or delete the first entry and + // uncomment the second to run NOAA-GFDL-AM4 only for the MJO_prop_amp POD, + // and likewise for the SM_ET_coupling POD. + { + "CASENAME" : "GFDL.Synthetic", + "convention" : "GFDL", + "FIRSTYR" : 1, + "LASTYR" : 10, + "pod_list": [ + // Optional: PODs to run for this model only (defaults to all) + "MJO_prop_amp" + ] + } + ], + // PATHS --------------------------------------------------------------------- + // Location of supporting data downloaded when the framework was installed. + + // If a relative path is given, it's resolved relative to the MDTF-diagnostics + // code directory. Environment variables (eg, $HOME) can be referenced with a + // "$" and will be expended to their current values when the framework runs. + + // Parent directory containing observational data used by individual PODs. + //"OBS_DATA_ROOT": "../travis_test/obs_data", + "OBS_DATA_ROOT": "../inputdata/obs_data", + + // Parent directory containing results from different models. + //"MODEL_DATA_ROOT": "../travis_test/model/", + "MODEL_DATA_ROOT": "/proj/inputdata/mdtf_test_data", + + // Working directory. Defaults to working directory if blank. + "WORKING_DIR": "../wkdir", + + // Directory to write output. The results of each run of the framework will be + // put in a subdirectory of this directory. + "OUTPUT_DIR": "../wkdir", + + // Location of the Anaconda/miniconda installation to use for managing + // dependencies (path returned lsby running `conda info --base`.) If empty, + // framework will attempt to determine location of system's conda installation. + "conda_root": "/opt/conda", + + // Directory containing the framework-specific conda environments. This should + // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left + // blank, the framework will look for its environments in the system default + // location. + "conda_env_root": "/opt/conda/envs/", + + // SETTINGS ------------------------------------------------------------------ + // Any command-line option recognized by the mdtf script (type `mdtf --help`) + // can be set here, in the form "flag name": "desired setting". + + // Method used to fetch model data. + "data_manager": "Local_File", + + // Method used to manage dependencies. + "environment_manager": "Conda", + + // Settings affecting what output is generated: + + // Set to true to have PODs save postscript figures in addition to bitmaps. + "save_ps": false, + + // Set to true to have PODs save netCDF files of processed data. + "save_nc": false, + + // Set to true to save HTML and bitmap plots in a .tar file. + "make_variab_tar": false, + + // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved + // under a unique name. + "overwrite": false, + + // Settings used in debugging: + + // Log verbosity level. + "verbose": 1, + + // Set to true for framework test. Data is fetched but PODs are not run. + "test_mode": false, + + // Set to true for framework test. No external commands are run and no remote + // data is copied. Implies test_mode. + "dry_run": false + } From 1b4c9e89f4d04684cfdedeaa32092f4cfe353b68 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 26 Sep 2023 10:28:09 -0400 Subject: [PATCH 2/7] Update Dockerfile micromamba path --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 749a8a17a..e1ebd838c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,5 +44,5 @@ ENV PATH="${PATH}:/proj/MDTF-diagnostics/" #RUN /proj/MDTF-diagnostics/mdtf_framework.py --help # Run mdtf on src/default_tests.jsonc # CMD ["${CODE_ROOT}/mdtf", "-f","${CODE_ROOT}/src/default_tests.jsonc"] -ENTRYPOINT ["micromamba activate _MDTF_base"] +ENTRYPOINT ["/usr/bin/micromamba activate _MDTF_base"] CMD ["/bin/bash"] From 617a6ce82c79edcf3f5cdbd2fd9e9fa4e9873f9c Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 26 Sep 2023 10:42:02 -0400 Subject: [PATCH 3/7] Update Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1ebd838c..d7670d61b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,5 +44,5 @@ ENV PATH="${PATH}:/proj/MDTF-diagnostics/" #RUN /proj/MDTF-diagnostics/mdtf_framework.py --help # Run mdtf on src/default_tests.jsonc # CMD ["${CODE_ROOT}/mdtf", "-f","${CODE_ROOT}/src/default_tests.jsonc"] -ENTRYPOINT ["/usr/bin/micromamba activate _MDTF_base"] -CMD ["/bin/bash"] +ENTRYPOINT ["/bin/bash"] +CMD ["source /proj/MDTF-diagnostics/src/conda/micromamba_init.sh;micromamba activate /opt/conda/envs/_MDTF_base"] From a29d18cf1dfeaf3a1e8c9c791c08271780574bd7 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 26 Sep 2023 11:05:46 -0400 Subject: [PATCH 4/7] Update Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d7670d61b..f4311761b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,10 +39,10 @@ RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml ENV PATH="${PATH}:/proj/MDTF-diagnostics/" -#cRUN micromamba activate _MDTF_base +RUN micromamba activate _MDTF_base # Verify installation #RUN /proj/MDTF-diagnostics/mdtf_framework.py --help # Run mdtf on src/default_tests.jsonc # CMD ["${CODE_ROOT}/mdtf", "-f","${CODE_ROOT}/src/default_tests.jsonc"] ENTRYPOINT ["/bin/bash"] -CMD ["source /proj/MDTF-diagnostics/src/conda/micromamba_init.sh;micromamba activate /opt/conda/envs/_MDTF_base"] +#CMD ["source /proj/MDTF-diagnostics/src/conda/micromamba_init.sh;micromamba activate /opt/conda/envs/_MDTF_base"] From 8d0023b665bbd8a91789dac31359cb97ea7902bb Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 26 Sep 2023 11:30:51 -0400 Subject: [PATCH 5/7] Update Dockerfile --- Dockerfile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4311761b..97bd5cb07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,17 +10,13 @@ RUN micromamba info RUN micromamba shell hook --shell bash -#RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml ## # Container Metadata -LABEL maintainer="20195932+wrongkindofdoctor@users.noreply.github.com" -LABEL version="alpha-01" +LABEL maintainer="mdtf-leads-AR" +LABEL version="test-01" LABEL description="This is a docker image for the MDTF-diagnostics package" -# Disable Prompt During Packages Installation -#ARG DEBIAN_FRONTEND=noninteractive - -# Copy the MDTF-diagnostics package contents from local machine to image +# Copy the MDTF-diagnostics package contents from local machine to image ENV CODE_ROOT=/proj/MDTF-diagnostics COPY src ${CODE_ROOT}/src COPY data ${CODE_ROOT}/data @@ -39,7 +35,9 @@ RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml ENV PATH="${PATH}:/proj/MDTF-diagnostics/" -RUN micromamba activate _MDTF_base +#RUN micromamba activate _MDTF_base +RUN "source /proj/MDTF-diagnostics/src/conda/micromamba_init.sh;micromamba activate /opt/conda/envs/_MDTF_base" + # Verify installation #RUN /proj/MDTF-diagnostics/mdtf_framework.py --help # Run mdtf on src/default_tests.jsonc From bb7a83e98d962c279b2b93946830c6d45c26eba0 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 26 Sep 2023 11:50:50 -0400 Subject: [PATCH 6/7] Update Dockerfile --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 97bd5cb07..1fdc73713 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ USER root RUN micromamba info -RUN micromamba shell hook --shell bash +#RUN micromamba shell hook --shell bash ## # Container Metadata @@ -35,12 +35,11 @@ RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml ENV PATH="${PATH}:/proj/MDTF-diagnostics/" -#RUN micromamba activate _MDTF_base -RUN "source /proj/MDTF-diagnostics/src/conda/micromamba_init.sh;micromamba activate /opt/conda/envs/_MDTF_base" +RUN eval "$(micromamba shell hook --shell bash )";micromamba activate _MDTF_base # Verify installation #RUN /proj/MDTF-diagnostics/mdtf_framework.py --help # Run mdtf on src/default_tests.jsonc # CMD ["${CODE_ROOT}/mdtf", "-f","${CODE_ROOT}/src/default_tests.jsonc"] -ENTRYPOINT ["/bin/bash"] +#ENTRYPOINT ["/bin/bash"] #CMD ["source /proj/MDTF-diagnostics/src/conda/micromamba_init.sh;micromamba activate /opt/conda/envs/_MDTF_base"] From 2bed954c43633ffae20da9ca0cec4cf2622548b5 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 26 Sep 2023 14:29:21 -0400 Subject: [PATCH 7/7] Update Dockerfile --- Dockerfile | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1fdc73713..8a97b11d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,6 @@ USER root RUN micromamba info - -#RUN micromamba shell hook --shell bash - ## # Container Metadata LABEL maintainer="mdtf-leads-AR" @@ -33,13 +30,7 @@ ENV CONDA_ENV_DIR=/opt/conda/envs RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml - ENV PATH="${PATH}:/proj/MDTF-diagnostics/" -RUN eval "$(micromamba shell hook --shell bash )";micromamba activate _MDTF_base +#RUN eval "$(micromamba shell hook --shell bash )";micromamba activate _MDTF_base +ENV PATH=$PATH:"/opt/conda/envs/_MDTF_NCL_base/bin/" -# Verify installation -#RUN /proj/MDTF-diagnostics/mdtf_framework.py --help -# Run mdtf on src/default_tests.jsonc -# CMD ["${CODE_ROOT}/mdtf", "-f","${CODE_ROOT}/src/default_tests.jsonc"] -#ENTRYPOINT ["/bin/bash"] -#CMD ["source /proj/MDTF-diagnostics/src/conda/micromamba_init.sh;micromamba activate /opt/conda/envs/_MDTF_base"]