Replace singularity with apptainer

Except where used in combination with python ewatercycle package
As eWaterCycle/ewatercycle#324 has not been merged/released yet

research-cloud-plugin.yml

@@ -51,57 +51,57 @@
     # Container engines
     - name: apptainer
       include_role:
-        name: ewatercycle.apptainer
+        name: apptainer
 
     # Install Conda + mamba
-    - name: Install conda
-      include_role:
-        name: conda
-
-    # TODO mount a home and scratch disk, see https://github.com/eWaterCycle/infra/issues/89
-    # - name: Scratch disk
-    #   mount:
-    #     path: /scratch
-    #     src: # TODO find correct value, possibly extracted from SRC API or ansible vars/facts
-    #     state: present
-    # - name: Home disk
-    #   mount:
-    #     path: /home
-    #     src: # TODO find correct value, possibly extracted from SRC API or ansible vars/facts
-    #     state: present
-
-    - name: Mount shared data dcache with rclone
-      include_role:
-        name: rclone
-        tasks_from: mount
-
-    # https://lab.ewatercycle.org/ functionality
-    - name: Welcome page
-      include_role:
-        name: labstart
-
-    # https://explore.ewatercycle.org/ functionality
-    - name: Experiment launcher
-      include_role:
-        name: launcher
-
-    - name: Explorer
-      include_role:
-        name: terria
-
-    # https://jupyter.ewatercycle.org/ functionality
-    - name: Create eWaterCycle conda env
-      include_role:
-        name: ewatercycle
-
-    - name: Set up Jupyter lab/hub
-      include_role:
-        name: jupyter
-
-    - name: Clean apt cache
-      apt:
-        autoclean: yes
-        autoremove: yes
+    # - name: Install conda
+    #   include_role:
+    #     name: conda
+
+    # # TODO mount a home and scratch disk, see https://github.com/eWaterCycle/infra/issues/89
+    # # - name: Scratch disk
+    # #   mount:
+    # #     path: /scratch
+    # #     src: # TODO find correct value, possibly extracted from SRC API or ansible vars/facts
+    # #     state: present
+    # # - name: Home disk
+    # #   mount:
+    # #     path: /home
+    # #     src: # TODO find correct value, possibly extracted from SRC API or ansible vars/facts
+    # #     state: present
+
+    # - name: Mount shared data dcache with rclone
+    #   include_role:
+    #     name: rclone
+    #     tasks_from: mount
+
+    # # https://lab.ewatercycle.org/ functionality
+    # - name: Welcome page
+    #   include_role:
+    #     name: labstart
+
+    # # https://explore.ewatercycle.org/ functionality
+    # - name: Experiment launcher
+    #   include_role:
+    #     name: launcher
+
+    # - name: Explorer
+    #   include_role:
+    #     name: terria
+
+    # # https://jupyter.ewatercycle.org/ functionality
+    # - name: Create eWaterCycle conda env
+    #   include_role:
+    #     name: ewatercycle
+
+    # - name: Set up Jupyter lab/hub
+    #   include_role:
+    #     name: jupyter
+
+    # - name: Clean apt cache
+    #   apt:
+    #     autoclean: yes
+    #     autoremove: yes
 
     - name: Debug
       debug:

roles/apptainer/tests/test.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: localhost
+- name: Test
+  hosts: localhost
   remote_user: root
   roles:
     - apptainer

roles/ewatercycle/README.md

@@ -41,8 +41,8 @@ climate_data_root_dir: '{{ data_root }}/climate-data'
 grdc_location: '{{ data_root }}/observation/grdc/dailies'
 # Location where example parameter sets should be downloaded and where any other read-only pararmeter set can be put
 parameterset_dir: '{{ data_root }}/parameter-sets'
-# Location where Singularity image files (*sif) of hydrological models should be stored
-singularity_image_root: '{{ data_root }}/singularity-images'
+# Location where Apptainer image files (*sif) of hydrological models should be stored
+apptainer_image_root: '{{ data_root }}/singularity-images'
 # Location where all home-directories are located
 home_root: /home
 ```
@@ -52,7 +52,7 @@ Dependencies
 
 Conda should be installed in `conda_root` directory. Use [../conda](../conda) role to install conda.
 
-Requires singularity executable which can be installed with [../singularity](../singularity) role.
+Requires apptainer executable which can be installed with [../apptainer](../apptainer) role.
 
 Example Playbook
 ----------------

roles/ewatercycle/defaults/main.yml

@@ -13,7 +13,7 @@ climate_data_root_dir: '{{ data_root }}/climate-data'
 grdc_location: '{{ data_root }}/observation/grdc/dailies'
 # Location where example parameter sets should be downloaded and where any other read-only pararmeter set can be put
 parameterset_dir: '{{ data_root }}/parameter-sets'
-# Location where Singularity image files (*sif) of hydrological models should be stored
-singularity_image_root: '{{ data_root }}/singularity-images'
+# Location where Apptainer image files (*sif) of hydrological models should be stored
+apptainer_image_root: '{{ data_root }}/singularity-images'
 # Location where all home-directories are located
 home_root: /home

roles/ewatercycle/tasks/main.yml

@@ -15,19 +15,6 @@
     line: conda activate {{ conda_environment }}
     regexp: '^conda activate {{ conda_environment }}'
 
-# Checklist system setup https://ewatercycle.readthedocs.io/en/latest/system_setup.html :
-# Conda environment -> done on line 10
-# Install ewatercycle package -> done on line 18
-# Configure ESMValTool -> done on line 42-49
-# Download climate data -> from dcache mounted on /mnt/data
-# Install container engine -> done in singularity role
-# Configure ewatercycle -> done on line 51-54
-# Model container images -> from dcache mounted on /mnt/data
-# Download example parameter sets -> from dcache mounted on /mnt/data
-# Prepare other parameter sets -> from dcache mounted on /mnt/data
-# Download example forcing -> from dcache mounted on /mnt/data
-# Download observation data -> from dcache mounted on /mnt/data
-
 # Loop over users in /home to copy .esmvaltool/config-user.yml using home_root=/home var
 - name: List my users
   shell: set -o pipefail && ls -1 | grep -v lost+found

roles/ewatercycle/templates/ewatercycle.yaml.j2

@@ -102,4 +102,4 @@ parameter_sets:
     target_model: wflow
     supported_model_versions: !!set {2020.1.1: null, 2020.1.2: null, 2020.1.3: null}
 parameterset_dir: {{ parameterset_dir }}
-singularity_dir: {{ singularity_image_root }}
+singularity_dir: {{ apptainer_image_root }}

roles/prep_shared_data/README.md

@@ -47,29 +47,29 @@ era5_variables:
 Variables for hydrological models
 
 ```yaml
-# Docker container images that will be downloaded and converted to singularity image files
+# Docker container images that will be downloaded and converted to Apptainer image files
 grpc4bmi_images:
     # - docker: ewatercycle/walrus-grpc4bmi
-    #   singularity: ewatercycle-walrus-grpc4bmi.sif
+    #   apptainer: ewatercycle-walrus-grpc4bmi.sif
     - docker: ewatercycle/pcrg-grpc4bmi:setters
-      singularity: ewatercycle-pcrg-grpc4bmi_setters.sif
+      apptainer: ewatercycle-pcrg-grpc4bmi_setters.sif
     - docker: ewatercycle/wflow-grpc4bmi:2020.1.1
-      singularity: ewatercycle-wflow-grpc4bmi_2020.1.1.sif
+      apptainer: ewatercycle-wflow-grpc4bmi_2020.1.1.sif
     - docker:  ewatercycle/marrmot-grpc4bmi:2020.11
-      singularity: ewatercycle-marrmot-grpc4bmi_2020.11.sif
+      apptainer: ewatercycle-marrmot-grpc4bmi_2020.11.sif
     # - docker: ewatercycle/hype-grpc4bmi
-    #   singularity: ewatercycle-hype-grpc4bmi.sif
+    #   apptainer: ewatercycle-hype-grpc4bmi.sif
     - docker: ewatercycle/lisflood-grpc4bmi:20.10
-      singularity: ewatercycle-lisflood-grpc4bmi_20.10.sif
+      apptainer: ewatercycle-lisflood-grpc4bmi_20.10.sif
 ```
 
 Other optional variables
 
 ```yaml
 # Where all data should be put
 data_root: /mnt/data
-# Directory where singularity sif files can be stored and read by other
-singularity_image_root: '{{ data_root }}/singularity-images'
+# Directory where apptainer sif files can be stored and read by other
+apptainer_image_root: '{{ data_root }}/singularity-images'
 # Location where conda is installed
 conda_root: /opt/conda
 # Name of conda environment to use

roles/prep_shared_data/defaults/main.yml

@@ -1,26 +1,26 @@
 ---
 # Where all data should be put
 data_root: /mnt/data
-# Directory where singularity sif files can be stored and read by other
-singularity_image_root: '{{ data_root }}/singularity-images'
-# Docker container images that will be downloaded and converted to singularity image files
+# Directory where Apptainer sif files can be stored and read by other
+apptainer_image_root: '{{ data_root }}/singularity-images'
+# Docker container images that will be downloaded and converted to apptainer image files
 grpc4bmi_images:
     # - docker: ewatercycle/walrus-grpc4bmi
-    #   singularity: ewatercycle-walrus-grpc4bmi.sif
+    #   apptainer: ewatercycle-walrus-grpc4bmi.sif
     - docker: ewatercycle/pcrg-grpc4bmi:setters
-      singularity: ewatercycle-pcrg-grpc4bmi_setters.sif
+      apptainer: ewatercycle-pcrg-grpc4bmi_setters.sif
     - docker: ewatercycle/wflow-grpc4bmi:2020.1.1
-      singularity: ewatercycle-wflow-grpc4bmi_2020.1.1.sif
+      apptainer: ewatercycle-wflow-grpc4bmi_2020.1.1.sif
     - docker: ewatercycle/wflow-grpc4bmi:2020.1.2
-      singularity: ewatercycle-wflow-grpc4bmi_2020.1.2.sif
+      apptainer: ewatercycle-wflow-grpc4bmi_2020.1.2.sif
     - docker: ewatercycle/wflow-grpc4bmi:2020.1.3
-      singularity: ewatercycle-wflow-grpc4bmi_2020.1.3.sif
+      apptainer: ewatercycle-wflow-grpc4bmi_2020.1.3.sif
     - docker:  ewatercycle/marrmot-grpc4bmi:2020.11
-      singularity: ewatercycle-marrmot-grpc4bmi_2020.11.sif
+      apptainer: ewatercycle-marrmot-grpc4bmi_2020.11.sif
     - docker: ewatercycle/lisflood-grpc4bmi:20.10
-      singularity: ewatercycle-lisflood-grpc4bmi_20.10.sif
+      apptainer: ewatercycle-lisflood-grpc4bmi_20.10.sif
     - docker: ewatercycle/hype-grpc4bmi:feb2021
-      singularity: ewatercycle-hype-grpc4bmi_feb2021.sif
+      apptainer: ewatercycle-hype-grpc4bmi_feb2021.sif
 # Location where conda is installed
 conda_root: /opt/conda
 # Name of conda environment to use

roles/prep_shared_data/tasks/apptainer-images.yml

@@ -0,0 +1,11 @@
+---
+- name: Apptainer image dir
+  file:
+    path: '{{ apptainer_image_root }}'
+    state: directory
+    mode: 0755
+- name: grpc4bmi Apptainer images
+  command: apptainer build {{ apptainer_image_root }}/{{ item.apptainer }} docker://{{ item.docker }}
+  args:
+    creates: '{{ apptainer_image_root }}/{{ item.apptainer }}'
+  loop: '{{ grpc4bmi_images }}'

roles/prep_shared_data/tasks/main.yml

@@ -3,8 +3,8 @@
   include: climate-data.yml
 - name: ESMValTool aux data
   include: esmvaltool-aux-data.yml
-- name: Build singularity image files (sif) for each model
-  include: singularity-images.yml
+- name: Build apptainer image files (sif) for each model
+  include: apptainer-images.yml
 - name: Download example parameter sets
   include: example-parameter-sets.yml
 - name: Download example forcing