Fix interactive tools on the secondary HTCondor cluster

Make the environment variables available to the Galaxy handlers also available to the systemd-nspawn HTCondor container (submit role).

This fixes interactive tools on the secondary HTCondor cluster. They were not working because the IT monitoring script was running on system's Python instead of on the virtualenv's Python.

group_vars/htcondor-secondary-submit-host.yml

@@ -28,6 +28,17 @@ nspawn_config: |
   [Network]
   VirtualEthernet=no
 
+nspawn_galaxy_environment_file: "/etc/profile.d/galaxy.sh"
+nspawn_galaxy_environment_vars: |
+  HOME={{ galaxy_root }}
+  VIRTUAL_ENV={{ galaxy_venv_dir }}
+  PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin
+  DOCUTILSCONFIG=''
+  PYTHONPATH={{ galaxy_server_dir }}/lib/galaxy/jobs/rules
+  {% for var in galaxy_systemd_handler_env | split %}
+  {{ var }}
+  {% endfor %}
+
 nspawn_enable: false
 nspawn_start: false
 
@@ -61,7 +72,7 @@ nspawn_ssh_authorized_keys:
   - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDV7gfNbNN5O8vH6/tM/iOFXKBP2YKRHXOmdfV8ogvu9BdVV0IPmDzk2EooVpThDE1VMv1hz3811tvBhHRJ6IgNhVIV/61w/+RazQD/AU27X8bX+Hb9EQ/bP4DW+6ySd/z5vdDLzpH5dbiMhzPEDkXVsylUT+hkQnas6cHspDhHmtKQ5MWOgDe3D/IEudTDJQe8hxxaU4TaZUmFzn7eYp9HvuK8qW0yCy4NWOxJJHA+G5wSCyLuKnaKo4AitUIzSKF1AB94oq7b96KONhPxgRptAk4OYIUTdNFbrI5HDaSNzHLnF5FbjQvG+Eu6m5nY5yvJMogE+jiuWeIXCZTCFljg287FUo0ohmbZpd802L6VXun14VumRC+rRgPrvBALo/CsyCsPIoBSTKhVElxKVOcRjmTLNfrUZM0GQxqJhIvah8BV+JTExkipPwkrKTdMAWIXvCoehxV+WMpBWqtEEzAzEoqJpaiec7HfriwsHTGESZWAPYEbFjzbHXQZtqBkbOvtokPMRmTWfWKxaplCMN6ddJeeY6faorD0w/e6lszWES1Q1ieajiPKDy37UvybKKvPTk4o3MzyzYOS4c8HQj+jnGeR5Q3ETuyz4psLyOfuBtIrfOeuxV42rFDmkYM3IrrRR+F9oklFG6Ig8DVfgQEzSG36NkgvpF4OdFvigYqXvw== cloud@vgcn"
 nspawn_ssh_host_trust_container: yes
 
-nspawn_condor_systemd_run: "/usr/bin/systemd-run --uid={{ galaxy_user.uid }} --gid={{ galaxy_group.gid }} --pipe --quiet --machine {{ nspawn_name }}"
+nspawn_condor_systemd_run: "/usr/bin/systemd-run -p EnvironmentFile={{ nspawn_galaxy_environment_file }} --uid={{ galaxy_user.uid }} --gid={{ galaxy_group.gid }} --pipe --quiet --machine {{ nspawn_name }}"
 nspawn_condor_rm_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_rm"
 nspawn_condor_ssh_to_job_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_ssh_to_job"
 nspawn_condor_submit_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_submit"

htcondor.yml

@@ -11,6 +11,7 @@
       changed_when: true
   vars_files:
     - mounts/mountpoints.yml
+    - secret_group_vars/all.yml
     - secret_group_vars/htcondor-secondary-submit-host.yml
   pre_tasks:
     # Because it is already disabled for sn06 and this setup is needed just
@@ -22,19 +23,19 @@
   roles:
     - kysrpex.systemd_nspawn
   post_tasks:
+    - name: Get the location of the container image.
+      ansible.builtin.shell:
+        executable: /bin/bash
+        cmd: |
+          set -o pipefail
+          machinectl image-status htcondor | grep "Path: " | awk '{$1=$1};1' | cut -d' ' -f2
+      register: nspawn_image
+      changed_when: false
+
     - name: Configure the container's sshd.
       when: nspawn_ssh and (nspawn_ssh_config is defined or nspawn_ssh_config_path is defined)
       become: true
       block:
-        - name: Get the location of the container image.
-          ansible.builtin.shell:
-            executable: /bin/bash
-            cmd: |
-              set -o pipefail
-              machinectl image-status htcondor | grep "Path: " | awk '{$1=$1};1' | cut -d' ' -f2
-          register: nspawn_image
-          changed_when: false
-
         - name: Write the sshd configuration to sshd_config.
           ansible.builtin.lineinfile:
             path: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
@@ -192,6 +193,14 @@
           - "{{ nspawn_condor_ssh_to_job_command }} *"
           - "{{ nspawn_condor_submit_command }} *"
 
+    - name: Make the environment variables available to the Galaxy handlers also available to the container.
+      ansible.builtin.copy:
+        content: "{{ nspawn_galaxy_environment_vars }}"
+        dest: "{{ (nspawn_image.stdout, nspawn_galaxy_environment_file | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}"
+        owner: root
+        group: "{{ galaxy_group.name }}"
+        mode: "0440"
+
 - name: HTCondor cluster.
   hosts: htcondor:!sn06.galaxyproject.eu
   handlers: