From c84981c20123da393a1637ab6f2b3b5d04638aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Tue, 21 Nov 2023 17:05:43 +0100 Subject: [PATCH 1/2] Fix interactive tools on the secondary HTCondor cluster Make the environment variables available to the Galaxy handlers also available to the systemd-nspawn HTCondor container (submit role). This fixes interactive tools on the secondary HTCondor cluster. They were not working because the IT monitoring script was running on system's Python instead of on the virtualenv's Python. --- group_vars/htcondor-secondary-submit-host.yml | 13 ++++++++- htcondor.yml | 27 ++++++++++++------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/group_vars/htcondor-secondary-submit-host.yml b/group_vars/htcondor-secondary-submit-host.yml index fbcbc9b36..e0a2c32e9 100644 --- a/group_vars/htcondor-secondary-submit-host.yml +++ b/group_vars/htcondor-secondary-submit-host.yml @@ -28,6 +28,17 @@ nspawn_config: | [Network] VirtualEthernet=no +nspawn_galaxy_environment_file: "/etc/profile.d/galaxy.sh" +nspawn_galaxy_environment_vars: | + HOME={{ galaxy_root }} + VIRTUAL_ENV={{ galaxy_venv_dir }} + PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin + DOCUTILSCONFIG='' + PYTHONPATH={{ galaxy_server_dir }}/lib/galaxy/jobs/rules + {% for var in galaxy_systemd_handler_env | split %} + {{ var }} + {% endfor %} + nspawn_enable: false nspawn_start: false @@ -61,7 +72,7 @@ nspawn_ssh_authorized_keys: - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDV7gfNbNN5O8vH6/tM/iOFXKBP2YKRHXOmdfV8ogvu9BdVV0IPmDzk2EooVpThDE1VMv1hz3811tvBhHRJ6IgNhVIV/61w/+RazQD/AU27X8bX+Hb9EQ/bP4DW+6ySd/z5vdDLzpH5dbiMhzPEDkXVsylUT+hkQnas6cHspDhHmtKQ5MWOgDe3D/IEudTDJQe8hxxaU4TaZUmFzn7eYp9HvuK8qW0yCy4NWOxJJHA+G5wSCyLuKnaKo4AitUIzSKF1AB94oq7b96KONhPxgRptAk4OYIUTdNFbrI5HDaSNzHLnF5FbjQvG+Eu6m5nY5yvJMogE+jiuWeIXCZTCFljg287FUo0ohmbZpd802L6VXun14VumRC+rRgPrvBALo/CsyCsPIoBSTKhVElxKVOcRjmTLNfrUZM0GQxqJhIvah8BV+JTExkipPwkrKTdMAWIXvCoehxV+WMpBWqtEEzAzEoqJpaiec7HfriwsHTGESZWAPYEbFjzbHXQZtqBkbOvtokPMRmTWfWKxaplCMN6ddJeeY6faorD0w/e6lszWES1Q1ieajiPKDy37UvybKKvPTk4o3MzyzYOS4c8HQj+jnGeR5Q3ETuyz4psLyOfuBtIrfOeuxV42rFDmkYM3IrrRR+F9oklFG6Ig8DVfgQEzSG36NkgvpF4OdFvigYqXvw== cloud@vgcn" nspawn_ssh_host_trust_container: yes -nspawn_condor_systemd_run: "/usr/bin/systemd-run --uid={{ galaxy_user.uid }} --gid={{ galaxy_group.gid }} --pipe --quiet --machine {{ nspawn_name }}" +nspawn_condor_systemd_run: "/usr/bin/systemd-run -p EnvironmentFile={{ nspawn_galaxy_environment_file }} --uid={{ galaxy_user.uid }} --gid={{ galaxy_group.gid }} --pipe --quiet --machine {{ nspawn_name }}" nspawn_condor_rm_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_rm" nspawn_condor_ssh_to_job_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_ssh_to_job" nspawn_condor_submit_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_submit" diff --git a/htcondor.yml b/htcondor.yml index ad799d8ee..6fae12afd 100644 --- a/htcondor.yml +++ b/htcondor.yml @@ -11,6 +11,7 @@ changed_when: true vars_files: - mounts/mountpoints.yml + - secret_group_vars/all.yml - secret_group_vars/htcondor-secondary-submit-host.yml pre_tasks: # Because it is already disabled for sn06 and this setup is needed just @@ -22,19 +23,19 @@ roles: - kysrpex.systemd_nspawn post_tasks: + - name: Get the location of the container image. + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -o pipefail + machinectl image-status htcondor | grep "Path: " | awk '{$1=$1};1' | cut -d' ' -f2 + register: nspawn_image + changed_when: false + - name: Configure the container's sshd. when: nspawn_ssh and (nspawn_ssh_config is defined or nspawn_ssh_config_path is defined) become: true block: - - name: Get the location of the container image. - ansible.builtin.shell: - executable: /bin/bash - cmd: | - set -o pipefail - machinectl image-status htcondor | grep "Path: " | awk '{$1=$1};1' | cut -d' ' -f2 - register: nspawn_image - changed_when: false - - name: Write the sshd configuration to sshd_config. ansible.builtin.lineinfile: path: "{{ (nspawn_image.stdout, nspawn_ssh_config_path | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" @@ -192,6 +193,14 @@ - "{{ nspawn_condor_ssh_to_job_command }} *" - "{{ nspawn_condor_submit_command }} *" + - name: Make the environment variables available to the Galaxy handlers also available to the container. + ansible.builtin.copy: + content: "{{ nspawn_galaxy_environment_vars }}" + dest: "{{ (nspawn_image.stdout, nspawn_galaxy_environment_file | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" + owner: root + group: "{{ galaxy_group.name }}" + mode: "0440" + - name: HTCondor cluster. hosts: htcondor:!sn06.galaxyproject.eu handlers: From f32c676f43f743df7068c703a9051115f3ebbdb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Wed, 22 Nov 2023 09:08:44 +0100 Subject: [PATCH 2/2] Change location of Galaxy env file Place the file containing the environment variables needed for condor_submit to work properly in the home directory of the Galaxy user, set its owner to the Galaxy user. --- group_vars/htcondor-secondary-submit-host.yml | 2 +- htcondor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/htcondor-secondary-submit-host.yml b/group_vars/htcondor-secondary-submit-host.yml index e0a2c32e9..139662807 100644 --- a/group_vars/htcondor-secondary-submit-host.yml +++ b/group_vars/htcondor-secondary-submit-host.yml @@ -28,7 +28,7 @@ nspawn_config: | [Network] VirtualEthernet=no -nspawn_galaxy_environment_file: "/etc/profile.d/galaxy.sh" +nspawn_galaxy_environment_file: "{{ galaxy_user.home }}/env" nspawn_galaxy_environment_vars: | HOME={{ galaxy_root }} VIRTUAL_ENV={{ galaxy_venv_dir }} diff --git a/htcondor.yml b/htcondor.yml index 6fae12afd..5253aacac 100644 --- a/htcondor.yml +++ b/htcondor.yml @@ -197,7 +197,7 @@ ansible.builtin.copy: content: "{{ nspawn_galaxy_environment_vars }}" dest: "{{ (nspawn_image.stdout, nspawn_galaxy_environment_file | regex_replace('(\\/*)?(.*)', '\\2')) | path_join }}" - owner: root + owner: "{{ galaxy_user.name }}" group: "{{ galaxy_group.name }}" mode: "0440"