From 1da06153da2f4c8b427c22b2ce35b3f36d6e5db9 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Fri, 3 Nov 2023 16:37:35 -0500 Subject: [PATCH 01/12] changes corresponding to mode 3 cvmfsexec PR --- creation/lib/cgWConsts.py | 7 +++++-- creation/lib/cgWDictFile.py | 32 +++++++++++++++++++++++++------- creation/lib/cgWParamDict.py | 30 ++++++++++++++++++++++++------ 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/creation/lib/cgWConsts.py b/creation/lib/cgWConsts.py index 317a8d5ee..d851acbfc 100644 --- a/creation/lib/cgWConsts.py +++ b/creation/lib/cgWConsts.py @@ -9,8 +9,9 @@ from . import cWConsts # these are in the stage dir, so they need to be renamed if changed -AFTER_FILE_LISTFILE = "after_%s" % cWConsts.FILE_LISTFILE -AT_FILE_LISTFILE = "at_%s" % cWConsts.FILE_LISTFILE +AFTER_FILE_LISTFILE = f"after_{cWConsts.FILE_LISTFILE}" +AT_FILE_LISTFILE = f"at_{cWConsts.FILE_LISTFILE}" +PRECVMFS_FILE_LISTFILE = f"precvmfs_{cWConsts.FILE_LISTFILE}" CONDOR_FILE = "condor_bin_%s.tgz" CONDOR_DIR = "condor" @@ -23,6 +24,8 @@ CVMFSEXEC_DIR = "cvmfsexec" CVMFSEXEC_ATTR = "CVMFSEXEC_DIR" +# constant that defines the priority values used by the factory/glidein +PRIORITY_SETTINGS = ["file_list", "precvmfs_file_list", "at_file_list", "after_file_list"] # these are in the submit dir, so they can be changed SUBMIT_ATTRS_FILE = "submit_attrs.cfg" diff --git a/creation/lib/cgWDictFile.py b/creation/lib/cgWDictFile.py index 4b8e22770..7dbc4dad3 100644 --- a/creation/lib/cgWDictFile.py +++ b/creation/lib/cgWDictFile.py @@ -290,6 +290,9 @@ def get_main_dicts(submit_dir, stage_dir): main_dicts["gridmap"] = cWDictFile.GridMapDict( stage_dir, cWConsts.insert_timestr(cWConsts.GRIDMAP_FILE) ) # TODO: versioned but no fname_idx? + main_dicts["precvmfs_file_list"] = cWDictFile.FileDictFile( + stage_dir, cWConsts.insert_timestr(cgWConsts.PRECVMFS_FILE_LISTFILE), fname_idx=cgWConsts.PRECVMFS_FILE_LISTFILE + ) main_dicts["at_file_list"] = cWDictFile.FileDictFile( stage_dir, cWConsts.insert_timestr(cgWConsts.AT_FILE_LISTFILE), fname_idx=cgWConsts.AT_FILE_LISTFILE ) @@ -370,10 +373,15 @@ def load_main_dicts(main_dicts): # update in place # print "\ndebug %s main_dicts['description'].keys2 = %s" % (__file__, main_dicts['description'].keys2) # print "\ndebug %s dir(main_dicts['description']) = %s" % (__file__, dir(main_dicts['description'])) # TODO: To remove if upgrade from older versions is not a problem + try: + main_dicts["precvmfs_file_list"].load(fname=main_dicts["description"].vals2["precvmfs_file_list"]) + except KeyError: + # when upgrading from older version the new precvmfs_file_list may not be in the description + main_dicts["precvmfs_file_list"].load() try: main_dicts["at_file_list"].load(fname=main_dicts["description"].vals2["at_file_list"]) except KeyError: - # when upgrading form older version the new at_file_list may not be in the description + # when upgrading from older version the new at_file_list may not be in the description main_dicts["at_file_list"].load() main_dicts["after_file_list"].load(fname=main_dicts["description"].vals2["after_file_list"]) load_common_dicts(main_dicts, main_dicts["description"]) @@ -409,7 +417,7 @@ def load_entry_dicts(entry_dicts, entry_name, summary_signature): # update in p def refresh_description(dicts): # update in place description_dict = dicts["description"] description_dict.add(dicts["signature"].get_fname(), "signature", allow_overwrite=True) - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.PRIORITY_SETTINGS: if k in dicts: description_dict.add(dicts[k].get_fname(), k, allow_overwrite=True) @@ -455,11 +463,21 @@ def refresh_file_list(dicts, is_main, files_set_readonly=True, files_reset_chang # dictionaries must have been written to disk before using this def refresh_signature(dicts): # update in place signature_dict = dicts["signature"] - for k in ("consts", "vars", "untar_cfg", "gridmap", "file_list", "at_file_list", "after_file_list", "description"): + for k in ( + "consts", + "vars", + "untar_cfg", + "gridmap", + "file_list", + "precvmfs_file_list", + "at_file_list", + "after_file_list", + "description", + ): if k in dicts: signature_dict.add_from_file(dicts[k].get_filepath(), allow_overwrite=True) # add signatures of all the files linked in the lists - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.PRIORITY_SETTINGS: if k in dicts: filedict = dicts[k] for fname in filedict.get_immutable_files(): @@ -493,11 +511,11 @@ def save_common_dicts(dicts, is_main, set_readonly=True): # 'consts','untar_cfg','vars' will be loaded refresh_file_list(dicts, is_main) # save files in the file lists - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.PRIORITY_SETTINGS: if k in dicts: dicts[k].save_files(allow_overwrite=True) # then save the lists - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.PRIORITY_SETTINGS: if k in dicts: dicts[k].save(set_readonly=set_readonly) # calc and save the signatures @@ -592,7 +610,7 @@ def reuse_common_dicts(dicts, other_dicts, is_main, all_reused): # since the file names may have changed, refresh the file_list refresh_file_list(dicts, is_main) # check file-based dictionaries - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.PRIORITY_SETTINGS: if k in dicts: all_reused = reuse_file_dict(dicts, other_dicts, k) and all_reused diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 4cbb34afc..466beed08 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -10,6 +10,8 @@ import os.path import shutil +from collections import Counter + from glideinwms.lib import pubCrypto, subprocessSupport from glideinwms.lib.util import str2bool @@ -219,6 +221,8 @@ def populate(self, other=None): self.dicts["params"].add("GLIDEIN_Factory_Collector", str(factory_monitoring_collector)) populate_gridmap(self.conf, self.dicts["gridmap"]) + # the following list will be a megalist containing all the scripts; used for duplication check logic subsequently + all_scripts = list() # NOTE that all the files in these _scripts lists are added as executables (i.e. must report with error_gen) file_list_scripts = [ "collector_setup.sh", @@ -226,8 +230,17 @@ def populate(self, other=None): "gwms-python", cgWConsts.CONDOR_STARTUP_FILE, ] + # add the above list to the megalist created before + all_scripts.extend(file_list_scripts) + + # singularity_setup should be performed after cvmfs_setup; condor_chirp's order does not matter + precvmfs_file_list_scripts = ["cvmfs_setup.sh"] + all_scripts.extend(precvmfs_file_list_scripts) # add this list to the megalist + # These are right after the entry, before some VO scripts. The order in the following list is important at_file_list_scripts = ["singularity_setup.sh", "condor_chirp", "gconfig.py"] + all_scripts.extend(at_file_list_scripts) # adding the above list to the megalist as before + # The order in the following list is important after_file_list_scripts = [ "check_proxy.sh", @@ -240,15 +253,14 @@ def populate(self, other=None): "glidein_sitewms_setup.sh", "script_wrapper.sh", "smart_partitionable.sh", - "cvmfs_setup.sh", "cvmfs_umount.sh", ] - # Only execute scripts once - duplicate_scripts = list(set(file_list_scripts).intersection(after_file_list_scripts)) - duplicate_scripts += list(set(file_list_scripts).intersection(at_file_list_scripts)) - duplicate_scripts += list(set(at_file_list_scripts).intersection(after_file_list_scripts)) + all_scripts.extend(after_file_list_scripts) # adding the above list to the megalist as before + # Scripts need to be only executed once, so check for duplicates + count_duplicates = Counter(all_scripts) + duplicate_scripts = [scr for scr, cnt in count_duplicates.items() if cnt > 1] if duplicate_scripts: - raise RuntimeError("Duplicates found in the list of files to execute '%s'" % ",".join(duplicate_scripts)) + raise RuntimeError(f"Duplicates found in the list of files to execute: {', '.join(duplicate_scripts)}") # Load more system scripts for script_name in file_list_scripts: @@ -414,6 +426,12 @@ def populate(self, other=None): ) # add additional system scripts + for script_name in precvmfs_file_list_scripts: + self.dicts["precvmfs_file_list"].add_from_file( + script_name, + cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec:r"), + os.path.join(cgWConsts.WEB_BASE_DIR, script_name), + ) for script_name in at_file_list_scripts: self.dicts["at_file_list"].add_from_file( script_name, From 3b8a229a0bf5184412b2e52aee25566c9f36ae36 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Fri, 3 Nov 2023 16:44:08 -0500 Subject: [PATCH 02/12] revised changes to scripts related to cvmfs setup --- creation/web_base/cvmfs_helper_funcs.sh | 879 +++++++++++++----------- creation/web_base/cvmfs_setup.sh | 236 ++++--- creation/web_base/cvmfs_umount.sh | 42 +- creation/web_base/singularity_lib.sh | 8 + 4 files changed, 644 insertions(+), 521 deletions(-) mode change 100755 => 100644 creation/web_base/cvmfs_setup.sh diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index 00a14925f..cf6aa7d13 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -18,9 +18,6 @@ # Author: # Namratha Urs # -# Version: -# 1.0 -# # to implement custom logging @@ -30,476 +27,532 @@ #exec &> $LOGFILE variables_reset() { - # DESCRIPTION: This function lists and initializes the common variables - # to empty strings. These variables also become available to scripts - # that import functions defined in this script. - # - # INPUT(S): None - # RETURN(S): Variables initialized to empty strings - - # indicates whether the perform_system_check function has been run - GWMS_SYSTEM_CHECK= - - # following set of variables used to store operating system and kernel info - GWMS_OS_DISTRO= - GWMS_OS_VERSION= - GWMS_OS_KRNL_ARCH= - GWMS_OS_KRNL_NUM= - GWMS_OS_KRNL_VER= - GWMS_OS_KRNL_MAJOR_REV= - GWMS_OS_KRNL_MINOR_REV= - GWMS_OS_KRNL_PATCH_NUM= - - # indicates whether CVMFS is locally mounted on the node - GWMS_IS_CVMFS_MNT= - # to indicate the status of mounting CVMFS by the glidein after evaluating the worker node - GWMS_IS_CVMFS= - - # indicates if unpriv userns is available (or supported); not if it is enabled - GWMS_IS_UNPRIV_USERNS_SUPPORTED= - # indicates if unpriv userns is enabled (and available) - GWMS_IS_UNPRIV_USERNS_ENABLED= - - # following variables store FUSE-related information - GWMS_IS_FUSE_INSTALLED= - GWMS_IS_FUSERMOUNT= - GWMS_IS_USR_IN_FUSE_GRP= + # DESCRIPTION: This function lists and initializes the common variables + # to empty strings. These variables also become available to scripts + # that import functions defined in this script. + # + # INPUT(S): None + # RETURN(S): Variables initialized to empty strings + + # indicates whether the perform_system_check function has been run + GWMS_SYSTEM_CHECK= + + # following set of variables used to store operating system and kernel info + GWMS_OS_DISTRO= + GWMS_OS_VERSION= + GWMS_OS_KRNL_ARCH= + GWMS_OS_KRNL_NUM= + GWMS_OS_KRNL_VER= + GWMS_OS_KRNL_MAJOR_REV= + GWMS_OS_KRNL_MINOR_REV= + GWMS_OS_KRNL_PATCH_NUM= + + # indicates whether CVMFS is locally mounted on the worker node (CE) + GWMS_IS_CVMFS_LOCAL_MNT= + # to indicate the status of on-demand mounting of CVMFS by the glidein after evaluating the worker node (CE) + GWMS_IS_CVMFS= + + # indicates if unpriv userns is available (or supported); not if it is enabled + GWMS_IS_UNPRIV_USERNS_SUPPORTED= + # indicates if unpriv userns is enabled (and available) + GWMS_IS_UNPRIV_USERNS_ENABLED= + + # following variables store FUSE-related information + GWMS_IS_FUSE_INSTALLED= + GWMS_IS_FUSERMOUNT= + GWMS_IS_USR_IN_FUSE_GRP= } loginfo() { - # DESCRIPTION: This function prints informational messages to STDOUT - # along with hostname and date/time. - # - # INPUT(S): String containing the message - # RETURN(S): Prints message to STDOUT + # DESCRIPTION: This function prints informational messages to STDOUT + # along with hostname and date/time. + # + # INPUT(S): String containing the message + # RETURN(S): Prints message to STDOUT - echo -e "$(hostname -s) $(date +%m-%d-%Y\ %T\ %Z) \t INFO: $1" >&2 + echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t INFO: $1" >&2 } logwarn(){ - # DESCRIPTION: This function prints warning messages to STDOUT along - # with hostname and date/time. - # - # INPUT(S): String containing the message - # RETURN(S): Prints message to STDOUT + # DESCRIPTION: This function prints warning messages to STDOUT along + # with hostname and date/time. + # + # INPUT(S): String containing the message + # RETURN(S): Prints message to STDOUT - echo -e "$(hostname -s) $(date +%m-%d-%Y\ %T\ %Z) \t WARNING: $1" >&2 + echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t WARNING: $1" >&2 } logerror() { - # DESCRIPTION: This function prints error messages to STDOUT along with - # hostname and date/time. - # - # INPUT(S): String containing the message - # RETURN(S): Prints message to STDOUT + # DESCRIPTION: This function prints error messages to STDOUT along with + # hostname and date/time. + # + # INPUT(S): String containing the message + # RETURN(S): Prints message to STDOUT - echo -e "$(hostname -s) $(date +%m-%d-%Y\ %T\ %Z) \t ERROR: $1" >&2 + echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t ERROR: $1" >&2 } -check_exit_status () { - # DESCRIPTION: This function prints an appropriate message to the - # console to indicate what the exit status means. - # - # INPUT(S): Number (exit status of a previously run command) - # RETURN(S): Prints "yes" or "no" to indicate the result of the command +print_exit_status () { + # DESCRIPTION: This function prints an appropriate message to the + # console to indicate what the exit status means. + # + # INPUT(S): Number (exit status of a previously run command) + # RETURN(S): Prints "yes" or "no" to indicate the result of the command - [[ $1 -eq 0 ]] && echo yes || echo no + [[ $1 -eq 0 ]] && echo yes || echo no } detect_local_cvmfs() { - CVMFS_ROOT="/cvmfs" - repo_name=oasis.opensciencegrid.org - # Second check... - if [[ -f $CVMFS_ROOT/$repo_name/.cvmfsdirtab || "$(ls -A $CVMFS_ROOT/$repo_name)" ]] &>/dev/null - then - loginfo "Validating CVMFS with ${repo_name}..." - true - else - logwarn "Validating CVMFS with ${repo_name}: directory empty or does not have .cvmfsdirtab" - false - fi - - GWMS_IS_CVMFS_MNT=$? - loginfo "CVMFS locally installed: $(check_exit_status $GWMS_IS_CVMFS_MNT)" + CVMFS_ROOT="/cvmfs" + repo_name=oasis.opensciencegrid.org + # Second check... + GWMS_IS_CVMFS_LOCAL_MNT=0 + if [[ -f $CVMFS_ROOT/$repo_name/.cvmfsdirtab || "$(ls -A $CVMFS_ROOT/$repo_name)" ]] &>/dev/null + then + loginfo "Validating CVMFS with ${repo_name}..." + else + logwarn "Validating CVMFS with ${repo_name}: directory empty or does not have .cvmfsdirtab" + GWMS_IS_CVMFS_LOCAL_MNT=1 + fi + + loginfo "CVMFS locally installed: $(print_exit_status $GWMS_IS_CVMFS_LOCAL_MNT)" } perform_system_check() { - # DESCRIPTION: This functions performs required system checks (such as - # operating system and kernel info, unprivileged user namespaces, FUSE - # status) and stores the results in the common variables for later use. - # - # INPUT(S): None - # RETURN(S): - # -> common variables containing the exit status of the - # corresponding commands - # -> results from running the check_exit_status function - # for logging purposes (variables starting with res_) - # -> assigns "yes" to GWMS_SYSTEM_CHECK to indicate this function - # has been run. - - if [ -f '/etc/redhat-release' ]; then - GWMS_OS_DISTRO=rhel - else - GWMS_OS_DISTRO=non-rhel - fi - - GWMS_OS_VERSION=`lsb_release -r | awk -F'\t' '{print $2}'` - GWMS_OS_KRNL_ARCH=`arch` - GWMS_OS_KRNL_NUM=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " ` - GWMS_OS_KRNL_VER=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $1}'` - GWMS_OS_KRNL_MAJOR_REV=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $2}'` - GWMS_OS_KRNL_MINOR_REV=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $3}'` - GWMS_OS_KRNL_PATCH_NUM=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 2 -d " "` - - #df -h | grep /cvmfs &>/dev/null - #GWMS_IS_CVMFS_MNT=$? - # call function to detect local CVMFS only if the GWMS_IS_CVMFS_MNT variable is not set; if the variable is not empty, do nothing - [[ -z "${GWMS_IS_CVMFS_MNT}" ]] && detect_local_cvmfs || : - - sysctl user.max_user_namespaces &>/dev/null - GWMS_IS_UNPRIV_USERNS_SUPPORTED=$? - - unshare -U true &>/dev/null - GWMS_IS_UNPRIV_USERNS_ENABLED=$? - - yum list installed *fuse* &>/dev/null - GWMS_IS_FUSE_INSTALLED=$? - - fusermount -V &>/dev/null - GWMS_IS_FUSERMOUNT=$? - - getent group fuse | grep $USER &>/dev/null - GWMS_IS_USR_IN_FUSE_GRP=$? - - # set the variable indicating this function has been run - GWMS_SYSTEM_CHECK=yes - + # DESCRIPTION: This functions performs required system checks (such as + # operating system and kernel info, unprivileged user namespaces, FUSE + # status) and stores the results in the common variables for later use. + # + # INPUT(S): None + # RETURN(S): + # -> common variables containing the exit status of the + # corresponding commands + # -> results from running the print_exit_status function + # for logging purposes (variables starting with res_) + # -> assigns "yes" to GWMS_SYSTEM_CHECK to indicate this function + # has been run. + + if [[ -f "/etc/redhat-release" ]]; then + GWMS_OS_DISTRO=rhel + else + GWMS_OS_DISTRO=non-rhel + fi + + GWMS_OS_VERSION=$(lsb_release -r | awk -F'\t' '{print $2}') + GWMS_OS_KRNL_ARCH=$(arch) + GWMS_OS_KRNL_NUM=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " ) + GWMS_OS_KRNL_VER=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $1}') + GWMS_OS_KRNL_MAJOR_REV=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $2}') + GWMS_OS_KRNL_MINOR_REV=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $3}') + GWMS_OS_KRNL_PATCH_NUM=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 2 -d " ") + + #df -h | grep /cvmfs &>/dev/null + #GWMS_IS_CVMFS_LOCAL_MNT=$? + # call function to detect local CVMFS only if the GWMS_IS_CVMFS_LOCAL_MNT variable is not set; if the variable is not empty, do nothing + [[ -z "${GWMS_IS_CVMFS_LOCAL_MNT}" ]] && detect_local_cvmfs || : + + sysctl user.max_user_namespaces &>/dev/null + GWMS_IS_UNPRIV_USERNS_SUPPORTED=$? + + unshare -U true &>/dev/null + GWMS_IS_UNPRIV_USERNS_ENABLED=$? + + yum list installed *fuse* &>/dev/null + GWMS_IS_FUSE_INSTALLED=$? + + fusermount -V &>/dev/null + GWMS_IS_FUSERMOUNT=$? + + getent group fuse | grep $USER &>/dev/null + GWMS_IS_USR_IN_FUSE_GRP=$? + + # set the variable indicating this function has been run + GWMS_SYSTEM_CHECK=yes } print_os_info () { - # DESCRIPTION: This functions prints operating system and kernel - # information to STDOUT. - # - # INPUT(S): None - # RETURN(S): Prints a message containing OS and kernel details + # DESCRIPTION: This functions prints operating system and kernel + # information to STDOUT. + # + # INPUT(S): None + # RETURN(S): Prints a message containing OS and kernel details - loginfo "Found $GWMS_OS_DISTRO${GWMS_OS_VERSION}-${GWMS_OS_KRNL_ARCH} with kernel $GWMS_OS_KRNL_NUM-$GWMS_OS_KRNL_PATCH_NUM" + loginfo "Found $GWMS_OS_DISTRO${GWMS_OS_VERSION}-${GWMS_OS_KRNL_ARCH} with kernel $GWMS_OS_KRNL_NUM-$GWMS_OS_KRNL_PATCH_NUM" } log_all_system_info () { - # DESCRIPTION: This function prints all the necessary system information - # stored in common and result variables (see perform_system_check - # function) for easy debugging. This has been done as collecting - # information about the worker node can be useful for troubleshooting - # and gathering stats about what is out there. - # - # INPUT(S): None - # RETURN(S): Prints user-friendly messages to STDOUT - - loginfo "..." - loginfo "Worker node details: " - loginfo "Operating system distro: $GWMS_OS_DISTRO" - loginfo "Operating System version: $GWMS_OS_VERSION" - loginfo "Kernel Architecture: $GWMS_OS_KRNL_ARCH" - loginfo "Kernel version: $GWMS_OS_KRNL_VER" - loginfo "Kernel major revision: $GWMS_OS_KRNL_MAJOR_REV" - loginfo "Kernel minor revision: $GWMS_OS_KRNL_MINOR_REV" - loginfo "Kernel patch number: $GWMS_OS_KRNL_PATCH_NUM" - - loginfo "CVMFS locally installed: $(check_exit_status $GWMS_IS_CVMFS_MNT)" - loginfo "Unprivileged user namespaces supported: $(check_exit_status $GWMS_IS_UNPRIV_USERNS_SUPPORTED)" - loginfo "Unprivileged user namespaces enabled: $(check_exit_status $GWMS_IS_UNPRIV_USERNS_ENABLED)" - loginfo "FUSE installed: $(check_exit_status $GWMS_IS_FUSE_INSTALLED)" - loginfo "fusermount available: $(check_exit_status $GWMS_IS_FUSERMOUNT)" - loginfo "Is the user in 'fuse' group: $(check_exit_status $GWMS_IS_USR_IN_FUSE_GRP)" - loginfo "..." + # DESCRIPTION: This function prints all the necessary system information + # stored in common and result variables (see perform_system_check + # function) for easy debugging. This has been done as collecting + # information about the worker node can be useful for troubleshooting + # and gathering stats about what is out there. + # + # INPUT(S): None + # RETURN(S): Prints user-friendly messages to STDOUT + + loginfo "..." + loginfo "Worker node details: " + loginfo "Hostname: $(hostname)" + loginfo "Operating system distro: $GWMS_OS_DISTRO" + loginfo "Operating System version: $GWMS_OS_VERSION" + loginfo "Kernel Architecture: $GWMS_OS_KRNL_ARCH" + loginfo "Kernel version: $GWMS_OS_KRNL_VER" + loginfo "Kernel major revision: $GWMS_OS_KRNL_MAJOR_REV" + loginfo "Kernel minor revision: $GWMS_OS_KRNL_MINOR_REV" + loginfo "Kernel patch number: $GWMS_OS_KRNL_PATCH_NUM" + + loginfo "CVMFS locally installed: $(print_exit_status $GWMS_IS_CVMFS_LOCAL_MNT)" + loginfo "Unprivileged user namespaces supported: $(print_exit_status $GWMS_IS_UNPRIV_USERNS_SUPPORTED)" + loginfo "Unprivileged user namespaces enabled: $(print_exit_status $GWMS_IS_UNPRIV_USERNS_ENABLED)" + loginfo "FUSE installed: $(print_exit_status $GWMS_IS_FUSE_INSTALLED)" + loginfo "fusermount available: $(print_exit_status $GWMS_IS_FUSERMOUNT)" + loginfo "Is the user in 'fuse' group: $(print_exit_status $GWMS_IS_USR_IN_FUSE_GRP)" + loginfo "..." } mount_cvmfs_repos () { - # DESCRIPTION: This function mounts all the required and additional - # CVMFS repositories that would be needed for user jobs. - # - # INPUT(S): 1. CVMFS configuration repository (string); 2. Additional CVMFS - # repositories (colon-delimited string) - # RETURN(S): Mounts the defined repositories on the worker node filesystem - - $glidein_cvmfsexec_dir/$dist_file $1 -- echo "setting up mount utilities..." &> /dev/null - if [[ $(df -h|grep /cvmfs|wc -l) -eq 1 ]]; then - loginfo "CVMFS config repo already mounted!" - continue - else - # mounting the configuration repo (pre-requisite) - loginfo "Mounting CVMFS config repo now..." - $glidein_cvmfsexec_dir/.cvmfsexec/mountrepo $1 - fi + # DESCRIPTION: This function mounts all the required and additional + # CVMFS repositories that would be needed for user jobs. + # + # INPUT(S): + # 1. cvmfsexec mode (integer) + # 2. CVMFS configuration repository (string) + # 3. Additional CVMFS repositories (colon-delimited string) + # RETURN(S): Mounts the defined repositories on the worker node filesystem + local cvmfsexec_mode=$1 + local config_repository=$2 + local additional_repos=$3 + + if [[ $cvmfsexec_mode -eq 1 ]]; then + "$glidein_cvmfsexec_dir/$dist_file" "$1" -- echo "setting up mount utilities..." &> /dev/null + fi + # at this point in the execution flow, it would have been determined that cvmfs is not locally available + # which implies no repositories are mounted but only config repo will be mounted + if [[ $(df -h|grep /cvmfs|wc -l) -eq 1 ]]; then + loginfo "CVMFS config repo already mounted!" + else + # mounting the configuration repo (pre-requisite) + loginfo "Mounting CVMFS config repo now..." + [[ $cvmfsexec_mode -eq 1 ]] && "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$config_repository" + [[ $cvmfsexec_mode -eq 3 ]] && $CVMFSMOUNT "$config_repository" + fi + + # using an array to unpack the names of additional CVMFS repositories + # from the colon-delimited string + repos=($(echo $additional_repos | tr ":" "\n")) + + loginfo "Mounting additional CVMFS repositories..." + # mount every repository that was previously unpacked + [[ "$cvmfsexec_mode" -ne 1 && "$cvmfsexec_mode" -ne 3 ]] && { logerror "Invalid cvmfsexec mode: mode $cvmfsexec_mode; aborting!"; return 1; } + for repo in "${repos[@]}" + do + case $cvmfsexec_mode in + 1) + "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$repo" + ;; + 3) + $CVMFSMOUNT "$repo" + ;; + esac + done + + # see if all the repositories got mounted + num_repos_mntd=$(df -h | grep /cvmfs | wc -l) + total_num_repos=$(( ${#repos[@]} + 1 )) + GWMS_IS_CVMFS=0 + if [[ "$num_repos_mntd" -eq "$total_num_repos" ]]; then + loginfo "All CVMFS repositories mounted successfully on the worker node" + echo 0 + else + logwarn "One or more CVMFS repositories might not be mounted on the worker node" + GWMS_IS_CVMFS=1 + echo 1 + fi + + # export this info to the glidein environment after CVMFS is provisioned on demand + gconfig_add GWMS_IS_CVMFS $(print_exit_status $GWMS_IS_CVMFS) + + get_mount_point +} - # using an array to unpack the names of additional CVMFS repositories - # from the colon-delimited string - declare -a cvmfs_repos - repos=($(echo $2 | tr ":" "\n")) - #echo ${repos[@]} - - loginfo "Mounting additional CVMFS repositories..." - # mount every repository that was previously unpacked - for repo in "${repos[@]}" - do - $glidein_cvmfsexec_dir/.cvmfsexec/mountrepo $repo - done - - # see if all the repositories got mounted - num_repos_mntd=`df -h | grep /cvmfs | wc -l` - total_num_repos=$(( ${#repos[@]} + 1 )) - if [ "$num_repos_mntd" -eq "$total_num_repos" ]; then - loginfo "All CVMFS repositories mounted successfully on the worker node" - true - else - logwarn "One or more CVMFS repositories might not be mounted on the worker node" - false - fi - GWMS_IS_CVMFS=$? +get_mount_point() { + # TODO: Verify the findmnt ... will always find the correct CVMFS mount + mount_point=$(findmnt -t fuse -S /dev/fuse | tail -n 1 | cut -d ' ' -f 1 ) + if [[ -n "$mount_point" && "$mount_point" != TARGET* ]]; then + mount_point=$(dirname "$mount_point") + if [[ -n "$mount_point" && "$mount_point" != /cvmfs ]]; then + CVMFS_MOUNT_DIR="$mount_point" + export CVMFS_MOUNT_DIR=$mount_point + gconfig_add CVMFS_MOUNT_DIR "$mount_point" + fi + fi } has_unpriv_userns() { - # DESCRIPTION: This function checks the status of unprivileged user - # namespaces being supported and enabled on the worker node. Depending - # - # INPUT(S): None - # RETURN(S): - # -> true (0) if unpriv userns can be used (supported and enabled), - # false otherwise - # -> status of unpriv userns (unavailable, disabled, enabled, - # error) to stdout - - # make sure that perform_system_check has run - [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - - # determine whether unprivileged user namespaces are supported and/or enabled... - if [[ "${GWMS_IS_UNPRIV_USERNS_ENABLED}" -eq 0 ]]; then - # unprivileged user namespaces is enabled - if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then - # unprivileged user namespaces is supported - loginfo "Unprivileged user namespaces supported and enabled" - echo enabled - else - # unprivileged user namespaces is not supported - logerror "Inconsistent system configuration: unprivileged userns is enabled but not supported" - echo error - fi - true - else - # unprivileged user namespaces is disabled - if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then - # unprivileged user namespaces is supported - logwarn "Unprivileged user namespaces disabled: can be enabled by the root user via sysctl" - echo disabled - else - # unprivileged user namespaces is not supported - logwarn "Unprivileged user namespaces disabled and unsupported: can be supported/enabled only after a system upgrade" - echo unavailable - fi - false + # DESCRIPTION: This function checks the status of unprivileged user + # namespaces being supported and enabled on the worker node. Depending + # + # INPUT(S): None + # RETURN(S): + # -> true (0) if unpriv userns can be used (supported and enabled), + # false otherwise + # -> status of unpriv userns (unavailable, disabled, enabled, + # error) to stdout + + # make sure that perform_system_check has run + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # determine whether unprivileged user namespaces are supported and/or enabled... + if [[ "${GWMS_IS_UNPRIV_USERNS_ENABLED}" -eq 0 ]]; then + # unprivileged user namespaces is enabled + if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then + # unprivileged user namespaces is supported + loginfo "Unprivileged user namespaces supported and enabled" + echo enabled + return 0 fi - + # unprivileged user namespaces is not supported + logerror "Inconsistent system configuration: unprivileged userns is enabled but not supported" + echo error + else + # unprivileged user namespaces is disabled + if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then + # unprivileged user namespaces is supported + logwarn "Unprivileged user namespaces disabled: can be enabled by the root user via sysctl" + echo disabled + else + # unprivileged user namespaces is not supported + logwarn "Unprivileged user namespaces disabled and unsupported: can be supported/enabled only after a system upgrade" + echo unavailable + fi + fi + return 1 } has_fuse() { - # DESCRIPTION: This function checks FUSE-related configurations on the - # worker node. This is a pre-requisite before evaluating whether CVMFS - # is mounted on the filesystem. - # - # FUSE Documentation references: - # https://www.kernel.org/doc/html/latest/filesystems/fuse.html - # https://en.wikipedia.org/wiki/Filesystem_in_Userspace - # - # INPUT(S): None - # RETURN(S): string denoting fuse availability (yes, no, error) - - # make sure that perform_system_check has run - [[ -n "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - - # check what specific configuration of unprivileged user namespaces exists in the system (worker node) - unpriv_userns_config=$(has_unpriv_userns) - - # exit from the script if unprivileged namespaces are not supported but enabled in the kernel - if [[ "${unpriv_userns_config}" == error ]]; then - "$error_gen" -error "`basename $0`" "WN_Resource" "Unprivileged user namespaces are not supported but enabled in the kernel! Check system configuration." - exit 1 - # determine if mountrepo/umountrepo could be used by checking availability of fuse, fusermount and user being in fuse group... - elif [[ "${GWMS_IS_FUSE_INSTALLED}" -eq 0 ]]; then - # fuse is installed - if [[ $unpriv_userns_config == unavailable ]]; then - # unprivileged user namespaces unsupported, i.e. kernels 2.x (scenarios 5b,6b) - if [[ "${GWMS_IS_USR_IN_FUSE_GRP}" -eq 0 ]]; then - # user is in fuse group -> fusermount is available (scenario 6b) - if [[ "${GWMS_IS_FUSERMOUNT}" -ne 0 ]]; then - logwarn "Inconsistent system configuration: fusermount is available with fuse installed and when user is in fuse group" - echo error - else - loginfo "FUSE requirements met by the worker node" - echo yes - fi - else - # user is not in fuse group -> fusermount is unavailable (scenario 5b) - if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - logwarn "Inconsistent system configuration: fusermount is available only when user is in fuse group and fuse is installed" - echo error - else - loginfo "FUSE requirements not satisfied: user is not in fuse group" - echo no - fi - fi - else - # unprivileged user namespaces is either enabled or disabled - if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - # fusermount is available (scenarios 7,8) - loginfo "FUSE requirements met by the worker node" - echo yes - else - # fusermount is not available (scenarios 5a,6a) - logwarn "Inconsistent system configuration: fusermount is available when fuse is installed " - echo error - fi - fi - else - # fuse is not installed - if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - # fusermount is somehow available and user is/is not in fuse group (scenarios 3,4) - logwarn "Inconsistent system configuration: fusermount is only available with fuse and/or when user belongs to the fuse group" - echo error - else - # fusermount is not available and user is/is not in fuse group (scenarios case 1,2) - loginfo "FUSE requirements not satisfied: fusermount is not available" - echo no - fi - fi - + # DESCRIPTION: This function checks FUSE-related configurations on the + # worker node. This is a pre-requisite before evaluating whether CVMFS + # is mounted on the filesystem. + # + # FUSE Documentation references: + # https://www.kernel.org/doc/html/latest/filesystems/fuse.html + # https://en.wikipedia.org/wiki/Filesystem_in_Userspace + # + # INPUT(S): None + # RETURN(S): string denoting fuse availability (yes, no, error) + + # make sure that perform_system_check has run + [[ -n "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # check what specific configuration of unprivileged user namespaces exists in the system (worker node) + local unpriv_userns_status + unpriv_userns_status=$(has_unpriv_userns) + + # exit from the script if unprivileged namespaces are not supported but enabled in the kernel + if [[ "${unpriv_userns_status}" == error ]]; then + "$error_gen" -error "$(basename $0)" "WN_Resource" "Unprivileged user namespaces are not supported but enabled in the kernel! Check system configuration." + exit 1 + fi + + # determine if mountrepo/umountrepo could be used by checking availability of fuse, fusermount and user being in fuse group... + if [[ "${GWMS_IS_FUSE_INSTALLED}" -eq 0 ]]; then + # fuse is not installed + if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then + # fusermount is somehow available and user is/is not in fuse group (scenarios 3,4) + logwarn "Inconsistent system configuration: fusermount is only available with fuse and/or when user belongs to the fuse group" + echo error + else + # fusermount is not available and user is/is not in fuse group (scenarios case 1,2) + loginfo "FUSE requirements not satisfied: fusermount is not available" + echo no + fi + return 1 + fi + + # fuse is installed + local ret_state + if [[ $unpriv_userns_status == unavailable ]]; then + # unprivileged user namespaces unsupported, i.e. kernels 2.x (scenarios 5b,6b) + if [[ "${GWMS_IS_USR_IN_FUSE_GRP}" -eq 0 ]]; then + # user is in fuse group -> fusermount is available (scenario 6b) + if [[ "${GWMS_IS_FUSERMOUNT}" -ne 0 ]]; then + logwarn "Inconsistent system configuration: fusermount is available with fuse installed and when user is in fuse group" + ret_state=error + else + loginfo "FUSE requirements met by the worker node" + ret_state=yes + fi + else + # user is not in fuse group -> fusermount is unavailable (scenario 5b) + if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then + logwarn "Inconsistent system configuration: fusermount is available only when user is in fuse group and fuse is installed" + ret_state=error + else + loginfo "FUSE requirements not satisfied: user is not in fuse group" + ret_state=no + fi + fi + else + # unprivileged user namespaces is either enabled or disabled + if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then + # fusermount is available (scenarios 7,8) + loginfo "FUSE requirements met by the worker node" + ret_state=yes + else + # fusermount is not available (scenarios 5a,6a) + logwarn "Inconsistent system configuration: fusermount is not available when fuse is installed " + ret_state=error + fi + fi + echo ret_state + [[ "$ret_state" == "yes"]] + return } evaluate_worker_node_config () { - # DESCRIPTION: This function evaluates the worker using FUSE and - # unpriv. userns configurations to determine whether CVMFS can be - # mounted using mountrepo utility. - # - # INPUT(S): None - # RETURN(S): string message whether CVMFS can be mounted - - # collect info about FUSE configuration on the worker node - fuse_config_status=$(has_fuse) - - # check fuse related configurations in the system (worker node) - if [[ $fuse_config_status == yes ]]; then - # success; - loginfo "CVMFS can be mounted and unmounted on the worker node using mountrepo/umountrepo utility" - true - elif [[ $fuse_config_status == no ]]; then - # failure; - logerror "CVMFS cannot be mounted on the worker node using mountrepo utility" - false - elif [[ $fuse_config_status == error ]]; then - # inconsistent system configurations detected in the worker node - logerror "Detected inconsistent configurations on the worker node. mountrepo utility cannot be used!!" - false - fi - + # DESCRIPTION: This function evaluates the worker using FUSE and + # unpriv. userns configurations to determine whether CVMFS can be + # mounted using mountrepo utility. + # + # INPUT(S): None + # RETURN(S): string message whether CVMFS can be mounted + + # collect info about FUSE configuration on the worker node + fuse_config_status=$(has_fuse) + + # check fuse related configurations in the system (worker node) + if [[ $fuse_config_status == yes ]]; then + # success; + loginfo "CVMFS can be mounted and unmounted on the worker node using mountrepo/umountrepo utility" + return 0 + fi + if [[ $fuse_config_status == no ]]; then + # failure; + logerror "CVMFS cannot be mounted on the worker node using mountrepo utility" + elif [[ $fuse_config_status == error ]]; then + # inconsistent system configurations detected in the worker node + logerror "Detected inconsistent configurations on the worker node. mountrepo utility cannot be used!!" + fi + return 1 } -perform_cvmfs_mount () { - # reset all variables used in this script's namespace before executing the rest of the script - variables_reset - - # perform checks on the worker node that will be used to assess whether CVMFS can be mounted or not - perform_system_check +prepare_for_cvmfs_mount () { + # reset all variables used in this script's namespace before executing the rest of the script + variables_reset + + # perform checks on the worker node that will be used to assess whether CVMFS can be mounted or not + perform_system_check + + # print/display all information pertaining to system checks performed previously (facilitates easy troubleshooting) + log_all_system_info + + # get the CVMFS source information from in the glidein configuration + cvmfs_source=$(gconfig_get CVMFS_SRC "$glidein_config") + + # get the directory where cvmfsexec is unpacked + glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") + + # get the CVMFS requirement setting passed as one of the factory attributes + glidein_cvmfs=$(gconfig_get GLIDEIN_CVMFS "$glidein_config") + + # # gather the worker node information; perform_system_check sets a few variables that can be helpful here + os_like=$GWMS_OS_DISTRO + os_ver=$(echo "$GWMS_OS_VERSION" | awk -F'.' '{print $1}') + arch=$GWMS_OS_KRNL_ARCH + # construct the name of the cvmfsexec distribution file based on the worker node specs + dist_file=cvmfsexec-${cvmfs_source}-${os_like}${os_ver}-${arch} + # the appropriate distribution file does not have to manually untarred as the glidein setup takes care of this automatically + + loginfo "CVMFS Source = $cvmfs_source" + # initializing CVMFS repositories to a variable for easy modification in the future + case $cvmfs_source in + osg) + GLIDEIN_CVMFS_CONFIG_REPO=config-osg.opensciencegrid.org + GLIDEIN_CVMFS_REPOS=singularity.opensciencegrid.org:cms.cern.ch:oasis.opensciencegrid.org + ;; + egi) + GLIDEIN_CVMFS_CONFIG_REPO=config-egi.egi.eu + GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch:oasis.opensciencegrid.org + ;; + default) + GLIDEIN_CVMFS_CONFIG_REPO=cvmfs-config.cern.ch + GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch:oasis.opensciencegrid.org + ;; + *) + "$error_gen" -error "$(basename "$0")" "WN_Resource" "Invalid factory attribute value specified for CVMFS source." + exit 1 + esac + # (optional) set an environment variable that suggests additional repos to be mounted after config repos are mounted + loginfo "CVMFS Config Repo = $GLIDEIN_CVMFS_CONFIG_REPO" + loginfo "CVMFS Repos = $GLIDEIN_CVMFS_REPOS" +} - # print/display all information pertaining to system checks performed previously (facilitates easy troubleshooting) - log_all_system_info - loginfo "CVMFS Source = $cvmfs_source" - # initializing CVMFS repositories to a variable for easy modification in the future - case $cvmfs_source in - osg) - GLIDEIN_CVMFS_CONFIG_REPO=config-osg.opensciencegrid.org - GLIDEIN_CVMFS_REPOS=singularity.opensciencegrid.org:cms.cern.ch - ;; - egi) - GLIDEIN_CVMFS_CONFIG_REPO=config-egi.egi.eu - GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch - ;; - default) - GLIDEIN_CVMFS_CONFIG_REPO=cvmfs-config.cern.ch - GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch - ;; - *) - "$error_gen" -error "`basename $0`" "WN_Resource" "Invalid factory attribute value specified for CVMFS source." - exit 1 - esac - # (optional) set an environment variable that suggests additional repos to be mounted after config repos are mounted - loginfo "CVMFS Config Repo = $GLIDEIN_CVMFS_CONFIG_REPO" - - # by this point, it would have been established that CVMFS is not locally available - # so, install CVMFS via mountrepo or cvmfsexec - loginfo "CVMFS is NOT locally mounted on the worker node! Mounting now..." - # check the operating system distribution - #if [[ $GWMS_OS_DISTRO = RHEL ]]; then - # evaluate the worker node's system configurations to decide whether CVMFS can be mounted or not - loginfo "Evaluating the worker node..." - # display operating system information - print_os_info - - # assess the worker node based on its existing system configurations and perform next steps accordingly - if evaluate_worker_node_config ; then - # if evaluation was true, then proceed to mount CVMFS - if [[ $glidein_cvmfs = never ]]; then - # do nothing; test the node and print the results but do not even try to mount CVMFS - # just continue with glidein startup - echo $? - "$error_gen" -ok "`basename $0`" "msg" "Not trying to install CVMFS." +perform_cvmfs_mount () { + # by this point, it would have been established that CVMFS is not locally available + # so, install CVMFS via mode 1 of cvmfsexec (mountrepo) or mode 3 of cvmfsexec (cvmfsexec) + loginfo "CVMFS is NOT locally mounted on the worker node! Mounting now..." + # check the operating system distribution + #if [[ $GWMS_OS_DISTRO = RHEL ]]; then + # evaluate the worker node's system configurations to decide whether CVMFS can be mounted or not + loginfo "Evaluating the worker node..." + # display operating system information + print_os_info + + local cvmfsexec_mode=$1 + # check whether the strict requirement of CVMFS mounting is defined (in the factory configuration) + if [[ $glidein_cvmfs = never ]]; then + # do nothing; test the node and print the results but do not even try to mount CVMFS + # just continue with glidein startup + "$error_gen" -ok "$(basename $0)" "msg" "Not trying to install CVMFS." + return 0 + fi + + # if strict requirement of CVMFS mounting is not set to 'never' (i.e. 'required', 'preferred', or 'optional') + # assess the worker node based on its existing system configurations and perform next steps accordingly + if evaluate_worker_node_config ; then + # if evaluation was true, then proceed to mount CVMFS + loginfo "Mounting CVMFS repositories..." + if ! mount_cvmfs_repos $cvmfsexec_mode $GLIDEIN_CVMFS_CONFIG_REPO $GLIDEIN_CVMFS_REPOS ; then + if [[ $glidein_cvmfs = preferred || $glidein_cvmfs = optional ]]; then + # if mount CVMFS is not successful, report a warning/error in the logs and continue with glidein startup + # script status must be OK, otherwise the glidein will fail + "$error_gen" -ok "$(basename $0)" "WN_Resource" "Unable to mount required CVMFS on the worker node. Continuing without CVMFS." + exit 0 + fi + if [[ $glidein_cvmfs = required ]]; then + # if mount CVMFS is not successful, report an error and exit with failure exit code + "$error_gen" -error "$(basename $0)" "WN_Resource" "CVMFS is required but unable to mount CVMFS on the worker node." else - loginfo "Mounting CVMFS repositories..." - if mount_cvmfs_repos $GLIDEIN_CVMFS_CONFIG_REPO $GLIDEIN_CVMFS_REPOS ; then - : - else - if [[ $glidein_cvmfs = required ]]; then - # if mount CVMFS is not successful, report an error and exit with failure exit code - echo $? - "$error_gen" -error "`basename $0`" "WN_Resource" "CVMFS is required but unable to mount CVMFS on the worker node." - exit 1 - elif [[ $glidein_cvmfs = preferred || $glidein_cvmfs = optional ]]; then - # if mount CVMFS is not successful, report a warning/error in the logs and continue with glidein startup - # script status must be OK, otherwise the glidein will fail - echo $? - "$error_gen" -ok "`basename $0`" "WN_Resource" "Unable to mount required CVMFS on the worker node. Continuing without CVMFS." - else - "$error_gen" -error "`basename $0`" "WN_Resource" "Invalid factory attribute value specified for CVMFS requirement." - exit 1 - fi - fi + "$error_gen" -error "$(basename $0)" "WN_Resource" "Invalid factory attribute value specified for CVMFS requirement." fi - else - # if evaluation was false, then exit from this activity of mounting CVMFS - "$error_gen" -error "`basename $0`" "WN_Resource" "Worker node configuration did not pass the evaluation checks. CVMFS will not be mounted." exit 1 fi - #else - # if operating system distribution is non-RHEL (any non-rhel OS) - # display operating system information and a user-friendly message - #print_os_info - #logwarn "This is a non-RHEL OS and is not covered in the implementation yet!" - # ----- Further Implementation: TBD (To Be Done) ----- # - #fi - - #fi - - #loginfo "End log for mounting CVMFS" + else + # if evaluation of the worker node was false, then exit from this activity of mounting CVMFS + "$error_gen" -error "$(basename $0)" "WN_Resource" "Worker node configuration did not pass the evaluation checks. CVMFS will not be mounted." + exit 1 + fi + #else + # if operating system distribution is non-RHEL (any non-rhel OS) + # display operating system information and a user-friendly message + #print_os_info + #logwarn "This is a non-RHEL OS and is not covered in the implementation yet!" + # ----- Further Implementation: TBD (To Be Done) ----- # + #fi + + #fi } diff --git a/creation/web_base/cvmfs_setup.sh b/creation/web_base/cvmfs_setup.sh old mode 100755 new mode 100644 index 982c7add1..96360b7b0 --- a/creation/web_base/cvmfs_setup.sh +++ b/creation/web_base/cvmfs_setup.sh @@ -1,105 +1,163 @@ -#!/bin/bash +#!/usr/bin/env bash # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 -# first parameter passed to this script will always be the glidein configuration file (glidein_config) -glidein_config=$1 - -# import add_config_line function -add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | awk '{print $2}') -# shellcheck source=./add_config_line.source -. "$add_config_line_source" - -# fetch the error reporting helper script -error_gen=$(gconfig_get ERROR_GEN_PATH "$glidein_config") - -# get the cvmfsexec attribute switch value from the config file -use_cvmfsexec=$(gconfig_get GLIDEIN_USE_CVMFSEXEC "$glidein_config") -# TODO: int or string?? if string, make the attribute value case insensitive -#use_cvmfsexec=${use_cvmfsexec,,} - -if [[ $use_cvmfsexec -ne 1 ]]; then - "$error_gen" -ok "$(basename $0)" "msg" "Not using cvmfsexec; skipping setup." - exit 0 -fi - -# if GLIDEIN_USE_CVMFSEXEC is set to 1 - check if CVMFS is locally available in the node -# validate CVMFS by examining the directories within CVMFS... checking just one directory should be sufficient? -# get the glidein work directory location from glidein_config file -work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$glidein_config") -# $PWD=/tmp/glide_xxx and every path is referenced with respect to $PWD - -# source the helper script -# TODO: Is this file somewhere in the source tree? use: # shellcheck source=./cvmfs_helper_funcs.sh -. $work_dir/cvmfs_helper_funcs.sh +printinfo() { + # DESCRIPTION: This function prints informational messages to STDOUT + # along with date/time. + # + # INPUT(S): String containing the message + # RETURN(S): Prints message to STDOUT + + echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t INFO: $1" >&2 +} + +determine_cvmfsexec_mode_usage() { + if [[ $GWMS_IS_UNPRIV_USERNS_SUPPORTED && $GWMS_IS_UNPRIV_USERNS_ENABLED && $GWMS_IS_FUSERMOUNT ]]; then + if [[ $GWMS_OS_KRNL_VER -ge 4 && $GWMS_OS_KRNL_MAJOR_REV -ge 18 || $GWMS_OS_KRNL_VER -ge 3 && $GWMS_OS_KRNL_MAJOR_REV -ge 10 && $GWMS_OS_KRNL_MINOR_REV -ge 0 && $GWMS_OS_KRNL_PATCH_NUM -ge 1127 ]]; then + # cvmfsexec mode 3 can be used + echo 3 # true + else + # cvmfsexec mode 3 unavailable; use mode 1 of cvmfsexec instead + echo 1 # false + fi + else + # User namespaces and/or fuse mounts not available in unprivileged mode + # Defaulting to mode 1 of cvmfsexec + echo 1 # false + fi +} + +is_cvmfs_needed() { + # get the cvmfsexec attribute switch value from the config file + [[ -e "$1" ]] && use_cvmfsexec=$(gconfig_get GLIDEIN_USE_CVMFSEXEC "$1") + # TODO: change this variable to 'GLIDEIN_CVMFS' [convention for external variables] + # TODO: when changed, the GLIDEIN_CVMFS variable takes on possible values from {required, preferred, optional, never} + # TODO: int or string?? if string, make the attribute value case insensitive + #use_cvmfsexec=${use_cvmfsexec,,} + + # source the helper script if use_cvmfsexec variable is not empty + if [[ -z $use_cvmfsexec ]]; then + # printinfo used instead of loginfo (from cvmfs_helper_funcs.sh) because + # helper functions are designed to be downloaded based on conditional download logic; GLIDEIN_USE_CVMFSEXEC should be set to 1 + printinfo "On-demand CVMFS provisioning not requested. Skipping related setup." + "$error_gen" -ok "$(basename $0)" "msg" "On-demand CVMFS provisioning not requested; skipping related setup." + false + # exit 0 + elif [[ $use_cvmfsexec -ne 1 ]]; then + # printinfo used instead of loginfo (from cvmfs_helper_funcs.sh) because + # helper functions are designed to be downloaded based on conditional download logic; GLIDEIN_USE_CVMFSEXEC should be set to 1 + printinfo "Not using on-demand CVMFS provisioning; skipping related setup." + "$error_gen" -ok "$(basename $0)" "msg" "On-demand CVMFS provisioning requested and not used. Skipping related setup." + false + # exit 0 + else + # $use_cvmfsexec -eq 1 + [[ -e "$1" ]] && work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$1") + + # shellcheck source=./cvmfs_helper_funcs.sh + . "$work_dir"/cvmfs_helper_funcs.sh + true + fi +} -variables_reset +is_cvmfs_locally_mounted() { + variables_reset -detect_local_cvmfs + detect_local_cvmfs -# check if CVMFS is already locally mounted... -if [[ $GWMS_IS_CVMFS_MNT -eq 0 ]]; then - # if it is so... - "$error_gen" -ok "$(basename $0)" "msg" "CVMFS is locally mounted on the node; skipping setup." - exit 0 -fi + # check if CVMFS is already locally mounted... + if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then + # if it is so... + loginfo "CVMFS is found locally; skipping on-demand CVMFS setup." + "$error_gen" -ok "$(basename $0)" "msg" "CVMFS is locally mounted on the node; skipping setup using cvmfsexec utilities." + exit 0 + fi -# if CVMFS is not found locally... -# get the CVMFS source information from in the glidein configuration -cvmfs_source=$(gconfig_get CVMFS_SRC "$glidein_config") + loginfo "CVMFS is not found locally on the worker node..." + false +} + +setup_cvmfsexec_use() { + gwms_cvmfsexec_mode=$(determine_cvmfsexec_mode_usage) + if [[ $gwms_cvmfsexec_mode -eq 3 ]]; then + loginfo "cvmfsexec can be used in mode 3" + elif [[ $gwms_cvmfsexec_mode -eq 1 ]]; then + loginfo "cvmfsexec will be used in mode 1 only" + else + logerror "invalid value for GWMS_CVMFSEXEC_MODE" + exit 1 + fi -# get the directory where cvmfsexec is unpacked -glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") + gconfig_add GWMS_CVMFSEXEC_MODE "$gwms_cvmfsexec_mode" +} -# get the CVMFS requirement setting passed as one of the factory attributes -glidein_cvmfs=$(gconfig_get GLIDEIN_CVMFS "$glidein_config") +################################## main ################################# -perform_system_check +# first parameter passed to this script will always be the glidein configuration file (glidein_config) +glidein_config=$1 -# gather the worker node information; perform_system_check sets a few variables that can be helpful here -os_like=$GWMS_OS_DISTRO -os_ver=$(echo $GWMS_OS_VERSION | awk -F'.' '{print $1}') -arch=$GWMS_OS_KRNL_ARCH -# construct the name of the cvmfsexec distribution file based on the worker node specs -dist_file=cvmfsexec-${cvmfs_source}-${os_like}${os_ver}-${arch} -# the appropriate distribution file does not have to manually untarred as the glidein setup takes care of this automatically +# import add_config_line function +add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | cut -d ' ' -f 2-) +# shellcheck source=./add_config_line.source +. "$add_config_line_source" -if [[ $use_cvmfsexec -eq 1 ]]; then - if [[ ! -d "$glidein_cvmfsexec_dir" && ! -f ${glidein_cvmfsexec_dir}/${dist_file} ]]; then - # neither the cvmfsexec directory nor the cvmfsexec distribution is found -- this happens when a directory named 'cvmfsexec' does not exist on the glidein because an appropriate distribution tarball is not found in the list of all the available tarballs and was not unpacked [trying to unpack osg-rhel8 on osg-rhel7 worker node] - # if use_cvmfsexec is set to 1, then warn that cvmfs will not be mounted and flag an error - logerror "Error occured during cvmfs setup: None of the available cvmfsexec distributions is compatible with the worker node specifications." - "$error_gen" -error "$(basename $0)" "WN_Resource" "Error occured during cvmfs setup... no matching cvmfsexec distribution available." - exit 1 - elif [[ -d "$glidein_cvmfsexec_dir" && ! -f ${glidein_cvmfsexec_dir}/${dist_file} ]]; then - # something might have gone wrong during the unpacking of the tarball into the glidein_cvmfsexec_dir - logerror "Something went wrong during the unpacking of the cvmfsexec distribution tarball!" - "$error_gen" -error "$(basename $0)" "WN_Resource" "Error: Something went wrong during the unpacking of the cvmfsexec distribution tarball" +# get the glidein work directory location from glidein_config file +[[ -e "$glidein_config" ]] && error_gen=$(gconfig_get ERROR_GEN_PATH "$glidein_config") + +if is_cvmfs_needed "$glidein_config" ; then + echo "On-demand CVMFS provisioning requested and is being setup..." + + is_cvmfs_locally_mounted + + prepare_for_cvmfs_mount + + setup_cvmfsexec_use + + printinfo "cvmfsexec mode $gwms_cvmfsexec_mode is being used..." + if [[ $gwms_cvmfsexec_mode -eq 3 ]]; then + # before exiting out of this block, do two things... + # one, set a variable indicating this script has been executed once + gwms_cvmfs_reexec="yes" + gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" + + # two, export required variables before reinvoking the glidein... + original_workspace=$(grep -m1 '^GLIDEIN_WORKSPACE_ORIG ' "$glidein_config" | cut -d ' ' -f 2-) + export GLIDEIN_WORKSPACE=$original_workspace + # export some necessary information for use inside cvmfsexec + export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec + export GWMS_CVMFSEXEC_MODE=$gwms_cvmfsexec_mode + export GLIDEIN_WORK_DIR="$work_dir" + export GLIDEIN_CVMFS_CONFIG_REPO="$GLIDEIN_CVMFS_CONFIG_REPO" + export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" + exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" + echo "!!WARNING!! Outside of reinvocation of glidein_startup" # this should not run; here as a safety check for debugging incorrect behavior of exec from previous line + + elif [[ $gwms_cvmfsexec_mode -eq 1 ]]; then + perform_cvmfs_mount $gwms_cvmfsexec_mode + + if [[ $GWMS_IS_CVMFS -ne 0 ]]; then + # Error occurred during mount of CVMFS repositories" + logerror "Error occured during mount of CVMFS repositories." + "$error_gen" -error "$(basename $0)" "WN_Resource" "Mount unsuccessful... CVMFS is still unavailable on the node." + exit 1 + fi + + gwms_cvmfs_reexec="no" + gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" + # exporting the variables as an environment variable for use in glidein reinvocation + export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec + export GWMS_CVMFSEXEC_MODE=$gwms_cvmfsexec_mode + + # CVMFS is now available on the worker node" + loginfo "Proceeding to execute the rest of the glidein setup..." + "$error_gen" -ok "$(basename $0)" "WN_Resource" "CVMFS mounted successfully and is now available." + else + logerror "Invalid value of gwms_cvmfsexec_mode!" exit 1 fi +else + # if CVMFS is not requested/not needed to be setup by the glidein + printinfo "Proceeding to execute the rest of the glidein setup..." fi - -perform_cvmfs_mount - -if [[ $GWMS_IS_CVMFS -ne 0 ]]; then - # Error occurred during mount of CVMFS repositories" - logerror "Error occured during mount of CVMFS repositories." - "$error_gen" -error "$(basename $0)" "WN_Resource" "Mount unsuccessful... CVMFS is still unavailable on the node." - exit 1 -fi - -# TODO: Verify the findmnt ... will always find the correct CVMFS mount -mount_point=$(findmnt -t fuse -S /dev/fuse | tail -n 1 | cut -d ' ' -f 1 ) -if [[ -n "$mount_point" && "$mount_point" != TARGET* ]]; then - mount_point=$(dirname "$mount_point") - if [[ -n "$mount_point" && "$mount_point" != /cvmfs ]]; then - CVMFS_MOUNT_DIR="$mount_point" - export CVMFS_MOUNT_DIR - gconfig_add CVMFS_MOUNT_DIR "$mount_point" - fi -fi - -# CVMFS is now available on the worker node" -loginfo "Proceeding to execute user job..." -"$error_gen" -ok "$(basename $0)" "WN_Resource" "CVMFS mounted successfully and is now available." diff --git a/creation/web_base/cvmfs_umount.sh b/creation/web_base/cvmfs_umount.sh index a34461d2a..273dfd09e 100755 --- a/creation/web_base/cvmfs_umount.sh +++ b/creation/web_base/cvmfs_umount.sh @@ -19,11 +19,8 @@ # Author: # Namratha Urs # -# Version: -# 1.0 -# -echo "Unmounting CVMFS as part of glidein cleanup..." +echo "Starting to unmount CVMFS as part of glidein cleanup..." glidein_config=$1 @@ -40,18 +37,19 @@ use_cvmfsexec=$(gconfig_get GLIDEIN_USE_CVMFSEXEC "$glidein_config") # TODO: int or string? if string, make the attribute value case insensitive #use_cvmfsexec=${use_cvmfsexec,,} -if [[ $use_cvmfsexec -ne 1 ]]; then - "$error_gen" -ok "$(basename $0)" "umnt_msg1" "Not using cvmfsexec; skipping cleanup." - exit 0 -fi - # get the glidein work directory location from glidein_config file work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$glidein_config") # $PWD=/tmp/glide_xxx and every path is referenced with respect to $PWD # source the helper script -# TODO: Is this file somewhere in the source tree? use: # shellcheck source=./cvmfs_helper_funcs.sh -. $work_dir/cvmfs_helper_funcs.sh +# shellcheck source=./cvmfs_helper_funcs.sh +. "$work_dir"/cvmfs_helper_funcs.sh + +if [[ $use_cvmfsexec -ne 1 ]]; then + loginfo "On-demand CVMFS provisioning not requested or not used; skipping cleanup." + "$error_gen" -ok "$(basename $0)" "umnt_msg1" "On-demand CVMFS not requested or not used; skipping cleanup." + exit 0 +fi # get the cvmfsexec directory location glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") @@ -66,20 +64,26 @@ loginfo "Start log for unmounting CVMFS" # check if CVMFS is locally mounted on the worker node detect_local_cvmfs -if [[ $GWMS_IS_CVMFS_MNT -eq 0 ]]; then +if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then # CVMFS is mounted locally in the filesystem; DO NOT UNMOUNT! loginfo "Skipping unmounting of CVMFS as it already is locally provisioned in the node!" "$error_gen" -ok "$(basename $0)" "umnt_msg2" "CVMFS is locally mounted on the node; skipping cleanup." exit 0 fi -loginfo "Unmounting CVMFS mounted by the glidein..." -$glidein_cvmfsexec_dir/.cvmfsexec/umountrepo -a - -if [[ -n "$CVMFS_MOUNT_DIR" ]]; then - CVMFS_MOUNT_DIR= - export CVMFS_MOUNT_DIR - gconfig_add CVMFS_MOUNT_DIR "" +gwms_cvmfsexec_mode=$(grep '^GWMS_CVMFSEXEC_MODE ' "$glidein_config" | awk '{print $2}') +loginfo "Unmounting CVMFS provisioned by the glidein..." +if [[ "$gwms_cvmfsexec_mode" -eq 1 ]]; then + "$glidein_cvmfsexec_dir"/.cvmfsexec/umountrepo -a + + if [[ -n "$CVMFS_MOUNT_DIR" ]]; then + CVMFS_MOUNT_DIR= + export CVMFS_MOUNT_DIR + gconfig_add CVMFS_MOUNT_DIR "" + fi +elif [[ "$gwms_cvmfsexec_mode" -eq 3 ]]; then + loginfo "CVMFS_MOUNT_DIR set to $CVMFS_MOUNT_DIR" + loginfo "CVMFSUMOUNT set to $CVMFSUMOUNT" fi # check again to ensure all CVMFS repositories were unmounted by umountrepo diff --git a/creation/web_base/singularity_lib.sh b/creation/web_base/singularity_lib.sh index 30b47a7a4..755e8ee42 100644 --- a/creation/web_base/singularity_lib.sh +++ b/creation/web_base/singularity_lib.sh @@ -1659,6 +1659,14 @@ singularity_get_image() { [[ -z "$singularity_image" ]] && singularity_image=$(dict_get_first SINGULARITY_IMAGES_DICT) fi + if [[ -n "$CVMFS_MOUNT_DIR" ]]; then + # set things up here since the path needs to be bindmounted inside the container + local mount_home=${CVMFS_MOUNT_DIR/\/dist\/cvmfs/} + local symlink_target + symlink_target=$(readlink $mount_home/dist/${singularity_image#/}) + singularity_image=$mount_home/dist/${symlink_target#/} + fi + # At this point, GWMS_SINGULARITY_IMAGE is still empty, something is wrong if [[ -z "$singularity_image" ]]; then [[ -z "$SINGULARITY_IMAGES_DICT" ]] && warn "No Singularity image available (SINGULARITY_IMAGES_DICT is empty)" || From e7479bca466ba6585e9d013e96e476cb154ab4a7 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Tue, 26 Dec 2023 11:42:04 -0600 Subject: [PATCH 03/12] updated for glidein reinvocation when in mode 3 - also updated syntax for the condition in the helper script --- creation/web_base/cvmfs_helper_funcs.sh | 2 +- creation/web_base/glidein_startup.sh | 1023 +++++++++++++++-------- 2 files changed, 692 insertions(+), 333 deletions(-) diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index cf6aa7d13..e00dcfb19 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -416,7 +416,7 @@ has_fuse() { fi fi echo ret_state - [[ "$ret_state" == "yes"]] + [[ "$ret_state" == "yes" ]] return } diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 640a05121..6c91e0077 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1174,6 +1174,10 @@ fetch_file_base() { START=$(date +%s) if [[ "${ffb_file_type}" = "exec:s" ]]; then "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" + elif [[ "${ffb_file_type}" = "exec:r" ]]; then + echo "INSIDE EXEC:r BLOCK..." + echo "###### Sourcing $ffb_outname script ############" + . "${ffb_outname}" glidein_config "${ffb_id}" else "${ffb_outname}" glidein_config "${ffb_id}" fi @@ -1317,405 +1321,756 @@ usage() { #################### Execution starts here.... ###################### ##################################################################### -# Variables initialized on top of the file - -# Command line options parsing. Storing all in global variables. -# params will contain the full list of parameters -# -param_XXX YYY will become "XXX YYY" -# TODO: can use an array instead? -params="" - -while [ $# -gt 0 ] -do case "$1" in - -factory) glidein_factory="$2";; - -name) glidein_name="$2";; - -entry) glidein_entry="$2";; - -clientname) client_name="$2";; - -clientgroup) client_group="$2";; - -web) repository_url="$2";; - -proxy) proxy_url="$2";; - -dir) work_dir="$2";; - -sign) sign_id="$2";; - -signtype) sign_type="$2";; - -signentry) sign_entry_id="$2";; - -cluster) condorg_cluster="$2";; - -subcluster) condorg_subcluster="$2";; - -submitcredid) glidein_cred_id="$2";; - -schedd) condorg_schedd="$2";; - -descript) descript_file="$2";; - -descriptentry) descript_entry_file="$2";; - -clientweb) client_repository_url="$2";; - -clientwebgroup) client_repository_group_url="$2";; - -clientsign) client_sign_id="$2";; - -clientsigntype) client_sign_type="$2";; - -clientsigngroup) client_sign_group_id="$2";; - -clientdescript) client_descript_file="$2";; - -clientdescriptgroup) client_descript_group_file="$2";; - -slotslayout) slots_layout="$2";; - -v) operation_mode="$2";; - -multiglidein) multi_glidein="$2";; - -multirestart) multi_glidein_restart="$2";; - -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (warn "Unknown option $1"; usage) 1>&2; exit 1 -esac -shift 2 -done - -# make sure we have a valid slots_layout -if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then - slots_layout="fixed" +printenv GWMS_CVMFS_REEXEC # > /dev/null +status=$? +if [[ "$status" -eq 0 ]]; then + echo ".....(RE)START OF GLIDEIN_STARTUP.SH....." + gwms_cvmfs_reexec=$(printenv GWMS_CVMFS_REEXEC | sed s"/ //g") + echo "GWMS_CVMFS_REEXEC (inside reinvocation) set to $gwms_cvmfs_reexec" + echo "work_dir variable (right after invocation): $work_dir" # should print nothing because the reinvocation is happening through an exec else - slots_layout="partitionable" + # echo "glidein_config set to $glidein_config" + echo ".....START OF GLIDEIN_STARTUP.SH....." + gwms_cvmfs_reexec=$(grep "^GWMS_CVMFS_REEXEC " "${glidein_config}" | cut -d ' ' -f 2-) + echo "GWMS_CVMFS_REEXEC set to $gwms_cvmfs_reexec" fi -#################################### -# Cleaup, print out message and exit -work_dir_created=0 -glide_local_tmp_dir_created=0 - -################ -# Parse and verify arguments - -# allow some parameters to change arguments -# multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein -tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") -[ -n "${tmp_par}" ] && multi_glidein=${tmp_par} - -case "${operation_mode}" in - nodebug) - sleep_time=1199 - set_debug=0;; - fast) - sleep_time=150 - set_debug=1;; - check) - sleep_time=150 - set -x - set_debug=2;; - *) - sleep_time=1199 - set_debug=1;; -esac - -if [ -z "${descript_file}" ]; then - warn "Missing descript fname." - usage -fi +if [[ -z "$gwms_cvmfs_reexec" ]]; then + # if GWMS_CVMFS_REEXEC is empty, then this script is being invoked the first time; so proceed with usual glidein setup... -if [ -z "${descript_entry_file}" ]; then - warn "Missing descript fname for entry." - usage -fi + # Variables initialized on top of the file -if [ -z "${glidein_name}" ]; then - warn "Missing gliden name." - usage -fi + # Command line options parsing. Storing all in global variables. + # params will contain the full list of parameters + # -param_XXX YYY will become "XXX YYY" + # TODO: can use an array instead? + params="" -if [ -z "${glidein_entry}" ]; then - warn "Missing glidein entry name." - usage -fi + while [ $# -gt 0 ] + do case "$1" in + -factory) glidein_factory="$2";; + -name) glidein_name="$2";; + -entry) glidein_entry="$2";; + -clientname) client_name="$2";; + -clientgroup) client_group="$2";; + -web) repository_url="$2";; + -proxy) proxy_url="$2";; + -dir) work_dir="$2";; + -sign) sign_id="$2";; + -signtype) sign_type="$2";; + -signentry) sign_entry_id="$2";; + -cluster) condorg_cluster="$2";; + -subcluster) condorg_subcluster="$2";; + -submitcredid) glidein_cred_id="$2";; + -schedd) condorg_schedd="$2";; + -descript) descript_file="$2";; + -descriptentry) descript_entry_file="$2";; + -clientweb) client_repository_url="$2";; + -clientwebgroup) client_repository_group_url="$2";; + -clientsign) client_sign_id="$2";; + -clientsigntype) client_sign_type="$2";; + -clientsigngroup) client_sign_group_id="$2";; + -clientdescript) client_descript_file="$2";; + -clientdescriptgroup) client_descript_group_file="$2";; + -slotslayout) slots_layout="$2";; + -v) operation_mode="$2";; + -multiglidein) multi_glidein="$2";; + -multirestart) multi_glidein_restart="$2";; + -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + *) (warn "Unknown option $1"; usage) 1>&2; exit 1 + esac + shift 2 + done + # make sure we have a valid slots_layout + if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then + slots_layout="fixed" + else + slots_layout="partitionable" + fi + + #################################### + # Cleaup, print out message and exit + work_dir_created=0 + glide_local_tmp_dir_created=0 + + ################ + # Parse and verify arguments + + # allow some parameters to change arguments + # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein + tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") + [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} + + case "${operation_mode}" in + nodebug) + sleep_time=1199 + set_debug=0;; + fast) + sleep_time=150 + set_debug=1;; + check) + sleep_time=150 + set -x + set_debug=2;; + *) + sleep_time=1199 + set_debug=1;; + esac -if [ -z "${repository_url}" ]; then - warn "Missing Web URL." - usage -fi + if [ -z "${descript_file}" ]; then + warn "Missing descript fname." + usage + fi -repository_entry_url="${repository_url}/entry_${glidein_entry}" + if [ -z "${descript_entry_file}" ]; then + warn "Missing descript fname for entry." + usage + fi -if [ -z "${proxy_url}" ]; then - proxy_url="None" -fi + if [ -z "${glidein_name}" ]; then + warn "Missing gliden name." + usage + fi -if [ "${proxy_url}" = "OSG" ]; then - if [ -z "${OSG_SQUID_LOCATION}" ]; then - # if OSG does not define a Squid, then don't use any - proxy_url="None" - warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 - else - proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" + if [ -z "${glidein_entry}" ]; then + warn "Missing glidein entry name." + usage fi -fi -if [ -z "${sign_id}" ]; then - warn "Missing signature." - usage -fi -if [ -z "${sign_entry_id}" ]; then - warn "Missing entry signature." - usage -fi + if [ -z "${repository_url}" ]; then + warn "Missing Web URL." + usage + fi -if [ -z "${sign_type}" ]; then - sign_type="sha1" -fi + repository_entry_url="${repository_url}/entry_${glidein_entry}" -if [ "${sign_type}" != "sha1" ]; then - warn "Unsupported signtype ${sign_type} found." - usage -fi + if [ -z "${proxy_url}" ]; then + proxy_url="None" + fi -if [ -n "${client_repository_url}" ]; then - # client data is optional, user url as a switch - if [ -z "${client_sign_type}" ]; then - client_sign_type="sha1" + if [ "${proxy_url}" = "OSG" ]; then + if [ -z "${OSG_SQUID_LOCATION}" ]; then + # if OSG does not define a Squid, then don't use any + proxy_url="None" + warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 + else + proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" + fi fi - if [ "${client_sign_type}" != "sha1" ]; then - warn "Unsupported clientsigntype ${client_sign_type} found." + if [ -z "${sign_id}" ]; then + warn "Missing signature." usage fi - if [ -z "${client_descript_file}" ]; then - warn "Missing client descript fname." + if [ -z "${sign_entry_id}" ]; then + warn "Missing entry signature." usage fi - if [ -n "${client_repository_group_url}" ]; then - # client group data is optional, user url as a switch - if [ -z "${client_group}" ]; then - warn "Missing client group name." + if [ -z "${sign_type}" ]; then + sign_type="sha1" + fi + + if [ "${sign_type}" != "sha1" ]; then + warn "Unsupported signtype ${sign_type} found." + usage + fi + + if [ -n "${client_repository_url}" ]; then + # client data is optional, user url as a switch + if [ -z "${client_sign_type}" ]; then + client_sign_type="sha1" + fi + + if [ "${client_sign_type}" != "sha1" ]; then + warn "Unsupported clientsigntype ${client_sign_type} found." usage fi - if [ -z "${client_descript_group_file}" ]; then - warn "Missing client descript fname for group." + if [ -z "${client_descript_file}" ]; then + warn "Missing client descript fname." usage fi + + if [ -n "${client_repository_group_url}" ]; then + # client group data is optional, user url as a switch + if [ -z "${client_group}" ]; then + warn "Missing client group name." + usage + fi + + if [ -z "${client_descript_group_file}" ]; then + warn "Missing client descript fname for group." + usage + fi + fi fi -fi -# Generate glidein UUID -if command -v uuidgen >/dev/null 2>&1; then - glidein_uuid="$(uuidgen)" -else - glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" -fi + # Generate glidein UUID + if command -v uuidgen >/dev/null 2>&1; then + glidein_uuid="$(uuidgen)" + else + glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" + fi + + # Print initial variables values (argumants and environment) + startup_time="$(date +%s)" + echo "Starting glidein_startup.sh at $(date) (${startup_time})" + + echo "script_checksum = '$(md5wrapper "$0")'" + echo "debug_mode = '${operation_mode}'" + echo "condorg_cluster = '${condorg_cluster}'" + echo "condorg_subcluster= '${condorg_subcluster}'" + echo "condorg_schedd = '${condorg_schedd}'" + echo "glidein_uuid = '${glidein_uuid}'" + echo "glidein_credential_id = '${glidein_cred_id}'" + echo "glidein_factory = '${glidein_factory}'" + echo "glidein_name = '${glidein_name}'" + echo "glidein_entry = '${glidein_entry}'" + if [ -n "${client_name}" ]; then + # client name not required as it is not used for anything but debug info + echo "client_name = '${client_name}'" + fi + if [ -n "${client_group}" ]; then + echo "client_group = '${client_group}'" + fi + echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" + echo "work_dir = '${work_dir}'" + echo "web_dir = '${repository_url}'" + echo "sign_type = '${sign_type}'" + echo "proxy_url = '${proxy_url}'" + echo "descript_fname = '${descript_file}'" + echo "descript_entry_fname = '${descript_entry_file}'" + echo "sign_id = '${sign_id}'" + echo "sign_entry_id = '${sign_entry_id}'" + if [ -n "${client_repository_url}" ]; then + echo "client_web_dir = '${client_repository_url}'" + echo "client_descript_fname = '${client_descript_file}'" + echo "client_sign_type = '${client_sign_type}'" + echo "client_sign_id = '${client_sign_id}'" + if [ -n "${client_repository_group_url}" ]; then + echo "client_web_group_dir = '${client_repository_group_url}'" + echo "client_descript_group_fname = '${client_descript_group_file}'" + echo "client_sign_group_id = '${client_sign_group_id}'" + fi + fi + echo + echo "Running on $(uname -n)" + echo "System: $(uname -a)" + if [ -e '/etc/redhat-release' ]; then + echo "Release: $(cat /etc/redhat-release 2>&1)" + fi + echo "As: $(id)" + echo "PID: $$" + echo + + if [ ${set_debug} -ne 0 ]; then + echo "------- Initial environment ---------------" 1>&2 + env 1>&2 + echo "------- =================== ---------------" 1>&2 + fi + + # Before anything else, spawn multiple glideins and wait, if asked to do so + if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then + # start multiple glideins + ON_DIE=0 + trap 'ignore_signal' SIGHUP + trap_with_arg 'on_die_multi' SIGTERM SIGINT SIGQUIT + do_start_all "${multi_glidein}" + # Wait for all glideins and exit 0 + # TODO: Summarize exit codes and status from all child glideins + echo "------ Multi-glidein parent waiting for child processes (${GWMS_MULTIGLIDEIN_CHILDS}) ----------" 1>&2 + wait + echo "------ Exiting multi-glidein parent ----------" 1>&2 + exit 0 + fi + + ######################################## + # make sure nobody else can write my files + # In the Grid world I cannot trust anybody + if ! umask 0022; then + early_glidein_failure "Failed in umask 0022" + fi + + ######################################## + # Setup OSG and/or Globus + if [ -r "${OSG_GRID}/setup.sh" ]; then + . "${OSG_GRID}/setup.sh" + else + if [ -r "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" ]; then + . "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" + fi + fi -# Print initial variables values (argumants and environment) -startup_time="$(date +%s)" -echo "Starting glidein_startup.sh at $(date) (${startup_time})" - -echo "script_checksum = '$(md5wrapper "$0")'" -echo "debug_mode = '${operation_mode}'" -echo "condorg_cluster = '${condorg_cluster}'" -echo "condorg_subcluster= '${condorg_subcluster}'" -echo "condorg_schedd = '${condorg_schedd}'" -echo "glidein_uuid = '${glidein_uuid}'" -echo "glidein_credential_id = '${glidein_cred_id}'" -echo "glidein_factory = '${glidein_factory}'" -echo "glidein_name = '${glidein_name}'" -echo "glidein_entry = '${glidein_entry}'" -if [ -n "${client_name}" ]; then - # client name not required as it is not used for anything but debug info - echo "client_name = '${client_name}'" -fi -if [ -n "${client_group}" ]; then - echo "client_group = '${client_group}'" -fi -echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" -echo "work_dir = '${work_dir}'" -echo "web_dir = '${repository_url}'" -echo "sign_type = '${sign_type}'" -echo "proxy_url = '${proxy_url}'" -echo "descript_fname = '${descript_file}'" -echo "descript_entry_fname = '${descript_entry_file}'" -echo "sign_id = '${sign_id}'" -echo "sign_entry_id = '${sign_entry_id}'" -if [ -n "${client_repository_url}" ]; then - echo "client_web_dir = '${client_repository_url}'" - echo "client_descript_fname = '${client_descript_file}'" - echo "client_sign_type = '${client_sign_type}'" - echo "client_sign_id = '${client_sign_id}'" - if [ -n "${client_repository_group_url}" ]; then - echo "client_web_group_dir = '${client_repository_group_url}'" - echo "client_descript_group_fname = '${client_descript_group_file}'" - echo "client_sign_group_id = '${client_sign_group_id}'" + if [ -z "${GLOBUS_PATH}" ]; then + if [ -z "${GLOBUS_LOCATION}" ]; then + # if GLOBUS_LOCATION not defined, try to guess it + if [ -r "/opt/globus/etc/globus-user-env.sh" ]; then + GLOBUS_LOCATION=/opt/globus + elif [ -r "/osgroot/osgcore/globus/etc/globus-user-env.sh" ]; then + GLOBUS_LOCATION=/osgroot/osgcore/globus + else + warn "GLOBUS_LOCATION not defined and could not guess it." + warn "Looked in:" + warn ' /opt/globus/etc/globus-user-env.sh' + warn ' /osgroot/osgcore/globus/etc/globus-user-env.sh' + warn 'Continuing like nothing happened' + fi + fi + + if [ -r "${GLOBUS_LOCATION}/etc/globus-user-env.sh" ]; then + . "${GLOBUS_LOCATION}/etc/globus-user-env.sh" + else + warn "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." + warn 'Continuing like nothing happened' + fi fi -fi -echo -echo "Running on $(uname -n)" -echo "System: $(uname -a)" -if [ -e '/etc/redhat-release' ]; then - echo "Release: $(cat /etc/redhat-release 2>&1)" -fi -echo "As: $(id)" -echo "PID: $$" -echo -if [ ${set_debug} -ne 0 ]; then - echo "------- Initial environment ---------------" 1>&2 - env 1>&2 - echo "------- =================== ---------------" 1>&2 -fi + [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath -# Before anything else, spawn multiple glideins and wait, if asked to do so -if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then - # start multiple glideins - ON_DIE=0 - trap 'ignore_signal' SIGHUP - trap_with_arg 'on_die_multi' SIGTERM SIGINT SIGQUIT - do_start_all "${multi_glidein}" - # Wait for all glideins and exit 0 - # TODO: Summarize exit codes and status from all child glideins - echo "------ Multi-glidein parent waiting for child processes (${GWMS_MULTIGLIDEIN_CHILDS}) ----------" 1>&2 - wait - echo "------ Exiting multi-glidein parent ----------" 1>&2 - exit 0 -fi + num_gct=0 -######################################## -# make sure nobody else can write my files -# In the Grid world I cannot trust anybody -if ! umask 0022; then - early_glidein_failure "Failed in umask 0022" -fi -######################################## -# Setup OSG and/or Globus -if [ -r "${OSG_GRID}/setup.sh" ]; then - . "${OSG_GRID}/setup.sh" -else - if [ -r "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" ]; then - . "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" + ######################################## + # prepare and move to the work directory + + # Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . + # Empty $work_dir means PWD (same as ".") + # A custom path could be provided (no "*)" in case) + tmp="${work_dir}" + if [ -z "${work_dir}" ]; then + work_dir="$(pwd)" + else + case "${work_dir}" in + Condor|CONDOR) work_dir="${_CONDOR_SCRATCH_DIR}";; + OSG) work_dir="${OSG_WN_TMP}";; + TMPDIR) work_dir="${TMPDIR}";; + AUTO) automatic_work_dir;; + .) work_dir="$(pwd)";; + esac + fi + + if [ -z "${work_dir}" ]; then + early_glidein_failure "Unable to identify Startup dir for the glidein ($tmp)." + fi + + if [ ! -e "${work_dir}" ]; then + early_glidein_failure "Startup dir '${work_dir}' ($tmp) does not exist." fi -fi -if [ -z "${GLOBUS_PATH}" ]; then - if [ -z "${GLOBUS_LOCATION}" ]; then - # if GLOBUS_LOCATION not defined, try to guess it - if [ -r "/opt/globus/etc/globus-user-env.sh" ]; then - GLOBUS_LOCATION=/opt/globus - elif [ -r "/osgroot/osgcore/globus/etc/globus-user-env.sh" ]; then - GLOBUS_LOCATION=/osgroot/osgcore/globus + start_dir="$(pwd)" + echo "Started in '${start_dir}' ($tmp)" + + work_dir_template="${work_dir}/glide_$(dir_id)XXXXXX" + if ! work_dir="$(mktemp -d "${work_dir_template}")"; then + early_glidein_failure "Cannot create word_dir '${work_dir_template}'" + else + if ! cd "${work_dir}"; then + early_glidein_failure "Work dir '${work_dir}' was created but cannot cd into it." else - warn "GLOBUS_LOCATION not defined and could not guess it." - warn "Looked in:" - warn ' /opt/globus/etc/globus-user-env.sh' - warn ' /osgroot/osgcore/globus/etc/globus-user-env.sh' - warn 'Continuing like nothing happened' + echo "Running in ${work_dir}" fi fi + work_dir_created=1 - if [ -r "${GLOBUS_LOCATION}/etc/globus-user-env.sh" ]; then - . "${GLOBUS_LOCATION}/etc/globus-user-env.sh" + # GWMS_SUBDIR defined on top + GWMS_DIR="${work_dir}/$GWMS_SUBDIR" + if ! mkdir "$GWMS_DIR" ; then + early_glidein_failure "Cannot create GWMS_DIR '$GWMS_DIR'" + fi + gwms_lib_dir="${GWMS_DIR}/lib" + if ! mkdir -p "$gwms_lib_dir" ; then + early_glidein_failure "Cannot create lib dir '$gwms_lib_dir'" + fi + gwms_bin_dir="${GWMS_DIR}/bin" + if ! mkdir -p "$gwms_bin_dir" ; then + early_glidein_failure "Cannot create bin dir '$gwms_bin_dir'" + fi + gwms_exec_dir="${GWMS_DIR}/exec" + if ! mkdir -p "$gwms_exec_dir" ; then + early_glidein_failure "Cannot create exec dir '$gwms_exec_dir'" else - warn "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." - warn 'Continuing like nothing happened' + for i in setup prejob postjob cleanup setup_singularity ; do + mkdir -p "$gwms_exec_dir"/$i + done fi -fi -[ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath + # mktemp makes it user readable by definition (ignores umask) + # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed + if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then + if ! chmod a+rx "${work_dir}"; then + early_glidein_failure "Failed chmod '${work_dir}'" + fi + fi -num_gct=0 + glide_local_tmp_dir_template="/tmp/glide_$(dir_id)$(id -u -n)_XXXXXX" + if ! glide_local_tmp_dir="$(mktemp -d "${glide_local_tmp_dir_template}")"; then + early_glidein_failure "Cannot create temp '${glide_local_tmp_dir_template}'" + fi + glide_local_tmp_dir_created=1 + glide_tmp_dir="${work_dir}/tmp" + if ! mkdir "${glide_tmp_dir}"; then + early_glidein_failure "Cannot create '${glide_tmp_dir}'" + fi -######################################## -# prepare and move to the work directory + if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then + # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed + # the tmpdirs should be world writable + # This way it will work even if the user spawned by the glidein is different than the glidein user + # This happened in GlExec, outside user stays the same in Singularity + if ! chmod 1777 "${glide_local_tmp_dir}"; then + early_glidein_failure "Failed chmod '${glide_local_tmp_dir}'" + fi -# Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . -# Empty $work_dir means PWD (same as ".") -# A custom path could be provided (no "*)" in case) -tmp="${work_dir}" -if [ -z "${work_dir}" ]; then - work_dir="$(pwd)" -else - case "${work_dir}" in - Condor|CONDOR) work_dir="${_CONDOR_SCRATCH_DIR}";; - OSG) work_dir="${OSG_WN_TMP}";; - TMPDIR) work_dir="${TMPDIR}";; - AUTO) automatic_work_dir;; - .) work_dir="$(pwd)";; - esac -fi + if ! chmod 1777 "${glide_tmp_dir}"; then + early_glidein_failure "Failed chmod '${glide_tmp_dir}'" + fi + fi -if [ -z "${work_dir}" ]; then - early_glidein_failure "Unable to identify Startup dir for the glidein ($tmp)." -fi + short_main_dir=main + main_dir="${work_dir}/${short_main_dir}" + if ! mkdir "${main_dir}"; then + early_glidein_failure "Cannot create '${main_dir}'" + fi -if [ ! -e "${work_dir}" ]; then - early_glidein_failure "Startup dir '${work_dir}' ($tmp) does not exist." -fi + short_entry_dir=entry_${glidein_entry} + entry_dir="${work_dir}/${short_entry_dir}" + if ! mkdir "${entry_dir}"; then + early_glidein_failure "Cannot create '${entry_dir}'" + fi -start_dir="$(pwd)" -echo "Started in '${start_dir}' ($tmp)" + if [ -n "${client_repository_url}" ]; then + short_client_dir=client + client_dir="${work_dir}/${short_client_dir}" + if ! mkdir "$client_dir"; then + early_glidein_failure "Cannot create '${client_dir}'" + fi -work_dir_template="${work_dir}/glide_$(dir_id)XXXXXX" -if ! work_dir="$(mktemp -d "${work_dir_template}")"; then - early_glidein_failure "Cannot create word_dir '${work_dir_template}'" -else - if ! cd "${work_dir}"; then - early_glidein_failure "Work dir '${work_dir}' was created but cannot cd into it." - else - echo "Running in ${work_dir}" + if [ -n "${client_repository_group_url}" ]; then + short_client_group_dir=client_group_${client_group} + client_group_dir="${work_dir}/${short_client_group_dir}" + if ! mkdir "${client_group_dir}"; then + early_glidein_failure "Cannot create '${client_group_dir}'" + fi + fi fi -fi -work_dir_created=1 -# GWMS_SUBDIR defined on top -GWMS_DIR="${work_dir}/$GWMS_SUBDIR" -if ! mkdir "$GWMS_DIR" ; then - early_glidein_failure "Cannot create GWMS_DIR '$GWMS_DIR'" -fi -gwms_lib_dir="${GWMS_DIR}/lib" -if ! mkdir -p "$gwms_lib_dir" ; then - early_glidein_failure "Cannot create lib dir '$gwms_lib_dir'" -fi -gwms_bin_dir="${GWMS_DIR}/bin" -if ! mkdir -p "$gwms_bin_dir" ; then - early_glidein_failure "Cannot create bin dir '$gwms_bin_dir'" -fi -gwms_exec_dir="${GWMS_DIR}/exec" -if ! mkdir -p "$gwms_exec_dir" ; then - early_glidein_failure "Cannot create exec dir '$gwms_exec_dir'" -else - for i in setup prejob postjob cleanup setup_singularity ; do - mkdir -p "$gwms_exec_dir"/$i + # Move the token files from condor to glidein workspace + # TODO: compare this w/ setup_x509.sh + # monitoring tokens, Should be using same credentials directory? + mv "${start_dir}/tokens.tgz" . + mv "${start_dir}/url_dirs.desc" . + # idtokens are handled in setup_x509.sh - TODO: remove once verified + #for idtk in ${start_dir}/*.idtoken; do + # if cp "${idtk}" . ; then + # echo "copied idtoken ${idtk} to $(pwd)" + # else + # echo "failed to copy idtoken ${idtk} to $(pwd)" 1>&2 + # fi + #done + #if [ -e "${GLIDEIN_CONDOR_TOKEN}" ]; then + # mkdir -p ticket + # tname="$(basename ${GLIDEIN_CONDOR_TOKEN})" + # cp "${GLIDEIN_CONDOR_TOKEN}" "ticket/${tname}" + # export GLIDEIN_CONDOR_TOKEN="$(pwd)/ticket/${tname}" + #fi + + # Extract and source all the data contained at the end of this script as tarball + extract_all_data + + wrapper_list="${PWD}/wrapper_list.lst" + touch "${wrapper_list}" + + # create glidein_config + glidein_config="${PWD}/glidein_config" + if ! echo > "${glidein_config}"; then + early_glidein_failure "Could not create '${glidein_config}'" + fi + if ! { + echo "# --- glidein_startup vals ---" + echo "GLIDEIN_UUID ${glidein_uuid}" + echo "GLIDEIN_Factory ${glidein_factory}" + echo "GLIDEIN_Name ${glidein_name}" + echo "GLIDEIN_Entry_Name ${glidein_entry}" + + if [ -n "${client_name}" ]; then + # client name not required as it is not used for anything but debug info + echo "GLIDECLIENT_Name ${client_name}" + fi + if [ -n "${client_group}" ]; then + # client group not required as it is not used for anything but debug info + echo "GLIDECLIENT_Group ${client_group}" + fi + echo "GLIDEIN_CredentialIdentifier ${glidein_cred_id}" + echo "CONDORG_CLUSTER ${condorg_cluster}" + echo "CONDORG_SUBCLUSTER ${condorg_subcluster}" + echo "CONDORG_SCHEDD ${condorg_schedd}" + echo "DEBUG_MODE ${set_debug}" + echo "GLIDEIN_STARTUP_PID $$" + echo "GLIDEIN_START_DIR_ORIG ${start_dir}" + echo "GLIDEIN_WORKSPACE_ORIG $(pwd)" + echo "GLIDEIN_WORK_DIR ${main_dir}" + echo "GLIDEIN_ENTRY_WORK_DIR ${entry_dir}" + echo "TMP_DIR ${glide_tmp_dir}" + echo "GLIDEIN_LOCAL_TMP_DIR ${glide_local_tmp_dir}" + echo "PROXY_URL ${proxy_url}" + echo "DESCRIPTION_FILE ${descript_file}" + echo "DESCRIPTION_ENTRY_FILE ${descript_entry_file}" + echo "GLIDEIN_Signature ${sign_id}" + echo "GLIDEIN_Entry_Signature ${sign_entry_id}" + + if [ -n "${client_repository_url}" ]; then + echo "GLIDECLIENT_WORK_DIR ${client_dir}" + echo "GLIDECLIENT_DESCRIPTION_FILE ${client_descript_file}" + echo "GLIDECLIENT_Signature ${client_sign_id}" + if [ -n "${client_repository_group_url}" ]; then + echo "GLIDECLIENT_GROUP_WORK_DIR ${client_group_dir}" + echo "GLIDECLIENT_DESCRIPTION_GROUP_FILE ${client_descript_group_file}" + echo "GLIDECLIENT_Group_Signature ${client_sign_group_id}" + fi + fi + echo "B64UUENCODE_SOURCE ${PWD}/b64uuencode.source" + echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" + echo "GET_ID_SELECTORS_SOURCE ${PWD}/get_id_selectors.source" + echo "LOGGING_UTILS_SOURCE ${PWD}/logging_utils.source" + echo "GLIDEIN_PATHS_SOURCE ${PWD}/glidein_paths.source" + echo "WRAPPER_LIST ${wrapper_list}" + echo "SLOTS_LAYOUT ${slots_layout}" + # Add a line saying we are still initializing... + echo "GLIDEIN_INITIALIZED 0" + # ...but be optimist, and leave advertise_only for the actual error handling script + echo "GLIDEIN_ADVERTISE_ONLY 0" + echo "GLIDEIN_CONDOR_TOKEN ${GLIDEIN_CONDOR_TOKEN}" + echo "# --- User Parameters ---" + } >> "${glidein_config}"; then + early_glidein_failure "Failed in updating '${glidein_config}'" + fi + # shellcheck disable=SC2086 + params2file ${params} + + ############################################ + # Setup logging + log_init "${glidein_uuid}" "${work_dir}" + # Remove these files, if they are still there + rm -rf tokens.tgz url_dirs.desc tokens + log_setup "${glidein_config}" + + echo "Downloading files from Factory and Frontend" + log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" + + ##################################### + # Fetch descript and signature files + + # disable signature check before I get the signature file itself + # disable_check_signature is global + disable_check_signature=yes + + for gs_id in main entry client client_group + do + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no cilent_repository + continue + fi + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no cilent_repository_group + continue + fi + fi + + gs_id_work_dir="$(get_work_dir ${gs_id})" + + # Fetch description file + gs_id_descript_file="$(get_descript_file ${gs_id})" + fetch_file_regular "${gs_id}" "${gs_id_descript_file}" + if ! signature_file_line="$(grep "^signature " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." + glidein_exit 1 + fi + signature_file=$(echo "${signature_file_line}" | cut -s -f 2-) + + # Fetch signature file + gs_id_signature="$(get_signature ${gs_id})" + fetch_file_regular "${gs_id}" "${signature_file}" + echo "${gs_id_signature} ${signature_file}" > "${gs_id_work_dir}/signature.sha1.test" + if ! (cd "${gs_id_work_dir}" && sha1sum -c signature.sha1.test) 1>&2 ; then + warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." + glidein_exit 1 + fi + # for simplicity use a fixed name for signature file + mv "${gs_id_work_dir}/${signature_file}" "${gs_id_work_dir}/signature.sha1" done -fi -# mktemp makes it user readable by definition (ignores umask) -# TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed -if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then - if ! chmod a+rx "${work_dir}"; then - early_glidein_failure "Failed chmod '${work_dir}'" - fi -fi + # re-enable for everything else + disable_check_signature= -glide_local_tmp_dir_template="/tmp/glide_$(dir_id)$(id -u -n)_XXXXXX" -if ! glide_local_tmp_dir="$(mktemp -d "${glide_local_tmp_dir_template}")"; then - early_glidein_failure "Cannot create temp '${glide_local_tmp_dir_template}'" -fi -glide_local_tmp_dir_created=1 + # Now verify the description was not tampered with + # doing it so late should be fine, since nobody should have been able + # to fake the signature file, even if it faked its name in + # the description file + for gs_id in main entry client client_group + do + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no cilent_repository + continue + fi + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no cilent_repository_group + continue + fi + fi -glide_tmp_dir="${work_dir}/tmp" -if ! mkdir "${glide_tmp_dir}"; then - early_glidein_failure "Cannot create '${glide_tmp_dir}'" -fi + gs_id_descript_file="$(get_descript_file ${gs_id})" + if ! check_file_signature "${gs_id}" "${gs_id_descript_file}"; then + gs_id_work_dir="$(get_work_dir ${gs_id})" + warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 + fi + done -if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then - # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed - # the tmpdirs should be world writable - # This way it will work even if the user spawned by the glidein is different than the glidein user - # This happened in GlExec, outside user stays the same in Singularity - if ! chmod 1777 "${glide_local_tmp_dir}"; then - early_glidein_failure "Failed chmod '${glide_local_tmp_dir}'" + ################################################### + # get last_script, as it is used by the fetch_file + gs_id_work_dir="$(get_work_dir main)" + gs_id_descript_file="$(get_descript_file main)" + last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" + if [ -z "${last_script}" ]; then + warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 fi + #cleanup_script="$(grep "^cleanup_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" + cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' ' -f 2-) - if ! chmod 1777 "${glide_tmp_dir}"; then - early_glidein_failure "Failed chmod '${glide_tmp_dir}'" - fi + + ############################## + # Fetch all the other files + for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main precvmfs_file_list" + do + gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" + + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no client_repository + continue + fi + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no client_repository_group + continue + fi + fi + + gs_file_list_id="$(echo "${gs_file_id}" |awk '{print $2}')" + + gs_id_work_dir="$(get_work_dir "${gs_id}")" + gs_id_descript_file="$(get_descript_file "${gs_id}")" + + # extract list file name + if ! gs_file_list_line="$(grep "^${gs_file_list_id} " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_file_list_id:0:11}" = "aftergroup_" ]; then + # afterfile_.. files optional when no client_repository_group + continue + fi + fi + warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 + fi + # space+tab separated file with multiple elements (was: awk '{print $2}', not safe for spaces in file name) + gs_file_list="$(echo "${gs_file_list_line}" | cut -s -f 2 | sed -e 's/[[:space:]]*$//')" + + # fetch list file + fetch_file_regular "${gs_id}" "${gs_file_list}" + + # Fetch files contained in list + # TODO: $file is actually a list, so it cannot be double-quoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here + # shellcheck disable=2086 + while read -r file + do + if [ "${file:0:1}" != "#" ]; then + fetch_file "${gs_id}" $file + fi + done < "${gs_id_work_dir}/${gs_file_list}" + + # Files to go into the GWMS_PATH + if [ "$gs_file_id" = "main at_file_list" ]; then + # setup here to make them available for other setup scripts + add_to_path "$gwms_bin_dir" + # all available now: gwms-python was in main,file_list; condor_chirp is in main,at_file_list + for file in "gwms-python" "condor_chirp" + do + cp "${gs_id_work_dir}/$file" "$gwms_bin_dir"/ + done + cp -r "${gs_id_work_dir}/lib"/* "$gwms_lib_dir"/ + cp "${gs_id_work_dir}/gconfig.py" "$gwms_lib_dir"/python/ + elif [ "$gs_file_id" = "main after_file_list" ]; then + # in case some library has been added/updated + rsync -ar "${gs_id_work_dir}/lib"/ "$gwms_lib_dir"/ + # new knowns binaries? add a loop like above: for file in ... + elif [[ "$gs_file_id" = client* ]]; then + # TODO: gwms25073 this is a workaround until there is an official designation for setup script fragments + [[ -e "${gs_id_work_dir}/setup_prejob.sh" ]] && { cp "${gs_id_work_dir}/setup_prejob.sh" "$gwms_exec_dir"/prejob/ ; chmod a-x "$gwms_exec_dir"/prejob/setup_prejob.sh ; } + fi + done fi -short_main_dir=main -main_dir="${work_dir}/${short_main_dir}" -if ! mkdir "${main_dir}"; then - early_glidein_failure "Cannot create '${main_dir}'" +if [[ -n "$gwms_cvmfs_reexec" && "$gwms_cvmfs_reexec" == "yes" ]]; then + # gwms_cvmfs_reexec is not empty; meaning this block is being run inside of cvmfsexec environment + echo "======================== control flow is inside reinvocation of glidein_startup.sh ========================" + printenv GLIDEIN_CONFIG > /dev/null + status=$? + if [[ ${status} -eq 0 ]]; then + work_dir=$(printenv GLIDEIN_WORK_DIR | sed "s/ //g") + glidein_config=$(printenv GLIDEIN_CONFIG | sed "s/ //g") + repository_url=$(printenv REPOSITORY_URL | sed "s/ //g") + main_dir=$(printenv GWMS_MAIN_DIR | sed "s/ //g") + last_script=$(printenv LAST_SCRIPT | sed "s/ //g") + check_signature=$(printenv CHECK_SIGNATURE | sed "s/ //g") + startup_time=$(printenv STARTUP_TIME | sed "s/ //g") + cvmfs_config_repo=$(printenv GLIDEIN_CVMFS_CONFIG_REPO | sed "s/ //g") + cvmfs_add_repos=$(printenv GLIDEIN_CVMFS_REPOS | sed "s/ //g") + gwms_cvmfsexec_mode=$(printenv GWMS_CVMFSEXEC_MODE | sed "s/ //g") + client_repository_url=$(printenv CLIENT_REPOSITORY_URL | sed "s/ //g") + client_repository_group_url=$(printenv CLIENT_REPOSITORY_GROUP_URL | sed "s/ //g") + wrapper_list=$(printenv WRAPPER_LIST | sed "s/ //g") + gwms_exec_dir=$(printenv GWMS_EXEC_DIR | sed "s/ //g") + fi + + # re-sourcing the helper script inside of cvmfsexec environment + . "$work_dir"/cvmfs_helper_funcs.sh + + mount_cvmfs_repos $gwms_cvmfsexec_mode $gwms_cvmfs_reexec $cvmfs_config_repo $cvmfs_add_repos + echo "GWMS_IS_CVMFS set to $GWMS_IS_CVMFS (mode 3)" + + # check if the cvmfs repos are still mounted inside of reinvocation + df -h + echo "CVMFS_MOUNT_DIR is $(printenv CVMFS_MOUNT_DIR) in mode 3" + + # re-source all the scripts as it'd have been done during the first invocation of this script + extract_all_data + + log_setup "${glidein_config}" fi +echo "*************** BEFORE THE SECOND FOR LOOP ***************" +glidein_debug_options=$(gconfig_get GLIDEIN_DEBUG_OPTIONS "$glidein_config") +glidein_debug_output=$(gconfig_get GLIDEIN_DEBUG_OUTPUT "$glidein_config") +echo "-*-*-*-*-*- glidein_debug_options: $glidein_debug_options -*-*-*-*-*-" +echo "-*-*-*-*-*- glidein_debug_output: $glidein_debug_output -*-*-*-*-*-" +export GLIDEIN_DEBUG_OPTIONS=$glidein_debug_options +export GLIDEIN_DEBUG_OUTPUT=$glidein_debug_output +echo "**********************************************************" + +<<<<<<< HEAD short_entry_dir=entry_${glidein_entry} entry_dir="${work_dir}/${short_entry_dir}" if ! mkdir "${entry_dir}"; then @@ -1934,6 +2289,11 @@ cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" do gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" +======= +for gs_file_id in "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" +do + gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" # one of four possibilities +>>>>>>> c39a07e72 (updated for glidein reinvocation when in mode 3) if [ -z "${client_repository_url}" ]; then if [ "${gs_id}" = "client" ]; then @@ -1969,7 +2329,6 @@ do # fetch list file fetch_file_regular "${gs_id}" "${gs_file_list}" - # Fetch files contained in list # TODO: $file is actually a list, so it cannot be double-quoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here # shellcheck disable=2086 From 661fa36bc0dca57d3843396e9fa005187661ef6c Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Mon, 25 Mar 2024 21:17:39 +0000 Subject: [PATCH 04/12] added changes post code review discussion - complete refactoring of code based on improved logic and better flow handling in exception cases - changed cvmfs-related helper scripts to always be downloaded regardless of whether CVMFS will be mounted or not - added documentation for the helper methods used towards the goal of mounting CVMFS on demand - updated the unmount logic based on the refactored code --- creation/lib/cgWParamDict.py | 16 +- creation/web_base/cvmfs_helper_funcs.sh | 344 ++++++++++++++---------- creation/web_base/cvmfs_setup.sh | 259 +++++++++--------- creation/web_base/cvmfs_umount.sh | 96 +++---- 4 files changed, 385 insertions(+), 330 deletions(-) diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 466beed08..1442b0812 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -303,13 +303,13 @@ def populate(self, other=None): ) self.dicts["untar_cfg"].add(pychirp_tarball, "lib/python/htchirp") - # Add helper scripts for on-demand cvmfs provisioning, conditional upon the attribute GLIDEIN_USE_CVMFSEXEC - # Add cvmfsexec helper script enabled by conditional download + # Add helper scripts for on-demand cvmfs provisioning + # Add cvmfsexec helper script cvmfs_helper = "cvmfs_helper_funcs.sh" self.dicts["file_list"].add_from_file( cvmfs_helper, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(cvmfs_helper), "exec", cond_download="GLIDEIN_USE_CVMFSEXEC" + cWConsts.insert_timestr(cvmfs_helper), "exec", ), os.path.join(cgWConsts.WEB_BASE_DIR, cvmfs_helper), ) @@ -319,7 +319,7 @@ def populate(self, other=None): self.dicts["file_list"].add_from_file( dist_select_script, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(dist_select_script), "exec", cond_download="GLIDEIN_USE_CVMFSEXEC" + cWConsts.insert_timestr(dist_select_script), "exec", ), os.path.join(cgWConsts.WEB_BASE_DIR, dist_select_script), ) @@ -409,7 +409,7 @@ def populate(self, other=None): # TODO: This check could be done in the XML, checking if the entries are consistent in the current version # fetch the on-demand cvmfs provisioning feature setting # if on-demand CVMFS not used at the global level; ignore and continue - ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFSEXEC", 0) + ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFS", 0) # check if on demand cvmfs provisioning is requested/enabled if ondemand_cvmfs != 0: # check the dir containing cvmfsexec distros to see if they were built previously @@ -422,7 +422,7 @@ def populate(self, other=None): else: # can be overridden at the entry level, so ignore and [entry supersedes global setting] print( - "...cvmfsexec distributions unavailable but on-demand CVMFS requested via GLIDEIN_USE_CVMFSEXEC; Continuing..." + "...cvmfsexec distributions unavailable but on-demand CVMFS requested via GLIDEIN_USE_CVMFS; Continuing..." ) # add additional system scripts @@ -734,7 +734,7 @@ def populate(self, entry, schedd, main_dicts): # TODO: This check could be done in the XML, checking if the entries are consistent in the current version # fetch the on-demand cvmfs provisioning feature setting # if on-demand CVMFS not used by entry, ignore and continue - ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFSEXEC", 0) + ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFS", 0) if ondemand_cvmfs != 0: # check the dir containing cvmfsexec distros to see if they were built previously if os.path.exists(os.path.join(self.work_dir, "../cvmfsexec/tarballs")) and os.listdir( @@ -745,7 +745,7 @@ def populate(self, entry, schedd, main_dicts): print("......RECOMMENDED: Rebuild distributions using the latest version of cvmfsexec.") else: print( - "...cvmfsexec distributions unavailable but on-demand CVMFS is requested via GLIDEIN_USE_CVMFSEXEC; Aborting!" + "...cvmfsexec distributions unavailable but on-demand CVMFS is requested via GLIDEIN_USE_CVMFS; Aborting!" ) exit(1) diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index e00dcfb19..6d8684529 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -39,7 +39,10 @@ variables_reset() { # following set of variables used to store operating system and kernel info GWMS_OS_DISTRO= - GWMS_OS_VERSION= + GWMS_OS_NAME= + GWMS_OS_VERSION_FULL= + GWMS_OS_VERSION_MAJOR= + GWMS_OS_VERSION_MINOR= GWMS_OS_KRNL_ARCH= GWMS_OS_KRNL_NUM= GWMS_OS_KRNL_VER= @@ -107,7 +110,13 @@ print_exit_status () { [[ $1 -eq 0 ]] && echo yes || echo no } + detect_local_cvmfs() { + # DESCRIPTION: This function detects whether CVMFS is natively (aka locally) available on the worker node. The result is stored in a common variable, i.e. GWMS_IS_CVMFS_LOCAL_MNT, and can be used downstream. + # + # INPUT(S): None + # RETURN(S): None + CVMFS_ROOT="/cvmfs" repo_name=oasis.opensciencegrid.org # Second check... @@ -120,9 +129,10 @@ detect_local_cvmfs() { GWMS_IS_CVMFS_LOCAL_MNT=1 fi - loginfo "CVMFS locally installed: $(print_exit_status $GWMS_IS_CVMFS_LOCAL_MNT)" + loginfo "Worker node has native CVMFS: $(print_exit_status $GWMS_IS_CVMFS_LOCAL_MNT)" } + perform_system_check() { # DESCRIPTION: This functions performs required system checks (such as # operating system and kernel info, unprivileged user namespaces, FUSE @@ -137,13 +147,28 @@ perform_system_check() { # -> assigns "yes" to GWMS_SYSTEM_CHECK to indicate this function # has been run. - if [[ -f "/etc/redhat-release" ]]; then - GWMS_OS_DISTRO=rhel - else - GWMS_OS_DISTRO=non-rhel + # reset all variables used in this script's namespace before executing the rest of the script + # variables_reset + + # local user_namespaces + if [[ -f '/etc/redhat-release' ]]; then + # rhel derivative; use /etc/redhat-release to fetch the release information + # NOTE: using /etc/redhat-release over /etc/os-release as it is more consistent to rely on + GWMS_OS_DISTRO="rhel" + # GWMS_OS_DISTRO="non-rhel" + #GWMS_OS_VERSION_FULL=$(cat /etc/redhat-release | cut -d " " -f 3) + else + # not a rhel derivative; use /etc/os-release instead [fallback option] + GWMS_OS_DISTRO="non-rhel" + #GWMS_OS_VERSION_FULL=$(cat /etc/os-release | egrep "VERSION_ID" | cut -d = -f 2 | tr -d '"') fi - GWMS_OS_VERSION=$(lsb_release -r | awk -F'\t' '{print $2}') + # source the os-release file to access the variables defined + . /etc/os-release + GWMS_OS_VERSION_FULL=$VERSION_ID + GWMS_OS_VERSION_MAJOR=$(echo "$GWMS_OS_VERSION_FULL" | awk -F'.' '{print $1}') + GWMS_OS_VERSION_MINOR=$(echo "$GWMS_OS_VERSION_FULL" | awk -F'.' '{print $2}') + GWMS_OS_NAME=${NAME,,} GWMS_OS_KRNL_ARCH=$(arch) GWMS_OS_KRNL_NUM=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " ) GWMS_OS_KRNL_VER=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $1}') @@ -151,21 +176,19 @@ perform_system_check() { GWMS_OS_KRNL_MINOR_REV=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $3}') GWMS_OS_KRNL_PATCH_NUM=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 2 -d " ") - #df -h | grep /cvmfs &>/dev/null - #GWMS_IS_CVMFS_LOCAL_MNT=$? # call function to detect local CVMFS only if the GWMS_IS_CVMFS_LOCAL_MNT variable is not set; if the variable is not empty, do nothing [[ -z "${GWMS_IS_CVMFS_LOCAL_MNT}" ]] && detect_local_cvmfs || : - sysctl user.max_user_namespaces &>/dev/null + cat /proc/sys/user/max_user_namespaces &>/dev/null GWMS_IS_UNPRIV_USERNS_SUPPORTED=$? unshare -U true &>/dev/null GWMS_IS_UNPRIV_USERNS_ENABLED=$? - yum list installed *fuse* &>/dev/null + [[ $GWMS_OS_VERSION_MAJOR -ge 9 ]] && dnf list installed fuse3* &>/dev/null || yum list installed fuse &>/dev/null GWMS_IS_FUSE_INSTALLED=$? - - fusermount -V &>/dev/null + + [[ $GWMS_OS_VERSION_MAJOR -ge 9 ]] && fusermount3 -V &>/dev/null || fusermount -V &>/dev/null GWMS_IS_FUSERMOUNT=$? getent group fuse | grep $USER &>/dev/null @@ -183,7 +206,7 @@ print_os_info () { # INPUT(S): None # RETURN(S): Prints a message containing OS and kernel details - loginfo "Found $GWMS_OS_DISTRO${GWMS_OS_VERSION}-${GWMS_OS_KRNL_ARCH} with kernel $GWMS_OS_KRNL_NUM-$GWMS_OS_KRNL_PATCH_NUM" + loginfo "Found $GWMS_OS_NAME [$GWMS_OS_DISTRO] ${GWMS_OS_VERSION_FULL}-${GWMS_OS_KRNL_ARCH} with kernel $GWMS_OS_KRNL_NUM-$GWMS_OS_KRNL_PATCH_NUM" } @@ -197,12 +220,16 @@ log_all_system_info () { # INPUT(S): None # RETURN(S): Prints user-friendly messages to STDOUT + # make sure that perform_system_check has run + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + print_os_info loginfo "..." loginfo "Worker node details: " loginfo "Hostname: $(hostname)" loginfo "Operating system distro: $GWMS_OS_DISTRO" - loginfo "Operating System version: $GWMS_OS_VERSION" - loginfo "Kernel Architecture: $GWMS_OS_KRNL_ARCH" + loginfo "Operating system name: $GWMS_OS_NAME" + loginfo "Operating system version: $GWMS_OS_VERSION_FULL" + loginfo "Kernel architecture: $GWMS_OS_KRNL_ARCH" loginfo "Kernel version: $GWMS_OS_KRNL_VER" loginfo "Kernel major revision: $GWMS_OS_KRNL_MAJOR_REV" loginfo "Kernel minor revision: $GWMS_OS_KRNL_MINOR_REV" @@ -230,20 +257,29 @@ mount_cvmfs_repos () { local cvmfsexec_mode=$1 local config_repository=$2 local additional_repos=$3 + local config_repo_mntd num_repos_mntd total_num_repos + # if using mode 3, config repo should have been mounted already + # otherwise if using mode 1, mount the config repo now... if [[ $cvmfsexec_mode -eq 1 ]]; then - "$glidein_cvmfsexec_dir/$dist_file" "$1" -- echo "setting up mount utilities..." &> /dev/null + "$glidein_cvmfsexec_dir/$dist_file" "$config_repository" -- echo "setting up mount utilities..." &> /dev/null fi # at this point in the execution flow, it would have been determined that cvmfs is not locally available - # which implies no repositories are mounted but only config repo will be mounted + # this implies no repositories should be mounted. However, only config repo will be mounted if in mode 1 or mode 3 by this point if [[ $(df -h|grep /cvmfs|wc -l) -eq 1 ]]; then loginfo "CVMFS config repo already mounted!" else - # mounting the configuration repo (pre-requisite) + # mounting the configuration repo (pre-requisite) in case something went wrong previously loginfo "Mounting CVMFS config repo now..." [[ $cvmfsexec_mode -eq 1 ]] && "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$config_repository" [[ $cvmfsexec_mode -eq 3 ]] && $CVMFSMOUNT "$config_repository" fi + # see if the config repository got mounted + config_repo_mntd=$(df -h | grep /cvmfs | wc -l) + if [[ config_repo_mntd -eq 0 ]]; then + logwarn "One or more CVMFS repositories might not be mounted on the worker node" + return 1 + fi # using an array to unpack the names of additional CVMFS repositories # from the colon-delimited string @@ -270,22 +306,24 @@ mount_cvmfs_repos () { GWMS_IS_CVMFS=0 if [[ "$num_repos_mntd" -eq "$total_num_repos" ]]; then loginfo "All CVMFS repositories mounted successfully on the worker node" - echo 0 + # export this info to the glidein environment after CVMFS is provisioned on demand + gconfig_add GWMS_IS_CVMFS $(print_exit_status $GWMS_IS_CVMFS) + get_mount_point + return 0 else logwarn "One or more CVMFS repositories might not be mounted on the worker node" GWMS_IS_CVMFS=1 - echo 1 + return 1 fi - - # export this info to the glidein environment after CVMFS is provisioned on demand - gconfig_add GWMS_IS_CVMFS $(print_exit_status $GWMS_IS_CVMFS) - - get_mount_point } get_mount_point() { - # TODO: Verify the findmnt ... will always find the correct CVMFS mount + # DESCRIPTION: This function is used to obtain the mount point information regarding where CVMFS is mounted on deman (when mounted). By default, CVMFS when mounted is at '/cvmfs'. Otherwise, CVMFS will be mounted at /.cvmfsexec/dist/cvmfs + # + # INPUT(S): None + # RETURN(S): None + mount_point=$(findmnt -t fuse -S /dev/fuse | tail -n 1 | cut -d ' ' -f 1 ) if [[ -n "$mount_point" && "$mount_point" != TARGET* ]]; then mount_point=$(dirname "$mount_point") @@ -300,7 +338,7 @@ get_mount_point() { has_unpriv_userns() { # DESCRIPTION: This function checks the status of unprivileged user - # namespaces being supported and enabled on the worker node. Depending + # namespaces being supported and enabled on the worker node. # # INPUT(S): None # RETURN(S): @@ -312,26 +350,26 @@ has_unpriv_userns() { # make sure that perform_system_check has run [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - # determine whether unprivileged user namespaces are supported and/or enabled... + # determine whether unprivileged user namespaces are supported and enabled... if [[ "${GWMS_IS_UNPRIV_USERNS_ENABLED}" -eq 0 ]]; then - # unprivileged user namespaces is enabled + # if unprivileged user namespaces is enabled in the system if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then - # unprivileged user namespaces is supported + # check if unprivileged user namespaces is supported by the system loginfo "Unprivileged user namespaces supported and enabled" echo enabled - return 0 - fi - # unprivileged user namespaces is not supported - logerror "Inconsistent system configuration: unprivileged userns is enabled but not supported" + return 0 + fi + # otherwise, if unprivileged user namespaces is not supported by the system + logerror "Inconsistent system configuration: unprivileged usernamespaces is enabled but not supported" echo error else - # unprivileged user namespaces is disabled + # if unprivileged user namespaces is found to be disabled if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then # unprivileged user namespaces is supported logwarn "Unprivileged user namespaces disabled: can be enabled by the root user via sysctl" echo disabled else - # unprivileged user namespaces is not supported + # otherwise, if unprivileged user namespaces is also not supported by the system logwarn "Unprivileged user namespaces disabled and unsupported: can be supported/enabled only after a system upgrade" echo unavailable fi @@ -341,48 +379,34 @@ has_unpriv_userns() { has_fuse() { - # DESCRIPTION: This function checks FUSE-related configurations on the - # worker node. This is a pre-requisite before evaluating whether CVMFS - # is mounted on the filesystem. - # - # FUSE Documentation references: - # https://www.kernel.org/doc/html/latest/filesystems/fuse.html - # https://en.wikipedia.org/wiki/Filesystem_in_Userspace + # DESCRIPTION: This function checks the status of FUSE configuration being available on the worker node. # # INPUT(S): None - # RETURN(S): string denoting fuse availability (yes, no, error) + # RETURN(S): + # -> true (0) if FUSE is available, false otherwise + # -> status of FUSE configuration (no, yes, error) to stdout # make sure that perform_system_check has run [[ -n "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - # check what specific configuration of unprivileged user namespaces exists in the system (worker node) - local unpriv_userns_status - unpriv_userns_status=$(has_unpriv_userns) - - # exit from the script if unprivileged namespaces are not supported but enabled in the kernel - if [[ "${unpriv_userns_status}" == error ]]; then - "$error_gen" -error "$(basename $0)" "WN_Resource" "Unprivileged user namespaces are not supported but enabled in the kernel! Check system configuration." - exit 1 - fi - - # determine if mountrepo/umountrepo could be used by checking availability of fuse, fusermount and user being in fuse group... - if [[ "${GWMS_IS_FUSE_INSTALLED}" -eq 0 ]]; then + # determine which cvmfsexec utilities can be used by checking availability of fuse, fusermount and user being in fuse group... + if [[ "${GWMS_IS_FUSE_INSTALLED}" -ne 0 ]]; then # fuse is not installed - if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - # fusermount is somehow available and user is/is not in fuse group (scenarios 3,4) - logwarn "Inconsistent system configuration: fusermount is only available with fuse and/or when user belongs to the fuse group" + if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then + # fusermount is somehow available and user is/is not in fuse group (scenarios 3,4) + logwarn "Inconsistent system configuration: fusermount is only available with fuse and/or when user belongs to the fuse group" echo error else # fusermount is not available and user is/is not in fuse group (scenarios case 1,2) loginfo "FUSE requirements not satisfied: fusermount is not available" echo no fi - return 1 + return 1 fi - # fuse is installed + # fuse rpm is installed local ret_state - if [[ $unpriv_userns_status == unavailable ]]; then + if [[ $unpriv_userns_status = "unavailable" ]]; then # unprivileged user namespaces unsupported, i.e. kernels 2.x (scenarios 5b,6b) if [[ "${GWMS_IS_USR_IN_FUSE_GRP}" -eq 0 ]]; then # user is in fuse group -> fusermount is available (scenario 6b) @@ -402,40 +426,71 @@ has_fuse() { loginfo "FUSE requirements not satisfied: user is not in fuse group" ret_state=no fi - fi + fi else # unprivileged user namespaces is either enabled or disabled if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - # fusermount is available (scenarios 7,8) + # fuse is installed with fusermount available (scenarios 7,8) loginfo "FUSE requirements met by the worker node" ret_state=yes else - # fusermount is not available (scenarios 5a,6a) + # fuse is installed but fusermount not available (scenarios 5a,6a) logwarn "Inconsistent system configuration: fusermount is not available when fuse is installed " ret_state=error fi fi - echo ret_state + echo $ret_state [[ "$ret_state" == "yes" ]] return } -evaluate_worker_node_config () { - # DESCRIPTION: This function evaluates the worker using FUSE and - # unpriv. userns configurations to determine whether CVMFS can be - # mounted using mountrepo utility. +determine_cvmfsexec_mode_usage() { + # DESCRIPTION: This function is used to determine the cvmfsexec mode that will be applicable based on the worker node specifications, including the status of unprivileged user namespaces and FUSE configuration. # # INPUT(S): None - # RETURN(S): string message whether CVMFS can be mounted + # RETURN(S): + # -> true (0) if it is determined that one of the three cvmfsexec modes can be used, false otherwise + # -> an integer indicating the mode of cvmfsexec that will be possible (0, 1, 2, 3) to stdout + + # make sure that perform_system_check has run + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # check what specific configuration of unprivileged user namespaces exists in the system (worker node) + unpriv_userns_status=$(has_unpriv_userns) - # collect info about FUSE configuration on the worker node + # check FUSE configuration on the worker node fuse_config_status=$(has_fuse) - # check fuse related configurations in the system (worker node) - if [[ $fuse_config_status == yes ]]; then - # success; - loginfo "CVMFS can be mounted and unmounted on the worker node using mountrepo/umountrepo utility" + if [[ "${unpriv_userns_status}" == "error" && "${fuse_config_status}" == "error" ]]; then + logwarn "User namespaces and fuse mounts not available in unprivileged mode" + echo 0 + return 1 + fi + # either 1. if unprivileged user namespaces are enabled (and therefore supported) + if [[ "${unpriv_userns_status}" == "enabled" ]]; then + # satisfies the minimum requirement for mode 3 + if [[ $GWMS_OS_KRNL_VER -ge 5 || $GWMS_OS_KRNL_VER -ge 4 && $GWMS_OS_KRNL_MAJOR_REV -ge 18 || $GWMS_OS_KRNL_VER -ge 3 && $GWMS_OS_KRNL_MAJOR_REV -ge 10 && $GWMS_OS_KRNL_MINOR_REV -ge 0 && $GWMS_OS_KRNL_PATCH_NUM -ge 1127 ]]; then + # if newer kernels >= 4.18 (RHEL8) or >= 3.10.0-1127 (RHEL 7.8), cvmfsexec can be used in mode 3 + echo 3 + return 0 + fi + # if RHEL <= 7.7, cvmfsexec can be used in mode 2 + echo 2 + return 0 + elif [[ "${unpriv_userns_status}" =~ ^(disabled|unavailable)$ ]]; then + # when unpriv. userns are disabled/not available, take FUSE status into consideration + if [[ "${fuse_config_status}" != "yes" ]]; then + # cvmfsexec cannot be used in either of the three modes + loginfo "cvmfsexec cannot be used in either of the three modes!" + echo 0 + return 1 + fi + fi + # or 2. solely based on fuse status on the worker node, determine whether any of the cvmfsexec modes can be used + if [[ "${fuse_config_status}" == "yes" ]]; then + # cvmfsexec can be used in mode 1 + echo 1 return 0 fi if [[ $fuse_config_status == no ]]; then @@ -445,37 +500,70 @@ evaluate_worker_node_config () { # inconsistent system configurations detected in the worker node logerror "Detected inconsistent configurations on the worker node. mountrepo utility cannot be used!!" fi + echo 0 return 1 } -prepare_for_cvmfs_mount () { - # reset all variables used in this script's namespace before executing the rest of the script - variables_reset +setup_cvmfsexec_use() { + # DESCRIPTION: This function performs the necessary setup prior to using cvmfsexec, if possible. If cvmfsexec can be used in either of the three modes, the specific mode information is written to the glidein configuration file. + # + # INPUT(S): None + # RETURN(S): an integer depicting the cvmfsexec mode that is applicable for the worker node. - # perform checks on the worker node that will be used to assess whether CVMFS can be mounted or not - perform_system_check + local gwms_cvmfsexec_mode + # first we perform checks on the worker node that will be used to assess whether cvmfsexec can be used and if yes, which mode of cvmfsexec can be used + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - # print/display all information pertaining to system checks performed previously (facilitates easy troubleshooting) + # second, log the results of the checks that were performed in the previous step log_all_system_info + # finally, using the results obtained from the checks, determine which mode of cvmfsexec can be used + gwms_cvmfsexec_mode=$(determine_cvmfsexec_mode_usage) + if [[ $gwms_cvmfsexec_mode -ne 0 ]]; then + gconfig_add GWMS_CVMFSEXEC_MODE $gwms_cvmfsexec_mode + fi + echo $gwms_cvmfsexec_mode +} + + +prepare_for_cvmfs_mount () { + # DESCRIPTION: This function is used to prepare the necessary items and keep them ready/accessible right before mounting CVMFS on demand. + # + # INPUT(S): None + # RETURN(S): None + + # if not previously performed, perform checks on the worker node that will be used to assess whether CVMFS can be mounted or not + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + # get the CVMFS source information from in the glidein configuration cvmfs_source=$(gconfig_get CVMFS_SRC "$glidein_config") # get the directory where cvmfsexec is unpacked glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") - # get the CVMFS requirement setting passed as one of the factory attributes - glidein_cvmfs=$(gconfig_get GLIDEIN_CVMFS "$glidein_config") - - # # gather the worker node information; perform_system_check sets a few variables that can be helpful here + # gather the worker node information to construct the name of the cvmfsexec distribution file based on the worker node specs + # perform_system_check sets a few variables that can be helpful here os_like=$GWMS_OS_DISTRO - os_ver=$(echo "$GWMS_OS_VERSION" | awk -F'.' '{print $1}') + os_ver=$GWMS_OS_VERSION_MAJOR arch=$GWMS_OS_KRNL_ARCH # construct the name of the cvmfsexec distribution file based on the worker node specs dist_file=cvmfsexec-${cvmfs_source}-${os_like}${os_ver}-${arch} # the appropriate distribution file does not have to manually untarred as the glidein setup takes care of this automatically + if [[ ! -d "$glidein_cvmfsexec_dir" && ! -f "${glidein_cvmfsexec_dir}/${dist_file}" ]]; then + # neither the cvmfsexec directory nor the cvmfsexec distribution is found -- this happens when a directory named 'cvmfsexec' does not exist on the glidein because an appropriate distribution tarball is not found in the list of all the available tarballs and was not unpacked [trying to unpack osg-rhel8 on osg-rhel7 worker node] + # if use_cvmfsexec is set to 1, then warn that cvmfs will not be mounted and flag an error + logerror "Error occured during preparing for cvmfs setup: None of the available cvmfsexec distributions is compatible with the worker node specifications." + "$error_gen" -error "$(basename $0)" "WN_Resource" "Error occured during cvmfs setup... no matching cvmfsexec distribution available." + exit 1 + elif [[ -d "$glidein_cvmfsexec_dir" && ! -f "${glidein_cvmfsexec_dir}/${dist_file}" ]]; then + # something might have gone wrong during the unpacking of the tarball into the glidein_cvmfsexec_dir + logerror "Something went wrong during the unpacking of the cvmfsexec distribution tarball!" + "$error_gen" -error "$(basename $0)" "WN_Resource" "Error: Something went wrong during the unpacking of the cvmfsexec distribution tarball" + exit 1 + fi + loginfo "CVMFS Source = $cvmfs_source" # initializing CVMFS repositories to a variable for easy modification in the future case $cvmfs_source in @@ -502,57 +590,37 @@ prepare_for_cvmfs_mount () { perform_cvmfs_mount () { - # by this point, it would have been established that CVMFS is not locally available - # so, install CVMFS via mode 1 of cvmfsexec (mountrepo) or mode 3 of cvmfsexec (cvmfsexec) - loginfo "CVMFS is NOT locally mounted on the worker node! Mounting now..." - # check the operating system distribution - #if [[ $GWMS_OS_DISTRO = RHEL ]]; then - # evaluate the worker node's system configurations to decide whether CVMFS can be mounted or not - loginfo "Evaluating the worker node..." - # display operating system information - print_os_info + # DESCRIPTION: This function serves as a wrapper for performing mounting of CVMFS on demand depending on a few factors. + # + # INPUT(S): an integer denoting the selected cvmfsexec mode + # RETURN(S): to stdout one of the following values: + # -> true (0) if CVMFS was mounted successfully without any errors + # -> false (1) if CVMFS was not mounted successfully due to errors + # -> false (2) if CVMFS was not mounted because the OS distribution found on the worker node is not rhel-based (only RHEL distros are supported as of now). + + local mode=$1 + # if not previously performed, assess the worker node based on its existing system configurations and perform next steps to mount CVMFS + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - local cvmfsexec_mode=$1 - # check whether the strict requirement of CVMFS mounting is defined (in the factory configuration) - if [[ $glidein_cvmfs = never ]]; then - # do nothing; test the node and print the results but do not even try to mount CVMFS - # just continue with glidein startup - "$error_gen" -ok "$(basename $0)" "msg" "Not trying to install CVMFS." - return 0 + # if strict requirement of CVMFS mounting is not set to 'never' (i.e. 'required' or 'preferred') + # by this point, it would have been established that CVMFS is not locally available, so install CVMFS via one of the three modes of cvmfsexec + loginfo "Mounting CVMFS on demand using mode $mode of cvmfsexec" + # check the operating system distribution + if [[ "${GWMS_OS_DISTRO}" != "rhel" ]]; then + # if operating system distribution is non-RHEL (any non-rhel OS) + print_os_info + logwarn "This is a non-RHEL OS and is not covered in the implementation yet!" + return 2 + # ----- Further Implementation: TBD (To Be Done) ----- # fi - - # if strict requirement of CVMFS mounting is not set to 'never' (i.e. 'required', 'preferred', or 'optional') - # assess the worker node based on its existing system configurations and perform next steps accordingly - if evaluate_worker_node_config ; then - # if evaluation was true, then proceed to mount CVMFS - loginfo "Mounting CVMFS repositories..." - if ! mount_cvmfs_repos $cvmfsexec_mode $GLIDEIN_CVMFS_CONFIG_REPO $GLIDEIN_CVMFS_REPOS ; then - if [[ $glidein_cvmfs = preferred || $glidein_cvmfs = optional ]]; then - # if mount CVMFS is not successful, report a warning/error in the logs and continue with glidein startup - # script status must be OK, otherwise the glidein will fail - "$error_gen" -ok "$(basename $0)" "WN_Resource" "Unable to mount required CVMFS on the worker node. Continuing without CVMFS." - exit 0 - fi - if [[ $glidein_cvmfs = required ]]; then - # if mount CVMFS is not successful, report an error and exit with failure exit code - "$error_gen" -error "$(basename $0)" "WN_Resource" "CVMFS is required but unable to mount CVMFS on the worker node." - else - "$error_gen" -error "$(basename $0)" "WN_Resource" "Invalid factory attribute value specified for CVMFS requirement." - fi - exit 1 - fi - else - # if evaluation of the worker node was false, then exit from this activity of mounting CVMFS - "$error_gen" -error "$(basename $0)" "WN_Resource" "Worker node configuration did not pass the evaluation checks. CVMFS will not be mounted." - exit 1 + + prepare_for_cvmfs_mount + if [[ $mode -eq 3 || $mode -eq 2 ]]; then + return # only prepare but do not actually mount (later in glidein reinvocation, mounting will be performed) + fi + loginfo "Mounting CVMFS repositories..." + if ! mount_cvmfs_repos $mode $GLIDEIN_CVMFS_CONFIG_REPO $GLIDEIN_CVMFS_REPOS ; then + return 1 fi - #else - # if operating system distribution is non-RHEL (any non-rhel OS) - # display operating system information and a user-friendly message - #print_os_info - #logwarn "This is a non-RHEL OS and is not covered in the implementation yet!" - # ----- Further Implementation: TBD (To Be Done) ----- # - #fi - - #fi + return 0 } diff --git a/creation/web_base/cvmfs_setup.sh b/creation/web_base/cvmfs_setup.sh index 96360b7b0..d44818bff 100644 --- a/creation/web_base/cvmfs_setup.sh +++ b/creation/web_base/cvmfs_setup.sh @@ -3,94 +3,15 @@ # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 -printinfo() { - # DESCRIPTION: This function prints informational messages to STDOUT - # along with date/time. - # - # INPUT(S): String containing the message - # RETURN(S): Prints message to STDOUT - - echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t INFO: $1" >&2 -} - -determine_cvmfsexec_mode_usage() { - if [[ $GWMS_IS_UNPRIV_USERNS_SUPPORTED && $GWMS_IS_UNPRIV_USERNS_ENABLED && $GWMS_IS_FUSERMOUNT ]]; then - if [[ $GWMS_OS_KRNL_VER -ge 4 && $GWMS_OS_KRNL_MAJOR_REV -ge 18 || $GWMS_OS_KRNL_VER -ge 3 && $GWMS_OS_KRNL_MAJOR_REV -ge 10 && $GWMS_OS_KRNL_MINOR_REV -ge 0 && $GWMS_OS_KRNL_PATCH_NUM -ge 1127 ]]; then - # cvmfsexec mode 3 can be used - echo 3 # true - else - # cvmfsexec mode 3 unavailable; use mode 1 of cvmfsexec instead - echo 1 # false - fi - else - # User namespaces and/or fuse mounts not available in unprivileged mode - # Defaulting to mode 1 of cvmfsexec - echo 1 # false - fi -} - -is_cvmfs_needed() { - # get the cvmfsexec attribute switch value from the config file - [[ -e "$1" ]] && use_cvmfsexec=$(gconfig_get GLIDEIN_USE_CVMFSEXEC "$1") - # TODO: change this variable to 'GLIDEIN_CVMFS' [convention for external variables] - # TODO: when changed, the GLIDEIN_CVMFS variable takes on possible values from {required, preferred, optional, never} - # TODO: int or string?? if string, make the attribute value case insensitive - #use_cvmfsexec=${use_cvmfsexec,,} - - # source the helper script if use_cvmfsexec variable is not empty - if [[ -z $use_cvmfsexec ]]; then - # printinfo used instead of loginfo (from cvmfs_helper_funcs.sh) because - # helper functions are designed to be downloaded based on conditional download logic; GLIDEIN_USE_CVMFSEXEC should be set to 1 - printinfo "On-demand CVMFS provisioning not requested. Skipping related setup." - "$error_gen" -ok "$(basename $0)" "msg" "On-demand CVMFS provisioning not requested; skipping related setup." - false - # exit 0 - elif [[ $use_cvmfsexec -ne 1 ]]; then - # printinfo used instead of loginfo (from cvmfs_helper_funcs.sh) because - # helper functions are designed to be downloaded based on conditional download logic; GLIDEIN_USE_CVMFSEXEC should be set to 1 - printinfo "Not using on-demand CVMFS provisioning; skipping related setup." - "$error_gen" -ok "$(basename $0)" "msg" "On-demand CVMFS provisioning requested and not used. Skipping related setup." - false - # exit 0 - else - # $use_cvmfsexec -eq 1 - [[ -e "$1" ]] && work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$1") - - # shellcheck source=./cvmfs_helper_funcs.sh - . "$work_dir"/cvmfs_helper_funcs.sh - true - fi -} - is_cvmfs_locally_mounted() { + # checking if CVMFS is natively available variables_reset - detect_local_cvmfs - - # check if CVMFS is already locally mounted... if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then # if it is so... - loginfo "CVMFS is found locally; skipping on-demand CVMFS setup." - "$error_gen" -ok "$(basename $0)" "msg" "CVMFS is locally mounted on the node; skipping setup using cvmfsexec utilities." - exit 0 - fi - - loginfo "CVMFS is not found locally on the worker node..." - false -} - -setup_cvmfsexec_use() { - gwms_cvmfsexec_mode=$(determine_cvmfsexec_mode_usage) - if [[ $gwms_cvmfsexec_mode -eq 3 ]]; then - loginfo "cvmfsexec can be used in mode 3" - elif [[ $gwms_cvmfsexec_mode -eq 1 ]]; then - loginfo "cvmfsexec will be used in mode 1 only" - else - logerror "invalid value for GWMS_CVMFSEXEC_MODE" - exit 1 + return 0 fi - - gconfig_add GWMS_CVMFSEXEC_MODE "$gwms_cvmfsexec_mode" + return 1 } ################################## main ################################# @@ -106,58 +27,138 @@ add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | # get the glidein work directory location from glidein_config file [[ -e "$glidein_config" ]] && error_gen=$(gconfig_get ERROR_GEN_PATH "$glidein_config") -if is_cvmfs_needed "$glidein_config" ; then - echo "On-demand CVMFS provisioning requested and is being setup..." - - is_cvmfs_locally_mounted - - prepare_for_cvmfs_mount - - setup_cvmfsexec_use - - printinfo "cvmfsexec mode $gwms_cvmfsexec_mode is being used..." - if [[ $gwms_cvmfsexec_mode -eq 3 ]]; then - # before exiting out of this block, do two things... - # one, set a variable indicating this script has been executed once - gwms_cvmfs_reexec="yes" - gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" - - # two, export required variables before reinvoking the glidein... - original_workspace=$(grep -m1 '^GLIDEIN_WORKSPACE_ORIG ' "$glidein_config" | cut -d ' ' -f 2-) - export GLIDEIN_WORKSPACE=$original_workspace - # export some necessary information for use inside cvmfsexec - export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec - export GWMS_CVMFSEXEC_MODE=$gwms_cvmfsexec_mode - export GLIDEIN_WORK_DIR="$work_dir" - export GLIDEIN_CVMFS_CONFIG_REPO="$GLIDEIN_CVMFS_CONFIG_REPO" - export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" - exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" - echo "!!WARNING!! Outside of reinvocation of glidein_startup" # this should not run; here as a safety check for debugging incorrect behavior of exec from previous line +[[ -e "$glidein_config" ]] && work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$1") +# shellcheck source=./cvmfs_helper_funcs.sh +. "$work_dir"/cvmfs_helper_funcs.sh + +# get the use_cvmfs attribute value; passed as one of the frontend attributes +use_cvmfs=$(gconfig_get GLIDEIN_USE_CVMFS "$1") +if [[ -z $use_cvmfs ]]; then + loginfo "CVMFS not requested (GLIDEIN_USE_CVMFS: $use_cvmfs); skipping CVMFS setup." + "$error_gen" -ok "$(basename $0)" "mnt_msg1" "CVMFS not requested; skipping setup." + return 0 +elif ! [[ $use_cvmfs =~ ^[0-1]$ ]]; then + # TODO: add this check at the xml level maybe? + logerror "Invalid attribute value: GLIDEIN_USE_CVMFS = ${use_cvmfs}" + "$error_gen" -error "$(basename $0)" "mnt_msg2" "Invalid attribute value: GLIDEIN_USE_CVMFS = ${use_cvmfs}" + exit 1 +fi - elif [[ $gwms_cvmfsexec_mode -eq 1 ]]; then - perform_cvmfs_mount $gwms_cvmfsexec_mode +# get the CVMFS requirement setting; passed as one of the factory attributes +glidein_cvmfs_require=$(gconfig_get GLIDEIN_CVMFS_REQUIRE "$glidein_config") +glidein_cvmfs_require=${glidein_cvmfs_require,,} +# check whether glidein_cvmfs_require value is valid +# TODO: add this check at the xml level perhaps? +if ! [[ "${glidein_cvmfs_require}" =~ ^(required|preferred|never)$ ]]; then + logerror "Invalid attribute value: GLIDEIN_CVMFS_REQUIRE = ${glidein_cvmfs_require}" + "$error_gen" -error "$(basename $0)" "mnt_msg3" "Invalid attribute value: GLIDEIN_CVMFS_REQUIRE = ${glidein_cvmfs_require}" + exit 1 +fi - if [[ $GWMS_IS_CVMFS -ne 0 ]]; then - # Error occurred during mount of CVMFS repositories" - logerror "Error occured during mount of CVMFS repositories." - "$error_gen" -error "$(basename $0)" "WN_Resource" "Mount unsuccessful... CVMFS is still unavailable on the node." - exit 1 - fi +if [[ $use_cvmfs -ne 1 ]]; then + if ! [[ "${glidein_cvmfs_require}" =~ ^(required|preferred)$ ]]; then + loginfo "CVMFS not used; skipping related setup." + "$error_gen" -ok "$(basename $0)" "mnt_msg4" "CVMFS not used. Skipping related setup." + return 0 + fi +else + # use_cvmfs is set to true, then do the following + if [[ "${glidein_cvmfs_require}" == "never" ]]; then + loginfo "CVMFS to be used (GLIDEIN_USE_CVMFS: $use_cvmfs) but GLIDEIN_CVMFS_REQUIRE set to $glidein_cvmfs_require; skipping related setup." + "$error_gen" -ok "$(basename $0)" "mnt_msg5" "CVMFS to be used but GLIDEIN_CVMFS_REQUIRE set to ${glidein_cvmfs_require}" + return 0 + fi +fi - gwms_cvmfs_reexec="no" - gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" - # exporting the variables as an environment variable for use in glidein reinvocation - export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec - export GWMS_CVMFSEXEC_MODE=$gwms_cvmfsexec_mode +# following block runs attempting to add CVMFS when either: +# 1. use_cvmfs is false and glidein_cvmfs_require is required or preferred (OR) +# 2. use_cvmfs is true +if is_cvmfs_locally_mounted; then + loginfo "CVMFS found locally; skipping CVMFS setup via cvmfsexec." + loginfo "Continuing to execute the rest of the glidein setup..." + "$error_gen" -ok "$(basename $0)" "mnt_msg6" "CVMFS is natively available on the node; skipping setup using cvmfsexec utilities." + return 0 +fi +# if native CVMFS not there, do the following +loginfo "Starting on-demand CVMFS setup..." +# make sure that perform_system_check has run +[[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check +cvmfsexec_mode=$(setup_cvmfsexec_use) +if ! [[ $cvmfsexec_mode =~ ^[1-3]$ ]]; then + if [[ $use_cvmfs -eq 1 || "${glidein_cvmfs_require}" == "required" ]]; then + # when (1) use_cvmfs is 1, or (2) use_cvmfs is 0 and glidein_cvmfs_require is set to required + logerror "GLIDEIN_USE_CVMFS set to $use_cvmfs but GLIDEIN_CVMFS_REQUIRE is $glidein_cvmfs_require; aborting glidein setup." + "$error_gen" -error "$(basename $0)" "mnt_msg7" "cvmfsexec cannot be used (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" + exit 1 + fi + # when use_cvmfs is 0 and glidein_cvmfs_require is set to preferred, just warn the user + logwarn "GLIDEIN_USE_CVMFS set to $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require" + loginfo "Proceeding to execute the remainder of the glidein setup (without CVMFS)..." + "$error_gen" -ok "$(basename $0)" "mnt_msg8" "cvmfsexec cannot be used but still continuing (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" + return 0 +fi - # CVMFS is now available on the worker node" - loginfo "Proceeding to execute the rest of the glidein setup..." - "$error_gen" -ok "$(basename $0)" "WN_Resource" "CVMFS mounted successfully and is now available." - else - logerror "Invalid value of gwms_cvmfsexec_mode!" +loginfo "GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require" +loginfo "cvmfsexec mode $cvmfsexec_mode is being used..." +echo "" +if [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]]; then + perform_cvmfs_mount $cvmfsexec_mode $glidein_cvmfs_require + # before exiting out of this block, do two things... + # one, set a variable indicating this script has been executed once + gwms_cvmfs_reexec="yes" + gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" + + # two, export required variables with some necessary information for use inside cvmfsexec before reinvoking the glidein... + original_workspace=$(gconfig_get GLIDEIN_WORKSPACE_ORIG "$glidein_config") + export GLIDEIN_WORKSPACE=$original_workspace + export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec + export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode + export GLIDEIN_WORK_DIR="$work_dir" + export GLIDEIN_CVMFS_CONFIG_REPO="$GLIDEIN_CVMFS_CONFIG_REPO" + export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" + + exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" + echo "!!WARNING!! Outside of reinvocation of glidein_startup" + # the above line of code should not run; but is here as a safety check for debugging incorrect behavior of exec from previous line +fi +# the following is run if cvmfsexec cannot be used in mode 3/2 +perform_cvmfs_mount $cvmfsexec_mode +if [[ $? -eq 0 ]]; then + if [[ $GWMS_IS_CVMFS -ne 0 ]]; then + # Error occurred during mount of CVMFS repositories" + logerror "Error occured during mount of CVMFS repositories." + "$error_gen" -error "$(basename $0)" "mnt_msg9" "Mount unsuccessful... CVMFS is still unavailable on the node." exit 1 fi + # CVMFS is available on the worker node now" + gwms_cvmfs_reexec="no" + gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" + # exporting the variables as an environment variable for use in glidein reinvocation + export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec + export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode + loginfo "Proceeding to execute the remainder of glidein setup..." + "$error_gen" -ok "$(basename $0)" "mnt_msg10" "CVMFS mounted successfully and is now available." + return 0 +elif [[ $? -eq 1 ]]; then + # if exit status is 1 + if [[ $cvmfs_required == "required" ]]; then + # if mount CVMFS is not successful, report an error and exit with failure exit code + loginfo "CVMFS is required but unable to mount CVMFS on worker node." + "$error_gen" -error "$(basename $0)" "WN_Resource" "CVMFS is required but unable to mount CVMFS on the worker node." + exit 1 + fi + # if cvmfs_required is set to preferred and mount CVMFS is not successful, report a warning/error in the logs and continue with glidein startup + # script status must be OK, otherwise the glidein will fail + "$error_gen" -ok "$(basename $0)" "WN_Resource" "Unable to mount required CVMFS on the worker node. Continuing without CVMFS." + return 0 else - # if CVMFS is not requested/not needed to be setup by the glidein - printinfo "Proceeding to execute the rest of the glidein setup..." + # if exit status is 2 + if [[ $cvmfs_required == "required" ]]; then + logerror "Non-RHEL OS found but not supported; aborting glidein setup!" + "$error_gen" -error "$(basename $0)" "WN_Resource" "Non-RHEL OS found but not supported; aborting glidein startup" + exit 1 + fi + # if CVMFS is not required, display operating system information and a user-friendly message + "$error_gen" -ok "$(basename $0)" "WN_Resource" "Non-RHEL OS found but not supported; continuing without CVMFS setup" + return 0 fi diff --git a/creation/web_base/cvmfs_umount.sh b/creation/web_base/cvmfs_umount.sh index 273dfd09e..6aafb8261 100755 --- a/creation/web_base/cvmfs_umount.sh +++ b/creation/web_base/cvmfs_umount.sh @@ -20,23 +20,17 @@ # Namratha Urs # -echo "Starting to unmount CVMFS as part of glidein cleanup..." - +echo "Unmounting on-demand CVMFS as part of glidein cleanup..." glidein_config=$1 # import add_config_line to use gconfig_ utilities add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | awk '{print $2}') # shellcheck source=./add_config_line.source . "$add_config_line_source" - # import error_gen error_gen=$(gconfig_get ERROR_GEN_PATH "$glidein_config") - # get the cvmfsexec attribute switch value from the config file -use_cvmfsexec=$(gconfig_get GLIDEIN_USE_CVMFSEXEC "$glidein_config") -# TODO: int or string? if string, make the attribute value case insensitive -#use_cvmfsexec=${use_cvmfsexec,,} - +use_cvmfs=$(gconfig_get GLIDEIN_USE_CVMFS "$glidein_config") # get the glidein work directory location from glidein_config file work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$glidein_config") # $PWD=/tmp/glide_xxx and every path is referenced with respect to $PWD @@ -45,55 +39,47 @@ work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$glidein_config") # shellcheck source=./cvmfs_helper_funcs.sh . "$work_dir"/cvmfs_helper_funcs.sh -if [[ $use_cvmfsexec -ne 1 ]]; then - loginfo "On-demand CVMFS provisioning not requested or not used; skipping cleanup." - "$error_gen" -ok "$(basename $0)" "umnt_msg1" "On-demand CVMFS not requested or not used; skipping cleanup." - exit 0 -fi - -# get the cvmfsexec directory location -glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") - -######################################################################################################## -# Start: main program -######################################################################################################## - loginfo "..." -loginfo "Start log for unmounting CVMFS" - -# check if CVMFS is locally mounted on the worker node -detect_local_cvmfs - -if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then - # CVMFS is mounted locally in the filesystem; DO NOT UNMOUNT! - loginfo "Skipping unmounting of CVMFS as it already is locally provisioned in the node!" - "$error_gen" -ok "$(basename $0)" "umnt_msg2" "CVMFS is locally mounted on the node; skipping cleanup." - exit 0 -fi - -gwms_cvmfsexec_mode=$(grep '^GWMS_CVMFSEXEC_MODE ' "$glidein_config" | awk '{print $2}') -loginfo "Unmounting CVMFS provisioned by the glidein..." -if [[ "$gwms_cvmfsexec_mode" -eq 1 ]]; then - "$glidein_cvmfsexec_dir"/.cvmfsexec/umountrepo -a - - if [[ -n "$CVMFS_MOUNT_DIR" ]]; then - CVMFS_MOUNT_DIR= - export CVMFS_MOUNT_DIR - gconfig_add CVMFS_MOUNT_DIR "" +is_cvmfs_mntd=$(gconfig_get GWMS_IS_CVMFS "$glidein_config") +loginfo "CVMFS mounted on demand: $is_cvmfs_mntd" +if [[ "${is_cvmfs_mntd}" == "yes" ]]; then + gwms_cvmfsexec_mode=$(gconfig_get GWMS_CVMFSEXEC_MODE "$glidein_config") + # get the cvmfsexec directory location + glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") + loginfo "Unmounting CVMFS provisioned by the glidein..." + if [[ "$gwms_cvmfsexec_mode" -eq 1 ]]; then + "$glidein_cvmfsexec_dir"/.cvmfsexec/umountrepo -a + + if [[ -n "$CVMFS_MOUNT_DIR" ]]; then + CVMFS_MOUNT_DIR= + export CVMFS_MOUNT_DIR + gconfig_add CVMFS_MOUNT_DIR "" + fi + elif [[ "$gwms_cvmfsexec_mode" -eq 3 || "$gwms_cvmfsexec_mode" -eq 2 ]]; then + loginfo "CVMFS_MOUNT_DIR set to $CVMFS_MOUNT_DIR" + loginfo "CVMFSUMOUNT set to $CVMFSUMOUNT" + fi + # check again to ensure all CVMFS repositories were unmounted by umountrepo + # searching for "/dev/fuse" since "/cvmfs" might return false positives (/etc/auto.fs /cvmfs line) + cat /proc/$$/mounts | grep /dev/fuse &> /dev/null && logerror "One or more CVMFS repositories might not be completely unmounted" || loginfo "CVMFS repositories unmounted" + "$error_gen" -ok "$(basename $0)" "umnt_msg1" "Glidein-based CVMFS unmount was successful." + # returning 0 to indicate the unmount process was successful + return 0 +else + # CVMFS was not mounted on-demand, so check if CVMFS is locally mounted on the worker node + detect_local_cvmfs + if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then + # CVMFS is mounted locally in the filesystem; DO NOT UNMOUNT! + loginfo "Skipping unmounting of CVMFS as it already is locally provisioned in the node!" + "$error_gen" -ok "$(basename $0)" "umnt_msg2" "CVMFS is locally mounted on the node; skipping cleanup." + else + # CVMFS is not found natively either + loginfo "CVMFS not found natively; so nothing to do here... continuing." + "$error_gen" -ok "$(basename $0)" "umnt_msg3" "No native CVMFS; nothing to cleanup before glidein shutdown." fi -elif [[ "$gwms_cvmfsexec_mode" -eq 3 ]]; then - loginfo "CVMFS_MOUNT_DIR set to $CVMFS_MOUNT_DIR" - loginfo "CVMFSUMOUNT set to $CVMFSUMOUNT" + return 0 fi -# check again to ensure all CVMFS repositories were unmounted by umountrepo -# searching for "/dev/fuse" since "/cvmfs" returns false positives (/etc/auto.fs /cvmfs line) -cat /proc/$$/mounts | grep /dev/fuse &> /dev/null && logerror "One or more CVMFS repositories might not be completely unmounted" || loginfo "CVMFS repositories unmounted" - -"$error_gen" -ok "$(basename $0)" "umnt_msg3" "Glidein-based CVMFS unmount was successful." -# returning 0 to indicate the unmount process was successful -true - -######################################################################################################## +############################################################################ # End: main program -######################################################################################################## +############################################################################ From ed46ff20482a7b35947809cd72245875e60219ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 21:24:28 +0000 Subject: [PATCH 05/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- creation/lib/cgWParamDict.py | 6 ++++-- creation/web_base/cvmfs_helper_funcs.sh | 12 ++++++------ creation/web_base/cvmfs_setup.sh | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 1442b0812..b4f7c94f4 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -309,7 +309,8 @@ def populate(self, other=None): self.dicts["file_list"].add_from_file( cvmfs_helper, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(cvmfs_helper), "exec", + cWConsts.insert_timestr(cvmfs_helper), + "exec", ), os.path.join(cgWConsts.WEB_BASE_DIR, cvmfs_helper), ) @@ -319,7 +320,8 @@ def populate(self, other=None): self.dicts["file_list"].add_from_file( dist_select_script, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(dist_select_script), "exec", + cWConsts.insert_timestr(dist_select_script), + "exec", ), os.path.join(cgWConsts.WEB_BASE_DIR, dist_select_script), ) diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index 6d8684529..c8cd00aae 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -187,7 +187,7 @@ perform_system_check() { [[ $GWMS_OS_VERSION_MAJOR -ge 9 ]] && dnf list installed fuse3* &>/dev/null || yum list installed fuse &>/dev/null GWMS_IS_FUSE_INSTALLED=$? - + [[ $GWMS_OS_VERSION_MAJOR -ge 9 ]] && fusermount3 -V &>/dev/null || fusermount -V &>/dev/null GWMS_IS_FUSERMOUNT=$? @@ -426,7 +426,7 @@ has_fuse() { loginfo "FUSE requirements not satisfied: user is not in fuse group" ret_state=no fi - fi + fi else # unprivileged user namespaces is either enabled or disabled if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then @@ -455,7 +455,7 @@ determine_cvmfsexec_mode_usage() { # make sure that perform_system_check has run [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - + # check what specific configuration of unprivileged user namespaces exists in the system (worker node) unpriv_userns_status=$(has_unpriv_userns) @@ -528,7 +528,7 @@ setup_cvmfsexec_use() { prepare_for_cvmfs_mount () { - # DESCRIPTION: This function is used to prepare the necessary items and keep them ready/accessible right before mounting CVMFS on demand. + # DESCRIPTION: This function is used to prepare the necessary items and keep them ready/accessible right before mounting CVMFS on demand. # # INPUT(S): None # RETURN(S): None @@ -590,7 +590,7 @@ prepare_for_cvmfs_mount () { perform_cvmfs_mount () { - # DESCRIPTION: This function serves as a wrapper for performing mounting of CVMFS on demand depending on a few factors. + # DESCRIPTION: This function serves as a wrapper for performing mounting of CVMFS on demand depending on a few factors. # # INPUT(S): an integer denoting the selected cvmfsexec mode # RETURN(S): to stdout one of the following values: @@ -613,7 +613,7 @@ perform_cvmfs_mount () { return 2 # ----- Further Implementation: TBD (To Be Done) ----- # fi - + prepare_for_cvmfs_mount if [[ $mode -eq 3 || $mode -eq 2 ]]; then return # only prepare but do not actually mount (later in glidein reinvocation, mounting will be performed) diff --git a/creation/web_base/cvmfs_setup.sh b/creation/web_base/cvmfs_setup.sh index d44818bff..a68750fbe 100644 --- a/creation/web_base/cvmfs_setup.sh +++ b/creation/web_base/cvmfs_setup.sh @@ -118,7 +118,7 @@ if [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]]; then export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" - echo "!!WARNING!! Outside of reinvocation of glidein_startup" + echo "!!WARNING!! Outside of reinvocation of glidein_startup" # the above line of code should not run; but is here as a safety check for debugging incorrect behavior of exec from previous line fi # the following is run if cvmfsexec cannot be used in mode 3/2 @@ -160,5 +160,5 @@ else fi # if CVMFS is not required, display operating system information and a user-friendly message "$error_gen" -ok "$(basename $0)" "WN_Resource" "Non-RHEL OS found but not supported; continuing without CVMFS setup" - return 0 + return 0 fi From ec959fa7a8ed43d5b55ac17563df8254f14d6b41 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Mon, 25 Mar 2024 21:33:18 +0000 Subject: [PATCH 06/12] removed debugging statements added previously --- creation/web_base/glidein_startup.sh | 284 ++++----------------------- 1 file changed, 36 insertions(+), 248 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 6c91e0077..c82244306 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -8,6 +8,15 @@ # This scripts runs all the others ############################ + +printenv GWMS_CVMFS_REEXEC > /dev/null +status=$? +if [[ "$status" -eq 0 ]]; then + echo "Now reinvoking glidein startup..." + gwms_cvmfs_reexec=$(printenv GWMS_CVMFS_REEXEC | sed s"/ //g") +else + gwms_cvmfs_reexec=$(grep "^GWMS_CVMFS_REEXEC " "${glidein_config}" | cut -d ' ' -f 2-) +fi # Customizable and initialization variables # Default IFS, to protect against unusual environment, better than "unset IFS" because works with restoring old one @@ -16,11 +25,14 @@ IFS=$' \t\n' # Some sites empty PATH. Setting a reasonable default if [[ -z "$PATH" ]]; then export PATH="/bin:/usr/bin" +elif [[ -z "$gwms_cvmfs_reexec" ]]; then + export PATH="/usr/sbin:$PATH" fi global_args="$*" # GWMS_STARTUP_SCRIPT=$0 GWMS_STARTUP_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" +export GWMS_STARTUP_SCRIPT=${GWMS_STARTUP_SCRIPT} # for glidein reinvocation GWMS_PATH="" # Relative to the work directory (GWMS_DIR, gwms_lib_dir, gwms_bin_dir and gwms_exec_dir will be the absolute paths) # bin (utilities), lib (libraries), exec (aux scripts to be executed/sourced, e.g. pre-job) @@ -1175,8 +1187,6 @@ fetch_file_base() { if [[ "${ffb_file_type}" = "exec:s" ]]; then "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" elif [[ "${ffb_file_type}" = "exec:r" ]]; then - echo "INSIDE EXEC:r BLOCK..." - echo "###### Sourcing $ffb_outname script ############" . "${ffb_outname}" glidein_config "${ffb_id}" else "${ffb_outname}" glidein_config "${ffb_id}" @@ -1265,7 +1275,6 @@ add_to_path() { export PATH="${tmp_path%:}" } - fixup_condor_dir() { # All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped # However the (not used anymore) gwms create_condor_tarball removes that dir @@ -1321,20 +1330,6 @@ usage() { #################### Execution starts here.... ###################### ##################################################################### -printenv GWMS_CVMFS_REEXEC # > /dev/null -status=$? -if [[ "$status" -eq 0 ]]; then - echo ".....(RE)START OF GLIDEIN_STARTUP.SH....." - gwms_cvmfs_reexec=$(printenv GWMS_CVMFS_REEXEC | sed s"/ //g") - echo "GWMS_CVMFS_REEXEC (inside reinvocation) set to $gwms_cvmfs_reexec" - echo "work_dir variable (right after invocation): $work_dir" # should print nothing because the reinvocation is happening through an exec -else - # echo "glidein_config set to $glidein_config" - echo ".....START OF GLIDEIN_STARTUP.SH....." - gwms_cvmfs_reexec=$(grep "^GWMS_CVMFS_REEXEC " "${glidein_config}" | cut -d ' ' -f 2-) - echo "GWMS_CVMFS_REEXEC set to $gwms_cvmfs_reexec" -fi - if [[ -z "$gwms_cvmfs_reexec" ]]; then # if GWMS_CVMFS_REEXEC is empty, then this script is being invoked the first time; so proceed with usual glidein setup... @@ -1444,6 +1439,7 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then usage fi + export REPOSITORY_URL=$repository_url # for glidein reinvocation repository_entry_url="${repository_url}/entry_${glidein_entry}" if [ -z "${proxy_url}" ]; then @@ -1506,7 +1502,9 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then warn "Missing client descript fname for group." usage fi + export CLIENT_REPOSITORY_GROUP_URL=$client_repository_group_url # for glidein reinvocation fi + export CLIENT_REPOSITORY_URL=$client_repository_url # for glidein reinvocation fi # Generate glidein UUID @@ -1519,6 +1517,7 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then # Print initial variables values (argumants and environment) startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" + export STARTUP_TIME=$startup_time # for glidein reinvocation echo "script_checksum = '$(md5wrapper "$0")'" echo "debug_mode = '${operation_mode}'" @@ -1675,6 +1674,7 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then fi fi work_dir_created=1 + export GLIDEIN_WORK_DIR=${GLIDEIN_WORK_DIR} # for glidein reinvocation # GWMS_SUBDIR defined on top GWMS_DIR="${work_dir}/$GWMS_SUBDIR" @@ -1685,10 +1685,12 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then if ! mkdir -p "$gwms_lib_dir" ; then early_glidein_failure "Cannot create lib dir '$gwms_lib_dir'" fi + export GWMS_LIB_DIR=${gwms_lib_dir} # for glidein reinvocation gwms_bin_dir="${GWMS_DIR}/bin" if ! mkdir -p "$gwms_bin_dir" ; then early_glidein_failure "Cannot create bin dir '$gwms_bin_dir'" fi + export GWMS_BIN_DIR=${gwms_bin_dir} # for glidein reinvocation gwms_exec_dir="${GWMS_DIR}/exec" if ! mkdir -p "$gwms_exec_dir" ; then early_glidein_failure "Cannot create exec dir '$gwms_exec_dir'" @@ -1696,7 +1698,11 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then for i in setup prejob postjob cleanup setup_singularity ; do mkdir -p "$gwms_exec_dir"/$i done + export GWMS_EXEC_DIR=${gwms_exec_dir} # for glidein reinvocation fi + # below lines for glidein reinvocation + export GWMS_SUBDIR=${GWMS_SUBDIR} + export GWMS_DIR=${GWMS_DIR} # mktemp makes it user readable by definition (ignores umask) # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed @@ -1736,12 +1742,14 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then if ! mkdir "${main_dir}"; then early_glidein_failure "Cannot create '${main_dir}'" fi + export GWMS_MAIN_DIR=${main_dir} # for glidein reinvocation short_entry_dir=entry_${glidein_entry} entry_dir="${work_dir}/${short_entry_dir}" if ! mkdir "${entry_dir}"; then early_glidein_failure "Cannot create '${entry_dir}'" fi + export GWMS_ENTRY_DIR=${entry_dir} # for glidein reinvocation if [ -n "${client_repository_url}" ]; then short_client_dir=client @@ -1749,6 +1757,7 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then if ! mkdir "$client_dir"; then early_glidein_failure "Cannot create '${client_dir}'" fi + export GWMS_CLIENT_DIR=${client_dir} # for glidein reinvocation if [ -n "${client_repository_group_url}" ]; then short_client_group_dir=client_group_${client_group} @@ -1757,6 +1766,7 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then early_glidein_failure "Cannot create '${client_group_dir}'" fi fi + export GWMS_CLIENTGROUP_DIR=${client_group_dir} # for glidein reinvocation fi # Move the token files from condor to glidein workspace @@ -1784,12 +1794,14 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then wrapper_list="${PWD}/wrapper_list.lst" touch "${wrapper_list}" + export WRAPPER_LIST=${wrapper_list} # for glidein reinvocation # create glidein_config glidein_config="${PWD}/glidein_config" if ! echo > "${glidein_config}"; then early_glidein_failure "Could not create '${glidein_config}'" fi + export GLIDEIN_CONFIG=${glidein_config} # for glidein reinvocation if ! { echo "# --- glidein_startup vals ---" echo "GLIDEIN_UUID ${glidein_uuid}" @@ -1942,6 +1954,7 @@ if [[ -z "$gwms_cvmfs_reexec" ]]; then gs_id_work_dir="$(get_work_dir main)" gs_id_descript_file="$(get_descript_file main)" last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" + export LAST_SCRIPT="$last_script" # for glidein reinvocation if [ -z "${last_script}" ]; then warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." glidein_exit 1 @@ -2025,7 +2038,6 @@ fi if [[ -n "$gwms_cvmfs_reexec" && "$gwms_cvmfs_reexec" == "yes" ]]; then # gwms_cvmfs_reexec is not empty; meaning this block is being run inside of cvmfsexec environment - echo "======================== control flow is inside reinvocation of glidein_startup.sh ========================" printenv GLIDEIN_CONFIG > /dev/null status=$? if [[ ${status} -eq 0 ]]; then @@ -2045,15 +2057,16 @@ if [[ -n "$gwms_cvmfs_reexec" && "$gwms_cvmfs_reexec" == "yes" ]]; then gwms_exec_dir=$(printenv GWMS_EXEC_DIR | sed "s/ //g") fi + # import add_config_line function + add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | cut -d ' ' -f 2-) + # shellcheck source=./add_config_line.source + . "$add_config_line_source" + # re-sourcing the helper script inside of cvmfsexec environment . "$work_dir"/cvmfs_helper_funcs.sh - - mount_cvmfs_repos $gwms_cvmfsexec_mode $gwms_cvmfs_reexec $cvmfs_config_repo $cvmfs_add_repos - echo "GWMS_IS_CVMFS set to $GWMS_IS_CVMFS (mode 3)" - + mount_cvmfs_repos $gwms_cvmfsexec_mode $cvmfs_config_repo $cvmfs_add_repos # check if the cvmfs repos are still mounted inside of reinvocation df -h - echo "CVMFS_MOUNT_DIR is $(printenv CVMFS_MOUNT_DIR) in mode 3" # re-source all the scripts as it'd have been done during the first invocation of this script extract_all_data @@ -2061,239 +2074,14 @@ if [[ -n "$gwms_cvmfs_reexec" && "$gwms_cvmfs_reexec" == "yes" ]]; then log_setup "${glidein_config}" fi -echo "*************** BEFORE THE SECOND FOR LOOP ***************" glidein_debug_options=$(gconfig_get GLIDEIN_DEBUG_OPTIONS "$glidein_config") glidein_debug_output=$(gconfig_get GLIDEIN_DEBUG_OUTPUT "$glidein_config") -echo "-*-*-*-*-*- glidein_debug_options: $glidein_debug_options -*-*-*-*-*-" -echo "-*-*-*-*-*- glidein_debug_output: $glidein_debug_output -*-*-*-*-*-" export GLIDEIN_DEBUG_OPTIONS=$glidein_debug_options export GLIDEIN_DEBUG_OUTPUT=$glidein_debug_output -echo "**********************************************************" - -<<<<<<< HEAD -short_entry_dir=entry_${glidein_entry} -entry_dir="${work_dir}/${short_entry_dir}" -if ! mkdir "${entry_dir}"; then - early_glidein_failure "Cannot create '${entry_dir}'" -fi - -if [ -n "${client_repository_url}" ]; then - short_client_dir=client - client_dir="${work_dir}/${short_client_dir}" - if ! mkdir "$client_dir"; then - early_glidein_failure "Cannot create '${client_dir}'" - fi - if [ -n "${client_repository_group_url}" ]; then - short_client_group_dir=client_group_${client_group} - client_group_dir="${work_dir}/${short_client_group_dir}" - if ! mkdir "${client_group_dir}"; then - early_glidein_failure "Cannot create '${client_group_dir}'" - fi - fi -fi - -# Move the token files from condor to glidein workspace -# TODO: compare this w/ setup_x509.sh -# monitoring tokens, Should be using same credentials directory? -mv "${start_dir}/tokens.tgz" . -mv "${start_dir}/url_dirs.desc" . -# idtokens are handled in setup_x509.sh - TODO: remove once verified -#for idtk in ${start_dir}/*.idtoken; do -# if cp "${idtk}" . ; then -# echo "copied idtoken ${idtk} to $(pwd)" -# else -# echo "failed to copy idtoken ${idtk} to $(pwd)" 1>&2 -# fi -#done -#if [ -e "${GLIDEIN_CONDOR_TOKEN}" ]; then -# mkdir -p ticket -# tname="$(basename ${GLIDEIN_CONDOR_TOKEN})" -# cp "${GLIDEIN_CONDOR_TOKEN}" "ticket/${tname}" -# export GLIDEIN_CONDOR_TOKEN="$(pwd)/ticket/${tname}" -#fi - -# Extract and source all the data contained at the end of this script as tarball -extract_all_data - -wrapper_list="${PWD}/wrapper_list.lst" -touch "${wrapper_list}" - -# create glidein_config -glidein_config="${PWD}/glidein_config" -if ! echo > "${glidein_config}"; then - early_glidein_failure "Could not create '${glidein_config}'" -fi -if ! { - echo "# --- glidein_startup vals ---" - echo "GLIDEIN_UUID ${glidein_uuid}" - echo "GLIDEIN_Factory ${glidein_factory}" - echo "GLIDEIN_Name ${glidein_name}" - echo "GLIDEIN_Entry_Name ${glidein_entry}" - - if [ -n "${client_name}" ]; then - # client name not required as it is not used for anything but debug info - echo "GLIDECLIENT_Name ${client_name}" - fi - if [ -n "${client_group}" ]; then - # client group not required as it is not used for anything but debug info - echo "GLIDECLIENT_Group ${client_group}" - fi - echo "GLIDEIN_CredentialIdentifier ${glidein_cred_id}" - echo "CONDORG_CLUSTER ${condorg_cluster}" - echo "CONDORG_SUBCLUSTER ${condorg_subcluster}" - echo "CONDORG_SCHEDD ${condorg_schedd}" - echo "DEBUG_MODE ${set_debug}" - echo "GLIDEIN_STARTUP_PID $$" - echo "GLIDEIN_START_DIR_ORIG ${start_dir}" - echo "GLIDEIN_WORKSPACE_ORIG $(pwd)" - echo "GLIDEIN_WORK_DIR ${main_dir}" - echo "GLIDEIN_ENTRY_WORK_DIR ${entry_dir}" - echo "TMP_DIR ${glide_tmp_dir}" - echo "GLIDEIN_LOCAL_TMP_DIR ${glide_local_tmp_dir}" - echo "PROXY_URL ${proxy_url}" - echo "DESCRIPTION_FILE ${descript_file}" - echo "DESCRIPTION_ENTRY_FILE ${descript_entry_file}" - echo "GLIDEIN_Signature ${sign_id}" - echo "GLIDEIN_Entry_Signature ${sign_entry_id}" - - if [ -n "${client_repository_url}" ]; then - echo "GLIDECLIENT_WORK_DIR ${client_dir}" - echo "GLIDECLIENT_DESCRIPTION_FILE ${client_descript_file}" - echo "GLIDECLIENT_Signature ${client_sign_id}" - if [ -n "${client_repository_group_url}" ]; then - echo "GLIDECLIENT_GROUP_WORK_DIR ${client_group_dir}" - echo "GLIDECLIENT_DESCRIPTION_GROUP_FILE ${client_descript_group_file}" - echo "GLIDECLIENT_Group_Signature ${client_sign_group_id}" - fi - fi - echo "B64UUENCODE_SOURCE ${PWD}/b64uuencode.source" - echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" - echo "GET_ID_SELECTORS_SOURCE ${PWD}/get_id_selectors.source" - echo "LOGGING_UTILS_SOURCE ${PWD}/logging_utils.source" - echo "GLIDEIN_PATHS_SOURCE ${PWD}/glidein_paths.source" - echo "WRAPPER_LIST ${wrapper_list}" - echo "SLOTS_LAYOUT ${slots_layout}" - # Add a line saying we are still initializing... - echo "GLIDEIN_INITIALIZED 0" - # ...but be optimist, and leave advertise_only for the actual error handling script - echo "GLIDEIN_ADVERTISE_ONLY 0" - echo "GLIDEIN_CONDOR_TOKEN ${GLIDEIN_CONDOR_TOKEN}" - echo "# --- User Parameters ---" -} >> "${glidein_config}"; then - early_glidein_failure "Failed in updating '${glidein_config}'" -fi -# shellcheck disable=SC2086 -params2file ${params} - -############################################ -# Setup logging -log_init "${glidein_uuid}" "${work_dir}" -# Remove these files, if they are still there -rm -rf tokens.tgz url_dirs.desc tokens -log_setup "${glidein_config}" - -echo "Downloading files from Factory and Frontend" -log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" - -##################################### -# Fetch descript and signature files - -# disable signature check before I get the signature file itself -# disable_check_signature is global -disable_check_signature=yes - -for gs_id in main entry client client_group -do - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no cilent_repository - continue - fi - fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no cilent_repository_group - continue - fi - fi - - gs_id_work_dir="$(get_work_dir ${gs_id})" - - # Fetch description file - gs_id_descript_file="$(get_descript_file ${gs_id})" - fetch_file_regular "${gs_id}" "${gs_id_descript_file}" - if ! signature_file_line="$(grep "^signature " "${gs_id_work_dir}/${gs_id_descript_file}")"; then - warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." - glidein_exit 1 - fi - signature_file=$(echo "${signature_file_line}" | cut -s -f 2-) - - # Fetch signature file - gs_id_signature="$(get_signature ${gs_id})" - fetch_file_regular "${gs_id}" "${signature_file}" - echo "${gs_id_signature} ${signature_file}" > "${gs_id_work_dir}/signature.sha1.test" - if ! (cd "${gs_id_work_dir}" && sha1sum -c signature.sha1.test) 1>&2 ; then - warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." - glidein_exit 1 - fi - # for simplicity use a fixed name for signature file - mv "${gs_id_work_dir}/${signature_file}" "${gs_id_work_dir}/signature.sha1" -done - -# re-enable for everything else -disable_check_signature= - -# Now verify the description was not tampered with -# doing it so late should be fine, since nobody should have been able -# to fake the signature file, even if it faked its name in -# the description file -for gs_id in main entry client client_group -do - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no cilent_repository - continue - fi - fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no cilent_repository_group - continue - fi - fi - - gs_id_descript_file="$(get_descript_file ${gs_id})" - if ! check_file_signature "${gs_id}" "${gs_id_descript_file}"; then - gs_id_work_dir="$(get_work_dir ${gs_id})" - warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." - glidein_exit 1 - fi -done - -################################################### -# get last_script, as it is used by the fetch_file -gs_id_work_dir="$(get_work_dir main)" -gs_id_descript_file="$(get_descript_file main)" -last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" -if [ -z "${last_script}" ]; then - warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." - glidein_exit 1 -fi -#cleanup_script="$(grep "^cleanup_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" -cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' ' -f 2-) - - -############################## -# Fetch all the other files -for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" -do - gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" -======= for gs_file_id in "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" do gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" # one of four possibilities ->>>>>>> c39a07e72 (updated for glidein reinvocation when in mode 3) if [ -z "${client_repository_url}" ]; then if [ "${gs_id}" = "client" ]; then From aefeffcae5a18837462d588f6b4a7749caac105f Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Wed, 3 Apr 2024 09:25:24 -0500 Subject: [PATCH 07/12] some updates after the final round of EL9 testing - minor changes (formatting, indentation, improved user facing messages etc.) - improvements to the existing unmounting process now that mode 3 will also be supported - code refactoring in cvmfs_setup script for better flow and readability - improvements in cvmfs_helper_funcs to allow for different behaviors as a result of using cvmfsexec modes --- creation/lib/cgWParamDict.py | 5 +- creation/web_base/cvmfs_helper_funcs.sh | 90 ++++++++++--------- creation/web_base/cvmfs_setup.sh | 87 +++++++++--------- creation/web_base/cvmfs_umount.sh | 87 ++++++++++-------- .../web_base/cvmfsexec_platform_select.sh | 7 +- creation/web_base/glidein_startup.sh | 3 +- creation/web_base/singularity_lib.sh | 2 +- 7 files changed, 144 insertions(+), 137 deletions(-) diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index b4f7c94f4..4b5f7cccf 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -319,10 +319,7 @@ def populate(self, other=None): dist_select_script = "cvmfsexec_platform_select.sh" self.dicts["file_list"].add_from_file( dist_select_script, - cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(dist_select_script), - "exec", - ), + cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(dist_select_script), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, dist_select_script), ) diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index c8cd00aae..08e54c316 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -259,60 +259,58 @@ mount_cvmfs_repos () { local additional_repos=$3 local config_repo_mntd num_repos_mntd total_num_repos - # if using mode 3, config repo should have been mounted already - # otherwise if using mode 1, mount the config repo now... - if [[ $cvmfsexec_mode -eq 1 ]]; then - "$glidein_cvmfsexec_dir/$dist_file" "$config_repository" -- echo "setting up mount utilities..." &> /dev/null - fi # at this point in the execution flow, it would have been determined that cvmfs is not locally available - # this implies no repositories should be mounted. However, only config repo will be mounted if in mode 1 or mode 3 by this point - if [[ $(df -h|grep /cvmfs|wc -l) -eq 1 ]]; then + # this implies no repositories should have been mounted. However, only config repo will be mounted if in mode 1 or mode 3 by this point + + # if using mode 3/2, config repo should have been mounted already by now + if [[ $(cat /proc/$$/mounts | grep /dev/fuse | grep ' /cvmfs' | wc -l) -eq 1 ]]; then loginfo "CVMFS config repo already mounted!" else - # mounting the configuration repo (pre-requisite) in case something went wrong previously + # mounting the configuration repo (pre-requisite) in case something went wrong previously or when using mode 1 + # if using mode 1, first setup the mount utility under .cvmfsexec + if [[ $cvmfsexec_mode -eq 1 ]]; then + "$glidein_cvmfsexec_dir/$dist_file" "$config_repository" -- echo "setting up mount utilities..." &> /dev/null + fi loginfo "Mounting CVMFS config repo now..." [[ $cvmfsexec_mode -eq 1 ]] && "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$config_repository" - [[ $cvmfsexec_mode -eq 3 ]] && $CVMFSMOUNT "$config_repository" - fi - # see if the config repository got mounted - config_repo_mntd=$(df -h | grep /cvmfs | wc -l) - if [[ config_repo_mntd -eq 0 ]]; then - logwarn "One or more CVMFS repositories might not be mounted on the worker node" - return 1 + [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]] && $CVMFSMOUNT "$config_repository" + # see if the config repository got mounted this time around + if [[ $(cat /proc/$$/mounts | grep /dev/fuse | wc -l) -eq 0 ]]; then + logwarn "CVMFS config repository might still not be mounted on the worker node" + return 1 + fi fi # using an array to unpack the names of additional CVMFS repositories # from the colon-delimited string repos=($(echo $additional_repos | tr ":" "\n")) - loginfo "Mounting additional CVMFS repositories..." - # mount every repository that was previously unpacked - [[ "$cvmfsexec_mode" -ne 1 && "$cvmfsexec_mode" -ne 3 ]] && { logerror "Invalid cvmfsexec mode: mode $cvmfsexec_mode; aborting!"; return 1; } + # mount every repository in the array for repo in "${repos[@]}" do - case $cvmfsexec_mode in - 1) - "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$repo" - ;; - 3) - $CVMFSMOUNT "$repo" - ;; - esac + [[ $cvmfsexec_mode -eq 1 ]] && "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$repo" + [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]] && $CVMFSMOUNT "$repo" done - - # see if all the repositories got mounted - num_repos_mntd=$(df -h | grep /cvmfs | wc -l) + # verify if all the repositories got mounted + if [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]]; then + # since mode 3 mounting shows original mounts as well as the bind mounts, only consider one of those sets for verification + num_repos_mntd=$(cat /proc/$$/mounts | grep /dev/fuse | grep ' /cvmfs' | wc -l) + else + num_repos_mntd=$(cat /proc/$$/mounts | grep /dev/fuse | wc -l) + fi total_num_repos=$(( ${#repos[@]} + 1 )) GWMS_IS_CVMFS=0 if [[ "$num_repos_mntd" -eq "$total_num_repos" ]]; then loginfo "All CVMFS repositories mounted successfully on the worker node" - # export this info to the glidein environment after CVMFS is provisioned on demand + # export this info to the glidein environment after CVMFS mounting was a success gconfig_add GWMS_IS_CVMFS $(print_exit_status $GWMS_IS_CVMFS) get_mount_point return 0 else logwarn "One or more CVMFS repositories might not be mounted on the worker node" GWMS_IS_CVMFS=1 + # export this info to indicate to the glidein environment that something went wrong during the CVMFS mounting + gconfig_add GWMS_IS_CVMFS $(print_exit_status $GWMS_IS_CVMFS) return 1 fi } @@ -549,19 +547,13 @@ prepare_for_cvmfs_mount () { arch=$GWMS_OS_KRNL_ARCH # construct the name of the cvmfsexec distribution file based on the worker node specs dist_file=cvmfsexec-${cvmfs_source}-${os_like}${os_ver}-${arch} - # the appropriate distribution file does not have to manually untarred as the glidein setup takes care of this automatically - - if [[ ! -d "$glidein_cvmfsexec_dir" && ! -f "${glidein_cvmfsexec_dir}/${dist_file}" ]]; then - # neither the cvmfsexec directory nor the cvmfsexec distribution is found -- this happens when a directory named 'cvmfsexec' does not exist on the glidein because an appropriate distribution tarball is not found in the list of all the available tarballs and was not unpacked [trying to unpack osg-rhel8 on osg-rhel7 worker node] - # if use_cvmfsexec is set to 1, then warn that cvmfs will not be mounted and flag an error - logerror "Error occured during preparing for cvmfs setup: None of the available cvmfsexec distributions is compatible with the worker node specifications." - "$error_gen" -error "$(basename $0)" "WN_Resource" "Error occured during cvmfs setup... no matching cvmfsexec distribution available." - exit 1 - elif [[ -d "$glidein_cvmfsexec_dir" && ! -f "${glidein_cvmfsexec_dir}/${dist_file}" ]]; then - # something might have gone wrong during the unpacking of the tarball into the glidein_cvmfsexec_dir - logerror "Something went wrong during the unpacking of the cvmfsexec distribution tarball!" - "$error_gen" -error "$(basename $0)" "WN_Resource" "Error: Something went wrong during the unpacking of the cvmfsexec distribution tarball" - exit 1 + # the appropriate distribution file does not have to be manually untarred as the glidein setup takes care of this automatically + # but check if the unpacking was okay and the distribution file is accessible + if [[ ! -d "$glidein_cvmfsexec_dir" || ! -f "${glidein_cvmfsexec_dir}/${dist_file}" ]]; then + # 1. the cvmfsexec directory containing platform-specific cvmfsexec distribution does not exist in the glidein -- this happens when (a). cvmfsexec distributions are not built at all with cvmfsexec_distro factory knob (unless they are already existing from a previous reconfig/upgrade that used the knob), or (b). when the build process for the cvmfsexec distributions failed but was requested to be used + # 2. the relevant cvmfsexec distribution is not found -- this happens when there may not be a cvmfsexec distribution that is compatible with the worker node system configuration (e., wrong file name) + logwarn "Could not find the unpacked cvmfsexec distribution ${dist_file}!" + return 1 fi loginfo "CVMFS Source = $cvmfs_source" @@ -615,6 +607,18 @@ perform_cvmfs_mount () { fi prepare_for_cvmfs_mount + if [[ $? -ne 0 ]]; then + # something went wrong during the prep for mounting + # if CVMFS is required, then abort from this and also the glidein setup by flagging an error + if [[ $use_cvmfs -eq 1 || "${glidein_cvmfs_require}" == "required" ]]; then + logerror "Aborting glidein setup (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" + "$error_gen" -error "$(basename $0)" "WN_Resource" "Error finding cvmfsexec distribution... aborting glidein setup!" + exit 1 + fi + # if CVMFS not required, just warn and continue but without mounting + logwarn "Unable to find an appropriate cvmfsexec distribution to mount CVMFS" + return 1 + fi if [[ $mode -eq 3 || $mode -eq 2 ]]; then return # only prepare but do not actually mount (later in glidein reinvocation, mounting will be performed) fi diff --git a/creation/web_base/cvmfs_setup.sh b/creation/web_base/cvmfs_setup.sh index a68750fbe..30b5a10a4 100644 --- a/creation/web_base/cvmfs_setup.sh +++ b/creation/web_base/cvmfs_setup.sh @@ -34,7 +34,7 @@ add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | # get the use_cvmfs attribute value; passed as one of the frontend attributes use_cvmfs=$(gconfig_get GLIDEIN_USE_CVMFS "$1") if [[ -z $use_cvmfs ]]; then - loginfo "CVMFS not requested (GLIDEIN_USE_CVMFS: $use_cvmfs); skipping CVMFS setup." + loginfo "CVMFS not requested (GLIDEIN_USE_CVMFS not used); skipping CVMFS setup." "$error_gen" -ok "$(basename $0)" "mnt_msg1" "CVMFS not requested; skipping setup." return 0 elif ! [[ $use_cvmfs =~ ^[0-1]$ ]]; then @@ -57,7 +57,7 @@ fi if [[ $use_cvmfs -ne 1 ]]; then if ! [[ "${glidein_cvmfs_require}" =~ ^(required|preferred)$ ]]; then - loginfo "CVMFS not used; skipping related setup." + loginfo "GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require; skipping related setup." "$error_gen" -ok "$(basename $0)" "mnt_msg4" "CVMFS not used. Skipping related setup." return 0 fi @@ -71,8 +71,8 @@ else fi # following block runs attempting to add CVMFS when either: -# 1. use_cvmfs is false and glidein_cvmfs_require is required or preferred (OR) -# 2. use_cvmfs is true +# 1. use_cvmfs is false (0) and glidein_cvmfs_require is required|preferred (OR) +# 2. use_cvmfs is true (1) if is_cvmfs_locally_mounted; then loginfo "CVMFS found locally; skipping CVMFS setup via cvmfsexec." loginfo "Continuing to execute the rest of the glidein setup..." @@ -85,80 +85,75 @@ loginfo "Starting on-demand CVMFS setup..." [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check cvmfsexec_mode=$(setup_cvmfsexec_use) if ! [[ $cvmfsexec_mode =~ ^[1-3]$ ]]; then + logwarn "cvmfsexec cannot be used in any of the three modes" if [[ $use_cvmfs -eq 1 || "${glidein_cvmfs_require}" == "required" ]]; then - # when (1) use_cvmfs is 1, or (2) use_cvmfs is 0 and glidein_cvmfs_require is set to required + # when (1) use_cvmfs is true (1), or (2) use_cvmfs is false (0) and glidein_cvmfs_require is set to required logerror "GLIDEIN_USE_CVMFS set to $use_cvmfs but GLIDEIN_CVMFS_REQUIRE is $glidein_cvmfs_require; aborting glidein setup." - "$error_gen" -error "$(basename $0)" "mnt_msg7" "cvmfsexec cannot be used (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" + "$error_gen" -error "$(basename $0)" "mnt_msg7" "cvmfsexec cannot be used, therefore aborting (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" exit 1 fi # when use_cvmfs is 0 and glidein_cvmfs_require is set to preferred, just warn the user - logwarn "GLIDEIN_USE_CVMFS set to $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require" - loginfo "Proceeding to execute the remainder of the glidein setup (without CVMFS)..." + logwarn "GLIDEIN_USE_CVMFS set to $use_cvmfs and GLIDEIN_CVMFS_REQUIRE is $glidein_cvmfs_require; continuing glidein setup (without CVMFS)..." "$error_gen" -ok "$(basename $0)" "mnt_msg8" "cvmfsexec cannot be used but still continuing (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" return 0 fi loginfo "GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require" loginfo "cvmfsexec mode $cvmfsexec_mode is being used..." -echo "" -if [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]]; then - perform_cvmfs_mount $cvmfsexec_mode $glidein_cvmfs_require - # before exiting out of this block, do two things... - # one, set a variable indicating this script has been executed once - gwms_cvmfs_reexec="yes" - gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" - - # two, export required variables with some necessary information for use inside cvmfsexec before reinvoking the glidein... - original_workspace=$(gconfig_get GLIDEIN_WORKSPACE_ORIG "$glidein_config") - export GLIDEIN_WORKSPACE=$original_workspace - export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec - export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode - export GLIDEIN_WORK_DIR="$work_dir" - export GLIDEIN_CVMFS_CONFIG_REPO="$GLIDEIN_CVMFS_CONFIG_REPO" - export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" - - exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" - echo "!!WARNING!! Outside of reinvocation of glidein_startup" - # the above line of code should not run; but is here as a safety check for debugging incorrect behavior of exec from previous line -fi # the following is run if cvmfsexec cannot be used in mode 3/2 -perform_cvmfs_mount $cvmfsexec_mode +perform_cvmfs_mount $cvmfsexec_mode $glidein_cvmfs_require if [[ $? -eq 0 ]]; then - if [[ $GWMS_IS_CVMFS -ne 0 ]]; then - # Error occurred during mount of CVMFS repositories" - logerror "Error occured during mount of CVMFS repositories." - "$error_gen" -error "$(basename $0)" "mnt_msg9" "Mount unsuccessful... CVMFS is still unavailable on the node." - exit 1 + # the following is run if cvmfsexec can be used in mode 3/2 + if [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]]; then + # before exiting out of this block, do two things... + # one, set a variable indicating this script has been executed once + gwms_cvmfs_reexec="yes" + gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" + + # two, export required variables with some necessary information for use inside cvmfsexec before reinvoking the glidein... + original_workspace=$(gconfig_get GLIDEIN_WORKSPACE_ORIG "$glidein_config") + export GLIDEIN_WORKSPACE=$original_workspace + export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec + export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode + export GLIDEIN_WORK_DIR="$work_dir" + export GLIDEIN_CVMFS_CONFIG_REPO="$GLIDEIN_CVMFS_CONFIG_REPO" + export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" + echo "Reinvoking glidein now..." + exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" + echo "!!WARNING!! Outside of reinvocation of glidein_startup" + # the above line of code should not run; but is here as a safety check for debugging incorrect behavior of exec from previous line fi - # CVMFS is available on the worker node now" + # the following is run if cvmfsexec can be used in mode 1 only + # CVMFS is available on the worker node now gwms_cvmfs_reexec="no" gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" # exporting the variables as an environment variable for use in glidein reinvocation export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode - loginfo "Proceeding to execute the remainder of glidein setup..." + loginfo "Proceeding to complete the remainder of glidein setup..." "$error_gen" -ok "$(basename $0)" "mnt_msg10" "CVMFS mounted successfully and is now available." return 0 elif [[ $? -eq 1 ]]; then # if exit status is 1 - if [[ $cvmfs_required == "required" ]]; then + if [[ "${glidein_cvmfs_require}" == "required" ]]; then # if mount CVMFS is not successful, report an error and exit with failure exit code - loginfo "CVMFS is required but unable to mount CVMFS on worker node." - "$error_gen" -error "$(basename $0)" "WN_Resource" "CVMFS is required but unable to mount CVMFS on the worker node." + logerror "Unable to mount CVMFS on worker node; aborting glidein setup (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -error "$(basename $0)" "WN_Resource" "CVMFS required but unable to mount CVMFS on the worker node." exit 1 fi # if cvmfs_required is set to preferred and mount CVMFS is not successful, report a warning/error in the logs and continue with glidein startup - # script status must be OK, otherwise the glidein will fail - "$error_gen" -ok "$(basename $0)" "WN_Resource" "Unable to mount required CVMFS on the worker node. Continuing without CVMFS." + logwarn "Unable to mount CVMFS on worker node; continuing without CVMFS (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -ok "$(basename $0)" "WN_Resource" "CVMFS preferred but could not be mounted. Continuing without CVMFS." return 0 else # if exit status is 2 - if [[ $cvmfs_required == "required" ]]; then - logerror "Non-RHEL OS found but not supported; aborting glidein setup!" - "$error_gen" -error "$(basename $0)" "WN_Resource" "Non-RHEL OS found but not supported; aborting glidein startup" + if [[ $glidein_cvmfs_require == "required" ]]; then + logerror "Non-RHEL OS found but not supported; aborting glidein setup! (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -error "$(basename $0)" "mnt_msg13" "Non-RHEL OS found but not supported; aborting glidein startup" exit 1 fi # if CVMFS is not required, display operating system information and a user-friendly message - "$error_gen" -ok "$(basename $0)" "WN_Resource" "Non-RHEL OS found but not supported; continuing without CVMFS setup" + loginfo "Found non-RHEL OS which is not supported; continuing without CVMFS (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -ok "$(basename $0)" "mnt_msg14" "Non-RHEL OS found but not supported; continuing without CVMFS setup" return 0 fi diff --git a/creation/web_base/cvmfs_umount.sh b/creation/web_base/cvmfs_umount.sh index 6aafb8261..522000077 100755 --- a/creation/web_base/cvmfs_umount.sh +++ b/creation/web_base/cvmfs_umount.sh @@ -20,7 +20,6 @@ # Namratha Urs # -echo "Unmounting on-demand CVMFS as part of glidein cleanup..." glidein_config=$1 # import add_config_line to use gconfig_ utilities @@ -39,46 +38,56 @@ work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$glidein_config") # shellcheck source=./cvmfs_helper_funcs.sh . "$work_dir"/cvmfs_helper_funcs.sh -loginfo "..." +# first check if CVMFS is locally mounted on the worker node +detect_local_cvmfs +if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then + # CVMFS is mounted locally in the filesystem; DO NOT UNMOUNT! + loginfo "Skipping unmounting of CVMFS as it already is locally provisioned in the node!" + "$error_gen" -ok "$(basename $0)" "umnt_msg2" "CVMFS is locally mounted on the node; skipping cleanup." + exit 0 +fi +# if not, CVMFS was mounted on-demand, so unmount based on the cvmfsexec mode is_cvmfs_mntd=$(gconfig_get GWMS_IS_CVMFS "$glidein_config") -loginfo "CVMFS mounted on demand: $is_cvmfs_mntd" -if [[ "${is_cvmfs_mntd}" == "yes" ]]; then - gwms_cvmfsexec_mode=$(gconfig_get GWMS_CVMFSEXEC_MODE "$glidein_config") - # get the cvmfsexec directory location - glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") - loginfo "Unmounting CVMFS provisioned by the glidein..." - if [[ "$gwms_cvmfsexec_mode" -eq 1 ]]; then - "$glidein_cvmfsexec_dir"/.cvmfsexec/umountrepo -a - - if [[ -n "$CVMFS_MOUNT_DIR" ]]; then - CVMFS_MOUNT_DIR= - export CVMFS_MOUNT_DIR - gconfig_add CVMFS_MOUNT_DIR "" - fi - elif [[ "$gwms_cvmfsexec_mode" -eq 3 || "$gwms_cvmfsexec_mode" -eq 2 ]]; then - loginfo "CVMFS_MOUNT_DIR set to $CVMFS_MOUNT_DIR" - loginfo "CVMFSUMOUNT set to $CVMFSUMOUNT" - fi - # check again to ensure all CVMFS repositories were unmounted by umountrepo - # searching for "/dev/fuse" since "/cvmfs" might return false positives (/etc/auto.fs /cvmfs line) - cat /proc/$$/mounts | grep /dev/fuse &> /dev/null && logerror "One or more CVMFS repositories might not be completely unmounted" || loginfo "CVMFS repositories unmounted" - "$error_gen" -ok "$(basename $0)" "umnt_msg1" "Glidein-based CVMFS unmount was successful." - # returning 0 to indicate the unmount process was successful - return 0 -else - # CVMFS was not mounted on-demand, so check if CVMFS is locally mounted on the worker node - detect_local_cvmfs - if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then - # CVMFS is mounted locally in the filesystem; DO NOT UNMOUNT! - loginfo "Skipping unmounting of CVMFS as it already is locally provisioned in the node!" - "$error_gen" -ok "$(basename $0)" "umnt_msg2" "CVMFS is locally mounted on the node; skipping cleanup." - else - # CVMFS is not found natively either - loginfo "CVMFS not found natively; so nothing to do here... continuing." - "$error_gen" -ok "$(basename $0)" "umnt_msg3" "No native CVMFS; nothing to cleanup before glidein shutdown." - fi - return 0 +loginfo "MyDebug (cvmfs_umount.sh): is_cvmfs_mntd = $is_cvmfs_mntd" +if [[ -z "${is_cvmfs_mntd}" || $is_cvmfs_mntd -ne 0 ]]; then + loginfo "CVMFS was not found mounted, skipping CVMFS cleanup..." + exit 0 +fi +# if is_cvmfs_mntd is 0, then do the following +loginfo "Found CVMFS that has been mounted on demand..." +# check which mode was used to mount CVMFS on demand +gwms_cvmfsexec_mode=$(gconfig_get GWMS_CVMFSEXEC_MODE "$glidein_config") +# get the cvmfsexec directory location +glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") +loginfo "Unmounting CVMFS provisioned by the glidein..." +mnt_dir=$(gconfig_get CVMFS_MOUNT_DIR "$glidein_config") +[[ -n "$CVMFS_MOUNT_DIR" ]] && loginfo "CVMFS_MOUNT_DIR set to $CVMFS_MOUNT_DIR" +if [[ $gwms_cvmfsexec_mode -eq 1 ]]; then + "$glidein_cvmfsexec_dir"/.cvmfsexec/umountrepo -a +elif [[ $gwms_cvmfsexec_mode -eq 3 || $gwms_cvmfsexec_mode -eq 2 ]]; then + repos=($(echo $GLIDEIN_CVMFS_REPOS | tr ":" "\n")) + loginfo "Unmounting CVMFS repositories..." + # mount every repository that was previously unpacked + for repo in "${repos[@]}" + do + $CVMFSUMOUNT "$repo" + done + loginfo "Unmounting CVMFS config repo now..." + $CVMFSUMOUNT "${GLIDEIN_CVMFS_CONFIG_REPO}" +fi +# clear the mount_dir variable if it was set during the mounting of CVMFS regardless of the mode +if [[ -n "$CVMFS_MOUNT_DIR" ]]; then + CVMFS_MOUNT_DIR= + export CVMFS_MOUNT_DIR + gconfig_add CVMFS_MOUNT_DIR "" fi +cat /proc/$$/mounts | grep /dev/fuse +# check again to ensure all CVMFS repositories were unmounted by umountrepo +# searching for "/dev/fuse" since "/cvmfs" might return false positives (/etc/auto.fs /cvmfs line) +cat /proc/$$/mounts | grep /dev/fuse &> /dev/null && logerror "One or more CVMFS repositories might not be completely unmounted" || loginfo "CVMFS repositories unmounted" +"$error_gen" -ok "$(basename $0)" "umnt_msg1" "Glidein-based CVMFS unmount was successful." +# returning 0 to indicate the unmount process was successful +exit 0 ############################################################################ # End: main program diff --git a/creation/web_base/cvmfsexec_platform_select.sh b/creation/web_base/cvmfsexec_platform_select.sh index 06991720f..e7f901453 100755 --- a/creation/web_base/cvmfsexec_platform_select.sh +++ b/creation/web_base/cvmfsexec_platform_select.sh @@ -32,14 +32,15 @@ if [[ ! $cvmfs_src =~ ^(osg|egi|default)$ ]]; then fi # TODO: is it possible to reuse cvmfs_helper_funcs.sh by sourcing it during the execution of this file???? -if [ -f '/etc/redhat-release' ]; then +if [[ -f '/etc/redhat-release' ]]; then os_distro=rhel + os_ver=$(cat /etc/redhat-release | cut -d " " -f 3 | awk -F'.' '{print $1}') else os_distro=non-rhel + os_ver=$(cat /etc/os-release | egrep "VERSION_ID" | cut -d = -f 2 | tr -d '"' | awk -F "." '{print $1}') fi -os_ver=`lsb_release -r | awk -F'\t' '{print $2}' | awk -F"." '{print $1}'` -krnl_arch=`arch` +krnl_arch=$(arch) mach_type=${os_distro}${os_ver}-${krnl_arch} cvmfsexec_platform="${cvmfs_src}-${mach_type}" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index c82244306..deabb5a9c 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -12,9 +12,10 @@ printenv GWMS_CVMFS_REEXEC > /dev/null status=$? if [[ "$status" -eq 0 ]]; then - echo "Now reinvoking glidein startup..." + # glidein being reinvoked gwms_cvmfs_reexec=$(printenv GWMS_CVMFS_REEXEC | sed s"/ //g") else + # regular glidein invocation gwms_cvmfs_reexec=$(grep "^GWMS_CVMFS_REEXEC " "${glidein_config}" | cut -d ' ' -f 2-) fi # Customizable and initialization variables diff --git a/creation/web_base/singularity_lib.sh b/creation/web_base/singularity_lib.sh index 755e8ee42..e6bfac890 100644 --- a/creation/web_base/singularity_lib.sh +++ b/creation/web_base/singularity_lib.sh @@ -1663,7 +1663,7 @@ singularity_get_image() { # set things up here since the path needs to be bindmounted inside the container local mount_home=${CVMFS_MOUNT_DIR/\/dist\/cvmfs/} local symlink_target - symlink_target=$(readlink $mount_home/dist/${singularity_image#/}) + symlink_target=$(readlink $mount_home/dist/${singularity_image#/}) singularity_image=$mount_home/dist/${symlink_target#/} fi From 5ebcb35b9c00f210e7a9a4aa6abbf1d2f8742181 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Wed, 3 Apr 2024 17:09:01 -0500 Subject: [PATCH 08/12] documentation updates for cvmfs support --- CHANGELOG.md | 25 ++++++ doc/factory/custom_vars.html | 162 ++++++++++++++++++++++------------- 2 files changed, 129 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 268aed4f2..e51b89c25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,31 @@ Changes since the last release ### Known Issues +## v3.11.0 \[2024-03-dd\] + +This is the very first version in the development series of GlideinWMS releases. + +### New features / functionalities + +- Supports all available modes of cvmfsexec when provisioning CVMFS on demand. (PR #372) + +### Changed defaults / behaviours + +### Deprecated / removed options and commands + +### Security Related Fixes + +### Bug Fixes + +### Testing / Development + +### Known Issues + +- When generating cvmfsexec distribution for EL9 machine type on an EL7 machine, the factory reconfig and/or upgrade fails as a result of an error in `create_cvmfsexec_distros.sh`. This is possibly due to the tools for EL7 being unable to handle EL9 files (as per Dave Dykstra). Please exercise caution if using `rhel9-x86_64` in the `mtypes` list for the `cvmfsexec_distro` tag in factory configuration. + - Our suggested workaround is to remove the EL9 machine type from the default list of machine types supported by the custom distros creation script. Add it back if you are running on an EL9 system and want an EL9 cvmfsexec distribution. (PR #312) +- When using on-demand CVMFS, all Glideins after the first one on a node are failing (Issue #287) + This happens because mountrepo and umountrepo work at the node level and subsequent Glideins see the mounts done by the first one and abort. To avoid problems use only whole-node Glideins when using on-demand CVMFS. All versions with on-demand CVMFS are affected. + ## v3.10.6 \[2024-01-25\] Minor new features, mostly a bug fix release diff --git a/doc/factory/custom_vars.html b/doc/factory/custom_vars.html index f0ec911c1..244adb779 100644 --- a/doc/factory/custom_vars.html +++ b/doc/factory/custom_vars.html @@ -2715,13 +2715,19 @@

Singularity Variables (dynamic)

-

CVMFS Variables

+

On-demand CVMFS Provisioning

- This section describes the variables that can be configured for use - with the glidein to allow CVMFS to be provisioned on-demand. These - variables are configured at the glidein level so that it applies to - all the entries that are configured in the factory's configuration - file. + GlideinWMS supports provisioning of CVMFS on demand when sites may not + have CVMFS natively. GlideinWMS uses the + cvmfsexec utility to provision CVMFS as needed when CVMFS is + unavailable (more details about cvmfsexec + here). The + following table describes the knobs that can be used to configure + GlideinWMS to allow provisioning of CVMFS when a native installation + of CVMFS is unavailable on a compute element (CE). The configuration + knobs identified for use in factory can be configured such that the + configuration applies to all the defined entries defined, or a + specific entry.

@@ -2742,18 +2748,17 @@

CVMFS Variables

- + - + @@ -2764,53 +2769,85 @@

CVMFS Variables

- +
GLIDEIN_USE_CVMFSEXECGLIDEIN_USE_CVMFS Int FactoryFrontend

- Not set by default. This variable controls whether - cvmfsexec utility is used to mount CVMFS. Possible values - are 0 or 1. A value of 0 indicates that glidein will not use - cvmfsexec to mount CVMFS on the worker noden. A value of - 1 indicates cvmfsexec will be used to mount CVMFS on the - worker node by the glidein. + Not set by default. This attribute controls whether the user/job + requires CVMFS. Possible values are 0 or 1. A value of 0 + indicates that the job does not need to use CVMFS during its + execution on the compute element. A value of 1 indicates that + the job needs to use CVMFS when it runs on the compute element.

Factory

- Not set by default. This variable controls the origin (source) + Not set by default. This attribute controls the origin (source) from which to download the CVMFS configuration repository from. Possible values are:
- osg - configuration for cvmfs-config-osg;
- egi - configuration for cvmfs-config-egi;
- default - configuration for cvmfs-config-default
+ osg - configuration for cvmfs-config-osg
+ egi - configuration for cvmfs-config-egi
+ default - configuration for cvmfs-config-default
GLIDEIN_CVMFSGLIDEIN_CVMFS_REQUIRE String Factory

- Not set by default. This variable controls the error handling - behavior of the glidein in case of failures during the mounting - of CVMFS. Possible values are:
- REQUIRED - if unable to mount, report an error and abort - mounting);
- PREFERRED - if unable to mount, notify and continue normally;
- OPTIONAL - continue if unable to mount (similar to - PREFERRED);
- NEVER - only exit if mounting fails
+ Not set by default. This attribute, in conjunction with + GLIDEIN_USE_CVMFS (defined in frontend), specifies the + site/factory policy to be used when provisioning CVMFS. This + also helps in determining the behavior of the glidein behavior + in case failures are encountered during the provisioning of + CVMFS.

+

+ This configuration attribute takes one of the following three + values:
+

+
    +
  • + REQUIRED - if unable to mount CVMFS on demand, report an error + and abort the glidein setup +
    • +
  • + PREFERRED - if unable to mount CVMFS on demand, report a + warning and continue with the glidein setup normally +
    • +
  • + NEVER - do not even attempt to mount CVMFS on demand even if + the user job requests via GLIDEIN_USE_CVMFS +
    • +
+

+ Assuming that CVMFS is not found natively, the policy driving + glidein behavior is as follows: +

+
    +
  • when GLIDEIN_USE_CVMFS is 1:
    • +
      +
    • + GLIDEIN_CVMFS_REQUIRE is NEVER: glidein will not match sites + (and therefore, not provision CVMFS on those sites). +
      • +
    • + GLIDEIN_CVMFS_REQUIRE is REQUIRED|PREFERRED: glidein will + try to provision CVMFS if not found natively. Glidein will + fail when unable to provision CVMFS irrespective of + GLIDEIN_CVMFS_REQUIRE value. +
      • +
    +
  • when GLIDEIN_USE_CVMFS is 0:
    • +
      +
    • + GLIDEIN_CVMFS_REQUIRE is NEVER: does not matter as + GLIEIN_USE_CVMFS says that the job does not want CVMFS. +
      • +
    • + GLIDEIN_CVMFS_REQUIRE is REQUIRED|PREFERRED: glidein will + try to provision CVMFS if not found natively. Glidein will + fail when unable to provision CVMFS only when + GLIDEIN_CVMFS_REQUIRE is REQUIRED. Otherwise, glidein will + continue without CVMFS (when GLIDEIN_CVMFS_REQUIRE is + PREFERRED). +
      • +
    +

-

- Dynamic creation and selection of cvmfsexec platform-specific - distribution -

-

- GlideinWMS factory includes the capability to mount and unmount CVMFS - on demand, i.e. for sites that do not have a local installation of - CVMFS available. This is achieved via the scripts - cvmfs_setup.sh and cvmfs_umount.sh. Two additional shell - scripts cvmfs_helper_funcs.sh and cvmfs_mount .sh are - used as helper scripts during the mounting process. On demand mounting - and unmounting is achieved via the cvmfsexec utility. - The helper scripts are enabled for conditional download based on the - GLIDEIN_USE_CVMFSEXEC variable. When the GLIDEIN_USE_CVMFSEXEC - variable is set (value of 1), the helper scripts are downloaded for - facilitating the mounting of CVMFS. -

cvmfsexec can be packaged as self-contained distribution specific to an operating system and platform with required configuration and @@ -2822,21 +2859,30 @@

distribution for use by the worker node. However, rather than having a tarball containing multiple cvmfsexec distribution files corresponding to a (CVMFS source, system platform, architecture) combination, - GlideinWMS dynamically creates and selects the appropriate cvmfsexec - distribution from the list of distributions. + GlideinWMS allows the dynamic creation and selection of the + appropriate cvmfsexec distribution from the list of distributions as + explained next. +

+

+ cvmfsexec distributions are tied to a CVMFS source, system platform + and architecture. As needed for provisioning CVMFS, the distributions + can be created as part of the factory reconfiguration and/or upgrade + by using the configuration knob <cvmfsexec_distro> tag in the + factory configuration file. When this knob is enabled, the script + `/bin/create_cvmfsexec_distros.sh` is invoked dynamically. Depending + on the CVMFS source ('sources' attribute for the knob) and the + platform requirements ('platforms' attribute for the knob), the + distributions are packaged as individual tarballs and are added to the + default list of uploads during factory reconfiguration and/or upgrade + when necessary.

- Specific to a CVMFS source, system platform and architecture, - cvmfsexec distribution files are created automatically by the script - named create_cvmfsexec_distros.sh. The script is placed in - hooks.reconfig.pre directory for dynamic execution. Each of - these distributions are packaged as individual tarballs and are added - to the default list of uploads at the time of factory reconfiguration - and/or upgrade. Another script, cvmfsexec_platform_select.sh, - automatically selects the appropriate distribution tarball based on - the specifics of the worker node and ships the distribution with the - glidein. This script is invoked during the customization of the worker - node as part of the glidein setup. + Once the cvmfsexec distributions are built, the + cvmfsexec_platform_select.sh script then automatically selects + the appropriate distribution tarball based on the specifics of the + worker node and ships the distribution with the glidein. This script + is invoked during the glidein startup process prior to the script that + actually performs all the needed steps towards CVMFS setup.

From c81d55a2f2e23fb2f9e82ae99fa6b7b5268cd581 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Tue, 23 Apr 2024 18:09:12 -0500 Subject: [PATCH 09/12] addressing most of the review comments --- creation/lib/cgWConsts.py | 4 +- creation/lib/cgWDictFile.py | 22 +- creation/web_base/cvmfs_helper_funcs.sh | 60 +- creation/web_base/cvmfs_setup.sh | 2 +- creation/web_base/glidein_startup.sh | 1238 ++++++++++++----------- 5 files changed, 652 insertions(+), 674 deletions(-) diff --git a/creation/lib/cgWConsts.py b/creation/lib/cgWConsts.py index d851acbfc..bfe80e650 100644 --- a/creation/lib/cgWConsts.py +++ b/creation/lib/cgWConsts.py @@ -24,8 +24,8 @@ CVMFSEXEC_DIR = "cvmfsexec" CVMFSEXEC_ATTR = "CVMFSEXEC_DIR" -# constant that defines the priority values used by the factory/glidein -PRIORITY_SETTINGS = ["file_list", "precvmfs_file_list", "at_file_list", "after_file_list"] +# constant defining the priorities of file lists used by the factory/glidein +FILE_LISTS_PRIORITIES = ("file_list", "precvmfs_file_list", "at_file_list", "after_file_list") # these are in the submit dir, so they can be changed SUBMIT_ATTRS_FILE = "submit_attrs.cfg" diff --git a/creation/lib/cgWDictFile.py b/creation/lib/cgWDictFile.py index 7dbc4dad3..d4d9347c5 100644 --- a/creation/lib/cgWDictFile.py +++ b/creation/lib/cgWDictFile.py @@ -417,7 +417,7 @@ def load_entry_dicts(entry_dicts, entry_name, summary_signature): # update in p def refresh_description(dicts): # update in place description_dict = dicts["description"] description_dict.add(dicts["signature"].get_fname(), "signature", allow_overwrite=True) - for k in cgWConsts.PRIORITY_SETTINGS: + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: description_dict.add(dicts[k].get_fname(), k, allow_overwrite=True) @@ -463,21 +463,11 @@ def refresh_file_list(dicts, is_main, files_set_readonly=True, files_reset_chang # dictionaries must have been written to disk before using this def refresh_signature(dicts): # update in place signature_dict = dicts["signature"] - for k in ( - "consts", - "vars", - "untar_cfg", - "gridmap", - "file_list", - "precvmfs_file_list", - "at_file_list", - "after_file_list", - "description", - ): + for k in ("consts", "vars", "untar_cfg", "gridmap") + cgWConsts.FILE_LISTS_PRIORITIES + ("description"): if k in dicts: signature_dict.add_from_file(dicts[k].get_filepath(), allow_overwrite=True) # add signatures of all the files linked in the lists - for k in cgWConsts.PRIORITY_SETTINGS: + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: filedict = dicts[k] for fname in filedict.get_immutable_files(): @@ -511,11 +501,11 @@ def save_common_dicts(dicts, is_main, set_readonly=True): # 'consts','untar_cfg','vars' will be loaded refresh_file_list(dicts, is_main) # save files in the file lists - for k in cgWConsts.PRIORITY_SETTINGS: + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: dicts[k].save_files(allow_overwrite=True) # then save the lists - for k in cgWConsts.PRIORITY_SETTINGS: + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: dicts[k].save(set_readonly=set_readonly) # calc and save the signatures @@ -610,7 +600,7 @@ def reuse_common_dicts(dicts, other_dicts, is_main, all_reused): # since the file names may have changed, refresh the file_list refresh_file_list(dicts, is_main) # check file-based dictionaries - for k in cgWConsts.PRIORITY_SETTINGS: + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: all_reused = reuse_file_dict(dicts, other_dicts, k) and all_reused diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index 08e54c316..7eaf287db 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -27,17 +27,16 @@ #exec &> $LOGFILE variables_reset() { - # DESCRIPTION: This function lists and initializes the common variables - # to empty strings. These variables also become available to scripts - # that import functions defined in this script. + # Initializes the common variables to empty strings. These variables also become available to scripts that import functions defined in this script. # # INPUT(S): None # RETURN(S): Variables initialized to empty strings - # indicates whether the perform_system_check function has been run + # all of the variables in here are initialized to an empty string, where empty means not evaluated + # this variable indicates whether the perform_system_check function has been run; when evaluated value would be 'yes', otherwise remains empty GWMS_SYSTEM_CHECK= - # following set of variables used to store operating system and kernel info + # these variables are used to store operating system and kernel info GWMS_OS_DISTRO= GWMS_OS_NAME= GWMS_OS_VERSION_FULL= @@ -50,6 +49,7 @@ variables_reset() { GWMS_OS_KRNL_MINOR_REV= GWMS_OS_KRNL_PATCH_NUM= + # the following variables are also initialized to an empty string; when evaluated can take on values 1 (meaning false/no) or 0 (meaning true/yes) # indicates whether CVMFS is locally mounted on the worker node (CE) GWMS_IS_CVMFS_LOCAL_MNT= # to indicate the status of on-demand mounting of CVMFS by the glidein after evaluating the worker node (CE) @@ -68,8 +68,7 @@ variables_reset() { loginfo() { - # DESCRIPTION: This function prints informational messages to STDOUT - # along with hostname and date/time. + # Prints informational messages to STDOUT along with hostname and date/time. # # INPUT(S): String containing the message # RETURN(S): Prints message to STDOUT @@ -79,8 +78,7 @@ loginfo() { logwarn(){ - # DESCRIPTION: This function prints warning messages to STDOUT along - # with hostname and date/time. + # Prints warning messages to STDOUT along with hostname and date/time. # # INPUT(S): String containing the message # RETURN(S): Prints message to STDOUT @@ -90,8 +88,7 @@ logwarn(){ logerror() { - # DESCRIPTION: This function prints error messages to STDOUT along with - # hostname and date/time. + # Prints error messages to STDOUT along with hostname and date/time. # # INPUT(S): String containing the message # RETURN(S): Prints message to STDOUT @@ -101,8 +98,7 @@ logerror() { print_exit_status () { - # DESCRIPTION: This function prints an appropriate message to the - # console to indicate what the exit status means. + # Prints an appropriate message to the console to indicate what the exit status means. # # INPUT(S): Number (exit status of a previously run command) # RETURN(S): Prints "yes" or "no" to indicate the result of the command @@ -112,14 +108,13 @@ print_exit_status () { detect_local_cvmfs() { - # DESCRIPTION: This function detects whether CVMFS is natively (aka locally) available on the worker node. The result is stored in a common variable, i.e. GWMS_IS_CVMFS_LOCAL_MNT, and can be used downstream. + # Detects whether CVMFS is natively (aka locally) available on the worker node. The result is stored in a common variable, i.e GWMS_IS_CVMFS_LOCAL_MNT, and can be used downstream. # # INPUT(S): None # RETURN(S): None - CVMFS_ROOT="/cvmfs" - repo_name=oasis.opensciencegrid.org - # Second check... + local CVMFS_ROOT="/cvmfs" + local repo_name=oasis.opensciencegrid.org GWMS_IS_CVMFS_LOCAL_MNT=0 if [[ -f $CVMFS_ROOT/$repo_name/.cvmfsdirtab || "$(ls -A $CVMFS_ROOT/$repo_name)" ]] &>/dev/null then @@ -134,9 +129,7 @@ detect_local_cvmfs() { perform_system_check() { - # DESCRIPTION: This functions performs required system checks (such as - # operating system and kernel info, unprivileged user namespaces, FUSE - # status) and stores the results in the common variables for later use. + # Performs required system checks (such as operating system and kernel info, unprivileged user namespaces, FUSE status) and stores the results in the common variables for later use. # # INPUT(S): None # RETURN(S): @@ -200,8 +193,7 @@ perform_system_check() { print_os_info () { - # DESCRIPTION: This functions prints operating system and kernel - # information to STDOUT. + # Prints operating system and kernel information to STDOUT. # # INPUT(S): None # RETURN(S): Prints a message containing OS and kernel details @@ -211,11 +203,7 @@ print_os_info () { log_all_system_info () { - # DESCRIPTION: This function prints all the necessary system information - # stored in common and result variables (see perform_system_check - # function) for easy debugging. This has been done as collecting - # information about the worker node can be useful for troubleshooting - # and gathering stats about what is out there. + # Prints all the necessary system information stored in common and result variables (see perform_system_check function) for easy debugging. This has been done as collecting information about the worker node can be useful for troubleshooting and gathering stats about what is out there. # # INPUT(S): None # RETURN(S): Prints user-friendly messages to STDOUT @@ -246,8 +234,7 @@ log_all_system_info () { mount_cvmfs_repos () { - # DESCRIPTION: This function mounts all the required and additional - # CVMFS repositories that would be needed for user jobs. + # Mounts all the required and additional CVMFS repositories that would be needed for user jobs. # # INPUT(S): # 1. cvmfsexec mode (integer) @@ -317,7 +304,7 @@ mount_cvmfs_repos () { get_mount_point() { - # DESCRIPTION: This function is used to obtain the mount point information regarding where CVMFS is mounted on deman (when mounted). By default, CVMFS when mounted is at '/cvmfs'. Otherwise, CVMFS will be mounted at /.cvmfsexec/dist/cvmfs + # Obtain the mount point information regarding where CVMFS is mounted on demand (when mounted). By default, CVMFS when mounted is at '/cvmfs'. Otherwise, CVMFS will be mounted at /.cvmfsexec/dist/cvmfs # # INPUT(S): None # RETURN(S): None @@ -335,8 +322,7 @@ get_mount_point() { has_unpriv_userns() { - # DESCRIPTION: This function checks the status of unprivileged user - # namespaces being supported and enabled on the worker node. + # Checks the status of unprivileged user namespaces being supported and enabled on the worker node. # # INPUT(S): None # RETURN(S): @@ -377,7 +363,7 @@ has_unpriv_userns() { has_fuse() { - # DESCRIPTION: This function checks the status of FUSE configuration being available on the worker node. + # Check the status of FUSE configuration being available on the worker node. # # INPUT(S): None # RETURN(S): @@ -444,7 +430,7 @@ has_fuse() { determine_cvmfsexec_mode_usage() { - # DESCRIPTION: This function is used to determine the cvmfsexec mode that will be applicable based on the worker node specifications, including the status of unprivileged user namespaces and FUSE configuration. + # Determine the cvmfsexec mode that will be applicable based on the worker node specifications, including the status of unprivileged user namespaces and FUSE configuration. # # INPUT(S): None # RETURN(S): @@ -504,7 +490,7 @@ determine_cvmfsexec_mode_usage() { setup_cvmfsexec_use() { - # DESCRIPTION: This function performs the necessary setup prior to using cvmfsexec, if possible. If cvmfsexec can be used in either of the three modes, the specific mode information is written to the glidein configuration file. + # Performs the necessary setup prior to using cvmfsexec, if possible. If cvmfsexec can be used in either of the three modes, the specific mode information is written to the glidein configuration file. # # INPUT(S): None # RETURN(S): an integer depicting the cvmfsexec mode that is applicable for the worker node. @@ -526,7 +512,7 @@ setup_cvmfsexec_use() { prepare_for_cvmfs_mount () { - # DESCRIPTION: This function is used to prepare the necessary items and keep them ready/accessible right before mounting CVMFS on demand. + # Prepare the necessary items and keep them ready/accessible right before mounting CVMFS on demand. # # INPUT(S): None # RETURN(S): None @@ -582,7 +568,7 @@ prepare_for_cvmfs_mount () { perform_cvmfs_mount () { - # DESCRIPTION: This function serves as a wrapper for performing mounting of CVMFS on demand depending on a few factors. + # Wrapper for performing mounting of CVMFS on demand depending on a few factors. # # INPUT(S): an integer denoting the selected cvmfsexec mode # RETURN(S): to stdout one of the following values: diff --git a/creation/web_base/cvmfs_setup.sh b/creation/web_base/cvmfs_setup.sh index 30b5a10a4..748ec9841 100644 --- a/creation/web_base/cvmfs_setup.sh +++ b/creation/web_base/cvmfs_setup.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index deabb5a9c..a10637539 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1332,729 +1332,731 @@ usage() { ##################################################################### if [[ -z "$gwms_cvmfs_reexec" ]]; then - # if GWMS_CVMFS_REEXEC is empty, then this script is being invoked the first time; so proceed with usual glidein setup... +# TODO: not indenting to limit git conflicts, hold off until GWMS v3.12 +# if GWMS_CVMFS_REEXEC is empty, then this script is being invoked the first time; so proceed with usual glidein setup... + +# Variables initialized on top of the file + +# Command line options parsing. Storing all in global variables. +# params will contain the full list of parameters +# -param_XXX YYY will become "XXX YYY" +# TODO: can use an array instead? +params="" + +while [ $# -gt 0 ] +do case "$1" in + -factory) glidein_factory="$2";; + -name) glidein_name="$2";; + -entry) glidein_entry="$2";; + -clientname) client_name="$2";; + -clientgroup) client_group="$2";; + -web) repository_url="$2";; + -proxy) proxy_url="$2";; + -dir) work_dir="$2";; + -sign) sign_id="$2";; + -signtype) sign_type="$2";; + -signentry) sign_entry_id="$2";; + -cluster) condorg_cluster="$2";; + -subcluster) condorg_subcluster="$2";; + -submitcredid) glidein_cred_id="$2";; + -schedd) condorg_schedd="$2";; + -descript) descript_file="$2";; + -descriptentry) descript_entry_file="$2";; + -clientweb) client_repository_url="$2";; + -clientwebgroup) client_repository_group_url="$2";; + -clientsign) client_sign_id="$2";; + -clientsigntype) client_sign_type="$2";; + -clientsigngroup) client_sign_group_id="$2";; + -clientdescript) client_descript_file="$2";; + -clientdescriptgroup) client_descript_group_file="$2";; + -slotslayout) slots_layout="$2";; + -v) operation_mode="$2";; + -multiglidein) multi_glidein="$2";; + -multirestart) multi_glidein_restart="$2";; + -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + *) (warn "Unknown option $1"; usage) 1>&2; exit 1 +esac +shift 2 +done - # Variables initialized on top of the file +# make sure we have a valid slots_layout +if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then + slots_layout="fixed" +else + slots_layout="partitionable" +fi - # Command line options parsing. Storing all in global variables. - # params will contain the full list of parameters - # -param_XXX YYY will become "XXX YYY" - # TODO: can use an array instead? - params="" +#################################### +# Cleaup, print out message and exit +work_dir_created=0 +glide_local_tmp_dir_created=0 + +################ +# Parse and verify arguments + +# allow some parameters to change arguments +# multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein +tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") +[ -n "${tmp_par}" ] && multi_glidein=${tmp_par} + +case "${operation_mode}" in + nodebug) + sleep_time=1199 + set_debug=0;; + fast) + sleep_time=150 + set_debug=1;; + check) + sleep_time=150 + set -x + set_debug=2;; + *) + sleep_time=1199 + set_debug=1;; +esac + +if [ -z "${descript_file}" ]; then + warn "Missing descript fname." + usage +fi - while [ $# -gt 0 ] - do case "$1" in - -factory) glidein_factory="$2";; - -name) glidein_name="$2";; - -entry) glidein_entry="$2";; - -clientname) client_name="$2";; - -clientgroup) client_group="$2";; - -web) repository_url="$2";; - -proxy) proxy_url="$2";; - -dir) work_dir="$2";; - -sign) sign_id="$2";; - -signtype) sign_type="$2";; - -signentry) sign_entry_id="$2";; - -cluster) condorg_cluster="$2";; - -subcluster) condorg_subcluster="$2";; - -submitcredid) glidein_cred_id="$2";; - -schedd) condorg_schedd="$2";; - -descript) descript_file="$2";; - -descriptentry) descript_entry_file="$2";; - -clientweb) client_repository_url="$2";; - -clientwebgroup) client_repository_group_url="$2";; - -clientsign) client_sign_id="$2";; - -clientsigntype) client_sign_type="$2";; - -clientsigngroup) client_sign_group_id="$2";; - -clientdescript) client_descript_file="$2";; - -clientdescriptgroup) client_descript_group_file="$2";; - -slotslayout) slots_layout="$2";; - -v) operation_mode="$2";; - -multiglidein) multi_glidein="$2";; - -multirestart) multi_glidein_restart="$2";; - -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (warn "Unknown option $1"; usage) 1>&2; exit 1 - esac - shift 2 - done +if [ -z "${descript_entry_file}" ]; then + warn "Missing descript fname for entry." + usage +fi - # make sure we have a valid slots_layout - if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then - slots_layout="fixed" - else - slots_layout="partitionable" - fi - - #################################### - # Cleaup, print out message and exit - work_dir_created=0 - glide_local_tmp_dir_created=0 - - ################ - # Parse and verify arguments - - # allow some parameters to change arguments - # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein - tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") - [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} - - case "${operation_mode}" in - nodebug) - sleep_time=1199 - set_debug=0;; - fast) - sleep_time=150 - set_debug=1;; - check) - sleep_time=150 - set -x - set_debug=2;; - *) - sleep_time=1199 - set_debug=1;; - esac +if [ -z "${glidein_name}" ]; then + warn "Missing gliden name." + usage +fi - if [ -z "${descript_file}" ]; then - warn "Missing descript fname." - usage - fi +if [ -z "${glidein_entry}" ]; then + warn "Missing glidein entry name." + usage +fi - if [ -z "${descript_entry_file}" ]; then - warn "Missing descript fname for entry." - usage - fi - if [ -z "${glidein_name}" ]; then - warn "Missing gliden name." - usage - fi +if [ -z "${repository_url}" ]; then + warn "Missing Web URL." + usage +fi - if [ -z "${glidein_entry}" ]; then - warn "Missing glidein entry name." - usage - fi +export GWMS_REPOSITORY_URL=$repository_url # for glidein reinvocation +repository_entry_url="${repository_url}/entry_${glidein_entry}" +if [ -z "${proxy_url}" ]; then + proxy_url="None" +fi - if [ -z "${repository_url}" ]; then - warn "Missing Web URL." - usage +if [ "${proxy_url}" = "OSG" ]; then + if [ -z "${OSG_SQUID_LOCATION}" ]; then + # if OSG does not define a Squid, then don't use any + proxy_url="None" + warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 + else + proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" fi +fi - export REPOSITORY_URL=$repository_url # for glidein reinvocation - repository_entry_url="${repository_url}/entry_${glidein_entry}" +if [ -z "${sign_id}" ]; then + warn "Missing signature." + usage +fi - if [ -z "${proxy_url}" ]; then - proxy_url="None" - fi +if [ -z "${sign_entry_id}" ]; then + warn "Missing entry signature." + usage +fi - if [ "${proxy_url}" = "OSG" ]; then - if [ -z "${OSG_SQUID_LOCATION}" ]; then - # if OSG does not define a Squid, then don't use any - proxy_url="None" - warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 - else - proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" - fi - fi +if [ -z "${sign_type}" ]; then + sign_type="sha1" +fi - if [ -z "${sign_id}" ]; then - warn "Missing signature." - usage - fi +if [ "${sign_type}" != "sha1" ]; then + warn "Unsupported signtype ${sign_type} found." + usage +fi - if [ -z "${sign_entry_id}" ]; then - warn "Missing entry signature." - usage +if [ -n "${client_repository_url}" ]; then + # client data is optional, user url as a switch + if [ -z "${client_sign_type}" ]; then + client_sign_type="sha1" fi - if [ -z "${sign_type}" ]; then - sign_type="sha1" + if [ "${client_sign_type}" != "sha1" ]; then + warn "Unsupported clientsigntype ${client_sign_type} found." + usage fi - if [ "${sign_type}" != "sha1" ]; then - warn "Unsupported signtype ${sign_type} found." + if [ -z "${client_descript_file}" ]; then + warn "Missing client descript fname." usage fi - if [ -n "${client_repository_url}" ]; then - # client data is optional, user url as a switch - if [ -z "${client_sign_type}" ]; then - client_sign_type="sha1" - fi - - if [ "${client_sign_type}" != "sha1" ]; then - warn "Unsupported clientsigntype ${client_sign_type} found." + if [ -n "${client_repository_group_url}" ]; then + # client group data is optional, user url as a switch + if [ -z "${client_group}" ]; then + warn "Missing client group name." usage fi - if [ -z "${client_descript_file}" ]; then - warn "Missing client descript fname." + if [ -z "${client_descript_group_file}" ]; then + warn "Missing client descript fname for group." usage fi + export GWMS_CLIENT_REPOSITORY_GROUP_URL=$client_repository_group_url # for glidein reinvocation + fi + export GWMS_CLIENT_REPOSITORY_URL=$client_repository_url # for glidein reinvocation +fi - if [ -n "${client_repository_group_url}" ]; then - # client group data is optional, user url as a switch - if [ -z "${client_group}" ]; then - warn "Missing client group name." - usage - fi +# Generate glidein UUID +if command -v uuidgen >/dev/null 2>&1; then + glidein_uuid="$(uuidgen)" +else + glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" +fi - if [ -z "${client_descript_group_file}" ]; then - warn "Missing client descript fname for group." - usage - fi - export CLIENT_REPOSITORY_GROUP_URL=$client_repository_group_url # for glidein reinvocation - fi - export CLIENT_REPOSITORY_URL=$client_repository_url # for glidein reinvocation +# Print initial variables values (argumants and environment) +startup_time="$(date +%s)" +echo "Starting glidein_startup.sh at $(date) (${startup_time})" +export GWMS_STARTUP_TIME=$startup_time # for glidein reinvocation + +echo "script_checksum = '$(md5wrapper "$0")'" +echo "debug_mode = '${operation_mode}'" +echo "condorg_cluster = '${condorg_cluster}'" +echo "condorg_subcluster= '${condorg_subcluster}'" +echo "condorg_schedd = '${condorg_schedd}'" +echo "glidein_uuid = '${glidein_uuid}'" +echo "glidein_credential_id = '${glidein_cred_id}'" +echo "glidein_factory = '${glidein_factory}'" +echo "glidein_name = '${glidein_name}'" +echo "glidein_entry = '${glidein_entry}'" +if [ -n "${client_name}" ]; then + # client name not required as it is not used for anything but debug info + echo "client_name = '${client_name}'" +fi +if [ -n "${client_group}" ]; then + echo "client_group = '${client_group}'" +fi +echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" +echo "work_dir = '${work_dir}'" +echo "web_dir = '${repository_url}'" +echo "sign_type = '${sign_type}'" +echo "proxy_url = '${proxy_url}'" +echo "descript_fname = '${descript_file}'" +echo "descript_entry_fname = '${descript_entry_file}'" +echo "sign_id = '${sign_id}'" +echo "sign_entry_id = '${sign_entry_id}'" +if [ -n "${client_repository_url}" ]; then + echo "client_web_dir = '${client_repository_url}'" + echo "client_descript_fname = '${client_descript_file}'" + echo "client_sign_type = '${client_sign_type}'" + echo "client_sign_id = '${client_sign_id}'" + if [ -n "${client_repository_group_url}" ]; then + echo "client_web_group_dir = '${client_repository_group_url}'" + echo "client_descript_group_fname = '${client_descript_group_file}'" + echo "client_sign_group_id = '${client_sign_group_id}'" fi +fi +echo +echo "Running on $(uname -n)" +echo "System: $(uname -a)" +if [ -e '/etc/redhat-release' ]; then + echo "Release: $(cat /etc/redhat-release 2>&1)" +fi +echo "As: $(id)" +echo "PID: $$" +echo - # Generate glidein UUID - if command -v uuidgen >/dev/null 2>&1; then - glidein_uuid="$(uuidgen)" - else - glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" - fi - - # Print initial variables values (argumants and environment) - startup_time="$(date +%s)" - echo "Starting glidein_startup.sh at $(date) (${startup_time})" - export STARTUP_TIME=$startup_time # for glidein reinvocation - - echo "script_checksum = '$(md5wrapper "$0")'" - echo "debug_mode = '${operation_mode}'" - echo "condorg_cluster = '${condorg_cluster}'" - echo "condorg_subcluster= '${condorg_subcluster}'" - echo "condorg_schedd = '${condorg_schedd}'" - echo "glidein_uuid = '${glidein_uuid}'" - echo "glidein_credential_id = '${glidein_cred_id}'" - echo "glidein_factory = '${glidein_factory}'" - echo "glidein_name = '${glidein_name}'" - echo "glidein_entry = '${glidein_entry}'" - if [ -n "${client_name}" ]; then - # client name not required as it is not used for anything but debug info - echo "client_name = '${client_name}'" - fi - if [ -n "${client_group}" ]; then - echo "client_group = '${client_group}'" - fi - echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" - echo "work_dir = '${work_dir}'" - echo "web_dir = '${repository_url}'" - echo "sign_type = '${sign_type}'" - echo "proxy_url = '${proxy_url}'" - echo "descript_fname = '${descript_file}'" - echo "descript_entry_fname = '${descript_entry_file}'" - echo "sign_id = '${sign_id}'" - echo "sign_entry_id = '${sign_entry_id}'" - if [ -n "${client_repository_url}" ]; then - echo "client_web_dir = '${client_repository_url}'" - echo "client_descript_fname = '${client_descript_file}'" - echo "client_sign_type = '${client_sign_type}'" - echo "client_sign_id = '${client_sign_id}'" - if [ -n "${client_repository_group_url}" ]; then - echo "client_web_group_dir = '${client_repository_group_url}'" - echo "client_descript_group_fname = '${client_descript_group_file}'" - echo "client_sign_group_id = '${client_sign_group_id}'" - fi - fi - echo - echo "Running on $(uname -n)" - echo "System: $(uname -a)" - if [ -e '/etc/redhat-release' ]; then - echo "Release: $(cat /etc/redhat-release 2>&1)" - fi - echo "As: $(id)" - echo "PID: $$" - echo - - if [ ${set_debug} -ne 0 ]; then - echo "------- Initial environment ---------------" 1>&2 - env 1>&2 - echo "------- =================== ---------------" 1>&2 - fi - - # Before anything else, spawn multiple glideins and wait, if asked to do so - if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then - # start multiple glideins - ON_DIE=0 - trap 'ignore_signal' SIGHUP - trap_with_arg 'on_die_multi' SIGTERM SIGINT SIGQUIT - do_start_all "${multi_glidein}" - # Wait for all glideins and exit 0 - # TODO: Summarize exit codes and status from all child glideins - echo "------ Multi-glidein parent waiting for child processes (${GWMS_MULTIGLIDEIN_CHILDS}) ----------" 1>&2 - wait - echo "------ Exiting multi-glidein parent ----------" 1>&2 - exit 0 - fi - - ######################################## - # make sure nobody else can write my files - # In the Grid world I cannot trust anybody - if ! umask 0022; then - early_glidein_failure "Failed in umask 0022" - fi - - ######################################## - # Setup OSG and/or Globus - if [ -r "${OSG_GRID}/setup.sh" ]; then - . "${OSG_GRID}/setup.sh" - else - if [ -r "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" ]; then - . "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" - fi - fi +if [ ${set_debug} -ne 0 ]; then + echo "------- Initial environment ---------------" 1>&2 + env 1>&2 + echo "------- =================== ---------------" 1>&2 +fi - if [ -z "${GLOBUS_PATH}" ]; then - if [ -z "${GLOBUS_LOCATION}" ]; then - # if GLOBUS_LOCATION not defined, try to guess it - if [ -r "/opt/globus/etc/globus-user-env.sh" ]; then - GLOBUS_LOCATION=/opt/globus - elif [ -r "/osgroot/osgcore/globus/etc/globus-user-env.sh" ]; then - GLOBUS_LOCATION=/osgroot/osgcore/globus - else - warn "GLOBUS_LOCATION not defined and could not guess it." - warn "Looked in:" - warn ' /opt/globus/etc/globus-user-env.sh' - warn ' /osgroot/osgcore/globus/etc/globus-user-env.sh' - warn 'Continuing like nothing happened' - fi - fi +# Before anything else, spawn multiple glideins and wait, if asked to do so +if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then + # start multiple glideins + ON_DIE=0 + trap 'ignore_signal' SIGHUP + trap_with_arg 'on_die_multi' SIGTERM SIGINT SIGQUIT + do_start_all "${multi_glidein}" + # Wait for all glideins and exit 0 + # TODO: Summarize exit codes and status from all child glideins + echo "------ Multi-glidein parent waiting for child processes (${GWMS_MULTIGLIDEIN_CHILDS}) ----------" 1>&2 + wait + echo "------ Exiting multi-glidein parent ----------" 1>&2 + exit 0 +fi + +######################################## +# make sure nobody else can write my files +# In the Grid world I cannot trust anybody +if ! umask 0022; then + early_glidein_failure "Failed in umask 0022" +fi + +######################################## +# Setup OSG and/or Globus +if [ -r "${OSG_GRID}/setup.sh" ]; then + . "${OSG_GRID}/setup.sh" +else + if [ -r "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" ]; then + . "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" + fi +fi - if [ -r "${GLOBUS_LOCATION}/etc/globus-user-env.sh" ]; then - . "${GLOBUS_LOCATION}/etc/globus-user-env.sh" +if [ -z "${GLOBUS_PATH}" ]; then + if [ -z "${GLOBUS_LOCATION}" ]; then + # if GLOBUS_LOCATION not defined, try to guess it + if [ -r "/opt/globus/etc/globus-user-env.sh" ]; then + GLOBUS_LOCATION=/opt/globus + elif [ -r "/osgroot/osgcore/globus/etc/globus-user-env.sh" ]; then + GLOBUS_LOCATION=/osgroot/osgcore/globus else - warn "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." + warn "GLOBUS_LOCATION not defined and could not guess it." + warn "Looked in:" + warn ' /opt/globus/etc/globus-user-env.sh' + warn ' /osgroot/osgcore/globus/etc/globus-user-env.sh' warn 'Continuing like nothing happened' fi fi - [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath + if [ -r "${GLOBUS_LOCATION}/etc/globus-user-env.sh" ]; then + . "${GLOBUS_LOCATION}/etc/globus-user-env.sh" + else + warn "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." + warn 'Continuing like nothing happened' + fi +fi - num_gct=0 +[ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath +num_gct=0 - ######################################## - # prepare and move to the work directory - # Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . - # Empty $work_dir means PWD (same as ".") - # A custom path could be provided (no "*)" in case) - tmp="${work_dir}" - if [ -z "${work_dir}" ]; then - work_dir="$(pwd)" - else - case "${work_dir}" in - Condor|CONDOR) work_dir="${_CONDOR_SCRATCH_DIR}";; - OSG) work_dir="${OSG_WN_TMP}";; - TMPDIR) work_dir="${TMPDIR}";; - AUTO) automatic_work_dir;; - .) work_dir="$(pwd)";; - esac - fi +######################################## +# prepare and move to the work directory - if [ -z "${work_dir}" ]; then - early_glidein_failure "Unable to identify Startup dir for the glidein ($tmp)." - fi +# Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . +# Empty $work_dir means PWD (same as ".") +# A custom path could be provided (no "*)" in case) +tmp="${work_dir}" +if [ -z "${work_dir}" ]; then + work_dir="$(pwd)" +else + case "${work_dir}" in + Condor|CONDOR) work_dir="${_CONDOR_SCRATCH_DIR}";; + OSG) work_dir="${OSG_WN_TMP}";; + TMPDIR) work_dir="${TMPDIR}";; + AUTO) automatic_work_dir;; + .) work_dir="$(pwd)";; + esac +fi - if [ ! -e "${work_dir}" ]; then - early_glidein_failure "Startup dir '${work_dir}' ($tmp) does not exist." - fi +if [ -z "${work_dir}" ]; then + early_glidein_failure "Unable to identify Startup dir for the glidein ($tmp)." +fi + +if [ ! -e "${work_dir}" ]; then + early_glidein_failure "Startup dir '${work_dir}' ($tmp) does not exist." +fi - start_dir="$(pwd)" - echo "Started in '${start_dir}' ($tmp)" +start_dir="$(pwd)" +echo "Started in '${start_dir}' ($tmp)" - work_dir_template="${work_dir}/glide_$(dir_id)XXXXXX" - if ! work_dir="$(mktemp -d "${work_dir_template}")"; then - early_glidein_failure "Cannot create word_dir '${work_dir_template}'" +work_dir_template="${work_dir}/glide_$(dir_id)XXXXXX" +if ! work_dir="$(mktemp -d "${work_dir_template}")"; then + early_glidein_failure "Cannot create word_dir '${work_dir_template}'" +else + if ! cd "${work_dir}"; then + early_glidein_failure "Work dir '${work_dir}' was created but cannot cd into it." else - if ! cd "${work_dir}"; then - early_glidein_failure "Work dir '${work_dir}' was created but cannot cd into it." - else - echo "Running in ${work_dir}" - fi + echo "Running in ${work_dir}" fi - work_dir_created=1 - export GLIDEIN_WORK_DIR=${GLIDEIN_WORK_DIR} # for glidein reinvocation +fi +work_dir_created=1 +export GWMS_GLIDEIN_WORK_DIR=${GLIDEIN_WORK_DIR} # for glidein reinvocation - # GWMS_SUBDIR defined on top - GWMS_DIR="${work_dir}/$GWMS_SUBDIR" - if ! mkdir "$GWMS_DIR" ; then - early_glidein_failure "Cannot create GWMS_DIR '$GWMS_DIR'" - fi - gwms_lib_dir="${GWMS_DIR}/lib" - if ! mkdir -p "$gwms_lib_dir" ; then - early_glidein_failure "Cannot create lib dir '$gwms_lib_dir'" - fi - export GWMS_LIB_DIR=${gwms_lib_dir} # for glidein reinvocation - gwms_bin_dir="${GWMS_DIR}/bin" - if ! mkdir -p "$gwms_bin_dir" ; then - early_glidein_failure "Cannot create bin dir '$gwms_bin_dir'" - fi - export GWMS_BIN_DIR=${gwms_bin_dir} # for glidein reinvocation - gwms_exec_dir="${GWMS_DIR}/exec" - if ! mkdir -p "$gwms_exec_dir" ; then - early_glidein_failure "Cannot create exec dir '$gwms_exec_dir'" - else - for i in setup prejob postjob cleanup setup_singularity ; do - mkdir -p "$gwms_exec_dir"/$i - done - export GWMS_EXEC_DIR=${gwms_exec_dir} # for glidein reinvocation +# GWMS_SUBDIR defined on top +GWMS_DIR="${work_dir}/$GWMS_SUBDIR" +if ! mkdir "$GWMS_DIR" ; then + early_glidein_failure "Cannot create GWMS_DIR '$GWMS_DIR'" +fi +gwms_lib_dir="${GWMS_DIR}/lib" +if ! mkdir -p "$gwms_lib_dir" ; then + early_glidein_failure "Cannot create lib dir '$gwms_lib_dir'" +fi +export GWMS_LIB_DIR=${gwms_lib_dir} # for glidein reinvocation +gwms_bin_dir="${GWMS_DIR}/bin" +if ! mkdir -p "$gwms_bin_dir" ; then + early_glidein_failure "Cannot create bin dir '$gwms_bin_dir'" +fi +export GWMS_BIN_DIR=${gwms_bin_dir} # for glidein reinvocation +gwms_exec_dir="${GWMS_DIR}/exec" +if ! mkdir -p "$gwms_exec_dir" ; then + early_glidein_failure "Cannot create exec dir '$gwms_exec_dir'" +else + for i in setup prejob postjob cleanup setup_singularity ; do + mkdir -p "$gwms_exec_dir"/$i + done + export GWMS_EXEC_DIR=${gwms_exec_dir} # for glidein reinvocation +fi +# below lines for glidein reinvocation +export GWMS_SUBDIR=${GWMS_SUBDIR} +export GWMS_DIR=${GWMS_DIR} + +# mktemp makes it user readable by definition (ignores umask) +# TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed +if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then + if ! chmod a+rx "${work_dir}"; then + early_glidein_failure "Failed chmod '${work_dir}'" fi - # below lines for glidein reinvocation - export GWMS_SUBDIR=${GWMS_SUBDIR} - export GWMS_DIR=${GWMS_DIR} +fi + +glide_local_tmp_dir_template="/tmp/glide_$(dir_id)$(id -u -n)_XXXXXX" +if ! glide_local_tmp_dir="$(mktemp -d "${glide_local_tmp_dir_template}")"; then + early_glidein_failure "Cannot create temp '${glide_local_tmp_dir_template}'" +fi +glide_local_tmp_dir_created=1 - # mktemp makes it user readable by definition (ignores umask) +glide_tmp_dir="${work_dir}/tmp" +if ! mkdir "${glide_tmp_dir}"; then + early_glidein_failure "Cannot create '${glide_tmp_dir}'" +fi + +if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed - if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then - if ! chmod a+rx "${work_dir}"; then - early_glidein_failure "Failed chmod '${work_dir}'" - fi + # the tmpdirs should be world writable + # This way it will work even if the user spawned by the glidein is different than the glidein user + # This happened in GlExec, outside user stays the same in Singularity + if ! chmod 1777 "${glide_local_tmp_dir}"; then + early_glidein_failure "Failed chmod '${glide_local_tmp_dir}'" fi - glide_local_tmp_dir_template="/tmp/glide_$(dir_id)$(id -u -n)_XXXXXX" - if ! glide_local_tmp_dir="$(mktemp -d "${glide_local_tmp_dir_template}")"; then - early_glidein_failure "Cannot create temp '${glide_local_tmp_dir_template}'" + if ! chmod 1777 "${glide_tmp_dir}"; then + early_glidein_failure "Failed chmod '${glide_tmp_dir}'" fi - glide_local_tmp_dir_created=1 +fi - glide_tmp_dir="${work_dir}/tmp" - if ! mkdir "${glide_tmp_dir}"; then - early_glidein_failure "Cannot create '${glide_tmp_dir}'" - fi +short_main_dir=main +main_dir="${work_dir}/${short_main_dir}" +if ! mkdir "${main_dir}"; then + early_glidein_failure "Cannot create '${main_dir}'" +fi +export GWMS_MAIN_DIR=${main_dir} # for glidein reinvocation - if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then - # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed - # the tmpdirs should be world writable - # This way it will work even if the user spawned by the glidein is different than the glidein user - # This happened in GlExec, outside user stays the same in Singularity - if ! chmod 1777 "${glide_local_tmp_dir}"; then - early_glidein_failure "Failed chmod '${glide_local_tmp_dir}'" - fi +short_entry_dir=entry_${glidein_entry} +entry_dir="${work_dir}/${short_entry_dir}" +if ! mkdir "${entry_dir}"; then + early_glidein_failure "Cannot create '${entry_dir}'" +fi +export GWMS_ENTRY_DIR=${entry_dir} # for glidein reinvocation - if ! chmod 1777 "${glide_tmp_dir}"; then - early_glidein_failure "Failed chmod '${glide_tmp_dir}'" - fi +if [ -n "${client_repository_url}" ]; then + short_client_dir=client + client_dir="${work_dir}/${short_client_dir}" + if ! mkdir "$client_dir"; then + early_glidein_failure "Cannot create '${client_dir}'" fi + export GWMS_CLIENT_DIR=${client_dir} # for glidein reinvocation - short_main_dir=main - main_dir="${work_dir}/${short_main_dir}" - if ! mkdir "${main_dir}"; then - early_glidein_failure "Cannot create '${main_dir}'" + if [ -n "${client_repository_group_url}" ]; then + short_client_group_dir=client_group_${client_group} + client_group_dir="${work_dir}/${short_client_group_dir}" + if ! mkdir "${client_group_dir}"; then + early_glidein_failure "Cannot create '${client_group_dir}'" + fi fi - export GWMS_MAIN_DIR=${main_dir} # for glidein reinvocation + export GWMS_CLIENTGROUP_DIR=${client_group_dir} # for glidein reinvocation +fi + +# Move the token files from condor to glidein workspace +# TODO: compare this w/ setup_x509.sh +# monitoring tokens, Should be using same credentials directory? +mv "${start_dir}/tokens.tgz" . +mv "${start_dir}/url_dirs.desc" . +# idtokens are handled in setup_x509.sh - TODO: remove once verified +#for idtk in ${start_dir}/*.idtoken; do +# if cp "${idtk}" . ; then +# echo "copied idtoken ${idtk} to $(pwd)" +# else +# echo "failed to copy idtoken ${idtk} to $(pwd)" 1>&2 +# fi +#done +#if [ -e "${GLIDEIN_CONDOR_TOKEN}" ]; then +# mkdir -p ticket +# tname="$(basename ${GLIDEIN_CONDOR_TOKEN})" +# cp "${GLIDEIN_CONDOR_TOKEN}" "ticket/${tname}" +# export GLIDEIN_CONDOR_TOKEN="$(pwd)/ticket/${tname}" +#fi + +# Extract and source all the data contained at the end of this script as tarball +extract_all_data + +wrapper_list="${PWD}/wrapper_list.lst" +touch "${wrapper_list}" +export GWMS_WRAPPER_LIST=${wrapper_list} # for glidein reinvocation + +# create glidein_config +glidein_config="${PWD}/glidein_config" +if ! echo > "${glidein_config}"; then + early_glidein_failure "Could not create '${glidein_config}'" +fi +export GWMS_GLIDEIN_CONFIG=${glidein_config} # for glidein reinvocation +if ! { + echo "# --- glidein_startup vals ---" + echo "GLIDEIN_UUID ${glidein_uuid}" + echo "GLIDEIN_Factory ${glidein_factory}" + echo "GLIDEIN_Name ${glidein_name}" + echo "GLIDEIN_Entry_Name ${glidein_entry}" - short_entry_dir=entry_${glidein_entry} - entry_dir="${work_dir}/${short_entry_dir}" - if ! mkdir "${entry_dir}"; then - early_glidein_failure "Cannot create '${entry_dir}'" + if [ -n "${client_name}" ]; then + # client name not required as it is not used for anything but debug info + echo "GLIDECLIENT_Name ${client_name}" fi - export GWMS_ENTRY_DIR=${entry_dir} # for glidein reinvocation + if [ -n "${client_group}" ]; then + # client group not required as it is not used for anything but debug info + echo "GLIDECLIENT_Group ${client_group}" + fi + echo "GLIDEIN_CredentialIdentifier ${glidein_cred_id}" + echo "CONDORG_CLUSTER ${condorg_cluster}" + echo "CONDORG_SUBCLUSTER ${condorg_subcluster}" + echo "CONDORG_SCHEDD ${condorg_schedd}" + echo "DEBUG_MODE ${set_debug}" + echo "GLIDEIN_STARTUP_PID $$" + echo "GLIDEIN_START_DIR_ORIG ${start_dir}" + echo "GLIDEIN_WORKSPACE_ORIG $(pwd)" + echo "GLIDEIN_WORK_DIR ${main_dir}" + echo "GLIDEIN_ENTRY_WORK_DIR ${entry_dir}" + echo "TMP_DIR ${glide_tmp_dir}" + echo "GLIDEIN_LOCAL_TMP_DIR ${glide_local_tmp_dir}" + echo "PROXY_URL ${proxy_url}" + echo "DESCRIPTION_FILE ${descript_file}" + echo "DESCRIPTION_ENTRY_FILE ${descript_entry_file}" + echo "GLIDEIN_Signature ${sign_id}" + echo "GLIDEIN_Entry_Signature ${sign_entry_id}" if [ -n "${client_repository_url}" ]; then - short_client_dir=client - client_dir="${work_dir}/${short_client_dir}" - if ! mkdir "$client_dir"; then - early_glidein_failure "Cannot create '${client_dir}'" - fi - export GWMS_CLIENT_DIR=${client_dir} # for glidein reinvocation - + echo "GLIDECLIENT_WORK_DIR ${client_dir}" + echo "GLIDECLIENT_DESCRIPTION_FILE ${client_descript_file}" + echo "GLIDECLIENT_Signature ${client_sign_id}" if [ -n "${client_repository_group_url}" ]; then - short_client_group_dir=client_group_${client_group} - client_group_dir="${work_dir}/${short_client_group_dir}" - if ! mkdir "${client_group_dir}"; then - early_glidein_failure "Cannot create '${client_group_dir}'" - fi - fi - export GWMS_CLIENTGROUP_DIR=${client_group_dir} # for glidein reinvocation - fi - - # Move the token files from condor to glidein workspace - # TODO: compare this w/ setup_x509.sh - # monitoring tokens, Should be using same credentials directory? - mv "${start_dir}/tokens.tgz" . - mv "${start_dir}/url_dirs.desc" . - # idtokens are handled in setup_x509.sh - TODO: remove once verified - #for idtk in ${start_dir}/*.idtoken; do - # if cp "${idtk}" . ; then - # echo "copied idtoken ${idtk} to $(pwd)" - # else - # echo "failed to copy idtoken ${idtk} to $(pwd)" 1>&2 - # fi - #done - #if [ -e "${GLIDEIN_CONDOR_TOKEN}" ]; then - # mkdir -p ticket - # tname="$(basename ${GLIDEIN_CONDOR_TOKEN})" - # cp "${GLIDEIN_CONDOR_TOKEN}" "ticket/${tname}" - # export GLIDEIN_CONDOR_TOKEN="$(pwd)/ticket/${tname}" - #fi - - # Extract and source all the data contained at the end of this script as tarball - extract_all_data + echo "GLIDECLIENT_GROUP_WORK_DIR ${client_group_dir}" + echo "GLIDECLIENT_DESCRIPTION_GROUP_FILE ${client_descript_group_file}" + echo "GLIDECLIENT_Group_Signature ${client_sign_group_id}" + fi + fi + echo "B64UUENCODE_SOURCE ${PWD}/b64uuencode.source" + echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" + echo "GET_ID_SELECTORS_SOURCE ${PWD}/get_id_selectors.source" + echo "LOGGING_UTILS_SOURCE ${PWD}/logging_utils.source" + echo "GLIDEIN_PATHS_SOURCE ${PWD}/glidein_paths.source" + echo "WRAPPER_LIST ${wrapper_list}" + echo "SLOTS_LAYOUT ${slots_layout}" + # Add a line saying we are still initializing... + echo "GLIDEIN_INITIALIZED 0" + # ...but be optimist, and leave advertise_only for the actual error handling script + echo "GLIDEIN_ADVERTISE_ONLY 0" + echo "GLIDEIN_CONDOR_TOKEN ${GLIDEIN_CONDOR_TOKEN}" + echo "# --- User Parameters ---" +} >> "${glidein_config}"; then + early_glidein_failure "Failed in updating '${glidein_config}'" +fi +# shellcheck disable=SC2086 +params2file ${params} - wrapper_list="${PWD}/wrapper_list.lst" - touch "${wrapper_list}" - export WRAPPER_LIST=${wrapper_list} # for glidein reinvocation - - # create glidein_config - glidein_config="${PWD}/glidein_config" - if ! echo > "${glidein_config}"; then - early_glidein_failure "Could not create '${glidein_config}'" - fi - export GLIDEIN_CONFIG=${glidein_config} # for glidein reinvocation - if ! { - echo "# --- glidein_startup vals ---" - echo "GLIDEIN_UUID ${glidein_uuid}" - echo "GLIDEIN_Factory ${glidein_factory}" - echo "GLIDEIN_Name ${glidein_name}" - echo "GLIDEIN_Entry_Name ${glidein_entry}" - - if [ -n "${client_name}" ]; then - # client name not required as it is not used for anything but debug info - echo "GLIDECLIENT_Name ${client_name}" - fi - if [ -n "${client_group}" ]; then - # client group not required as it is not used for anything but debug info - echo "GLIDECLIENT_Group ${client_group}" - fi - echo "GLIDEIN_CredentialIdentifier ${glidein_cred_id}" - echo "CONDORG_CLUSTER ${condorg_cluster}" - echo "CONDORG_SUBCLUSTER ${condorg_subcluster}" - echo "CONDORG_SCHEDD ${condorg_schedd}" - echo "DEBUG_MODE ${set_debug}" - echo "GLIDEIN_STARTUP_PID $$" - echo "GLIDEIN_START_DIR_ORIG ${start_dir}" - echo "GLIDEIN_WORKSPACE_ORIG $(pwd)" - echo "GLIDEIN_WORK_DIR ${main_dir}" - echo "GLIDEIN_ENTRY_WORK_DIR ${entry_dir}" - echo "TMP_DIR ${glide_tmp_dir}" - echo "GLIDEIN_LOCAL_TMP_DIR ${glide_local_tmp_dir}" - echo "PROXY_URL ${proxy_url}" - echo "DESCRIPTION_FILE ${descript_file}" - echo "DESCRIPTION_ENTRY_FILE ${descript_entry_file}" - echo "GLIDEIN_Signature ${sign_id}" - echo "GLIDEIN_Entry_Signature ${sign_entry_id}" - - if [ -n "${client_repository_url}" ]; then - echo "GLIDECLIENT_WORK_DIR ${client_dir}" - echo "GLIDECLIENT_DESCRIPTION_FILE ${client_descript_file}" - echo "GLIDECLIENT_Signature ${client_sign_id}" - if [ -n "${client_repository_group_url}" ]; then - echo "GLIDECLIENT_GROUP_WORK_DIR ${client_group_dir}" - echo "GLIDECLIENT_DESCRIPTION_GROUP_FILE ${client_descript_group_file}" - echo "GLIDECLIENT_Group_Signature ${client_sign_group_id}" - fi - fi - echo "B64UUENCODE_SOURCE ${PWD}/b64uuencode.source" - echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" - echo "GET_ID_SELECTORS_SOURCE ${PWD}/get_id_selectors.source" - echo "LOGGING_UTILS_SOURCE ${PWD}/logging_utils.source" - echo "GLIDEIN_PATHS_SOURCE ${PWD}/glidein_paths.source" - echo "WRAPPER_LIST ${wrapper_list}" - echo "SLOTS_LAYOUT ${slots_layout}" - # Add a line saying we are still initializing... - echo "GLIDEIN_INITIALIZED 0" - # ...but be optimist, and leave advertise_only for the actual error handling script - echo "GLIDEIN_ADVERTISE_ONLY 0" - echo "GLIDEIN_CONDOR_TOKEN ${GLIDEIN_CONDOR_TOKEN}" - echo "# --- User Parameters ---" - } >> "${glidein_config}"; then - early_glidein_failure "Failed in updating '${glidein_config}'" - fi - # shellcheck disable=SC2086 - params2file ${params} - - ############################################ - # Setup logging - log_init "${glidein_uuid}" "${work_dir}" - # Remove these files, if they are still there - rm -rf tokens.tgz url_dirs.desc tokens - log_setup "${glidein_config}" +############################################ +# Setup logging +log_init "${glidein_uuid}" "${work_dir}" +# Remove these files, if they are still there +rm -rf tokens.tgz url_dirs.desc tokens +log_setup "${glidein_config}" - echo "Downloading files from Factory and Frontend" - log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" +echo "Downloading files from Factory and Frontend" +log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" - ##################################### - # Fetch descript and signature files +##################################### +# Fetch descript and signature files - # disable signature check before I get the signature file itself - # disable_check_signature is global - disable_check_signature=yes +# disable signature check before I get the signature file itself +# disable_check_signature is global +disable_check_signature=yes - for gs_id in main entry client client_group - do - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no cilent_repository - continue - fi +for gs_id in main entry client client_group +do + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no cilent_repository + continue fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no cilent_repository_group - continue - fi + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no cilent_repository_group + continue fi + fi - gs_id_work_dir="$(get_work_dir ${gs_id})" + gs_id_work_dir="$(get_work_dir ${gs_id})" - # Fetch description file - gs_id_descript_file="$(get_descript_file ${gs_id})" - fetch_file_regular "${gs_id}" "${gs_id_descript_file}" - if ! signature_file_line="$(grep "^signature " "${gs_id_work_dir}/${gs_id_descript_file}")"; then - warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." - glidein_exit 1 - fi - signature_file=$(echo "${signature_file_line}" | cut -s -f 2-) - - # Fetch signature file - gs_id_signature="$(get_signature ${gs_id})" - fetch_file_regular "${gs_id}" "${signature_file}" - echo "${gs_id_signature} ${signature_file}" > "${gs_id_work_dir}/signature.sha1.test" - if ! (cd "${gs_id_work_dir}" && sha1sum -c signature.sha1.test) 1>&2 ; then - warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." - glidein_exit 1 - fi - # for simplicity use a fixed name for signature file - mv "${gs_id_work_dir}/${signature_file}" "${gs_id_work_dir}/signature.sha1" - done + # Fetch description file + gs_id_descript_file="$(get_descript_file ${gs_id})" + fetch_file_regular "${gs_id}" "${gs_id_descript_file}" + if ! signature_file_line="$(grep "^signature " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." + glidein_exit 1 + fi + signature_file=$(echo "${signature_file_line}" | cut -s -f 2-) - # re-enable for everything else - disable_check_signature= + # Fetch signature file + gs_id_signature="$(get_signature ${gs_id})" + fetch_file_regular "${gs_id}" "${signature_file}" + echo "${gs_id_signature} ${signature_file}" > "${gs_id_work_dir}/signature.sha1.test" + if ! (cd "${gs_id_work_dir}" && sha1sum -c signature.sha1.test) 1>&2 ; then + warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." + glidein_exit 1 + fi + # for simplicity use a fixed name for signature file + mv "${gs_id_work_dir}/${signature_file}" "${gs_id_work_dir}/signature.sha1" +done - # Now verify the description was not tampered with - # doing it so late should be fine, since nobody should have been able - # to fake the signature file, even if it faked its name in - # the description file - for gs_id in main entry client client_group - do - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no cilent_repository - continue - fi - fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no cilent_repository_group - continue - fi - fi +# re-enable for everything else +disable_check_signature= +export GWMS_CHECK_SIGNATURE=$disable_check_signature # for glidein reinvocation - gs_id_descript_file="$(get_descript_file ${gs_id})" - if ! check_file_signature "${gs_id}" "${gs_id_descript_file}"; then - gs_id_work_dir="$(get_work_dir ${gs_id})" - warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." - glidein_exit 1 +# Now verify the description was not tampered with +# doing it so late should be fine, since nobody should have been able +# to fake the signature file, even if it faked its name in +# the description file +for gs_id in main entry client client_group +do + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no cilent_repository + continue fi - done + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no cilent_repository_group + continue + fi + fi - ################################################### - # get last_script, as it is used by the fetch_file - gs_id_work_dir="$(get_work_dir main)" - gs_id_descript_file="$(get_descript_file main)" - last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" - export LAST_SCRIPT="$last_script" # for glidein reinvocation - if [ -z "${last_script}" ]; then - warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." + gs_id_descript_file="$(get_descript_file ${gs_id})" + if ! check_file_signature "${gs_id}" "${gs_id_descript_file}"; then + gs_id_work_dir="$(get_work_dir ${gs_id})" + warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." glidein_exit 1 fi - #cleanup_script="$(grep "^cleanup_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" - cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' ' -f 2-) +done +################################################### +# get last_script, as it is used by the fetch_file +gs_id_work_dir="$(get_work_dir main)" +gs_id_descript_file="$(get_descript_file main)" +last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" +export GWMS_LAST_SCRIPT="$last_script" # for glidein reinvocation +if [ -z "${last_script}" ]; then + warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 +fi +#cleanup_script="$(grep "^cleanup_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" +cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' ' -f 2-) - ############################## - # Fetch all the other files - for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main precvmfs_file_list" - do - gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no client_repository - continue - fi +############################## +# Fetch all the other files +for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main precvmfs_file_list" +do + gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" + + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no client_repository + continue fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no client_repository_group - continue - fi + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no client_repository_group + continue fi + fi - gs_file_list_id="$(echo "${gs_file_id}" |awk '{print $2}')" + gs_file_list_id="$(echo "${gs_file_id}" |awk '{print $2}')" - gs_id_work_dir="$(get_work_dir "${gs_id}")" - gs_id_descript_file="$(get_descript_file "${gs_id}")" + gs_id_work_dir="$(get_work_dir "${gs_id}")" + gs_id_descript_file="$(get_descript_file "${gs_id}")" - # extract list file name - if ! gs_file_list_line="$(grep "^${gs_file_list_id} " "${gs_id_work_dir}/${gs_id_descript_file}")"; then - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_file_list_id:0:11}" = "aftergroup_" ]; then - # afterfile_.. files optional when no client_repository_group - continue - fi + # extract list file name + if ! gs_file_list_line="$(grep "^${gs_file_list_id} " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_file_list_id:0:11}" = "aftergroup_" ]; then + # afterfile_.. files optional when no client_repository_group + continue fi - warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." - glidein_exit 1 fi - # space+tab separated file with multiple elements (was: awk '{print $2}', not safe for spaces in file name) - gs_file_list="$(echo "${gs_file_list_line}" | cut -s -f 2 | sed -e 's/[[:space:]]*$//')" + warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 + fi + # space+tab separated file with multiple elements (was: awk '{print $2}', not safe for spaces in file name) + gs_file_list="$(echo "${gs_file_list_line}" | cut -s -f 2 | sed -e 's/[[:space:]]*$//')" - # fetch list file - fetch_file_regular "${gs_id}" "${gs_file_list}" + # fetch list file + fetch_file_regular "${gs_id}" "${gs_file_list}" - # Fetch files contained in list - # TODO: $file is actually a list, so it cannot be double-quoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here - # shellcheck disable=2086 - while read -r file - do - if [ "${file:0:1}" != "#" ]; then - fetch_file "${gs_id}" $file - fi - done < "${gs_id_work_dir}/${gs_file_list}" - - # Files to go into the GWMS_PATH - if [ "$gs_file_id" = "main at_file_list" ]; then - # setup here to make them available for other setup scripts - add_to_path "$gwms_bin_dir" - # all available now: gwms-python was in main,file_list; condor_chirp is in main,at_file_list - for file in "gwms-python" "condor_chirp" - do - cp "${gs_id_work_dir}/$file" "$gwms_bin_dir"/ - done - cp -r "${gs_id_work_dir}/lib"/* "$gwms_lib_dir"/ - cp "${gs_id_work_dir}/gconfig.py" "$gwms_lib_dir"/python/ - elif [ "$gs_file_id" = "main after_file_list" ]; then - # in case some library has been added/updated - rsync -ar "${gs_id_work_dir}/lib"/ "$gwms_lib_dir"/ - # new knowns binaries? add a loop like above: for file in ... - elif [[ "$gs_file_id" = client* ]]; then - # TODO: gwms25073 this is a workaround until there is an official designation for setup script fragments - [[ -e "${gs_id_work_dir}/setup_prejob.sh" ]] && { cp "${gs_id_work_dir}/setup_prejob.sh" "$gwms_exec_dir"/prejob/ ; chmod a-x "$gwms_exec_dir"/prejob/setup_prejob.sh ; } + # Fetch files contained in list + # TODO: $file is actually a list, so it cannot be double-quoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here + # shellcheck disable=2086 + while read -r file + do + if [ "${file:0:1}" != "#" ]; then + fetch_file "${gs_id}" $file fi - done + done < "${gs_id_work_dir}/${gs_file_list}" + + # Files to go into the GWMS_PATH + if [ "$gs_file_id" = "main at_file_list" ]; then + # setup here to make them available for other setup scripts + add_to_path "$gwms_bin_dir" + # all available now: gwms-python was in main,file_list; condor_chirp is in main,at_file_list + for file in "gwms-python" "condor_chirp" + do + cp "${gs_id_work_dir}/$file" "$gwms_bin_dir"/ + done + cp -r "${gs_id_work_dir}/lib"/* "$gwms_lib_dir"/ + cp "${gs_id_work_dir}/gconfig.py" "$gwms_lib_dir"/python/ + elif [ "$gs_file_id" = "main after_file_list" ]; then + # in case some library has been added/updated + rsync -ar "${gs_id_work_dir}/lib"/ "$gwms_lib_dir"/ + # new knowns binaries? add a loop like above: for file in ... + elif [[ "$gs_file_id" = client* ]]; then + # TODO: gwms25073 this is a workaround until there is an official designation for setup script fragments + [[ -e "${gs_id_work_dir}/setup_prejob.sh" ]] && { cp "${gs_id_work_dir}/setup_prejob.sh" "$gwms_exec_dir"/prejob/ ; chmod a-x "$gwms_exec_dir"/prejob/setup_prejob.sh ; } + fi +done fi if [[ -n "$gwms_cvmfs_reexec" && "$gwms_cvmfs_reexec" == "yes" ]]; then # gwms_cvmfs_reexec is not empty; meaning this block is being run inside of cvmfsexec environment - printenv GLIDEIN_CONFIG > /dev/null + printenv GWMS_GLIDEIN_CONFIG > /dev/null status=$? if [[ ${status} -eq 0 ]]; then - work_dir=$(printenv GLIDEIN_WORK_DIR | sed "s/ //g") - glidein_config=$(printenv GLIDEIN_CONFIG | sed "s/ //g") - repository_url=$(printenv REPOSITORY_URL | sed "s/ //g") + work_dir=$(printenv GWMS_GLIDEIN_WORK_DIR | sed "s/ //g") + glidein_config=$(printenv GWMS_GLIDEIN_CONFIG | sed "s/ //g") + repository_url=$(printenv GWMS_REPOSITORY_URL | sed "s/ //g") main_dir=$(printenv GWMS_MAIN_DIR | sed "s/ //g") - last_script=$(printenv LAST_SCRIPT | sed "s/ //g") - check_signature=$(printenv CHECK_SIGNATURE | sed "s/ //g") - startup_time=$(printenv STARTUP_TIME | sed "s/ //g") + last_script=$(printenv GWMS_LAST_SCRIPT | sed "s/ //g") + check_signature=$(printenv GWMS_CHECK_SIGNATURE | sed "s/ //g") + startup_time=$(printenv GWMS_STARTUP_TIME | sed "s/ //g") cvmfs_config_repo=$(printenv GLIDEIN_CVMFS_CONFIG_REPO | sed "s/ //g") cvmfs_add_repos=$(printenv GLIDEIN_CVMFS_REPOS | sed "s/ //g") gwms_cvmfsexec_mode=$(printenv GWMS_CVMFSEXEC_MODE | sed "s/ //g") - client_repository_url=$(printenv CLIENT_REPOSITORY_URL | sed "s/ //g") - client_repository_group_url=$(printenv CLIENT_REPOSITORY_GROUP_URL | sed "s/ //g") - wrapper_list=$(printenv WRAPPER_LIST | sed "s/ //g") + client_repository_url=$(printenv GWMS_CLIENT_REPOSITORY_URL | sed "s/ //g") + client_repository_group_url=$(printenv GWMS_CLIENT_REPOSITORY_GROUP_URL | sed "s/ //g") + wrapper_list=$(printenv GWMS_WRAPPER_LIST | sed "s/ //g") gwms_exec_dir=$(printenv GWMS_EXEC_DIR | sed "s/ //g") fi @@ -2082,7 +2084,7 @@ export GLIDEIN_DEBUG_OUTPUT=$glidein_debug_output for gs_file_id in "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" do - gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" # one of four possibilities + gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" if [ -z "${client_repository_url}" ]; then if [ "${gs_id}" = "client" ]; then From bd2fbdf91753a159857c770635e8fe117d0f0f53 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Fri, 31 May 2024 13:20:38 -0500 Subject: [PATCH 10/12] addressing the remaining PR review comments - fixed a runtime error when concatenating a set with only one element - added a hyperlink reference to the existing section that outlines the variables related to the on-demand CVMFS provisioning feature - updated name of the variable (GLIDEIN_WORK_DIR) to specify its usage being internal to GWMS - added `export` command to explicitly export the value of PATH when a glidein is reinvoked. This was done to resolve the `pivot_root: command not found` error --- creation/lib/cgWDictFile.py | 2 +- creation/web_base/cvmfs_setup.sh | 3 ++- creation/web_base/glidein_startup.sh | 6 +----- doc/factory/custom_vars.html | 1 + 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/creation/lib/cgWDictFile.py b/creation/lib/cgWDictFile.py index d4d9347c5..e093e5fa9 100644 --- a/creation/lib/cgWDictFile.py +++ b/creation/lib/cgWDictFile.py @@ -463,7 +463,7 @@ def refresh_file_list(dicts, is_main, files_set_readonly=True, files_reset_chang # dictionaries must have been written to disk before using this def refresh_signature(dicts): # update in place signature_dict = dicts["signature"] - for k in ("consts", "vars", "untar_cfg", "gridmap") + cgWConsts.FILE_LISTS_PRIORITIES + ("description"): + for k in ("consts", "vars", "untar_cfg", "gridmap") + cgWConsts.FILE_LISTS_PRIORITIES + ("description",): if k in dicts: signature_dict.add_from_file(dicts[k].get_filepath(), allow_overwrite=True) # add signatures of all the files linked in the lists diff --git a/creation/web_base/cvmfs_setup.sh b/creation/web_base/cvmfs_setup.sh index 748ec9841..35c89cf02 100644 --- a/creation/web_base/cvmfs_setup.sh +++ b/creation/web_base/cvmfs_setup.sh @@ -115,9 +115,10 @@ if [[ $? -eq 0 ]]; then export GLIDEIN_WORKSPACE=$original_workspace export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode - export GLIDEIN_WORK_DIR="$work_dir" + export GWMS_GLIDEIN_WORK_DIR="$work_dir" export GLIDEIN_CVMFS_CONFIG_REPO="$GLIDEIN_CVMFS_CONFIG_REPO" export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" + export PATH=$PATH echo "Reinvoking glidein now..." exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" echo "!!WARNING!! Outside of reinvocation of glidein_startup" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index a10637539..e18396e94 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -26,8 +26,6 @@ IFS=$' \t\n' # Some sites empty PATH. Setting a reasonable default if [[ -z "$PATH" ]]; then export PATH="/bin:/usr/bin" -elif [[ -z "$gwms_cvmfs_reexec" ]]; then - export PATH="/usr/sbin:$PATH" fi global_args="$*" @@ -1676,7 +1674,7 @@ else fi fi work_dir_created=1 -export GWMS_GLIDEIN_WORK_DIR=${GLIDEIN_WORK_DIR} # for glidein reinvocation +export GWMS_GLIDEIN_WORK_DIR=${work_dir} # for glidein reinvocation # GWMS_SUBDIR defined on top GWMS_DIR="${work_dir}/$GWMS_SUBDIR" @@ -2068,8 +2066,6 @@ if [[ -n "$gwms_cvmfs_reexec" && "$gwms_cvmfs_reexec" == "yes" ]]; then # re-sourcing the helper script inside of cvmfsexec environment . "$work_dir"/cvmfs_helper_funcs.sh mount_cvmfs_repos $gwms_cvmfsexec_mode $cvmfs_config_repo $cvmfs_add_repos - # check if the cvmfs repos are still mounted inside of reinvocation - df -h # re-source all the scripts as it'd have been done during the first invocation of this script extract_all_data diff --git a/doc/factory/custom_vars.html b/doc/factory/custom_vars.html index 244adb779..bdd83d493 100644 --- a/doc/factory/custom_vars.html +++ b/doc/factory/custom_vars.html @@ -106,6 +106,7 @@

Custom HTCondor Variables

  • Admin variables
  • Dynamic variables
  • Singularity variables
    • +
  • CVMFS variables
  • Set and discover available CPUs
    • From 8a7bee7aa1f6e1dce4081df7633b3926aa364584 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Fri, 31 May 2024 18:26:44 -0500 Subject: [PATCH 11/12] minor changes after testing on SL7 --- creation/web_base/cvmfs_helper_funcs.sh | 2 +- creation/web_base/cvmfsexec_platform_select.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index 7eaf287db..1d8274489 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -150,7 +150,7 @@ perform_system_check() { GWMS_OS_DISTRO="rhel" # GWMS_OS_DISTRO="non-rhel" #GWMS_OS_VERSION_FULL=$(cat /etc/redhat-release | cut -d " " -f 3) - else + else # not a rhel derivative; use /etc/os-release instead [fallback option] GWMS_OS_DISTRO="non-rhel" #GWMS_OS_VERSION_FULL=$(cat /etc/os-release | egrep "VERSION_ID" | cut -d = -f 2 | tr -d '"') diff --git a/creation/web_base/cvmfsexec_platform_select.sh b/creation/web_base/cvmfsexec_platform_select.sh index e7f901453..f602338ed 100755 --- a/creation/web_base/cvmfsexec_platform_select.sh +++ b/creation/web_base/cvmfsexec_platform_select.sh @@ -34,12 +34,13 @@ fi # TODO: is it possible to reuse cvmfs_helper_funcs.sh by sourcing it during the execution of this file???? if [[ -f '/etc/redhat-release' ]]; then os_distro=rhel - os_ver=$(cat /etc/redhat-release | cut -d " " -f 3 | awk -F'.' '{print $1}') else os_distro=non-rhel - os_ver=$(cat /etc/os-release | egrep "VERSION_ID" | cut -d = -f 2 | tr -d '"' | awk -F "." '{print $1}') fi - +# using os-release file to get OS-related info +. /etc/os-release +os_ver_full=$VERSION_ID +os_ver=$(echo "$os_ver_full" | awk -F'.' '{print $1}') krnl_arch=$(arch) mach_type=${os_distro}${os_ver}-${krnl_arch} From f6b18024f066ef3ff1c83c7e3319ac28f7c7acb9 Mon Sep 17 00:00:00 2001 From: Namratha Urs Date: Thu, 19 Sep 2024 15:15:00 -0500 Subject: [PATCH 12/12] addressing review comments --- creation/lib/cgWParamDict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 4b5f7cccf..a6a247ec4 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -222,7 +222,7 @@ def populate(self, other=None): populate_gridmap(self.conf, self.dicts["gridmap"]) # the following list will be a megalist containing all the scripts; used for duplication check logic subsequently - all_scripts = list() + all_scripts = [] # NOTE that all the files in these _scripts lists are added as executables (i.e. must report with error_gen) file_list_scripts = [ "collector_setup.sh",