diff --git a/CHANGELOG.md b/CHANGELOG.md index 268aed4f2..e51b89c25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,31 @@ Changes since the last release ### Known Issues +## v3.11.0 \[2024-03-dd\] + +This is the very first version in the development series of GlideinWMS releases. + +### New features / functionalities + +- Supports all available modes of cvmfsexec when provisioning CVMFS on demand. (PR #372) + +### Changed defaults / behaviours + +### Deprecated / removed options and commands + +### Security Related Fixes + +### Bug Fixes + +### Testing / Development + +### Known Issues + +- When generating cvmfsexec distribution for EL9 machine type on an EL7 machine, the factory reconfig and/or upgrade fails as a result of an error in `create_cvmfsexec_distros.sh`. This is possibly due to the tools for EL7 being unable to handle EL9 files (as per Dave Dykstra). Please exercise caution if using `rhel9-x86_64` in the `mtypes` list for the `cvmfsexec_distro` tag in factory configuration. + - Our suggested workaround is to remove the EL9 machine type from the default list of machine types supported by the custom distros creation script. Add it back if you are running on an EL9 system and want an EL9 cvmfsexec distribution. (PR #312) +- When using on-demand CVMFS, all Glideins after the first one on a node are failing (Issue #287) + This happens because mountrepo and umountrepo work at the node level and subsequent Glideins see the mounts done by the first one and abort. To avoid problems use only whole-node Glideins when using on-demand CVMFS. All versions with on-demand CVMFS are affected. + ## v3.10.6 \[2024-01-25\] Minor new features, mostly a bug fix release diff --git a/creation/lib/cgWConsts.py b/creation/lib/cgWConsts.py index 317a8d5ee..bfe80e650 100644 --- a/creation/lib/cgWConsts.py +++ b/creation/lib/cgWConsts.py @@ -9,8 +9,9 @@ from . import cWConsts # these are in the stage dir, so they need to be renamed if changed -AFTER_FILE_LISTFILE = "after_%s" % cWConsts.FILE_LISTFILE -AT_FILE_LISTFILE = "at_%s" % cWConsts.FILE_LISTFILE +AFTER_FILE_LISTFILE = f"after_{cWConsts.FILE_LISTFILE}" +AT_FILE_LISTFILE = f"at_{cWConsts.FILE_LISTFILE}" +PRECVMFS_FILE_LISTFILE = f"precvmfs_{cWConsts.FILE_LISTFILE}" CONDOR_FILE = "condor_bin_%s.tgz" CONDOR_DIR = "condor" @@ -23,6 +24,8 @@ CVMFSEXEC_DIR = "cvmfsexec" CVMFSEXEC_ATTR = "CVMFSEXEC_DIR" +# constant defining the priorities of file lists used by the factory/glidein +FILE_LISTS_PRIORITIES = ("file_list", "precvmfs_file_list", "at_file_list", "after_file_list") # these are in the submit dir, so they can be changed SUBMIT_ATTRS_FILE = "submit_attrs.cfg" diff --git a/creation/lib/cgWDictFile.py b/creation/lib/cgWDictFile.py index 4b8e22770..e093e5fa9 100644 --- a/creation/lib/cgWDictFile.py +++ b/creation/lib/cgWDictFile.py @@ -290,6 +290,9 @@ def get_main_dicts(submit_dir, stage_dir): main_dicts["gridmap"] = cWDictFile.GridMapDict( stage_dir, cWConsts.insert_timestr(cWConsts.GRIDMAP_FILE) ) # TODO: versioned but no fname_idx? + main_dicts["precvmfs_file_list"] = cWDictFile.FileDictFile( + stage_dir, cWConsts.insert_timestr(cgWConsts.PRECVMFS_FILE_LISTFILE), fname_idx=cgWConsts.PRECVMFS_FILE_LISTFILE + ) main_dicts["at_file_list"] = cWDictFile.FileDictFile( stage_dir, cWConsts.insert_timestr(cgWConsts.AT_FILE_LISTFILE), fname_idx=cgWConsts.AT_FILE_LISTFILE ) @@ -370,10 +373,15 @@ def load_main_dicts(main_dicts): # update in place # print "\ndebug %s main_dicts['description'].keys2 = %s" % (__file__, main_dicts['description'].keys2) # print "\ndebug %s dir(main_dicts['description']) = %s" % (__file__, dir(main_dicts['description'])) # TODO: To remove if upgrade from older versions is not a problem + try: + main_dicts["precvmfs_file_list"].load(fname=main_dicts["description"].vals2["precvmfs_file_list"]) + except KeyError: + # when upgrading from older version the new precvmfs_file_list may not be in the description + main_dicts["precvmfs_file_list"].load() try: main_dicts["at_file_list"].load(fname=main_dicts["description"].vals2["at_file_list"]) except KeyError: - # when upgrading form older version the new at_file_list may not be in the description + # when upgrading from older version the new at_file_list may not be in the description main_dicts["at_file_list"].load() main_dicts["after_file_list"].load(fname=main_dicts["description"].vals2["after_file_list"]) load_common_dicts(main_dicts, main_dicts["description"]) @@ -409,7 +417,7 @@ def load_entry_dicts(entry_dicts, entry_name, summary_signature): # update in p def refresh_description(dicts): # update in place description_dict = dicts["description"] description_dict.add(dicts["signature"].get_fname(), "signature", allow_overwrite=True) - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: description_dict.add(dicts[k].get_fname(), k, allow_overwrite=True) @@ -455,11 +463,11 @@ def refresh_file_list(dicts, is_main, files_set_readonly=True, files_reset_chang # dictionaries must have been written to disk before using this def refresh_signature(dicts): # update in place signature_dict = dicts["signature"] - for k in ("consts", "vars", "untar_cfg", "gridmap", "file_list", "at_file_list", "after_file_list", "description"): + for k in ("consts", "vars", "untar_cfg", "gridmap") + cgWConsts.FILE_LISTS_PRIORITIES + ("description",): if k in dicts: signature_dict.add_from_file(dicts[k].get_filepath(), allow_overwrite=True) # add signatures of all the files linked in the lists - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: filedict = dicts[k] for fname in filedict.get_immutable_files(): @@ -493,11 +501,11 @@ def save_common_dicts(dicts, is_main, set_readonly=True): # 'consts','untar_cfg','vars' will be loaded refresh_file_list(dicts, is_main) # save files in the file lists - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: dicts[k].save_files(allow_overwrite=True) # then save the lists - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: dicts[k].save(set_readonly=set_readonly) # calc and save the signatures @@ -592,7 +600,7 @@ def reuse_common_dicts(dicts, other_dicts, is_main, all_reused): # since the file names may have changed, refresh the file_list refresh_file_list(dicts, is_main) # check file-based dictionaries - for k in ("file_list", "at_file_list", "after_file_list"): + for k in cgWConsts.FILE_LISTS_PRIORITIES: if k in dicts: all_reused = reuse_file_dict(dicts, other_dicts, k) and all_reused diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 4cbb34afc..a6a247ec4 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -10,6 +10,8 @@ import os.path import shutil +from collections import Counter + from glideinwms.lib import pubCrypto, subprocessSupport from glideinwms.lib.util import str2bool @@ -219,6 +221,8 @@ def populate(self, other=None): self.dicts["params"].add("GLIDEIN_Factory_Collector", str(factory_monitoring_collector)) populate_gridmap(self.conf, self.dicts["gridmap"]) + # the following list will be a megalist containing all the scripts; used for duplication check logic subsequently + all_scripts = [] # NOTE that all the files in these _scripts lists are added as executables (i.e. must report with error_gen) file_list_scripts = [ "collector_setup.sh", @@ -226,8 +230,17 @@ def populate(self, other=None): "gwms-python", cgWConsts.CONDOR_STARTUP_FILE, ] + # add the above list to the megalist created before + all_scripts.extend(file_list_scripts) + + # singularity_setup should be performed after cvmfs_setup; condor_chirp's order does not matter + precvmfs_file_list_scripts = ["cvmfs_setup.sh"] + all_scripts.extend(precvmfs_file_list_scripts) # add this list to the megalist + # These are right after the entry, before some VO scripts. The order in the following list is important at_file_list_scripts = ["singularity_setup.sh", "condor_chirp", "gconfig.py"] + all_scripts.extend(at_file_list_scripts) # adding the above list to the megalist as before + # The order in the following list is important after_file_list_scripts = [ "check_proxy.sh", @@ -240,15 +253,14 @@ def populate(self, other=None): "glidein_sitewms_setup.sh", "script_wrapper.sh", "smart_partitionable.sh", - "cvmfs_setup.sh", "cvmfs_umount.sh", ] - # Only execute scripts once - duplicate_scripts = list(set(file_list_scripts).intersection(after_file_list_scripts)) - duplicate_scripts += list(set(file_list_scripts).intersection(at_file_list_scripts)) - duplicate_scripts += list(set(at_file_list_scripts).intersection(after_file_list_scripts)) + all_scripts.extend(after_file_list_scripts) # adding the above list to the megalist as before + # Scripts need to be only executed once, so check for duplicates + count_duplicates = Counter(all_scripts) + duplicate_scripts = [scr for scr, cnt in count_duplicates.items() if cnt > 1] if duplicate_scripts: - raise RuntimeError("Duplicates found in the list of files to execute '%s'" % ",".join(duplicate_scripts)) + raise RuntimeError(f"Duplicates found in the list of files to execute: {', '.join(duplicate_scripts)}") # Load more system scripts for script_name in file_list_scripts: @@ -291,13 +303,14 @@ def populate(self, other=None): ) self.dicts["untar_cfg"].add(pychirp_tarball, "lib/python/htchirp") - # Add helper scripts for on-demand cvmfs provisioning, conditional upon the attribute GLIDEIN_USE_CVMFSEXEC - # Add cvmfsexec helper script enabled by conditional download + # Add helper scripts for on-demand cvmfs provisioning + # Add cvmfsexec helper script cvmfs_helper = "cvmfs_helper_funcs.sh" self.dicts["file_list"].add_from_file( cvmfs_helper, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(cvmfs_helper), "exec", cond_download="GLIDEIN_USE_CVMFSEXEC" + cWConsts.insert_timestr(cvmfs_helper), + "exec", ), os.path.join(cgWConsts.WEB_BASE_DIR, cvmfs_helper), ) @@ -306,9 +319,7 @@ def populate(self, other=None): dist_select_script = "cvmfsexec_platform_select.sh" self.dicts["file_list"].add_from_file( dist_select_script, - cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(dist_select_script), "exec", cond_download="GLIDEIN_USE_CVMFSEXEC" - ), + cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(dist_select_script), "exec"), os.path.join(cgWConsts.WEB_BASE_DIR, dist_select_script), ) @@ -397,7 +408,7 @@ def populate(self, other=None): # TODO: This check could be done in the XML, checking if the entries are consistent in the current version # fetch the on-demand cvmfs provisioning feature setting # if on-demand CVMFS not used at the global level; ignore and continue - ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFSEXEC", 0) + ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFS", 0) # check if on demand cvmfs provisioning is requested/enabled if ondemand_cvmfs != 0: # check the dir containing cvmfsexec distros to see if they were built previously @@ -410,10 +421,16 @@ def populate(self, other=None): else: # can be overridden at the entry level, so ignore and [entry supersedes global setting] print( - "...cvmfsexec distributions unavailable but on-demand CVMFS requested via GLIDEIN_USE_CVMFSEXEC; Continuing..." + "...cvmfsexec distributions unavailable but on-demand CVMFS requested via GLIDEIN_USE_CVMFS; Continuing..." ) # add additional system scripts + for script_name in precvmfs_file_list_scripts: + self.dicts["precvmfs_file_list"].add_from_file( + script_name, + cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec:r"), + os.path.join(cgWConsts.WEB_BASE_DIR, script_name), + ) for script_name in at_file_list_scripts: self.dicts["at_file_list"].add_from_file( script_name, @@ -716,7 +733,7 @@ def populate(self, entry, schedd, main_dicts): # TODO: This check could be done in the XML, checking if the entries are consistent in the current version # fetch the on-demand cvmfs provisioning feature setting # if on-demand CVMFS not used by entry, ignore and continue - ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFSEXEC", 0) + ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFS", 0) if ondemand_cvmfs != 0: # check the dir containing cvmfsexec distros to see if they were built previously if os.path.exists(os.path.join(self.work_dir, "../cvmfsexec/tarballs")) and os.listdir( @@ -727,7 +744,7 @@ def populate(self, entry, schedd, main_dicts): print("......RECOMMENDED: Rebuild distributions using the latest version of cvmfsexec.") else: print( - "...cvmfsexec distributions unavailable but on-demand CVMFS is requested via GLIDEIN_USE_CVMFSEXEC; Aborting!" + "...cvmfsexec distributions unavailable but on-demand CVMFS is requested via GLIDEIN_USE_CVMFS; Aborting!" ) exit(1) diff --git a/creation/web_base/cvmfs_helper_funcs.sh b/creation/web_base/cvmfs_helper_funcs.sh index 00a14925f..1d8274489 100755 --- a/creation/web_base/cvmfs_helper_funcs.sh +++ b/creation/web_base/cvmfs_helper_funcs.sh @@ -18,9 +18,6 @@ # Author: # Namratha Urs # -# Version: -# 1.0 -# # to implement custom logging @@ -30,476 +27,590 @@ #exec &> $LOGFILE variables_reset() { - # DESCRIPTION: This function lists and initializes the common variables - # to empty strings. These variables also become available to scripts - # that import functions defined in this script. - # - # INPUT(S): None - # RETURN(S): Variables initialized to empty strings - - # indicates whether the perform_system_check function has been run - GWMS_SYSTEM_CHECK= - - # following set of variables used to store operating system and kernel info - GWMS_OS_DISTRO= - GWMS_OS_VERSION= - GWMS_OS_KRNL_ARCH= - GWMS_OS_KRNL_NUM= - GWMS_OS_KRNL_VER= - GWMS_OS_KRNL_MAJOR_REV= - GWMS_OS_KRNL_MINOR_REV= - GWMS_OS_KRNL_PATCH_NUM= - - # indicates whether CVMFS is locally mounted on the node - GWMS_IS_CVMFS_MNT= - # to indicate the status of mounting CVMFS by the glidein after evaluating the worker node - GWMS_IS_CVMFS= - - # indicates if unpriv userns is available (or supported); not if it is enabled - GWMS_IS_UNPRIV_USERNS_SUPPORTED= - # indicates if unpriv userns is enabled (and available) - GWMS_IS_UNPRIV_USERNS_ENABLED= - - # following variables store FUSE-related information - GWMS_IS_FUSE_INSTALLED= - GWMS_IS_FUSERMOUNT= - GWMS_IS_USR_IN_FUSE_GRP= + # Initializes the common variables to empty strings. These variables also become available to scripts that import functions defined in this script. + # + # INPUT(S): None + # RETURN(S): Variables initialized to empty strings + + # all of the variables in here are initialized to an empty string, where empty means not evaluated + # this variable indicates whether the perform_system_check function has been run; when evaluated value would be 'yes', otherwise remains empty + GWMS_SYSTEM_CHECK= + + # these variables are used to store operating system and kernel info + GWMS_OS_DISTRO= + GWMS_OS_NAME= + GWMS_OS_VERSION_FULL= + GWMS_OS_VERSION_MAJOR= + GWMS_OS_VERSION_MINOR= + GWMS_OS_KRNL_ARCH= + GWMS_OS_KRNL_NUM= + GWMS_OS_KRNL_VER= + GWMS_OS_KRNL_MAJOR_REV= + GWMS_OS_KRNL_MINOR_REV= + GWMS_OS_KRNL_PATCH_NUM= + + # the following variables are also initialized to an empty string; when evaluated can take on values 1 (meaning false/no) or 0 (meaning true/yes) + # indicates whether CVMFS is locally mounted on the worker node (CE) + GWMS_IS_CVMFS_LOCAL_MNT= + # to indicate the status of on-demand mounting of CVMFS by the glidein after evaluating the worker node (CE) + GWMS_IS_CVMFS= + + # indicates if unpriv userns is available (or supported); not if it is enabled + GWMS_IS_UNPRIV_USERNS_SUPPORTED= + # indicates if unpriv userns is enabled (and available) + GWMS_IS_UNPRIV_USERNS_ENABLED= + + # following variables store FUSE-related information + GWMS_IS_FUSE_INSTALLED= + GWMS_IS_FUSERMOUNT= + GWMS_IS_USR_IN_FUSE_GRP= } loginfo() { - # DESCRIPTION: This function prints informational messages to STDOUT - # along with hostname and date/time. - # - # INPUT(S): String containing the message - # RETURN(S): Prints message to STDOUT + # Prints informational messages to STDOUT along with hostname and date/time. + # + # INPUT(S): String containing the message + # RETURN(S): Prints message to STDOUT - echo -e "$(hostname -s) $(date +%m-%d-%Y\ %T\ %Z) \t INFO: $1" >&2 + echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t INFO: $1" >&2 } logwarn(){ - # DESCRIPTION: This function prints warning messages to STDOUT along - # with hostname and date/time. - # - # INPUT(S): String containing the message - # RETURN(S): Prints message to STDOUT + # Prints warning messages to STDOUT along with hostname and date/time. + # + # INPUT(S): String containing the message + # RETURN(S): Prints message to STDOUT - echo -e "$(hostname -s) $(date +%m-%d-%Y\ %T\ %Z) \t WARNING: $1" >&2 + echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t WARNING: $1" >&2 } logerror() { - # DESCRIPTION: This function prints error messages to STDOUT along with - # hostname and date/time. - # - # INPUT(S): String containing the message - # RETURN(S): Prints message to STDOUT + # Prints error messages to STDOUT along with hostname and date/time. + # + # INPUT(S): String containing the message + # RETURN(S): Prints message to STDOUT - echo -e "$(hostname -s) $(date +%m-%d-%Y\ %T\ %Z) \t ERROR: $1" >&2 + echo -e "$(date +%m-%d-%Y\ %T\ %Z) \t ERROR: $1" >&2 } -check_exit_status () { - # DESCRIPTION: This function prints an appropriate message to the - # console to indicate what the exit status means. - # - # INPUT(S): Number (exit status of a previously run command) - # RETURN(S): Prints "yes" or "no" to indicate the result of the command +print_exit_status () { + # Prints an appropriate message to the console to indicate what the exit status means. + # + # INPUT(S): Number (exit status of a previously run command) + # RETURN(S): Prints "yes" or "no" to indicate the result of the command - [[ $1 -eq 0 ]] && echo yes || echo no + [[ $1 -eq 0 ]] && echo yes || echo no } + detect_local_cvmfs() { - CVMFS_ROOT="/cvmfs" - repo_name=oasis.opensciencegrid.org - # Second check... - if [[ -f $CVMFS_ROOT/$repo_name/.cvmfsdirtab || "$(ls -A $CVMFS_ROOT/$repo_name)" ]] &>/dev/null - then - loginfo "Validating CVMFS with ${repo_name}..." - true - else - logwarn "Validating CVMFS with ${repo_name}: directory empty or does not have .cvmfsdirtab" - false - fi - - GWMS_IS_CVMFS_MNT=$? - loginfo "CVMFS locally installed: $(check_exit_status $GWMS_IS_CVMFS_MNT)" + # Detects whether CVMFS is natively (aka locally) available on the worker node. The result is stored in a common variable, i.e GWMS_IS_CVMFS_LOCAL_MNT, and can be used downstream. + # + # INPUT(S): None + # RETURN(S): None + + local CVMFS_ROOT="/cvmfs" + local repo_name=oasis.opensciencegrid.org + GWMS_IS_CVMFS_LOCAL_MNT=0 + if [[ -f $CVMFS_ROOT/$repo_name/.cvmfsdirtab || "$(ls -A $CVMFS_ROOT/$repo_name)" ]] &>/dev/null + then + loginfo "Validating CVMFS with ${repo_name}..." + else + logwarn "Validating CVMFS with ${repo_name}: directory empty or does not have .cvmfsdirtab" + GWMS_IS_CVMFS_LOCAL_MNT=1 + fi + + loginfo "Worker node has native CVMFS: $(print_exit_status $GWMS_IS_CVMFS_LOCAL_MNT)" } -perform_system_check() { - # DESCRIPTION: This functions performs required system checks (such as - # operating system and kernel info, unprivileged user namespaces, FUSE - # status) and stores the results in the common variables for later use. - # - # INPUT(S): None - # RETURN(S): - # -> common variables containing the exit status of the - # corresponding commands - # -> results from running the check_exit_status function - # for logging purposes (variables starting with res_) - # -> assigns "yes" to GWMS_SYSTEM_CHECK to indicate this function - # has been run. - - if [ -f '/etc/redhat-release' ]; then - GWMS_OS_DISTRO=rhel - else - GWMS_OS_DISTRO=non-rhel - fi - - GWMS_OS_VERSION=`lsb_release -r | awk -F'\t' '{print $2}'` - GWMS_OS_KRNL_ARCH=`arch` - GWMS_OS_KRNL_NUM=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " ` - GWMS_OS_KRNL_VER=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $1}'` - GWMS_OS_KRNL_MAJOR_REV=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $2}'` - GWMS_OS_KRNL_MINOR_REV=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $3}'` - GWMS_OS_KRNL_PATCH_NUM=`uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 2 -d " "` - - #df -h | grep /cvmfs &>/dev/null - #GWMS_IS_CVMFS_MNT=$? - # call function to detect local CVMFS only if the GWMS_IS_CVMFS_MNT variable is not set; if the variable is not empty, do nothing - [[ -z "${GWMS_IS_CVMFS_MNT}" ]] && detect_local_cvmfs || : - - sysctl user.max_user_namespaces &>/dev/null - GWMS_IS_UNPRIV_USERNS_SUPPORTED=$? - - unshare -U true &>/dev/null - GWMS_IS_UNPRIV_USERNS_ENABLED=$? - - yum list installed *fuse* &>/dev/null - GWMS_IS_FUSE_INSTALLED=$? - - fusermount -V &>/dev/null - GWMS_IS_FUSERMOUNT=$? - - getent group fuse | grep $USER &>/dev/null - GWMS_IS_USR_IN_FUSE_GRP=$? - - # set the variable indicating this function has been run - GWMS_SYSTEM_CHECK=yes +perform_system_check() { + # Performs required system checks (such as operating system and kernel info, unprivileged user namespaces, FUSE status) and stores the results in the common variables for later use. + # + # INPUT(S): None + # RETURN(S): + # -> common variables containing the exit status of the + # corresponding commands + # -> results from running the print_exit_status function + # for logging purposes (variables starting with res_) + # -> assigns "yes" to GWMS_SYSTEM_CHECK to indicate this function + # has been run. + + # reset all variables used in this script's namespace before executing the rest of the script + # variables_reset + + # local user_namespaces + if [[ -f '/etc/redhat-release' ]]; then + # rhel derivative; use /etc/redhat-release to fetch the release information + # NOTE: using /etc/redhat-release over /etc/os-release as it is more consistent to rely on + GWMS_OS_DISTRO="rhel" + # GWMS_OS_DISTRO="non-rhel" + #GWMS_OS_VERSION_FULL=$(cat /etc/redhat-release | cut -d " " -f 3) + else + # not a rhel derivative; use /etc/os-release instead [fallback option] + GWMS_OS_DISTRO="non-rhel" + #GWMS_OS_VERSION_FULL=$(cat /etc/os-release | egrep "VERSION_ID" | cut -d = -f 2 | tr -d '"') + fi + + # source the os-release file to access the variables defined + . /etc/os-release + GWMS_OS_VERSION_FULL=$VERSION_ID + GWMS_OS_VERSION_MAJOR=$(echo "$GWMS_OS_VERSION_FULL" | awk -F'.' '{print $1}') + GWMS_OS_VERSION_MINOR=$(echo "$GWMS_OS_VERSION_FULL" | awk -F'.' '{print $2}') + GWMS_OS_NAME=${NAME,,} + GWMS_OS_KRNL_ARCH=$(arch) + GWMS_OS_KRNL_NUM=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " ) + GWMS_OS_KRNL_VER=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $1}') + GWMS_OS_KRNL_MAJOR_REV=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $2}') + GWMS_OS_KRNL_MINOR_REV=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 1 -d " " | awk -F'.' '{print $3}') + GWMS_OS_KRNL_PATCH_NUM=$(uname -r | awk -F'-' '{split($2,a,"."); print $1,a[1]}' | cut -f 2 -d " ") + + # call function to detect local CVMFS only if the GWMS_IS_CVMFS_LOCAL_MNT variable is not set; if the variable is not empty, do nothing + [[ -z "${GWMS_IS_CVMFS_LOCAL_MNT}" ]] && detect_local_cvmfs || : + + cat /proc/sys/user/max_user_namespaces &>/dev/null + GWMS_IS_UNPRIV_USERNS_SUPPORTED=$? + + unshare -U true &>/dev/null + GWMS_IS_UNPRIV_USERNS_ENABLED=$? + + [[ $GWMS_OS_VERSION_MAJOR -ge 9 ]] && dnf list installed fuse3* &>/dev/null || yum list installed fuse &>/dev/null + GWMS_IS_FUSE_INSTALLED=$? + + [[ $GWMS_OS_VERSION_MAJOR -ge 9 ]] && fusermount3 -V &>/dev/null || fusermount -V &>/dev/null + GWMS_IS_FUSERMOUNT=$? + + getent group fuse | grep $USER &>/dev/null + GWMS_IS_USR_IN_FUSE_GRP=$? + + # set the variable indicating this function has been run + GWMS_SYSTEM_CHECK=yes } print_os_info () { - # DESCRIPTION: This functions prints operating system and kernel - # information to STDOUT. - # - # INPUT(S): None - # RETURN(S): Prints a message containing OS and kernel details + # Prints operating system and kernel information to STDOUT. + # + # INPUT(S): None + # RETURN(S): Prints a message containing OS and kernel details - loginfo "Found $GWMS_OS_DISTRO${GWMS_OS_VERSION}-${GWMS_OS_KRNL_ARCH} with kernel $GWMS_OS_KRNL_NUM-$GWMS_OS_KRNL_PATCH_NUM" + loginfo "Found $GWMS_OS_NAME [$GWMS_OS_DISTRO] ${GWMS_OS_VERSION_FULL}-${GWMS_OS_KRNL_ARCH} with kernel $GWMS_OS_KRNL_NUM-$GWMS_OS_KRNL_PATCH_NUM" } log_all_system_info () { - # DESCRIPTION: This function prints all the necessary system information - # stored in common and result variables (see perform_system_check - # function) for easy debugging. This has been done as collecting - # information about the worker node can be useful for troubleshooting - # and gathering stats about what is out there. - # - # INPUT(S): None - # RETURN(S): Prints user-friendly messages to STDOUT - - loginfo "..." - loginfo "Worker node details: " - loginfo "Operating system distro: $GWMS_OS_DISTRO" - loginfo "Operating System version: $GWMS_OS_VERSION" - loginfo "Kernel Architecture: $GWMS_OS_KRNL_ARCH" - loginfo "Kernel version: $GWMS_OS_KRNL_VER" - loginfo "Kernel major revision: $GWMS_OS_KRNL_MAJOR_REV" - loginfo "Kernel minor revision: $GWMS_OS_KRNL_MINOR_REV" - loginfo "Kernel patch number: $GWMS_OS_KRNL_PATCH_NUM" - - loginfo "CVMFS locally installed: $(check_exit_status $GWMS_IS_CVMFS_MNT)" - loginfo "Unprivileged user namespaces supported: $(check_exit_status $GWMS_IS_UNPRIV_USERNS_SUPPORTED)" - loginfo "Unprivileged user namespaces enabled: $(check_exit_status $GWMS_IS_UNPRIV_USERNS_ENABLED)" - loginfo "FUSE installed: $(check_exit_status $GWMS_IS_FUSE_INSTALLED)" - loginfo "fusermount available: $(check_exit_status $GWMS_IS_FUSERMOUNT)" - loginfo "Is the user in 'fuse' group: $(check_exit_status $GWMS_IS_USR_IN_FUSE_GRP)" - loginfo "..." + # Prints all the necessary system information stored in common and result variables (see perform_system_check function) for easy debugging. This has been done as collecting information about the worker node can be useful for troubleshooting and gathering stats about what is out there. + # + # INPUT(S): None + # RETURN(S): Prints user-friendly messages to STDOUT + + # make sure that perform_system_check has run + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + print_os_info + loginfo "..." + loginfo "Worker node details: " + loginfo "Hostname: $(hostname)" + loginfo "Operating system distro: $GWMS_OS_DISTRO" + loginfo "Operating system name: $GWMS_OS_NAME" + loginfo "Operating system version: $GWMS_OS_VERSION_FULL" + loginfo "Kernel architecture: $GWMS_OS_KRNL_ARCH" + loginfo "Kernel version: $GWMS_OS_KRNL_VER" + loginfo "Kernel major revision: $GWMS_OS_KRNL_MAJOR_REV" + loginfo "Kernel minor revision: $GWMS_OS_KRNL_MINOR_REV" + loginfo "Kernel patch number: $GWMS_OS_KRNL_PATCH_NUM" + + loginfo "CVMFS locally installed: $(print_exit_status $GWMS_IS_CVMFS_LOCAL_MNT)" + loginfo "Unprivileged user namespaces supported: $(print_exit_status $GWMS_IS_UNPRIV_USERNS_SUPPORTED)" + loginfo "Unprivileged user namespaces enabled: $(print_exit_status $GWMS_IS_UNPRIV_USERNS_ENABLED)" + loginfo "FUSE installed: $(print_exit_status $GWMS_IS_FUSE_INSTALLED)" + loginfo "fusermount available: $(print_exit_status $GWMS_IS_FUSERMOUNT)" + loginfo "Is the user in 'fuse' group: $(print_exit_status $GWMS_IS_USR_IN_FUSE_GRP)" + loginfo "..." } mount_cvmfs_repos () { - # DESCRIPTION: This function mounts all the required and additional - # CVMFS repositories that would be needed for user jobs. - # - # INPUT(S): 1. CVMFS configuration repository (string); 2. Additional CVMFS - # repositories (colon-delimited string) - # RETURN(S): Mounts the defined repositories on the worker node filesystem - - $glidein_cvmfsexec_dir/$dist_file $1 -- echo "setting up mount utilities..." &> /dev/null - if [[ $(df -h|grep /cvmfs|wc -l) -eq 1 ]]; then - loginfo "CVMFS config repo already mounted!" - continue - else - # mounting the configuration repo (pre-requisite) - loginfo "Mounting CVMFS config repo now..." - $glidein_cvmfsexec_dir/.cvmfsexec/mountrepo $1 - fi - - # using an array to unpack the names of additional CVMFS repositories - # from the colon-delimited string - declare -a cvmfs_repos - repos=($(echo $2 | tr ":" "\n")) - #echo ${repos[@]} - - loginfo "Mounting additional CVMFS repositories..." - # mount every repository that was previously unpacked - for repo in "${repos[@]}" - do - $glidein_cvmfsexec_dir/.cvmfsexec/mountrepo $repo - done - - # see if all the repositories got mounted - num_repos_mntd=`df -h | grep /cvmfs | wc -l` - total_num_repos=$(( ${#repos[@]} + 1 )) - if [ "$num_repos_mntd" -eq "$total_num_repos" ]; then - loginfo "All CVMFS repositories mounted successfully on the worker node" - true - else - logwarn "One or more CVMFS repositories might not be mounted on the worker node" - false - fi - - GWMS_IS_CVMFS=$? + # Mounts all the required and additional CVMFS repositories that would be needed for user jobs. + # + # INPUT(S): + # 1. cvmfsexec mode (integer) + # 2. CVMFS configuration repository (string) + # 3. Additional CVMFS repositories (colon-delimited string) + # RETURN(S): Mounts the defined repositories on the worker node filesystem + local cvmfsexec_mode=$1 + local config_repository=$2 + local additional_repos=$3 + local config_repo_mntd num_repos_mntd total_num_repos + + # at this point in the execution flow, it would have been determined that cvmfs is not locally available + # this implies no repositories should have been mounted. However, only config repo will be mounted if in mode 1 or mode 3 by this point + + # if using mode 3/2, config repo should have been mounted already by now + if [[ $(cat /proc/$$/mounts | grep /dev/fuse | grep ' /cvmfs' | wc -l) -eq 1 ]]; then + loginfo "CVMFS config repo already mounted!" + else + # mounting the configuration repo (pre-requisite) in case something went wrong previously or when using mode 1 + # if using mode 1, first setup the mount utility under .cvmfsexec + if [[ $cvmfsexec_mode -eq 1 ]]; then + "$glidein_cvmfsexec_dir/$dist_file" "$config_repository" -- echo "setting up mount utilities..." &> /dev/null + fi + loginfo "Mounting CVMFS config repo now..." + [[ $cvmfsexec_mode -eq 1 ]] && "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$config_repository" + [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]] && $CVMFSMOUNT "$config_repository" + # see if the config repository got mounted this time around + if [[ $(cat /proc/$$/mounts | grep /dev/fuse | wc -l) -eq 0 ]]; then + logwarn "CVMFS config repository might still not be mounted on the worker node" + return 1 + fi + fi + + # using an array to unpack the names of additional CVMFS repositories + # from the colon-delimited string + repos=($(echo $additional_repos | tr ":" "\n")) + loginfo "Mounting additional CVMFS repositories..." + # mount every repository in the array + for repo in "${repos[@]}" + do + [[ $cvmfsexec_mode -eq 1 ]] && "$glidein_cvmfsexec_dir"/.cvmfsexec/mountrepo "$repo" + [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]] && $CVMFSMOUNT "$repo" + done + # verify if all the repositories got mounted + if [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]]; then + # since mode 3 mounting shows original mounts as well as the bind mounts, only consider one of those sets for verification + num_repos_mntd=$(cat /proc/$$/mounts | grep /dev/fuse | grep ' /cvmfs' | wc -l) + else + num_repos_mntd=$(cat /proc/$$/mounts | grep /dev/fuse | wc -l) + fi + total_num_repos=$(( ${#repos[@]} + 1 )) + GWMS_IS_CVMFS=0 + if [[ "$num_repos_mntd" -eq "$total_num_repos" ]]; then + loginfo "All CVMFS repositories mounted successfully on the worker node" + # export this info to the glidein environment after CVMFS mounting was a success + gconfig_add GWMS_IS_CVMFS $(print_exit_status $GWMS_IS_CVMFS) + get_mount_point + return 0 + else + logwarn "One or more CVMFS repositories might not be mounted on the worker node" + GWMS_IS_CVMFS=1 + # export this info to indicate to the glidein environment that something went wrong during the CVMFS mounting + gconfig_add GWMS_IS_CVMFS $(print_exit_status $GWMS_IS_CVMFS) + return 1 + fi } -has_unpriv_userns() { - # DESCRIPTION: This function checks the status of unprivileged user - # namespaces being supported and enabled on the worker node. Depending - # - # INPUT(S): None - # RETURN(S): - # -> true (0) if unpriv userns can be used (supported and enabled), - # false otherwise - # -> status of unpriv userns (unavailable, disabled, enabled, - # error) to stdout - - # make sure that perform_system_check has run - [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - - # determine whether unprivileged user namespaces are supported and/or enabled... - if [[ "${GWMS_IS_UNPRIV_USERNS_ENABLED}" -eq 0 ]]; then - # unprivileged user namespaces is enabled - if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then - # unprivileged user namespaces is supported - loginfo "Unprivileged user namespaces supported and enabled" - echo enabled - else - # unprivileged user namespaces is not supported - logerror "Inconsistent system configuration: unprivileged userns is enabled but not supported" - echo error - fi - true - else - # unprivileged user namespaces is disabled - if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then - # unprivileged user namespaces is supported - logwarn "Unprivileged user namespaces disabled: can be enabled by the root user via sysctl" - echo disabled - else - # unprivileged user namespaces is not supported - logwarn "Unprivileged user namespaces disabled and unsupported: can be supported/enabled only after a system upgrade" - echo unavailable - fi - false - fi +get_mount_point() { + # Obtain the mount point information regarding where CVMFS is mounted on demand (when mounted). By default, CVMFS when mounted is at '/cvmfs'. Otherwise, CVMFS will be mounted at /.cvmfsexec/dist/cvmfs + # + # INPUT(S): None + # RETURN(S): None + + mount_point=$(findmnt -t fuse -S /dev/fuse | tail -n 1 | cut -d ' ' -f 1 ) + if [[ -n "$mount_point" && "$mount_point" != TARGET* ]]; then + mount_point=$(dirname "$mount_point") + if [[ -n "$mount_point" && "$mount_point" != /cvmfs ]]; then + CVMFS_MOUNT_DIR="$mount_point" + export CVMFS_MOUNT_DIR=$mount_point + gconfig_add CVMFS_MOUNT_DIR "$mount_point" + fi + fi +} + +has_unpriv_userns() { + # Checks the status of unprivileged user namespaces being supported and enabled on the worker node. + # + # INPUT(S): None + # RETURN(S): + # -> true (0) if unpriv userns can be used (supported and enabled), + # false otherwise + # -> status of unpriv userns (unavailable, disabled, enabled, + # error) to stdout + + # make sure that perform_system_check has run + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # determine whether unprivileged user namespaces are supported and enabled... + if [[ "${GWMS_IS_UNPRIV_USERNS_ENABLED}" -eq 0 ]]; then + # if unprivileged user namespaces is enabled in the system + if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then + # check if unprivileged user namespaces is supported by the system + loginfo "Unprivileged user namespaces supported and enabled" + echo enabled + return 0 + fi + # otherwise, if unprivileged user namespaces is not supported by the system + logerror "Inconsistent system configuration: unprivileged usernamespaces is enabled but not supported" + echo error + else + # if unprivileged user namespaces is found to be disabled + if [[ "${GWMS_IS_UNPRIV_USERNS_SUPPORTED}" -eq 0 ]]; then + # unprivileged user namespaces is supported + logwarn "Unprivileged user namespaces disabled: can be enabled by the root user via sysctl" + echo disabled + else + # otherwise, if unprivileged user namespaces is also not supported by the system + logwarn "Unprivileged user namespaces disabled and unsupported: can be supported/enabled only after a system upgrade" + echo unavailable + fi + fi + return 1 } has_fuse() { - # DESCRIPTION: This function checks FUSE-related configurations on the - # worker node. This is a pre-requisite before evaluating whether CVMFS - # is mounted on the filesystem. - # - # FUSE Documentation references: - # https://www.kernel.org/doc/html/latest/filesystems/fuse.html - # https://en.wikipedia.org/wiki/Filesystem_in_Userspace - # - # INPUT(S): None - # RETURN(S): string denoting fuse availability (yes, no, error) - - # make sure that perform_system_check has run - [[ -n "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check - - # check what specific configuration of unprivileged user namespaces exists in the system (worker node) - unpriv_userns_config=$(has_unpriv_userns) - - # exit from the script if unprivileged namespaces are not supported but enabled in the kernel - if [[ "${unpriv_userns_config}" == error ]]; then - "$error_gen" -error "`basename $0`" "WN_Resource" "Unprivileged user namespaces are not supported but enabled in the kernel! Check system configuration." - exit 1 - # determine if mountrepo/umountrepo could be used by checking availability of fuse, fusermount and user being in fuse group... - elif [[ "${GWMS_IS_FUSE_INSTALLED}" -eq 0 ]]; then - # fuse is installed - if [[ $unpriv_userns_config == unavailable ]]; then - # unprivileged user namespaces unsupported, i.e. kernels 2.x (scenarios 5b,6b) - if [[ "${GWMS_IS_USR_IN_FUSE_GRP}" -eq 0 ]]; then - # user is in fuse group -> fusermount is available (scenario 6b) - if [[ "${GWMS_IS_FUSERMOUNT}" -ne 0 ]]; then - logwarn "Inconsistent system configuration: fusermount is available with fuse installed and when user is in fuse group" - echo error - else - loginfo "FUSE requirements met by the worker node" - echo yes - fi - else - # user is not in fuse group -> fusermount is unavailable (scenario 5b) - if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - logwarn "Inconsistent system configuration: fusermount is available only when user is in fuse group and fuse is installed" - echo error - else - loginfo "FUSE requirements not satisfied: user is not in fuse group" - echo no - fi - fi - else - # unprivileged user namespaces is either enabled or disabled - if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - # fusermount is available (scenarios 7,8) - loginfo "FUSE requirements met by the worker node" - echo yes - else - # fusermount is not available (scenarios 5a,6a) - logwarn "Inconsistent system configuration: fusermount is available when fuse is installed " - echo error - fi - fi - else - # fuse is not installed - if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then - # fusermount is somehow available and user is/is not in fuse group (scenarios 3,4) - logwarn "Inconsistent system configuration: fusermount is only available with fuse and/or when user belongs to the fuse group" - echo error - else - # fusermount is not available and user is/is not in fuse group (scenarios case 1,2) - loginfo "FUSE requirements not satisfied: fusermount is not available" - echo no - fi - fi + # Check the status of FUSE configuration being available on the worker node. + # + # INPUT(S): None + # RETURN(S): + # -> true (0) if FUSE is available, false otherwise + # -> status of FUSE configuration (no, yes, error) to stdout + + # make sure that perform_system_check has run + [[ -n "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # determine which cvmfsexec utilities can be used by checking availability of fuse, fusermount and user being in fuse group... + if [[ "${GWMS_IS_FUSE_INSTALLED}" -ne 0 ]]; then + # fuse is not installed + if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then + # fusermount is somehow available and user is/is not in fuse group (scenarios 3,4) + logwarn "Inconsistent system configuration: fusermount is only available with fuse and/or when user belongs to the fuse group" + echo error + else + # fusermount is not available and user is/is not in fuse group (scenarios case 1,2) + loginfo "FUSE requirements not satisfied: fusermount is not available" + echo no + fi + return 1 + fi + + # fuse rpm is installed + local ret_state + if [[ $unpriv_userns_status = "unavailable" ]]; then + # unprivileged user namespaces unsupported, i.e. kernels 2.x (scenarios 5b,6b) + if [[ "${GWMS_IS_USR_IN_FUSE_GRP}" -eq 0 ]]; then + # user is in fuse group -> fusermount is available (scenario 6b) + if [[ "${GWMS_IS_FUSERMOUNT}" -ne 0 ]]; then + logwarn "Inconsistent system configuration: fusermount is available with fuse installed and when user is in fuse group" + ret_state=error + else + loginfo "FUSE requirements met by the worker node" + ret_state=yes + fi + else + # user is not in fuse group -> fusermount is unavailable (scenario 5b) + if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then + logwarn "Inconsistent system configuration: fusermount is available only when user is in fuse group and fuse is installed" + ret_state=error + else + loginfo "FUSE requirements not satisfied: user is not in fuse group" + ret_state=no + fi + fi + else + # unprivileged user namespaces is either enabled or disabled + if [[ "${GWMS_IS_FUSERMOUNT}" -eq 0 ]]; then + # fuse is installed with fusermount available (scenarios 7,8) + loginfo "FUSE requirements met by the worker node" + ret_state=yes + else + # fuse is installed but fusermount not available (scenarios 5a,6a) + logwarn "Inconsistent system configuration: fusermount is not available when fuse is installed " + ret_state=error + fi + fi + echo $ret_state + [[ "$ret_state" == "yes" ]] + return +} + +determine_cvmfsexec_mode_usage() { + # Determine the cvmfsexec mode that will be applicable based on the worker node specifications, including the status of unprivileged user namespaces and FUSE configuration. + # + # INPUT(S): None + # RETURN(S): + # -> true (0) if it is determined that one of the three cvmfsexec modes can be used, false otherwise + # -> an integer indicating the mode of cvmfsexec that will be possible (0, 1, 2, 3) to stdout + + # make sure that perform_system_check has run + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # check what specific configuration of unprivileged user namespaces exists in the system (worker node) + unpriv_userns_status=$(has_unpriv_userns) + + # check FUSE configuration on the worker node + fuse_config_status=$(has_fuse) + + if [[ "${unpriv_userns_status}" == "error" && "${fuse_config_status}" == "error" ]]; then + logwarn "User namespaces and fuse mounts not available in unprivileged mode" + echo 0 + return 1 + fi + # either 1. if unprivileged user namespaces are enabled (and therefore supported) + if [[ "${unpriv_userns_status}" == "enabled" ]]; then + # satisfies the minimum requirement for mode 3 + if [[ $GWMS_OS_KRNL_VER -ge 5 || $GWMS_OS_KRNL_VER -ge 4 && $GWMS_OS_KRNL_MAJOR_REV -ge 18 || $GWMS_OS_KRNL_VER -ge 3 && $GWMS_OS_KRNL_MAJOR_REV -ge 10 && $GWMS_OS_KRNL_MINOR_REV -ge 0 && $GWMS_OS_KRNL_PATCH_NUM -ge 1127 ]]; then + # if newer kernels >= 4.18 (RHEL8) or >= 3.10.0-1127 (RHEL 7.8), cvmfsexec can be used in mode 3 + echo 3 + return 0 + fi + # if RHEL <= 7.7, cvmfsexec can be used in mode 2 + echo 2 + return 0 + elif [[ "${unpriv_userns_status}" =~ ^(disabled|unavailable)$ ]]; then + # when unpriv. userns are disabled/not available, take FUSE status into consideration + if [[ "${fuse_config_status}" != "yes" ]]; then + # cvmfsexec cannot be used in either of the three modes + loginfo "cvmfsexec cannot be used in either of the three modes!" + echo 0 + return 1 + fi + fi + # or 2. solely based on fuse status on the worker node, determine whether any of the cvmfsexec modes can be used + if [[ "${fuse_config_status}" == "yes" ]]; then + # cvmfsexec can be used in mode 1 + echo 1 + return 0 + fi + if [[ $fuse_config_status == no ]]; then + # failure; + logerror "CVMFS cannot be mounted on the worker node using mountrepo utility" + elif [[ $fuse_config_status == error ]]; then + # inconsistent system configurations detected in the worker node + logerror "Detected inconsistent configurations on the worker node. mountrepo utility cannot be used!!" + fi + echo 0 + return 1 } -evaluate_worker_node_config () { - # DESCRIPTION: This function evaluates the worker using FUSE and - # unpriv. userns configurations to determine whether CVMFS can be - # mounted using mountrepo utility. - # - # INPUT(S): None - # RETURN(S): string message whether CVMFS can be mounted - - # collect info about FUSE configuration on the worker node - fuse_config_status=$(has_fuse) - - # check fuse related configurations in the system (worker node) - if [[ $fuse_config_status == yes ]]; then - # success; - loginfo "CVMFS can be mounted and unmounted on the worker node using mountrepo/umountrepo utility" - true - elif [[ $fuse_config_status == no ]]; then - # failure; - logerror "CVMFS cannot be mounted on the worker node using mountrepo utility" - false - elif [[ $fuse_config_status == error ]]; then - # inconsistent system configurations detected in the worker node - logerror "Detected inconsistent configurations on the worker node. mountrepo utility cannot be used!!" - false - fi +setup_cvmfsexec_use() { + # Performs the necessary setup prior to using cvmfsexec, if possible. If cvmfsexec can be used in either of the three modes, the specific mode information is written to the glidein configuration file. + # + # INPUT(S): None + # RETURN(S): an integer depicting the cvmfsexec mode that is applicable for the worker node. + + local gwms_cvmfsexec_mode + # first we perform checks on the worker node that will be used to assess whether cvmfsexec can be used and if yes, which mode of cvmfsexec can be used + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # second, log the results of the checks that were performed in the previous step + log_all_system_info + # finally, using the results obtained from the checks, determine which mode of cvmfsexec can be used + gwms_cvmfsexec_mode=$(determine_cvmfsexec_mode_usage) + if [[ $gwms_cvmfsexec_mode -ne 0 ]]; then + gconfig_add GWMS_CVMFSEXEC_MODE $gwms_cvmfsexec_mode + fi + echo $gwms_cvmfsexec_mode } -perform_cvmfs_mount () { - # reset all variables used in this script's namespace before executing the rest of the script - variables_reset - - # perform checks on the worker node that will be used to assess whether CVMFS can be mounted or not - perform_system_check - - # print/display all information pertaining to system checks performed previously (facilitates easy troubleshooting) - log_all_system_info - - loginfo "CVMFS Source = $cvmfs_source" - # initializing CVMFS repositories to a variable for easy modification in the future - case $cvmfs_source in - osg) - GLIDEIN_CVMFS_CONFIG_REPO=config-osg.opensciencegrid.org - GLIDEIN_CVMFS_REPOS=singularity.opensciencegrid.org:cms.cern.ch - ;; - egi) - GLIDEIN_CVMFS_CONFIG_REPO=config-egi.egi.eu - GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch - ;; - default) - GLIDEIN_CVMFS_CONFIG_REPO=cvmfs-config.cern.ch - GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch - ;; - *) - "$error_gen" -error "`basename $0`" "WN_Resource" "Invalid factory attribute value specified for CVMFS source." - exit 1 - esac - # (optional) set an environment variable that suggests additional repos to be mounted after config repos are mounted - loginfo "CVMFS Config Repo = $GLIDEIN_CVMFS_CONFIG_REPO" - - # by this point, it would have been established that CVMFS is not locally available - # so, install CVMFS via mountrepo or cvmfsexec - loginfo "CVMFS is NOT locally mounted on the worker node! Mounting now..." - # check the operating system distribution - #if [[ $GWMS_OS_DISTRO = RHEL ]]; then - # evaluate the worker node's system configurations to decide whether CVMFS can be mounted or not - loginfo "Evaluating the worker node..." - # display operating system information - print_os_info - - # assess the worker node based on its existing system configurations and perform next steps accordingly - if evaluate_worker_node_config ; then - # if evaluation was true, then proceed to mount CVMFS - if [[ $glidein_cvmfs = never ]]; then - # do nothing; test the node and print the results but do not even try to mount CVMFS - # just continue with glidein startup - echo $? - "$error_gen" -ok "`basename $0`" "msg" "Not trying to install CVMFS." - else - loginfo "Mounting CVMFS repositories..." - if mount_cvmfs_repos $GLIDEIN_CVMFS_CONFIG_REPO $GLIDEIN_CVMFS_REPOS ; then - : - else - if [[ $glidein_cvmfs = required ]]; then - # if mount CVMFS is not successful, report an error and exit with failure exit code - echo $? - "$error_gen" -error "`basename $0`" "WN_Resource" "CVMFS is required but unable to mount CVMFS on the worker node." - exit 1 - elif [[ $glidein_cvmfs = preferred || $glidein_cvmfs = optional ]]; then - # if mount CVMFS is not successful, report a warning/error in the logs and continue with glidein startup - # script status must be OK, otherwise the glidein will fail - echo $? - "$error_gen" -ok "`basename $0`" "WN_Resource" "Unable to mount required CVMFS on the worker node. Continuing without CVMFS." - else - "$error_gen" -error "`basename $0`" "WN_Resource" "Invalid factory attribute value specified for CVMFS requirement." - exit 1 - fi - fi - fi - else - # if evaluation was false, then exit from this activity of mounting CVMFS - "$error_gen" -error "`basename $0`" "WN_Resource" "Worker node configuration did not pass the evaluation checks. CVMFS will not be mounted." +prepare_for_cvmfs_mount () { + # Prepare the necessary items and keep them ready/accessible right before mounting CVMFS on demand. + # + # INPUT(S): None + # RETURN(S): None + + # if not previously performed, perform checks on the worker node that will be used to assess whether CVMFS can be mounted or not + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # get the CVMFS source information from in the glidein configuration + cvmfs_source=$(gconfig_get CVMFS_SRC "$glidein_config") + + # get the directory where cvmfsexec is unpacked + glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") + + # gather the worker node information to construct the name of the cvmfsexec distribution file based on the worker node specs + # perform_system_check sets a few variables that can be helpful here + os_like=$GWMS_OS_DISTRO + os_ver=$GWMS_OS_VERSION_MAJOR + arch=$GWMS_OS_KRNL_ARCH + # construct the name of the cvmfsexec distribution file based on the worker node specs + dist_file=cvmfsexec-${cvmfs_source}-${os_like}${os_ver}-${arch} + # the appropriate distribution file does not have to be manually untarred as the glidein setup takes care of this automatically + # but check if the unpacking was okay and the distribution file is accessible + if [[ ! -d "$glidein_cvmfsexec_dir" || ! -f "${glidein_cvmfsexec_dir}/${dist_file}" ]]; then + # 1. the cvmfsexec directory containing platform-specific cvmfsexec distribution does not exist in the glidein -- this happens when (a). cvmfsexec distributions are not built at all with cvmfsexec_distro factory knob (unless they are already existing from a previous reconfig/upgrade that used the knob), or (b). when the build process for the cvmfsexec distributions failed but was requested to be used + # 2. the relevant cvmfsexec distribution is not found -- this happens when there may not be a cvmfsexec distribution that is compatible with the worker node system configuration (e., wrong file name) + logwarn "Could not find the unpacked cvmfsexec distribution ${dist_file}!" + return 1 + fi + + loginfo "CVMFS Source = $cvmfs_source" + # initializing CVMFS repositories to a variable for easy modification in the future + case $cvmfs_source in + osg) + GLIDEIN_CVMFS_CONFIG_REPO=config-osg.opensciencegrid.org + GLIDEIN_CVMFS_REPOS=singularity.opensciencegrid.org:cms.cern.ch:oasis.opensciencegrid.org + ;; + egi) + GLIDEIN_CVMFS_CONFIG_REPO=config-egi.egi.eu + GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch:oasis.opensciencegrid.org + ;; + default) + GLIDEIN_CVMFS_CONFIG_REPO=cvmfs-config.cern.ch + GLIDEIN_CVMFS_REPOS=config-osg.opensciencegrid.org:singularity.opensciencegrid.org:cms.cern.ch:oasis.opensciencegrid.org + ;; + *) + "$error_gen" -error "$(basename "$0")" "WN_Resource" "Invalid factory attribute value specified for CVMFS source." exit 1 - fi - #else - # if operating system distribution is non-RHEL (any non-rhel OS) - # display operating system information and a user-friendly message - #print_os_info - #logwarn "This is a non-RHEL OS and is not covered in the implementation yet!" - # ----- Further Implementation: TBD (To Be Done) ----- # - #fi + esac + # (optional) set an environment variable that suggests additional repos to be mounted after config repos are mounted + loginfo "CVMFS Config Repo = $GLIDEIN_CVMFS_CONFIG_REPO" + loginfo "CVMFS Repos = $GLIDEIN_CVMFS_REPOS" +} - #fi - #loginfo "End log for mounting CVMFS" +perform_cvmfs_mount () { + # Wrapper for performing mounting of CVMFS on demand depending on a few factors. + # + # INPUT(S): an integer denoting the selected cvmfsexec mode + # RETURN(S): to stdout one of the following values: + # -> true (0) if CVMFS was mounted successfully without any errors + # -> false (1) if CVMFS was not mounted successfully due to errors + # -> false (2) if CVMFS was not mounted because the OS distribution found on the worker node is not rhel-based (only RHEL distros are supported as of now). + + local mode=$1 + # if not previously performed, assess the worker node based on its existing system configurations and perform next steps to mount CVMFS + [[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check + + # if strict requirement of CVMFS mounting is not set to 'never' (i.e. 'required' or 'preferred') + # by this point, it would have been established that CVMFS is not locally available, so install CVMFS via one of the three modes of cvmfsexec + loginfo "Mounting CVMFS on demand using mode $mode of cvmfsexec" + # check the operating system distribution + if [[ "${GWMS_OS_DISTRO}" != "rhel" ]]; then + # if operating system distribution is non-RHEL (any non-rhel OS) + print_os_info + logwarn "This is a non-RHEL OS and is not covered in the implementation yet!" + return 2 + # ----- Further Implementation: TBD (To Be Done) ----- # + fi + + prepare_for_cvmfs_mount + if [[ $? -ne 0 ]]; then + # something went wrong during the prep for mounting + # if CVMFS is required, then abort from this and also the glidein setup by flagging an error + if [[ $use_cvmfs -eq 1 || "${glidein_cvmfs_require}" == "required" ]]; then + logerror "Aborting glidein setup (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" + "$error_gen" -error "$(basename $0)" "WN_Resource" "Error finding cvmfsexec distribution... aborting glidein setup!" + exit 1 + fi + # if CVMFS not required, just warn and continue but without mounting + logwarn "Unable to find an appropriate cvmfsexec distribution to mount CVMFS" + return 1 + fi + if [[ $mode -eq 3 || $mode -eq 2 ]]; then + return # only prepare but do not actually mount (later in glidein reinvocation, mounting will be performed) + fi + loginfo "Mounting CVMFS repositories..." + if ! mount_cvmfs_repos $mode $GLIDEIN_CVMFS_CONFIG_REPO $GLIDEIN_CVMFS_REPOS ; then + return 1 + fi + return 0 } diff --git a/creation/web_base/cvmfs_setup.sh b/creation/web_base/cvmfs_setup.sh old mode 100755 new mode 100644 index 982c7add1..35c89cf02 --- a/creation/web_base/cvmfs_setup.sh +++ b/creation/web_base/cvmfs_setup.sh @@ -3,103 +3,158 @@ # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +is_cvmfs_locally_mounted() { + # checking if CVMFS is natively available + variables_reset + detect_local_cvmfs + if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then + # if it is so... + return 0 + fi + return 1 +} + +################################## main ################################# + # first parameter passed to this script will always be the glidein configuration file (glidein_config) glidein_config=$1 # import add_config_line function -add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | awk '{print $2}') +add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | cut -d ' ' -f 2-) # shellcheck source=./add_config_line.source . "$add_config_line_source" -# fetch the error reporting helper script -error_gen=$(gconfig_get ERROR_GEN_PATH "$glidein_config") - -# get the cvmfsexec attribute switch value from the config file -use_cvmfsexec=$(gconfig_get GLIDEIN_USE_CVMFSEXEC "$glidein_config") -# TODO: int or string?? if string, make the attribute value case insensitive -#use_cvmfsexec=${use_cvmfsexec,,} - -if [[ $use_cvmfsexec -ne 1 ]]; then - "$error_gen" -ok "$(basename $0)" "msg" "Not using cvmfsexec; skipping setup." - exit 0 -fi - -# if GLIDEIN_USE_CVMFSEXEC is set to 1 - check if CVMFS is locally available in the node -# validate CVMFS by examining the directories within CVMFS... checking just one directory should be sufficient? # get the glidein work directory location from glidein_config file -work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$glidein_config") -# $PWD=/tmp/glide_xxx and every path is referenced with respect to $PWD - -# source the helper script -# TODO: Is this file somewhere in the source tree? use: # shellcheck source=./cvmfs_helper_funcs.sh -. $work_dir/cvmfs_helper_funcs.sh - -variables_reset - -detect_local_cvmfs - -# check if CVMFS is already locally mounted... -if [[ $GWMS_IS_CVMFS_MNT -eq 0 ]]; then - # if it is so... - "$error_gen" -ok "$(basename $0)" "msg" "CVMFS is locally mounted on the node; skipping setup." - exit 0 +[[ -e "$glidein_config" ]] && error_gen=$(gconfig_get ERROR_GEN_PATH "$glidein_config") + +[[ -e "$glidein_config" ]] && work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$1") +# shellcheck source=./cvmfs_helper_funcs.sh +. "$work_dir"/cvmfs_helper_funcs.sh + +# get the use_cvmfs attribute value; passed as one of the frontend attributes +use_cvmfs=$(gconfig_get GLIDEIN_USE_CVMFS "$1") +if [[ -z $use_cvmfs ]]; then + loginfo "CVMFS not requested (GLIDEIN_USE_CVMFS not used); skipping CVMFS setup." + "$error_gen" -ok "$(basename $0)" "mnt_msg1" "CVMFS not requested; skipping setup." + return 0 +elif ! [[ $use_cvmfs =~ ^[0-1]$ ]]; then + # TODO: add this check at the xml level maybe? + logerror "Invalid attribute value: GLIDEIN_USE_CVMFS = ${use_cvmfs}" + "$error_gen" -error "$(basename $0)" "mnt_msg2" "Invalid attribute value: GLIDEIN_USE_CVMFS = ${use_cvmfs}" + exit 1 fi -# if CVMFS is not found locally... -# get the CVMFS source information from in the glidein configuration -cvmfs_source=$(gconfig_get CVMFS_SRC "$glidein_config") - -# get the directory where cvmfsexec is unpacked -glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") - -# get the CVMFS requirement setting passed as one of the factory attributes -glidein_cvmfs=$(gconfig_get GLIDEIN_CVMFS "$glidein_config") - -perform_system_check - -# gather the worker node information; perform_system_check sets a few variables that can be helpful here -os_like=$GWMS_OS_DISTRO -os_ver=$(echo $GWMS_OS_VERSION | awk -F'.' '{print $1}') -arch=$GWMS_OS_KRNL_ARCH -# construct the name of the cvmfsexec distribution file based on the worker node specs -dist_file=cvmfsexec-${cvmfs_source}-${os_like}${os_ver}-${arch} -# the appropriate distribution file does not have to manually untarred as the glidein setup takes care of this automatically +# get the CVMFS requirement setting; passed as one of the factory attributes +glidein_cvmfs_require=$(gconfig_get GLIDEIN_CVMFS_REQUIRE "$glidein_config") +glidein_cvmfs_require=${glidein_cvmfs_require,,} +# check whether glidein_cvmfs_require value is valid +# TODO: add this check at the xml level perhaps? +if ! [[ "${glidein_cvmfs_require}" =~ ^(required|preferred|never)$ ]]; then + logerror "Invalid attribute value: GLIDEIN_CVMFS_REQUIRE = ${glidein_cvmfs_require}" + "$error_gen" -error "$(basename $0)" "mnt_msg3" "Invalid attribute value: GLIDEIN_CVMFS_REQUIRE = ${glidein_cvmfs_require}" + exit 1 +fi -if [[ $use_cvmfsexec -eq 1 ]]; then - if [[ ! -d "$glidein_cvmfsexec_dir" && ! -f ${glidein_cvmfsexec_dir}/${dist_file} ]]; then - # neither the cvmfsexec directory nor the cvmfsexec distribution is found -- this happens when a directory named 'cvmfsexec' does not exist on the glidein because an appropriate distribution tarball is not found in the list of all the available tarballs and was not unpacked [trying to unpack osg-rhel8 on osg-rhel7 worker node] - # if use_cvmfsexec is set to 1, then warn that cvmfs will not be mounted and flag an error - logerror "Error occured during cvmfs setup: None of the available cvmfsexec distributions is compatible with the worker node specifications." - "$error_gen" -error "$(basename $0)" "WN_Resource" "Error occured during cvmfs setup... no matching cvmfsexec distribution available." - exit 1 - elif [[ -d "$glidein_cvmfsexec_dir" && ! -f ${glidein_cvmfsexec_dir}/${dist_file} ]]; then - # something might have gone wrong during the unpacking of the tarball into the glidein_cvmfsexec_dir - logerror "Something went wrong during the unpacking of the cvmfsexec distribution tarball!" - "$error_gen" -error "$(basename $0)" "WN_Resource" "Error: Something went wrong during the unpacking of the cvmfsexec distribution tarball" - exit 1 +if [[ $use_cvmfs -ne 1 ]]; then + if ! [[ "${glidein_cvmfs_require}" =~ ^(required|preferred)$ ]]; then + loginfo "GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require; skipping related setup." + "$error_gen" -ok "$(basename $0)" "mnt_msg4" "CVMFS not used. Skipping related setup." + return 0 + fi +else + # use_cvmfs is set to true, then do the following + if [[ "${glidein_cvmfs_require}" == "never" ]]; then + loginfo "CVMFS to be used (GLIDEIN_USE_CVMFS: $use_cvmfs) but GLIDEIN_CVMFS_REQUIRE set to $glidein_cvmfs_require; skipping related setup." + "$error_gen" -ok "$(basename $0)" "mnt_msg5" "CVMFS to be used but GLIDEIN_CVMFS_REQUIRE set to ${glidein_cvmfs_require}" + return 0 fi fi -perform_cvmfs_mount - -if [[ $GWMS_IS_CVMFS -ne 0 ]]; then - # Error occurred during mount of CVMFS repositories" - logerror "Error occured during mount of CVMFS repositories." - "$error_gen" -error "$(basename $0)" "WN_Resource" "Mount unsuccessful... CVMFS is still unavailable on the node." - exit 1 +# following block runs attempting to add CVMFS when either: +# 1. use_cvmfs is false (0) and glidein_cvmfs_require is required|preferred (OR) +# 2. use_cvmfs is true (1) +if is_cvmfs_locally_mounted; then + loginfo "CVMFS found locally; skipping CVMFS setup via cvmfsexec." + loginfo "Continuing to execute the rest of the glidein setup..." + "$error_gen" -ok "$(basename $0)" "mnt_msg6" "CVMFS is natively available on the node; skipping setup using cvmfsexec utilities." + return 0 fi - -# TODO: Verify the findmnt ... will always find the correct CVMFS mount -mount_point=$(findmnt -t fuse -S /dev/fuse | tail -n 1 | cut -d ' ' -f 1 ) -if [[ -n "$mount_point" && "$mount_point" != TARGET* ]]; then - mount_point=$(dirname "$mount_point") - if [[ -n "$mount_point" && "$mount_point" != /cvmfs ]]; then - CVMFS_MOUNT_DIR="$mount_point" - export CVMFS_MOUNT_DIR - gconfig_add CVMFS_MOUNT_DIR "$mount_point" +# if native CVMFS not there, do the following +loginfo "Starting on-demand CVMFS setup..." +# make sure that perform_system_check has run +[[ -z "${GWMS_SYSTEM_CHECK}" ]] && perform_system_check +cvmfsexec_mode=$(setup_cvmfsexec_use) +if ! [[ $cvmfsexec_mode =~ ^[1-3]$ ]]; then + logwarn "cvmfsexec cannot be used in any of the three modes" + if [[ $use_cvmfs -eq 1 || "${glidein_cvmfs_require}" == "required" ]]; then + # when (1) use_cvmfs is true (1), or (2) use_cvmfs is false (0) and glidein_cvmfs_require is set to required + logerror "GLIDEIN_USE_CVMFS set to $use_cvmfs but GLIDEIN_CVMFS_REQUIRE is $glidein_cvmfs_require; aborting glidein setup." + "$error_gen" -error "$(basename $0)" "mnt_msg7" "cvmfsexec cannot be used, therefore aborting (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" + exit 1 fi + # when use_cvmfs is 0 and glidein_cvmfs_require is set to preferred, just warn the user + logwarn "GLIDEIN_USE_CVMFS set to $use_cvmfs and GLIDEIN_CVMFS_REQUIRE is $glidein_cvmfs_require; continuing glidein setup (without CVMFS)..." + "$error_gen" -ok "$(basename $0)" "mnt_msg8" "cvmfsexec cannot be used but still continuing (GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require)" + return 0 fi -# CVMFS is now available on the worker node" -loginfo "Proceeding to execute user job..." -"$error_gen" -ok "$(basename $0)" "WN_Resource" "CVMFS mounted successfully and is now available." +loginfo "GLIDEIN_USE_CVMFS: $use_cvmfs, GLIDEIN_CVMFS_REQUIRE: $glidein_cvmfs_require" +loginfo "cvmfsexec mode $cvmfsexec_mode is being used..." +# the following is run if cvmfsexec cannot be used in mode 3/2 +perform_cvmfs_mount $cvmfsexec_mode $glidein_cvmfs_require +if [[ $? -eq 0 ]]; then + # the following is run if cvmfsexec can be used in mode 3/2 + if [[ $cvmfsexec_mode -eq 3 || $cvmfsexec_mode -eq 2 ]]; then + # before exiting out of this block, do two things... + # one, set a variable indicating this script has been executed once + gwms_cvmfs_reexec="yes" + gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" + + # two, export required variables with some necessary information for use inside cvmfsexec before reinvoking the glidein... + original_workspace=$(gconfig_get GLIDEIN_WORKSPACE_ORIG "$glidein_config") + export GLIDEIN_WORKSPACE=$original_workspace + export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec + export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode + export GWMS_GLIDEIN_WORK_DIR="$work_dir" + export GLIDEIN_CVMFS_CONFIG_REPO="$GLIDEIN_CVMFS_CONFIG_REPO" + export GLIDEIN_CVMFS_REPOS="$GLIDEIN_CVMFS_REPOS" + export PATH=$PATH + echo "Reinvoking glidein now..." + exec "$glidein_cvmfsexec_dir"/"$dist_file" -- "$GWMS_STARTUP_SCRIPT" + echo "!!WARNING!! Outside of reinvocation of glidein_startup" + # the above line of code should not run; but is here as a safety check for debugging incorrect behavior of exec from previous line + fi + # the following is run if cvmfsexec can be used in mode 1 only + # CVMFS is available on the worker node now + gwms_cvmfs_reexec="no" + gconfig_add GWMS_CVMFS_REEXEC "$gwms_cvmfs_reexec" + # exporting the variables as an environment variable for use in glidein reinvocation + export GWMS_CVMFS_REEXEC=$gwms_cvmfs_reexec + export GWMS_CVMFSEXEC_MODE=$cvmfsexec_mode + loginfo "Proceeding to complete the remainder of glidein setup..." + "$error_gen" -ok "$(basename $0)" "mnt_msg10" "CVMFS mounted successfully and is now available." + return 0 +elif [[ $? -eq 1 ]]; then + # if exit status is 1 + if [[ "${glidein_cvmfs_require}" == "required" ]]; then + # if mount CVMFS is not successful, report an error and exit with failure exit code + logerror "Unable to mount CVMFS on worker node; aborting glidein setup (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -error "$(basename $0)" "WN_Resource" "CVMFS required but unable to mount CVMFS on the worker node." + exit 1 + fi + # if cvmfs_required is set to preferred and mount CVMFS is not successful, report a warning/error in the logs and continue with glidein startup + logwarn "Unable to mount CVMFS on worker node; continuing without CVMFS (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -ok "$(basename $0)" "WN_Resource" "CVMFS preferred but could not be mounted. Continuing without CVMFS." + return 0 +else + # if exit status is 2 + if [[ $glidein_cvmfs_require == "required" ]]; then + logerror "Non-RHEL OS found but not supported; aborting glidein setup! (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -error "$(basename $0)" "mnt_msg13" "Non-RHEL OS found but not supported; aborting glidein startup" + exit 1 + fi + # if CVMFS is not required, display operating system information and a user-friendly message + loginfo "Found non-RHEL OS which is not supported; continuing without CVMFS (CVMFS ${glidein_cvmfs_require})" + "$error_gen" -ok "$(basename $0)" "mnt_msg14" "Non-RHEL OS found but not supported; continuing without CVMFS setup" + return 0 +fi diff --git a/creation/web_base/cvmfs_umount.sh b/creation/web_base/cvmfs_umount.sh index a34461d2a..522000077 100755 --- a/creation/web_base/cvmfs_umount.sh +++ b/creation/web_base/cvmfs_umount.sh @@ -19,11 +19,6 @@ # Author: # Namratha Urs # -# Version: -# 1.0 -# - -echo "Unmounting CVMFS as part of glidein cleanup..." glidein_config=$1 @@ -31,65 +26,69 @@ glidein_config=$1 add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | awk '{print $2}') # shellcheck source=./add_config_line.source . "$add_config_line_source" - # import error_gen error_gen=$(gconfig_get ERROR_GEN_PATH "$glidein_config") - # get the cvmfsexec attribute switch value from the config file -use_cvmfsexec=$(gconfig_get GLIDEIN_USE_CVMFSEXEC "$glidein_config") -# TODO: int or string? if string, make the attribute value case insensitive -#use_cvmfsexec=${use_cvmfsexec,,} - -if [[ $use_cvmfsexec -ne 1 ]]; then - "$error_gen" -ok "$(basename $0)" "umnt_msg1" "Not using cvmfsexec; skipping cleanup." - exit 0 -fi - +use_cvmfs=$(gconfig_get GLIDEIN_USE_CVMFS "$glidein_config") # get the glidein work directory location from glidein_config file work_dir=$(gconfig_get GLIDEIN_WORK_DIR "$glidein_config") # $PWD=/tmp/glide_xxx and every path is referenced with respect to $PWD # source the helper script -# TODO: Is this file somewhere in the source tree? use: # shellcheck source=./cvmfs_helper_funcs.sh -. $work_dir/cvmfs_helper_funcs.sh - -# get the cvmfsexec directory location -glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") - -######################################################################################################## -# Start: main program -######################################################################################################## +# shellcheck source=./cvmfs_helper_funcs.sh +. "$work_dir"/cvmfs_helper_funcs.sh -loginfo "..." -loginfo "Start log for unmounting CVMFS" - -# check if CVMFS is locally mounted on the worker node +# first check if CVMFS is locally mounted on the worker node detect_local_cvmfs - -if [[ $GWMS_IS_CVMFS_MNT -eq 0 ]]; then +if [[ $GWMS_IS_CVMFS_LOCAL_MNT -eq 0 ]]; then # CVMFS is mounted locally in the filesystem; DO NOT UNMOUNT! loginfo "Skipping unmounting of CVMFS as it already is locally provisioned in the node!" "$error_gen" -ok "$(basename $0)" "umnt_msg2" "CVMFS is locally mounted on the node; skipping cleanup." exit 0 fi - -loginfo "Unmounting CVMFS mounted by the glidein..." -$glidein_cvmfsexec_dir/.cvmfsexec/umountrepo -a - +# if not, CVMFS was mounted on-demand, so unmount based on the cvmfsexec mode +is_cvmfs_mntd=$(gconfig_get GWMS_IS_CVMFS "$glidein_config") +loginfo "MyDebug (cvmfs_umount.sh): is_cvmfs_mntd = $is_cvmfs_mntd" +if [[ -z "${is_cvmfs_mntd}" || $is_cvmfs_mntd -ne 0 ]]; then + loginfo "CVMFS was not found mounted, skipping CVMFS cleanup..." + exit 0 +fi +# if is_cvmfs_mntd is 0, then do the following +loginfo "Found CVMFS that has been mounted on demand..." +# check which mode was used to mount CVMFS on demand +gwms_cvmfsexec_mode=$(gconfig_get GWMS_CVMFSEXEC_MODE "$glidein_config") +# get the cvmfsexec directory location +glidein_cvmfsexec_dir=$(gconfig_get CVMFSEXEC_DIR "$glidein_config") +loginfo "Unmounting CVMFS provisioned by the glidein..." +mnt_dir=$(gconfig_get CVMFS_MOUNT_DIR "$glidein_config") +[[ -n "$CVMFS_MOUNT_DIR" ]] && loginfo "CVMFS_MOUNT_DIR set to $CVMFS_MOUNT_DIR" +if [[ $gwms_cvmfsexec_mode -eq 1 ]]; then + "$glidein_cvmfsexec_dir"/.cvmfsexec/umountrepo -a +elif [[ $gwms_cvmfsexec_mode -eq 3 || $gwms_cvmfsexec_mode -eq 2 ]]; then + repos=($(echo $GLIDEIN_CVMFS_REPOS | tr ":" "\n")) + loginfo "Unmounting CVMFS repositories..." + # mount every repository that was previously unpacked + for repo in "${repos[@]}" + do + $CVMFSUMOUNT "$repo" + done + loginfo "Unmounting CVMFS config repo now..." + $CVMFSUMOUNT "${GLIDEIN_CVMFS_CONFIG_REPO}" +fi +# clear the mount_dir variable if it was set during the mounting of CVMFS regardless of the mode if [[ -n "$CVMFS_MOUNT_DIR" ]]; then CVMFS_MOUNT_DIR= export CVMFS_MOUNT_DIR gconfig_add CVMFS_MOUNT_DIR "" fi - +cat /proc/$$/mounts | grep /dev/fuse # check again to ensure all CVMFS repositories were unmounted by umountrepo -# searching for "/dev/fuse" since "/cvmfs" returns false positives (/etc/auto.fs /cvmfs line) +# searching for "/dev/fuse" since "/cvmfs" might return false positives (/etc/auto.fs /cvmfs line) cat /proc/$$/mounts | grep /dev/fuse &> /dev/null && logerror "One or more CVMFS repositories might not be completely unmounted" || loginfo "CVMFS repositories unmounted" - -"$error_gen" -ok "$(basename $0)" "umnt_msg3" "Glidein-based CVMFS unmount was successful." +"$error_gen" -ok "$(basename $0)" "umnt_msg1" "Glidein-based CVMFS unmount was successful." # returning 0 to indicate the unmount process was successful -true +exit 0 -######################################################################################################## +############################################################################ # End: main program -######################################################################################################## +############################################################################ diff --git a/creation/web_base/cvmfsexec_platform_select.sh b/creation/web_base/cvmfsexec_platform_select.sh index 06991720f..f602338ed 100755 --- a/creation/web_base/cvmfsexec_platform_select.sh +++ b/creation/web_base/cvmfsexec_platform_select.sh @@ -32,14 +32,16 @@ if [[ ! $cvmfs_src =~ ^(osg|egi|default)$ ]]; then fi # TODO: is it possible to reuse cvmfs_helper_funcs.sh by sourcing it during the execution of this file???? -if [ -f '/etc/redhat-release' ]; then +if [[ -f '/etc/redhat-release' ]]; then os_distro=rhel else os_distro=non-rhel fi - -os_ver=`lsb_release -r | awk -F'\t' '{print $2}' | awk -F"." '{print $1}'` -krnl_arch=`arch` +# using os-release file to get OS-related info +. /etc/os-release +os_ver_full=$VERSION_ID +os_ver=$(echo "$os_ver_full" | awk -F'.' '{print $1}') +krnl_arch=$(arch) mach_type=${os_distro}${os_ver}-${krnl_arch} cvmfsexec_platform="${cvmfs_src}-${mach_type}" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 640a05121..e18396e94 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -8,6 +8,16 @@ # This scripts runs all the others ############################ + +printenv GWMS_CVMFS_REEXEC > /dev/null +status=$? +if [[ "$status" -eq 0 ]]; then + # glidein being reinvoked + gwms_cvmfs_reexec=$(printenv GWMS_CVMFS_REEXEC | sed s"/ //g") +else + # regular glidein invocation + gwms_cvmfs_reexec=$(grep "^GWMS_CVMFS_REEXEC " "${glidein_config}" | cut -d ' ' -f 2-) +fi # Customizable and initialization variables # Default IFS, to protect against unusual environment, better than "unset IFS" because works with restoring old one @@ -21,6 +31,7 @@ fi global_args="$*" # GWMS_STARTUP_SCRIPT=$0 GWMS_STARTUP_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" +export GWMS_STARTUP_SCRIPT=${GWMS_STARTUP_SCRIPT} # for glidein reinvocation GWMS_PATH="" # Relative to the work directory (GWMS_DIR, gwms_lib_dir, gwms_bin_dir and gwms_exec_dir will be the absolute paths) # bin (utilities), lib (libraries), exec (aux scripts to be executed/sourced, e.g. pre-job) @@ -1174,6 +1185,8 @@ fetch_file_base() { START=$(date +%s) if [[ "${ffb_file_type}" = "exec:s" ]]; then "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" + elif [[ "${ffb_file_type}" = "exec:r" ]]; then + . "${ffb_outname}" glidein_config "${ffb_id}" else "${ffb_outname}" glidein_config "${ffb_id}" fi @@ -1261,7 +1274,6 @@ add_to_path() { export PATH="${tmp_path%:}" } - fixup_condor_dir() { # All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped # However the (not used anymore) gwms create_condor_tarball removes that dir @@ -1317,6 +1329,10 @@ usage() { #################### Execution starts here.... ###################### ##################################################################### +if [[ -z "$gwms_cvmfs_reexec" ]]; then +# TODO: not indenting to limit git conflicts, hold off until GWMS v3.12 +# if GWMS_CVMFS_REEXEC is empty, then this script is being invoked the first time; so proceed with usual glidein setup... + # Variables initialized on top of the file # Command line options parsing. Storing all in global variables. @@ -1423,6 +1439,7 @@ if [ -z "${repository_url}" ]; then usage fi +export GWMS_REPOSITORY_URL=$repository_url # for glidein reinvocation repository_entry_url="${repository_url}/entry_${glidein_entry}" if [ -z "${proxy_url}" ]; then @@ -1485,7 +1502,9 @@ if [ -n "${client_repository_url}" ]; then warn "Missing client descript fname for group." usage fi + export GWMS_CLIENT_REPOSITORY_GROUP_URL=$client_repository_group_url # for glidein reinvocation fi + export GWMS_CLIENT_REPOSITORY_URL=$client_repository_url # for glidein reinvocation fi # Generate glidein UUID @@ -1498,6 +1517,7 @@ fi # Print initial variables values (argumants and environment) startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" +export GWMS_STARTUP_TIME=$startup_time # for glidein reinvocation echo "script_checksum = '$(md5wrapper "$0")'" echo "debug_mode = '${operation_mode}'" @@ -1654,6 +1674,7 @@ else fi fi work_dir_created=1 +export GWMS_GLIDEIN_WORK_DIR=${work_dir} # for glidein reinvocation # GWMS_SUBDIR defined on top GWMS_DIR="${work_dir}/$GWMS_SUBDIR" @@ -1664,10 +1685,12 @@ gwms_lib_dir="${GWMS_DIR}/lib" if ! mkdir -p "$gwms_lib_dir" ; then early_glidein_failure "Cannot create lib dir '$gwms_lib_dir'" fi +export GWMS_LIB_DIR=${gwms_lib_dir} # for glidein reinvocation gwms_bin_dir="${GWMS_DIR}/bin" if ! mkdir -p "$gwms_bin_dir" ; then early_glidein_failure "Cannot create bin dir '$gwms_bin_dir'" fi +export GWMS_BIN_DIR=${gwms_bin_dir} # for glidein reinvocation gwms_exec_dir="${GWMS_DIR}/exec" if ! mkdir -p "$gwms_exec_dir" ; then early_glidein_failure "Cannot create exec dir '$gwms_exec_dir'" @@ -1675,7 +1698,11 @@ else for i in setup prejob postjob cleanup setup_singularity ; do mkdir -p "$gwms_exec_dir"/$i done + export GWMS_EXEC_DIR=${gwms_exec_dir} # for glidein reinvocation fi +# below lines for glidein reinvocation +export GWMS_SUBDIR=${GWMS_SUBDIR} +export GWMS_DIR=${GWMS_DIR} # mktemp makes it user readable by definition (ignores umask) # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed @@ -1715,12 +1742,14 @@ main_dir="${work_dir}/${short_main_dir}" if ! mkdir "${main_dir}"; then early_glidein_failure "Cannot create '${main_dir}'" fi +export GWMS_MAIN_DIR=${main_dir} # for glidein reinvocation short_entry_dir=entry_${glidein_entry} entry_dir="${work_dir}/${short_entry_dir}" if ! mkdir "${entry_dir}"; then early_glidein_failure "Cannot create '${entry_dir}'" fi +export GWMS_ENTRY_DIR=${entry_dir} # for glidein reinvocation if [ -n "${client_repository_url}" ]; then short_client_dir=client @@ -1728,6 +1757,7 @@ if [ -n "${client_repository_url}" ]; then if ! mkdir "$client_dir"; then early_glidein_failure "Cannot create '${client_dir}'" fi + export GWMS_CLIENT_DIR=${client_dir} # for glidein reinvocation if [ -n "${client_repository_group_url}" ]; then short_client_group_dir=client_group_${client_group} @@ -1736,6 +1766,7 @@ if [ -n "${client_repository_url}" ]; then early_glidein_failure "Cannot create '${client_group_dir}'" fi fi + export GWMS_CLIENTGROUP_DIR=${client_group_dir} # for glidein reinvocation fi # Move the token files from condor to glidein workspace @@ -1763,12 +1794,14 @@ extract_all_data wrapper_list="${PWD}/wrapper_list.lst" touch "${wrapper_list}" +export GWMS_WRAPPER_LIST=${wrapper_list} # for glidein reinvocation # create glidein_config glidein_config="${PWD}/glidein_config" if ! echo > "${glidein_config}"; then early_glidein_failure "Could not create '${glidein_config}'" fi +export GWMS_GLIDEIN_CONFIG=${glidein_config} # for glidein reinvocation if ! { echo "# --- glidein_startup vals ---" echo "GLIDEIN_UUID ${glidein_uuid}" @@ -1888,6 +1921,7 @@ done # re-enable for everything else disable_check_signature= +export GWMS_CHECK_SIGNATURE=$disable_check_signature # for glidein reinvocation # Now verify the description was not tampered with # doing it so late should be fine, since nobody should have been able @@ -1921,6 +1955,7 @@ done gs_id_work_dir="$(get_work_dir main)" gs_id_descript_file="$(get_descript_file main)" last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" +export GWMS_LAST_SCRIPT="$last_script" # for glidein reinvocation if [ -z "${last_script}" ]; then warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." glidein_exit 1 @@ -1931,7 +1966,7 @@ cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' ############################## # Fetch all the other files -for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" +for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main precvmfs_file_list" do gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" @@ -2000,6 +2035,117 @@ do [[ -e "${gs_id_work_dir}/setup_prejob.sh" ]] && { cp "${gs_id_work_dir}/setup_prejob.sh" "$gwms_exec_dir"/prejob/ ; chmod a-x "$gwms_exec_dir"/prejob/setup_prejob.sh ; } fi done +fi + +if [[ -n "$gwms_cvmfs_reexec" && "$gwms_cvmfs_reexec" == "yes" ]]; then + # gwms_cvmfs_reexec is not empty; meaning this block is being run inside of cvmfsexec environment + printenv GWMS_GLIDEIN_CONFIG > /dev/null + status=$? + if [[ ${status} -eq 0 ]]; then + work_dir=$(printenv GWMS_GLIDEIN_WORK_DIR | sed "s/ //g") + glidein_config=$(printenv GWMS_GLIDEIN_CONFIG | sed "s/ //g") + repository_url=$(printenv GWMS_REPOSITORY_URL | sed "s/ //g") + main_dir=$(printenv GWMS_MAIN_DIR | sed "s/ //g") + last_script=$(printenv GWMS_LAST_SCRIPT | sed "s/ //g") + check_signature=$(printenv GWMS_CHECK_SIGNATURE | sed "s/ //g") + startup_time=$(printenv GWMS_STARTUP_TIME | sed "s/ //g") + cvmfs_config_repo=$(printenv GLIDEIN_CVMFS_CONFIG_REPO | sed "s/ //g") + cvmfs_add_repos=$(printenv GLIDEIN_CVMFS_REPOS | sed "s/ //g") + gwms_cvmfsexec_mode=$(printenv GWMS_CVMFSEXEC_MODE | sed "s/ //g") + client_repository_url=$(printenv GWMS_CLIENT_REPOSITORY_URL | sed "s/ //g") + client_repository_group_url=$(printenv GWMS_CLIENT_REPOSITORY_GROUP_URL | sed "s/ //g") + wrapper_list=$(printenv GWMS_WRAPPER_LIST | sed "s/ //g") + gwms_exec_dir=$(printenv GWMS_EXEC_DIR | sed "s/ //g") + fi + + # import add_config_line function + add_config_line_source=$(grep -m1 '^ADD_CONFIG_LINE_SOURCE ' "$glidein_config" | cut -d ' ' -f 2-) + # shellcheck source=./add_config_line.source + . "$add_config_line_source" + + # re-sourcing the helper script inside of cvmfsexec environment + . "$work_dir"/cvmfs_helper_funcs.sh + mount_cvmfs_repos $gwms_cvmfsexec_mode $cvmfs_config_repo $cvmfs_add_repos + + # re-source all the scripts as it'd have been done during the first invocation of this script + extract_all_data + + log_setup "${glidein_config}" +fi + +glidein_debug_options=$(gconfig_get GLIDEIN_DEBUG_OPTIONS "$glidein_config") +glidein_debug_output=$(gconfig_get GLIDEIN_DEBUG_OUTPUT "$glidein_config") +export GLIDEIN_DEBUG_OPTIONS=$glidein_debug_options +export GLIDEIN_DEBUG_OUTPUT=$glidein_debug_output + +for gs_file_id in "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" +do + gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" + + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no client_repository + continue + fi + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no client_repository_group + continue + fi + fi + + gs_file_list_id="$(echo "${gs_file_id}" |awk '{print $2}')" + + gs_id_work_dir="$(get_work_dir "${gs_id}")" + gs_id_descript_file="$(get_descript_file "${gs_id}")" + + # extract list file name + if ! gs_file_list_line="$(grep "^${gs_file_list_id} " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_file_list_id:0:11}" = "aftergroup_" ]; then + # afterfile_.. files optional when no client_repository_group + continue + fi + fi + warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 + fi + # space+tab separated file with multiple elements (was: awk '{print $2}', not safe for spaces in file name) + gs_file_list="$(echo "${gs_file_list_line}" | cut -s -f 2 | sed -e 's/[[:space:]]*$//')" + + # fetch list file + fetch_file_regular "${gs_id}" "${gs_file_list}" + # Fetch files contained in list + # TODO: $file is actually a list, so it cannot be double-quoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here + # shellcheck disable=2086 + while read -r file + do + if [ "${file:0:1}" != "#" ]; then + fetch_file "${gs_id}" $file + fi + done < "${gs_id_work_dir}/${gs_file_list}" + + # Files to go into the GWMS_PATH + if [ "$gs_file_id" = "main at_file_list" ]; then + # setup here to make them available for other setup scripts + add_to_path "$gwms_bin_dir" + # all available now: gwms-python was in main,file_list; condor_chirp is in main,at_file_list + for file in "gwms-python" "condor_chirp" + do + cp "${gs_id_work_dir}/$file" "$gwms_bin_dir"/ + done + cp -r "${gs_id_work_dir}/lib"/* "$gwms_lib_dir"/ + cp "${gs_id_work_dir}/gconfig.py" "$gwms_lib_dir"/python/ + elif [ "$gs_file_id" = "main after_file_list" ]; then + # in case some library has been added/updated + rsync -ar "${gs_id_work_dir}/lib"/ "$gwms_lib_dir"/ + # new knowns binaries? add a loop like above: for file in ... + elif [[ "$gs_file_id" = client* ]]; then + # TODO: gwms25073 this is a workaround until there is an official designation for setup script fragments + [[ -e "${gs_id_work_dir}/setup_prejob.sh" ]] && { cp "${gs_id_work_dir}/setup_prejob.sh" "$gwms_exec_dir"/prejob/ ; chmod a-x "$gwms_exec_dir"/prejob/setup_prejob.sh ; } + fi +done fixup_condor_dir diff --git a/creation/web_base/singularity_lib.sh b/creation/web_base/singularity_lib.sh index 30b47a7a4..e6bfac890 100644 --- a/creation/web_base/singularity_lib.sh +++ b/creation/web_base/singularity_lib.sh @@ -1659,6 +1659,14 @@ singularity_get_image() { [[ -z "$singularity_image" ]] && singularity_image=$(dict_get_first SINGULARITY_IMAGES_DICT) fi + if [[ -n "$CVMFS_MOUNT_DIR" ]]; then + # set things up here since the path needs to be bindmounted inside the container + local mount_home=${CVMFS_MOUNT_DIR/\/dist\/cvmfs/} + local symlink_target + symlink_target=$(readlink $mount_home/dist/${singularity_image#/}) + singularity_image=$mount_home/dist/${symlink_target#/} + fi + # At this point, GWMS_SINGULARITY_IMAGE is still empty, something is wrong if [[ -z "$singularity_image" ]]; then [[ -z "$SINGULARITY_IMAGES_DICT" ]] && warn "No Singularity image available (SINGULARITY_IMAGES_DICT is empty)" || diff --git a/doc/factory/custom_vars.html b/doc/factory/custom_vars.html index f0ec911c1..bdd83d493 100644 --- a/doc/factory/custom_vars.html +++ b/doc/factory/custom_vars.html @@ -106,6 +106,7 @@

Custom HTCondor Variables

  • Admin variables
  • Dynamic variables
  • Singularity variables
    • +
  • CVMFS variables
  • Set and discover available CPUs
    • @@ -2715,13 +2716,19 @@

    Singularity Variables (dynamic)

    -

    CVMFS Variables

    +

    On-demand CVMFS Provisioning

    - This section describes the variables that can be configured for use - with the glidein to allow CVMFS to be provisioned on-demand. These - variables are configured at the glidein level so that it applies to - all the entries that are configured in the factory's configuration - file. + GlideinWMS supports provisioning of CVMFS on demand when sites may not + have CVMFS natively. GlideinWMS uses the + cvmfsexec utility to provision CVMFS as needed when CVMFS is + unavailable (more details about cvmfsexec + here). The + following table describes the knobs that can be used to configure + GlideinWMS to allow provisioning of CVMFS when a native installation + of CVMFS is unavailable on a compute element (CE). The configuration + knobs identified for use in factory can be configured such that the + configuration applies to all the defined entries defined, or a + specific entry.

    @@ -2742,18 +2749,17 @@

    CVMFS Variables

    - + - + @@ -2764,53 +2770,85 @@

    CVMFS Variables

    - +
    GLIDEIN_USE_CVMFSEXECGLIDEIN_USE_CVMFS Int FactoryFrontend

    - Not set by default. This variable controls whether - cvmfsexec utility is used to mount CVMFS. Possible values - are 0 or 1. A value of 0 indicates that glidein will not use - cvmfsexec to mount CVMFS on the worker noden. A value of - 1 indicates cvmfsexec will be used to mount CVMFS on the - worker node by the glidein. + Not set by default. This attribute controls whether the user/job + requires CVMFS. Possible values are 0 or 1. A value of 0 + indicates that the job does not need to use CVMFS during its + execution on the compute element. A value of 1 indicates that + the job needs to use CVMFS when it runs on the compute element.

    Factory

    - Not set by default. This variable controls the origin (source) + Not set by default. This attribute controls the origin (source) from which to download the CVMFS configuration repository from. Possible values are:
    - osg - configuration for cvmfs-config-osg;
    - egi - configuration for cvmfs-config-egi;
    - default - configuration for cvmfs-config-default
    + osg - configuration for cvmfs-config-osg
    + egi - configuration for cvmfs-config-egi
    + default - configuration for cvmfs-config-default
    GLIDEIN_CVMFSGLIDEIN_CVMFS_REQUIRE String Factory

    - Not set by default. This variable controls the error handling - behavior of the glidein in case of failures during the mounting - of CVMFS. Possible values are:
    - REQUIRED - if unable to mount, report an error and abort - mounting);
    - PREFERRED - if unable to mount, notify and continue normally;
    - OPTIONAL - continue if unable to mount (similar to - PREFERRED);
    - NEVER - only exit if mounting fails
    + Not set by default. This attribute, in conjunction with + GLIDEIN_USE_CVMFS (defined in frontend), specifies the + site/factory policy to be used when provisioning CVMFS. This + also helps in determining the behavior of the glidein behavior + in case failures are encountered during the provisioning of + CVMFS.

    +

    + This configuration attribute takes one of the following three + values:
    +

    +
      +
    • + REQUIRED - if unable to mount CVMFS on demand, report an error + and abort the glidein setup +
      • +
    • + PREFERRED - if unable to mount CVMFS on demand, report a + warning and continue with the glidein setup normally +
      • +
    • + NEVER - do not even attempt to mount CVMFS on demand even if + the user job requests via GLIDEIN_USE_CVMFS +
      • +
    +

    + Assuming that CVMFS is not found natively, the policy driving + glidein behavior is as follows: +

    +
      +
    • when GLIDEIN_USE_CVMFS is 1:
      • +
        +
      • + GLIDEIN_CVMFS_REQUIRE is NEVER: glidein will not match sites + (and therefore, not provision CVMFS on those sites). +
        • +
      • + GLIDEIN_CVMFS_REQUIRE is REQUIRED|PREFERRED: glidein will + try to provision CVMFS if not found natively. Glidein will + fail when unable to provision CVMFS irrespective of + GLIDEIN_CVMFS_REQUIRE value. +
        • +
      +
    • when GLIDEIN_USE_CVMFS is 0:
      • +
        +
      • + GLIDEIN_CVMFS_REQUIRE is NEVER: does not matter as + GLIEIN_USE_CVMFS says that the job does not want CVMFS. +
        • +
      • + GLIDEIN_CVMFS_REQUIRE is REQUIRED|PREFERRED: glidein will + try to provision CVMFS if not found natively. Glidein will + fail when unable to provision CVMFS only when + GLIDEIN_CVMFS_REQUIRE is REQUIRED. Otherwise, glidein will + continue without CVMFS (when GLIDEIN_CVMFS_REQUIRE is + PREFERRED). +
        • +
      +

    -

    - Dynamic creation and selection of cvmfsexec platform-specific - distribution -

    -

    - GlideinWMS factory includes the capability to mount and unmount CVMFS - on demand, i.e. for sites that do not have a local installation of - CVMFS available. This is achieved via the scripts - cvmfs_setup.sh and cvmfs_umount.sh. Two additional shell - scripts cvmfs_helper_funcs.sh and cvmfs_mount .sh are - used as helper scripts during the mounting process. On demand mounting - and unmounting is achieved via the cvmfsexec utility. - The helper scripts are enabled for conditional download based on the - GLIDEIN_USE_CVMFSEXEC variable. When the GLIDEIN_USE_CVMFSEXEC - variable is set (value of 1), the helper scripts are downloaded for - facilitating the mounting of CVMFS. -

    cvmfsexec can be packaged as self-contained distribution specific to an operating system and platform with required configuration and @@ -2822,21 +2860,30 @@

    distribution for use by the worker node. However, rather than having a tarball containing multiple cvmfsexec distribution files corresponding to a (CVMFS source, system platform, architecture) combination, - GlideinWMS dynamically creates and selects the appropriate cvmfsexec - distribution from the list of distributions. + GlideinWMS allows the dynamic creation and selection of the + appropriate cvmfsexec distribution from the list of distributions as + explained next. +

    +

    + cvmfsexec distributions are tied to a CVMFS source, system platform + and architecture. As needed for provisioning CVMFS, the distributions + can be created as part of the factory reconfiguration and/or upgrade + by using the configuration knob <cvmfsexec_distro> tag in the + factory configuration file. When this knob is enabled, the script + `/bin/create_cvmfsexec_distros.sh` is invoked dynamically. Depending + on the CVMFS source ('sources' attribute for the knob) and the + platform requirements ('platforms' attribute for the knob), the + distributions are packaged as individual tarballs and are added to the + default list of uploads during factory reconfiguration and/or upgrade + when necessary.

    - Specific to a CVMFS source, system platform and architecture, - cvmfsexec distribution files are created automatically by the script - named create_cvmfsexec_distros.sh. The script is placed in - hooks.reconfig.pre directory for dynamic execution. Each of - these distributions are packaged as individual tarballs and are added - to the default list of uploads at the time of factory reconfiguration - and/or upgrade. Another script, cvmfsexec_platform_select.sh, - automatically selects the appropriate distribution tarball based on - the specifics of the worker node and ships the distribution with the - glidein. This script is invoked during the customization of the worker - node as part of the glidein setup. + Once the cvmfsexec distributions are built, the + cvmfsexec_platform_select.sh script then automatically selects + the appropriate distribution tarball based on the specifics of the + worker node and ships the distribution with the glidein. This script + is invoked during the glidein startup process prior to the script that + actually performs all the needed steps towards CVMFS setup.