Revised changes for mode 3 cvmfsexec support

CHANGELOG.md

@@ -34,6 +34,31 @@ Changes since the last release
 
 ### Known Issues
 
+## v3.11.0 \[2024-03-dd\]
+
+This is the very first version in the development series of GlideinWMS releases.
+
+### New features / functionalities
+
+-   Supports all available modes of cvmfsexec when provisioning CVMFS on demand. (PR #372)
+
+### Changed defaults / behaviours
+
+### Deprecated / removed options and commands
+
+### Security Related Fixes
+
+### Bug Fixes
+
+### Testing / Development
+
+### Known Issues
+
+-   When generating cvmfsexec distribution for EL9 machine type on an EL7 machine, the factory reconfig and/or upgrade fails as a result of an error in `create_cvmfsexec_distros.sh`. This is possibly due to the tools for EL7 being unable to handle EL9 files (as per Dave Dykstra). Please exercise caution if using `rhel9-x86_64` in the `mtypes` list for the `cvmfsexec_distro` tag in factory configuration.
+    -   Our suggested workaround is to remove the EL9 machine type from the default list of machine types supported by the custom distros creation script. Add it back if you are running on an EL9 system and want an EL9 cvmfsexec distribution. (PR #312)
+-   When using on-demand CVMFS, all Glideins after the first one on a node are failing (Issue #287)
+    This happens because mountrepo and umountrepo work at the node level and subsequent Glideins see the mounts done by the first one and abort. To avoid problems use only whole-node Glideins when using on-demand CVMFS. All versions with on-demand CVMFS are affected.
+
 ## v3.10.6 \[2024-01-25\]
 
 Minor new features, mostly a bug fix release

creation/lib/cgWConsts.py

@@ -9,8 +9,9 @@
 from . import cWConsts
 
 # these are in the stage dir, so they need to be renamed if changed
-AFTER_FILE_LISTFILE = "after_%s" % cWConsts.FILE_LISTFILE
-AT_FILE_LISTFILE = "at_%s" % cWConsts.FILE_LISTFILE
+AFTER_FILE_LISTFILE = f"after_{cWConsts.FILE_LISTFILE}"
+AT_FILE_LISTFILE = f"at_{cWConsts.FILE_LISTFILE}"
+PRECVMFS_FILE_LISTFILE = f"precvmfs_{cWConsts.FILE_LISTFILE}"
 
 CONDOR_FILE = "condor_bin_%s.tgz"
 CONDOR_DIR = "condor"
@@ -23,6 +24,8 @@
 CVMFSEXEC_DIR = "cvmfsexec"
 CVMFSEXEC_ATTR = "CVMFSEXEC_DIR"
 
+# constant defining the priorities of file lists used by the factory/glidein
+FILE_LISTS_PRIORITIES = ("file_list", "precvmfs_file_list", "at_file_list", "after_file_list")
 
 # these are in the submit dir, so they can be changed
 SUBMIT_ATTRS_FILE = "submit_attrs.cfg"

creation/lib/cgWDictFile.py

@@ -290,6 +290,9 @@ def get_main_dicts(submit_dir, stage_dir):
     main_dicts["gridmap"] = cWDictFile.GridMapDict(
         stage_dir, cWConsts.insert_timestr(cWConsts.GRIDMAP_FILE)
     )  # TODO: versioned but no fname_idx?
+    main_dicts["precvmfs_file_list"] = cWDictFile.FileDictFile(
+        stage_dir, cWConsts.insert_timestr(cgWConsts.PRECVMFS_FILE_LISTFILE), fname_idx=cgWConsts.PRECVMFS_FILE_LISTFILE
+    )
     main_dicts["at_file_list"] = cWDictFile.FileDictFile(
         stage_dir, cWConsts.insert_timestr(cgWConsts.AT_FILE_LISTFILE), fname_idx=cgWConsts.AT_FILE_LISTFILE
     )
@@ -370,10 +373,15 @@ def load_main_dicts(main_dicts):  # update in place
     # print "\ndebug %s main_dicts['description'].keys2 = %s" % (__file__, main_dicts['description'].keys2)
     # print "\ndebug %s dir(main_dicts['description']) = %s" % (__file__, dir(main_dicts['description']))
     # TODO: To remove if upgrade from older versions is not a problem
+    try:
+        main_dicts["precvmfs_file_list"].load(fname=main_dicts["description"].vals2["precvmfs_file_list"])
+    except KeyError:
+        # when upgrading from older version the new precvmfs_file_list may not be in the description
+        main_dicts["precvmfs_file_list"].load()
     try:
         main_dicts["at_file_list"].load(fname=main_dicts["description"].vals2["at_file_list"])
     except KeyError:
-        # when upgrading form older version the new at_file_list may not be in the description
+        # when upgrading from older version the new at_file_list may not be in the description
         main_dicts["at_file_list"].load()
     main_dicts["after_file_list"].load(fname=main_dicts["description"].vals2["after_file_list"])
     load_common_dicts(main_dicts, main_dicts["description"])
@@ -409,7 +417,7 @@ def load_entry_dicts(entry_dicts, entry_name, summary_signature):  # update in p
 def refresh_description(dicts):  # update in place
     description_dict = dicts["description"]
     description_dict.add(dicts["signature"].get_fname(), "signature", allow_overwrite=True)
-    for k in ("file_list", "at_file_list", "after_file_list"):
+    for k in cgWConsts.FILE_LISTS_PRIORITIES:
         if k in dicts:
             description_dict.add(dicts[k].get_fname(), k, allow_overwrite=True)
 
@@ -455,11 +463,11 @@ def refresh_file_list(dicts, is_main, files_set_readonly=True, files_reset_chang
 # dictionaries must have been written to disk before using this
 def refresh_signature(dicts):  # update in place
     signature_dict = dicts["signature"]
-    for k in ("consts", "vars", "untar_cfg", "gridmap", "file_list", "at_file_list", "after_file_list", "description"):
+    for k in ("consts", "vars", "untar_cfg", "gridmap") + cgWConsts.FILE_LISTS_PRIORITIES + ("description",):
         if k in dicts:
             signature_dict.add_from_file(dicts[k].get_filepath(), allow_overwrite=True)
     # add signatures of all the files linked in the lists
-    for k in ("file_list", "at_file_list", "after_file_list"):
+    for k in cgWConsts.FILE_LISTS_PRIORITIES:
         if k in dicts:
             filedict = dicts[k]
             for fname in filedict.get_immutable_files():
@@ -493,11 +501,11 @@ def save_common_dicts(dicts, is_main, set_readonly=True):
     # 'consts','untar_cfg','vars' will be loaded
     refresh_file_list(dicts, is_main)
     # save files in the file lists
-    for k in ("file_list", "at_file_list", "after_file_list"):
+    for k in cgWConsts.FILE_LISTS_PRIORITIES:
         if k in dicts:
             dicts[k].save_files(allow_overwrite=True)
     # then save the lists
-    for k in ("file_list", "at_file_list", "after_file_list"):
+    for k in cgWConsts.FILE_LISTS_PRIORITIES:
         if k in dicts:
             dicts[k].save(set_readonly=set_readonly)
     # calc and save the signatures
@@ -592,7 +600,7 @@ def reuse_common_dicts(dicts, other_dicts, is_main, all_reused):
     # since the file names may have changed, refresh the file_list
     refresh_file_list(dicts, is_main)
     # check file-based dictionaries
-    for k in ("file_list", "at_file_list", "after_file_list"):
+    for k in cgWConsts.FILE_LISTS_PRIORITIES:
         if k in dicts:
             all_reused = reuse_file_dict(dicts, other_dicts, k) and all_reused
 

creation/lib/cgWParamDict.py

@@ -10,6 +10,8 @@
 import os.path
 import shutil
 
+from collections import Counter
+
 from glideinwms.lib import pubCrypto, subprocessSupport
 from glideinwms.lib.util import str2bool
 
@@ -219,15 +221,26 @@ def populate(self, other=None):
             self.dicts["params"].add("GLIDEIN_Factory_Collector", str(factory_monitoring_collector))
         populate_gridmap(self.conf, self.dicts["gridmap"])
 
+        # the following list will be a megalist containing all the scripts; used for duplication check logic subsequently
+        all_scripts = list()
         # NOTE that all the files in these _scripts lists are added as executables (i.e. must report with error_gen)
         file_list_scripts = [
             "collector_setup.sh",
             "create_temp_mapfile.sh",
             "gwms-python",
             cgWConsts.CONDOR_STARTUP_FILE,
         ]
+        # add the above list to the megalist created before
+        all_scripts.extend(file_list_scripts)
+
+        # singularity_setup should be performed after cvmfs_setup; condor_chirp's order does not matter
+        precvmfs_file_list_scripts = ["cvmfs_setup.sh"]
+        all_scripts.extend(precvmfs_file_list_scripts)  # add this list to the megalist
+
         # These are right after the entry, before some VO scripts. The order in the following list is important
         at_file_list_scripts = ["singularity_setup.sh", "condor_chirp", "gconfig.py"]
+        all_scripts.extend(at_file_list_scripts)  # adding the above list to the megalist as before
+
         # The order in the following list is important
         after_file_list_scripts = [
             "check_proxy.sh",
@@ -240,15 +253,14 @@ def populate(self, other=None):
             "glidein_sitewms_setup.sh",
             "script_wrapper.sh",
             "smart_partitionable.sh",
-            "cvmfs_setup.sh",
             "cvmfs_umount.sh",
         ]
-        # Only execute scripts once
-        duplicate_scripts = list(set(file_list_scripts).intersection(after_file_list_scripts))
-        duplicate_scripts += list(set(file_list_scripts).intersection(at_file_list_scripts))
-        duplicate_scripts += list(set(at_file_list_scripts).intersection(after_file_list_scripts))
+        all_scripts.extend(after_file_list_scripts)  # adding the above list to the megalist as before
+        # Scripts need to be only executed once, so check for duplicates
+        count_duplicates = Counter(all_scripts)
+        duplicate_scripts = [scr for scr, cnt in count_duplicates.items() if cnt > 1]
         if duplicate_scripts:
-            raise RuntimeError("Duplicates found in the list of files to execute '%s'" % ",".join(duplicate_scripts))
+            raise RuntimeError(f"Duplicates found in the list of files to execute: {', '.join(duplicate_scripts)}")
 
         # Load more system scripts
         for script_name in file_list_scripts:
@@ -291,13 +303,14 @@ def populate(self, other=None):
         )
         self.dicts["untar_cfg"].add(pychirp_tarball, "lib/python/htchirp")
 
-        # Add helper scripts for on-demand cvmfs provisioning, conditional upon the attribute GLIDEIN_USE_CVMFSEXEC
-        # Add cvmfsexec helper script enabled by conditional download
+        # Add helper scripts for on-demand cvmfs provisioning
+        # Add cvmfsexec helper script
         cvmfs_helper = "cvmfs_helper_funcs.sh"
         self.dicts["file_list"].add_from_file(
             cvmfs_helper,
             cWDictFile.FileDictFile.make_val_tuple(
-                cWConsts.insert_timestr(cvmfs_helper), "exec", cond_download="GLIDEIN_USE_CVMFSEXEC"
+                cWConsts.insert_timestr(cvmfs_helper),
+                "exec",
             ),
             os.path.join(cgWConsts.WEB_BASE_DIR, cvmfs_helper),
         )
@@ -306,9 +319,7 @@ def populate(self, other=None):
         dist_select_script = "cvmfsexec_platform_select.sh"
         self.dicts["file_list"].add_from_file(
             dist_select_script,
-            cWDictFile.FileDictFile.make_val_tuple(
-                cWConsts.insert_timestr(dist_select_script), "exec", cond_download="GLIDEIN_USE_CVMFSEXEC"
-            ),
+            cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(dist_select_script), "exec"),
             os.path.join(cgWConsts.WEB_BASE_DIR, dist_select_script),
         )
 
@@ -397,7 +408,7 @@ def populate(self, other=None):
             # TODO: This check could be done in the XML, checking if the entries are consistent in the current version
             # fetch the on-demand cvmfs provisioning feature setting
             # if on-demand CVMFS not used at the global level; ignore and continue
-            ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFSEXEC", 0)
+            ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFS", 0)
             # check if on demand cvmfs provisioning is requested/enabled
             if ondemand_cvmfs != 0:
                 # check the dir containing cvmfsexec distros to see if they were built previously
@@ -410,10 +421,16 @@ def populate(self, other=None):
                 else:
                     # can be overridden at the entry level, so ignore and [entry supersedes global setting]
                     print(
-                        "...cvmfsexec distributions unavailable but on-demand CVMFS requested via GLIDEIN_USE_CVMFSEXEC; Continuing..."
+                        "...cvmfsexec distributions unavailable but on-demand CVMFS requested via GLIDEIN_USE_CVMFS; Continuing..."
                     )
 
         # add additional system scripts
+        for script_name in precvmfs_file_list_scripts:
+            self.dicts["precvmfs_file_list"].add_from_file(
+                script_name,
+                cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(script_name), "exec:r"),
+                os.path.join(cgWConsts.WEB_BASE_DIR, script_name),
+            )
         for script_name in at_file_list_scripts:
             self.dicts["at_file_list"].add_from_file(
                 script_name,
@@ -716,7 +733,7 @@ def populate(self, entry, schedd, main_dicts):
         # TODO: This check could be done in the XML, checking if the entries are consistent in the current version
         # fetch the on-demand cvmfs provisioning feature setting
         # if on-demand CVMFS not used by entry, ignore and continue
-        ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFSEXEC", 0)
+        ondemand_cvmfs = self.dicts["attrs"].get("GLIDEIN_USE_CVMFS", 0)
         if ondemand_cvmfs != 0:
             # check the dir containing cvmfsexec distros to see if they were built previously
             if os.path.exists(os.path.join(self.work_dir, "../cvmfsexec/tarballs")) and os.listdir(
@@ -727,7 +744,7 @@ def populate(self, entry, schedd, main_dicts):
                 print("......RECOMMENDED: Rebuild distributions using the latest version of cvmfsexec.")
             else:
                 print(
-                    "...cvmfsexec distributions unavailable but on-demand CVMFS is requested via GLIDEIN_USE_CVMFSEXEC; Aborting!"
+                    "...cvmfsexec distributions unavailable but on-demand CVMFS is requested via GLIDEIN_USE_CVMFS; Aborting!"
                 )
                 exit(1)