DAOS-16695 test: make useradd and groupadd robust

Make useradd and groupadd robust by using a common uid and gid across
all clients.

Test-tag: DfuseMUPerms HarnessLaunchSetupTest always_passes,vm
Skip-unit-tests: true
Skip-fault-injection-test: true

Signed-off-by: Dalton Bohning <[email protected]>

src/tests/ftest/util/launch_utils.py

@@ -25,7 +25,8 @@
 from util.slurm_utils import create_partition, delete_partition, show_partition
 from util.storage_utils import StorageException, StorageInfo
 from util.systemctl_utils import SystemctlFailure, create_override_config
-from util.user_utils import get_group_id, get_user_groups, groupadd, useradd, userdel
+from util.user_utils import (get_group_id, get_next_uid_gid, get_user_groups, groupadd, useradd,
+                             userdel)
 from util.yaml_utils import YamlUpdater, get_yaml_data
 
 D_TM_SHARED_MEMORY_KEY = 0x10242048
@@ -639,6 +640,15 @@ def _user_setup(self, logger, test, create=False):
         # Keep track of queried groups to avoid redundant work
         group_gid = {}
 
+        # Get the next common UID and GID amongst all clients
+        next_uid, next_gid = None, None
+        if create:
+            try:
+                next_uid, next_gid = get_next_uid_gid(logger, clients)
+            except Exception as error:  # pylint: disable=broad-except
+                self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info())
+                return 128
+
         # Query and optionally create all groups and users
         for _user in users:
             user, *group = _user.split(':')
@@ -647,29 +657,32 @@ def _user_setup(self, logger, test, create=False):
             # Save the group's gid
             if group and group not in group_gid:
                 try:
-                    group_gid[group] = self._query_create_group(logger, clients, group, create)
+                    group_gid[group] = self._query_create_group(
+                        logger, clients, group, create, next_gid)
+                    next_gid += 1
                 except LaunchException as error:
                     self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info())
                     return 128
 
-            gid = group_gid.get(group, None)
             try:
-                self._query_create_user(logger, clients, user, gid, create)
+                self._query_create_user(logger, clients, user, group_gid[group], create, next_uid)
+                next_uid += 1
             except LaunchException as error:
                 self.test_result.fail_test(logger, "Prepare", str(error), sys.exc_info())
                 return 128
 
         return 0
 
     @staticmethod
-    def _query_create_group(logger, hosts, group, create=False):
+    def _query_create_group(logger, hosts, group, create=False, gid=None):
         """Query and optionally create a group on remote hosts.
 
         Args:
             logger (Logger): logger for the messages produced by this method
             hosts (NodeSet): hosts on which to query and create the group
             group (str): group to query and create
             create (bool, optional): whether to create the group if non-existent
+            gid (int, optional): GID for the new group when creating. Default is None
 
         Raises:
             LaunchException: if there is an error querying or creating the group
@@ -689,7 +702,7 @@ def _query_create_group(logger, hosts, group, create=False):
 
         # Create the group
         logger.info('Creating group %s', group)
-        if not groupadd(logger, hosts, group, True).passed:
+        if not groupadd(logger, hosts, group, gid, True).passed:
             raise LaunchException(f'Error creating group {group}')
 
         # Get the group id on each node
@@ -702,7 +715,7 @@ def _query_create_group(logger, hosts, group, create=False):
         raise LaunchException(f'Group not setup correctly: {group}')
 
     @staticmethod
-    def _query_create_user(logger, hosts, user, gid=None, create=False):
+    def _query_create_user(logger, hosts, user, gid=None, create=False, uid=None):
         """Query and optionally create a user on remote hosts.
 
         Args:
@@ -711,6 +724,7 @@ def _query_create_user(logger, hosts, user, gid=None, create=False):
             user (str): user to query and create
             gid (str, optional): user's primary gid. Default is None
             create (bool, optional): whether to create the group if non-existent. Default is False
+            uid (int, optional): GID for the new group when creating. Default is None
 
         Raises:
             LaunchException: if there is an error querying or creating the user
@@ -731,7 +745,7 @@ def _query_create_user(logger, hosts, user, gid=None, create=False):
 
         logger.info('Creating user %s in group %s', user, gid)
         test_env = TestEnvironment()
-        if not useradd(logger, hosts, user, gid, test_env.user_dir).passed:
+        if not useradd(logger, hosts, user, gid, test_env.user_dir, uid).passed:
             raise LaunchException(f'Error creating user {user}')
 
     def _clear_mount_points(self, logger, test, clear_mounts):

src/tests/ftest/util/user_utils.py

@@ -13,6 +13,7 @@
 
 from ClusterShell.NodeSet import NodeSet
 # pylint: disable=import-error,no-name-in-module
+from util.exception_utils import CommandFailure
 from util.run_utils import command_as_user, run_remote
 
 
@@ -92,13 +93,52 @@ def getent(log, hosts, database, key, run_user=None):
     return run_remote(log, hosts, command_as_user(command, run_user))
 
 
-def groupadd(log, hosts, group, force=False, run_user="root"):
+def get_next_uid_gid(log, hosts):
+    """Get the next common UID and GID across some hosts.
+    Args:
+        log (logger): logger for the messages produced by this method
+        hosts (NodeSet): hosts on which to run the command
+    Returns:
+        (int, int): next UID, next GID common across hosts
+    Raises:
+        CommandFailure: if the command fails on one or more hosts
+        ValueError: if the command output is unexpected on one or more hosts
+    """
+    command = '''
+UID_MIN=$(grep -E '^UID_MIN' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+UID_MAX=$(grep -E '^UID_MAX' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+GID_MIN=$(grep -E '^GID_MIN' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+GID_MAX=$(grep -E '^GID_MAX' /etc/login.defs | tr -s ' ' | cut -d ' ' -f 2)
+NEXT_UID=$(cat /etc/passwd | cut -d ":" -f 3 | xargs -n 1 -I % sh -c \
+    "if [[ % -ge $UID_MIN ]] && [[ % -le $UID_MAX ]]; then echo %; fi" \
+        | sort -n | tail -n 1 | awk '{ print $1+1 }')
+NEXT_GID=$(cat /etc/group | cut -d ":" -f 3 | xargs -n 1 -I % sh -c \
+    "if [[ % -ge $GID_MIN ]] && [[ % -le $GID_MAX ]]; then echo %; fi" \
+        | sort -n | tail -n 1 | awk '{ print $1+1 }')
+echo "NEXT_UID=$NEXT_UID"
+echo "NEXT_GID=$NEXT_GID"
+'''
+    result = run_remote(log, hosts, command)
+    if not result.passed:
+        raise CommandFailure(f"Failed to get NEXT_UID and NEXT_GID on {result.failed_hosts}")
+    all_output = "\n".join(result.all_stdout.values())
+    all_uid = re.findall(r'NEXT_UID=([0-9]+)', all_output)
+    all_gid = re.findall(r'NEXT_GID=([0-9]+)', all_output)
+    if len(all_uid) != len(hosts) or len(all_gid) != len(hosts):
+        raise ValueError(f"Failed to get NEXT_UID and NEXT_GID on {hosts}")
+    max_uid = max(map(int, all_uid))
+    max_gid = max(map(int, all_gid))
+    return max_uid, max_gid
+
+
+def groupadd(log, hosts, group, gid=None, force=False, run_user="root"):
     """Run groupadd remotely.
 
     Args:
         log (logger): logger for the messages produced by this method
         hosts (NodeSet): hosts on which to run the command
         group (str): the group to create
+        gid (int, optional): GID for the new group. Defaults to None
         force (bool, optional): whether to use the force option. Default is False
         run_user (str, optional): user to run the command as. Default is root
 
@@ -108,6 +148,7 @@ def groupadd(log, hosts, group, force=False, run_user="root"):
     command = ' '.join(filter(None, [
         'groupadd',
         '-r',
+        f'-g {gid}' if gid else None,
         '-f' if force else None,
         group]))
     return run_remote(log, hosts, command_as_user(command, run_user))
@@ -133,7 +174,7 @@ def groupdel(log, hosts, group, force=False, run_user="root"):
     return run_remote(log, hosts, command_as_user(command, run_user))
 
 
-def useradd(log, hosts, user, group=None, parent_dir=None, run_user="root"):
+def useradd(log, hosts, user, group=None, parent_dir=None, uid=None, run_user="root"):
     """Run useradd remotely.
 
     Args:
@@ -142,6 +183,7 @@ def useradd(log, hosts, user, group=None, parent_dir=None, run_user="root"):
         user (str): user to create
         group (str, optional): user group. Default is None
         parent_dir (str, optional): parent home directory. Default is None
+        uid (int, optional): UID for the new user. Defaults to None
         run_user (str, optional): user to run the command as. Default is root
 
     Returns:
@@ -152,6 +194,7 @@ def useradd(log, hosts, user, group=None, parent_dir=None, run_user="root"):
         '-m',
         f'-g {group}' if group else None,
         f'-d {os.path.join(parent_dir, user)}' if parent_dir else None,
+        f'-u {uid}' if uid else None,
         user]))
     return run_remote(log, hosts, command_as_user(command, run_user))