Merge branch 'release/2.4' into mjmac/DAOS-14561-2.4

Change-Id: I589ae28f55c772f15bf10e49c1cda9364bb4ea8d

Jenkinsfile

@@ -1223,7 +1223,11 @@ pipeline {
                             functionalTest(
                                 inst_repos: daosRepos(),
                                 inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
-                                test_function: 'runTestFunctionalV2'))
+                                test_function: 'runTestFunctionalV2',
+                                ftest_arg: getFunctionalArgs(
+                                    pragma_suffix: '-hw-medium-ucx-provider',
+                                    default_nvme: 'auto:-3DNAND',
+                                    provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'))['ftest_arg']))
                     }
                     post {
                         always {

TAG

@@ -1 +1 @@
-2.4.1-rc1
+2.4.1-rc2

ci/provisioning/post_provision_config.sh

@@ -19,6 +19,9 @@ source ci/provisioning/post_provision_config_common_functions.sh
 # shellcheck disable=SC1091
 source ci/junit.sh
 
+
+: "${MLNX_VER_NUM:=latest-5.8}"
+
 : "${DISTRO:=EL_7}"
 DSL_REPO_var="DAOS_STACK_${DISTRO}_LOCAL_REPO"
 DSG_REPO_var="DAOS_STACK_${DISTRO}_GROUP_REPO"
@@ -42,32 +45,35 @@ if ! retry_cmd 2400 clush -B -S -l root -w "$NODESTRING" \
            DAOS_STACK_GROUP_REPO=\"${!DSG_REPO_var:-}\"
            DAOS_STACK_EL_8_APPSTREAM_REPO=\"${!DSA_REPO_var:-}\"
            DISTRO=\"$DISTRO\"
-           DAOS_STACK_RETRY_DELAY_SECONDS=\"${DAOS_STACK_RETRY_DELAY_SECONDS}\"
-           DAOS_STACK_RETRY_COUNT=\"${DAOS_STACK_RETRY_COUNT}\"
-           BUILD_URL=\"${BUILD_URL}\"
-           STAGE_NAME=\"${STAGE_NAME}\"
-           OPERATIONS_EMAIL=\"${OPERATIONS_EMAIL}\"
+           DAOS_STACK_RETRY_DELAY_SECONDS=\"$DAOS_STACK_RETRY_DELAY_SECONDS\"
+           DAOS_STACK_RETRY_COUNT=\"$DAOS_STACK_RETRY_COUNT\"
+           MLNX_VER_NUM=\"$MLNX_VER_NUM\"
+           BUILD_URL=\"$BUILD_URL\"
+           STAGE_NAME=\"$STAGE_NAME\"
+           OPERATIONS_EMAIL=\"$OPERATIONS_EMAIL\"
            COMMIT_MESSAGE=\"$sanitized_commit_message\"
            REPO_FILE_URL=\"$REPO_FILE_URL\"
            ARTIFACTORY_URL=\"${ARTIFACTORY_URL:-}\"
            BRANCH_NAME=\"${BRANCH_NAME:-}\"
            CHANGE_TARGET=\"${CHANGE_TARGET:-}\"
            CI_RPM_TEST_VERSION=\"${CI_RPM_TEST_VERSION:-}\"
            CI_PR_REPOS=\"${CI_PR_REPOS:-}\"
+           REPO_PATH=\"${REPO_PATH:-}\"
+           ARTIFACTS_URL=\"${ARTIFACTS_URL:-}\"
            $(cat ci/stacktrace.sh)
            $(cat ci/junit.sh)
            $(cat ci/provisioning/post_provision_config_common_functions.sh)
            $(cat ci/provisioning/post_provision_config_common.sh)
-           $(cat ci/provisioning/post_provision_config_nodes_"${DISTRO}".sh)
+           $(cat ci/provisioning/post_provision_config_nodes_"$DISTRO".sh)
            $(cat ci/provisioning/post_provision_config_nodes.sh)"; then
     report_junit post_provision_config.sh results.xml "$NODESTRING"
     exit 1
 fi
 
-git log --format=%s -n 1 HEAD | \
+git log --format=%B -n 1 HEAD | sed -ne '1s/^\([A-Z][A-Z]*-[0-9][0-9]*\) .*/\1/p' \
+                                     -e '/^Fixes:/{s/^Fixes: *//;s/ /\
+/g;p}' | \
   retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" \
-                                     "cat >/tmp/commit_title"
+                                     "cat >/tmp/commit_fixes"
 git log --pretty=format:%h --abbrev-commit --abbrev=7 |
   retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" "cat >/tmp/commit_list"
-retry_cmd 600 ssh root@"${NODELIST%%,*}" "mkdir -p /scratch && " \
-                                         "mount wolf-2:/export/scratch /scratch"

ci/provisioning/post_provision_config_common.sh

@@ -24,38 +24,32 @@ if [ -n "$repo_files_pr" ]; then
     REPO_FILE_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-do/job/repo-files/job/$branch/$build_number/artifact/"
 fi
 
-id=$(lsb_release -si)
-release=$(lsb_release -sr)
+. /etc/os-release
 # shellcheck disable=SC2034
 EXCLUDE_UPGRADE=mercury,daos,daos-\*
 if rpm -qa | grep mlnx; then
     # packages not to allow upgrading if MLNX OFED is installed
     EXCLUDE_UPGRADE+=,openmpi,\*mlnx\*,\*ucx\*
 fi
-case "$id" in
-    CentOS|Rocky|AlmaLinux|RedHatEnterpriseServer)
-        if [ "${release%%.*}" = 7 ]; then
-            DISTRO_NAME=centos${release%%.*}
+case "$ID_LIKE" in
+    *rhel*)
+        if [ "$VERSION_ID" = "7" ]; then
+            DISTRO_NAME=centos"$VERSION_ID"
             EXCLUDE_UPGRADE+=,fuse
         else
-            DISTRO_NAME=el${release%%.*}
+            DISTRO_NAME=el${VERSION_ID%%.*}
             EXCLUDE_UPGRADE+=,dpdk\*
         fi
         REPOS_DIR=/etc/yum.repos.d
         DISTRO_GENERIC=el
-        # shellcheck disable=SC2034
-        LSB_RELEASE=redhat-lsb-core
         ;;
-    openSUSE)
+    *suse*)
         # shellcheck disable=SC2034
-        DISTRO_NAME=leap${release%%.*}
+        DISTRO_NAME=leap${VERSION_ID%%.*}
         # shellcheck disable=SC2034
         DISTRO_GENERIC=sl
         # shellcheck disable=SC2034
         REPOS_DIR=/etc/dnf/repos.d
         EXCLUDE_UPGRADE+=,fuse,fuse-libs,fuse-devel
         ;;
 esac
-
-# shellcheck disable=SC2034
-MLNX_VER_NUM=5.8-3.0.7.0

ci/provisioning/post_provision_config_nodes_EL_8.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-#  (C) Copyright 2021-2022 Intel Corporation.
+#  (C) Copyright 2021-2023 Intel Corporation.
 #
 #  SPDX-License-Identifier: BSD-2-Clause-Patent
 
@@ -16,8 +16,14 @@ group_repo_post() {
 
 distro_custom() {
     # install avocado
-    dnf -y install python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} \
-                   clustershell
+    local avocado_rpms=(python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}})
+    if [ -z "$(dnf repoquery "${avocado_rpms[@]}")" ]; then
+        avocado_rpms=()
+        pip install "avocado-framework<83.0"
+        pip install "avocado-framework-plugin-result-html<83.0"
+        pip install "avocado-framework-plugin-varianter-yaml-to-mux<83.0"
+    fi
+    dnf -y install "${avocado_rpms[@]}" clustershell
 
     # for Launchable's pip install
     dnf -y install python3-setuptools.noarch
@@ -47,21 +53,23 @@ install_mofed() {
 
 
     stream=false
-    gversion="$(lsb_release -sr)"
+    gversion="$VERSION_ID"
     if [ "$gversion" == "8" ]; then
-        gversion="8.6"
+        # Mellanox does not have a release for 8.9 yet.
+        gversion="8.8"
         stream=true
      elif [[ $gversion = *.*.* ]]; then
         gversion="${gversion%.*}"
     fi
 
     # Add a repo to install MOFED RPMS
-    repo_url=https://artifactory.dc.hpdd.intel.com/artifactory/mlnx_ofed/"$MLNX_VER_NUM-rhel$gversion"-x86_64/
+    artifactory_base_url="https://artifactory.dc.hpdd.intel.com/artifactory/"
+    mellanox_proxy="${artifactory_base_url}mellanox-proxy/mlnx_ofed/"
+    mellanox_key_url="${artifactory_base_url}mlnx_ofed/RPM-GPG-KEY-Mellanox"
+    rpm --import "$mellanox_key_url"
+    repo_url="$mellanox_proxy$MLNX_VER_NUM/rhel$gversion/x86_64/"
     dnf -y config-manager --add-repo="$repo_url"
-    curl -L -O "$repo_url"RPM-GPG-KEY-Mellanox
     dnf -y config-manager --save --setopt="$(url_to_repo "$repo_url")".gpgcheck=1
-    rpm --import RPM-GPG-KEY-Mellanox
-    rm -f RPM-GPG-KEY-Mellanox
     dnf repolist || true
 
     time dnf -y install mlnx-ofed-basic ucx-cma ucx-ib ucx-knem ucx-rdmacm ucx-xpmem

debian/changelog

@@ -1,3 +1,10 @@
+daos (2.4.1-2) unstable; urgency=medium
+
+  [ Phillip Henderson ]
+  * Second release candidate for 2.4.1
+
+ -- Phillip Henderson <[email protected]>  Fri, 05 Jan 2024 12:51:00 -0500
+
 daos (2.4.1-1) unstable; urgency=medium
 
   [ Phillip Henderson ]

docs/release/release_notes.md

@@ -3,6 +3,40 @@
 We are pleased to announce the release of DAOS version 2.4.
 
 
+## DAOS Version 2.4.1 (2024-01-19)
+
+### Updates in this Release
+
+The DAOS 2.4.1 release contains the following updates on top of DAOS 2.4.0:
+
+* Operating System support for SLES 15.5 and Leap 15.5.
+
+* Operating System support for EL8.8 (RHEL, Rocky Linux, Alma Linux).
+
+* MLNX\_OFED Version 23.04 has been validated on InfiniBand fabrics.
+
+* The [UCX](https://docs.daos.io/v2.4/admin/ucx/) provider support on
+  InfiniBand fabrics has been expanded to include `ucx+ud_x`,
+  which is now the recommended provider for large InfiniBand fabrics.
+
+* The following prerequisite software packages that are included
+  in the DAOS RPM builds have been updated with DAOS 2.4.1:
+
+  - Argobots has been updated to 1.1-3
+  - DPDK has been updated to 21.11.2-2
+  - Libfabric has been updated to 1.19.0-1
+  - Mercury has been updated to 2.3.1-2
+  - Raft has been updated to 0.10.1-2
+  - SPDK has been update to 22.01.2-5
+
+### Bug fixes
+
+The DAOS 2.4.1 release includes fixes for several defects.
+For details, please refer to the Github
+[release/2.4 commit history](https://github.com/daos-stack/daos/commits/release/2.4)
+and the associated [Jira tickets](https://jira.daos.io/) as stated in the commit messages.
+
+
 ## DAOS Version 2.4.0 (2023-09-22)
 
 ### General Support