Merge remote-tracking branch 'origin/master' into ckochhof/dev/master…

…/daos-16501

.github/dependabot.yml

@@ -5,13 +5,28 @@ updates:
       - /utils/ansible/ftest/
       - /utils/cq/
       - /
+    ignore:
+      - dependency-name: "avocado-framework*"
     schedule:
       interval: daily
     groups:
       python-packages:
         patterns:
           - "*"
   - package-ecosystem: github-actions
+    target-branch: master
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      gha-versions:
+        patterns:
+          - "*"
+    assignees:
+      - daos-stack/actions-watchers
+
+  - package-ecosystem: github-actions
+    target-branch: release/2.6
     directory: /
     schedule:
       interval: weekly

.github/workflows/linting.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python environment
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38  # v5.4.0
         with:
           python-version: '3'
       - name: Install extra python packages
@@ -86,7 +86,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python environment
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38  # v5.4.0
         with:
           python-version: '3'
       - name: Add parser
@@ -139,7 +139,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38  # v5.4.0
         with:
           python-version: '3.11'
       - name: Install python packages
@@ -194,7 +194,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python environment
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b  # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38  # v5.4.0
         with:
           python-version: '3'
       - name: Install extra python packages

.github/workflows/ossf-scorecard.yml

@@ -71,6 +71,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4  # v3.28.5
+        uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0  # v3.28.9
         with:
           sarif_file: results.sarif

.github/workflows/trivy.yml

@@ -58,7 +58,7 @@ jobs:
           trivy-config: 'utils/trivy/trivy.yaml'
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@f6091c0113d1dcf9b98e269ee48e8a7e51b7bdd4  # v3.28.5
+        uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0  # v3.28.9
         with:
           sarif_file: 'trivy-results.sarif'
 

debian/changelog

@@ -1,8 +1,14 @@
-daos (2.7.101-4) unstable; urgency=medium
+daos (2.7.101-6) unstable; urgency=medium
   [ Cedric Koch-Hofer]
   * Add support of the libasan
 
- -- Cedric Koch-Hofer <[email protected]>  Mon, 20 Jan 2025 14:12:00 -0700
+ -- Cedric Koch-Hofer <[email protected]>  Wed, 12 Feb 2025 14:12:00 -0700
+
+daos (2.7.101-5) unstable; urgency=medium
+  [ Jan Michalski ]
+  * Add ddb_ut and dtx_ut to the server-tests package
+
+ -- Jan Michalski <[email protected]>  Wed, 22 Jan 2025 14:31:00 +0000
 
 daos (2.7.101-3) unstable; urgency=medium
   [ Jeff Olivier ]

debian/daos-server-tests.install

@@ -1,4 +1,5 @@
 usr/bin/dtx_tests
+usr/bin/dtx_ut
 usr/bin/jump_pl_map
 usr/bin/ring_pl_map
 usr/bin/evt_ctl
@@ -9,6 +10,7 @@ usr/bin/bio_ut
 usr/bin/vea_ut
 usr/bin/vos_tests
 usr/bin/ddb_tests
+usr/bin/ddb_ut
 usr/bin/vea_stress
 usr/bin/vos_perf
 usr/bin/obj_ctl

site_scons/prereq_tools/base.py

@@ -132,13 +132,14 @@ class MissingSystemLibs(Exception):
         component    -- component that has missing targets
     """
 
-    def __init__(self, component):
+    def __init__(self, component, prog):
         super().__init__()
         self.component = component
+        self.prog = prog
 
     def __str__(self):
         """Exception string"""
-        return f'{self.component} has unmet dependencies required for build'
+        return f"{self.component} requires {self.prog} for build"
 
 
 class DownloadRequired(Exception):
@@ -654,7 +655,7 @@ def _setup_compiler(self):
                 if self.__check_only:
                     continue
                 config.Finish()
-                raise MissingSystemLibs(prog)
+                raise MissingSystemLibs(compiler, prog)
             args = {name: prog}
             self.__env.Replace(**args)
 
@@ -1480,7 +1481,7 @@ def build(self, env, needed_libs):
         if build_dep:
 
             if self._has_missing_system_deps(self.prereqs.system_env):
-                raise MissingSystemLibs(self.name)
+                raise MissingSystemLibs(self.name, self.required_progs)
 
             self.get()
 

src/cart/README.env

@@ -213,3 +213,7 @@ This file lists the environment variables used in CaRT.
 
  . CRT_CXI_INIT_RETRY
    Retry count for HG_Init_opt2() when initializing the CXI provider (default = 3).
+
+   D_PROGRESS_BUSY
+   Force busy polling when progressing, preventing from sleeping when waiting for
+   new messages.

src/cart/crt_hg.c

@@ -845,7 +845,7 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_
 
 	init_info.na_init_info.auth_key = prov_data->cpg_na_config.noc_auth_key;
 
-	if (crt_provider_is_block_mode(provider))
+	if (crt_provider_is_block_mode(provider) && !prov_data->cpg_progress_busy)
 		init_info.na_init_info.progress_mode = 0;
 	else
 		init_info.na_init_info.progress_mode = NA_NO_BLOCK;

src/cart/crt_init.c

@@ -94,6 +94,8 @@ dump_opt(crt_init_options_t *opt)
 		D_INFO("auth_key is set\n");
 	if (opt->cio_thread_mode_single)
 		D_INFO("thread mode single is set\n");
+	if (opt->cio_progress_busy)
+		D_INFO("progress busy mode is set\n");
 }
 
 static int
@@ -199,6 +201,14 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, bool p
 	prov_data->cpg_max_unexp_size = max_unexpect_size;
 	prov_data->cpg_primary        = primary;
 
+	if (opt && opt->cio_progress_busy) {
+		prov_data->cpg_progress_busy = opt->cio_progress_busy;
+	} else {
+		bool progress_busy = false;
+		crt_env_get(D_PROGRESS_BUSY, &progress_busy);
+		prov_data->cpg_progress_busy = progress_busy;
+	}
+
 	for (i = 0; i < CRT_SRV_CONTEXT_NUM; i++)
 		prov_data->cpg_used_idx[i] = false;
 

src/cart/crt_internal_types.h

@@ -56,38 +56,39 @@ enum crt_traffic_class { CRT_TRAFFIC_CLASSES };
 
 struct crt_prov_gdata {
 	/** NA plugin type */
-	int			cpg_provider;
+	int                  cpg_provider;
 
-	struct crt_na_config	cpg_na_config;
+	struct crt_na_config cpg_na_config;
 	/** Context0 URI */
-	char			cpg_addr[CRT_ADDR_STR_MAX_LEN];
+	char                 cpg_addr[CRT_ADDR_STR_MAX_LEN];
 
 	/** CaRT contexts list */
-	d_list_t		cpg_ctx_list;
+	d_list_t             cpg_ctx_list;
 	/** actual number of items in CaRT contexts list */
-	int			cpg_ctx_num;
+	int                  cpg_ctx_num;
 	/** maximum number of contexts user wants to create */
-	uint32_t		cpg_ctx_max_num;
+	uint32_t             cpg_ctx_max_num;
 
 	/** free-list of indices */
-	bool			cpg_used_idx[CRT_SRV_CONTEXT_NUM];
+	bool                 cpg_used_idx[CRT_SRV_CONTEXT_NUM];
 
 	/** Hints to mercury/ofi for max expected/unexp sizes */
-	uint32_t		cpg_max_exp_size;
-	uint32_t		cpg_max_unexp_size;
+	uint32_t             cpg_max_exp_size;
+	uint32_t             cpg_max_unexp_size;
 
 	/** Number of remote tags */
-	uint32_t		cpg_num_remote_tags;
-	uint32_t		cpg_last_remote_tag;
+	uint32_t             cpg_num_remote_tags;
+	uint32_t             cpg_last_remote_tag;
 
 	/** Set of flags */
-	unsigned int		cpg_sep_mode		: 1,
-				cpg_primary		: 1,
-				cpg_contig_ports	: 1,
-				cpg_inited		: 1;
+	bool                 cpg_sep_mode;
+	bool                 cpg_primary;
+	bool                 cpg_contig_ports;
+	bool                 cpg_inited;
+	bool                 cpg_progress_busy;
 
 	/** Mutext to protect fields above */
-	pthread_mutex_t		cpg_mutex;
+	pthread_mutex_t      cpg_mutex;
 };
 
 #define MAX_NUM_SECONDARY_PROVS 2
@@ -230,6 +231,7 @@ struct crt_event_cb_priv {
 	ENV_STR(D_PORT)                                                                            \
 	ENV(D_PORT_AUTO_ADJUST)                                                                    \
 	ENV(D_THREAD_MODE_SINGLE)                                                                  \
+	ENV(D_PROGRESS_BUSY)                                                                       \
 	ENV(D_POST_INCR)                                                                           \
 	ENV(D_POST_INIT)                                                                           \
 	ENV(D_MRECV_BUF)                                                                           \

src/client/api/init.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -270,6 +271,7 @@ daos_init(void)
 	if (rc != 0)
 		D_GOTO(out_obj, rc);
 #endif
+	daos_array_env_init();
 	module_initialized++;
 	D_GOTO(unlock, rc = 0);
 

src/client/array/dc_array.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -69,6 +70,24 @@ struct io_params {
 	char			akey_val;
 };
 
+unsigned int array_list_io_limit;
+
+void
+daos_array_env_init()
+{
+	array_list_io_limit = DAOS_ARRAY_LIST_IO_LIMIT;
+	d_getenv_uint("DAOS_ARRAY_LIST_IO_LIMIT", &array_list_io_limit);
+	if (array_list_io_limit == 0) {
+		array_list_io_limit = UINT_MAX;
+	}
+	if (array_list_io_limit > DAOS_ARRAY_LIST_IO_LIMIT) {
+		D_WARN("Setting a high limit for list io descriptors (%u) is not recommended\n",
+		       array_list_io_limit);
+	} else {
+		D_DEBUG(DB_TRACE, "ARRAY List IO limit = %u\n", array_list_io_limit);
+	}
+}
+
 static void
 array_free(struct d_hlink *hlink)
 {
@@ -1436,6 +1455,38 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 		D_GOTO(err_task, rc = -DER_INVAL);
 	}
 
+	/*
+	 * If we are above the limit, check for small recx size. Just a best effort check for
+	 * extreme cases to reject.
+	 */
+	if (rg_iod->arr_nr > array_list_io_limit) {
+		daos_size_t i;
+		daos_size_t tiny_count = 0;
+
+		/* quick shortcut check */
+		for (i = 0; i < rg_iod->arr_nr; i = i * 2) {
+			if (rg_iod->arr_rgs[i].rg_len > DAOS_ARRAY_RG_LEN_THD)
+				break;
+			if (i == 0)
+				i++;
+		}
+
+		/** Full check if quick check fails */
+		if (i >= rg_iod->arr_nr) {
+			for (i = 0; i < rg_iod->arr_nr; i++) {
+				if (rg_iod->arr_rgs[i].rg_len <= DAOS_ARRAY_RG_LEN_THD)
+					tiny_count++;
+				if (tiny_count > array_list_io_limit)
+					break;
+			}
+			if (tiny_count > array_list_io_limit) {
+				D_ERROR("List io supports a max of %u offsets (using %zu)",
+					array_list_io_limit, rg_iod->arr_nr);
+				D_GOTO(err_task, rc = -DER_NOTSUPPORTED);
+			}
+		}
+	}
+
 	array = array_hdl2ptr(array_oh);
 	if (array == NULL) {
 		D_ERROR("Invalid array handle: "DF_RC"\n", DP_RC(-DER_NO_HDL));

src/client/dfuse/pil4dfs/int_dfs.c

@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2022-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -6129,14 +6130,18 @@ futimens(int fd, const struct timespec times[2])
 static int
 new_fcntl(int fd, int cmd, ...)
 {
-	int     fd_directed, param, OrgFunc = 1;
+	int     fd_directed, OrgFunc = 1;
 	int     next_dirfd, next_fd, rc;
+	void   *param;
 	va_list arg;
 
 	va_start(arg, cmd);
-	param = va_arg(arg, int);
+	param = va_arg(arg, void *);
 	va_end(arg);
 
+	if (!d_hook_enabled)
+		return libc_fcntl(fd, cmd, param);
+
 	if (fd < FD_FILE_BASE && d_compatible_mode)
 		return libc_fcntl(fd, cmd, param);
 
@@ -6155,9 +6160,6 @@ new_fcntl(int fd, int cmd, ...)
 	case F_ADD_SEALS:
 		fd_directed = d_get_fd_redirected(fd);
 
-		if (!d_hook_enabled)
-			return libc_fcntl(fd, cmd, param);
-
 		if (cmd == F_GETFL) {
 			if (fd_directed >= FD_DIR_BASE)
 				return dir_list[fd_directed - FD_DIR_BASE]->open_flag;
@@ -6206,12 +6208,15 @@ new_fcntl(int fd, int cmd, ...)
 	case F_OFD_GETLK:
 	case F_GETOWN_EX:
 	case F_SETOWN_EX:
-		if (!d_hook_enabled)
+		fd_directed = d_get_fd_redirected(fd);
+		if (fd_directed >= FD_FILE_BASE) {
+			errno = ENOTSUP;
+			return (-1);
+		} else {
 			return libc_fcntl(fd, cmd, param);
-
-		return libc_fcntl(fd, cmd, param);
+		}
 	default:
-		return libc_fcntl(fd, cmd);
+		return libc_fcntl(fd, cmd, param);
 	}
 }