DAOS-17055 client: add a soft limit of 4k to nr ranges for list-io

For dfs_readx/writex and array_read/write operations, add a limit for the number of IODs being passed to DAOS of 16k if the range lengths are under 16 bytes (best effort checking). Features: dfs Signed-off-by: Mohamad Chaarawi <[email protected]>
daos-stack · Feb 10, 2025 · 0faab49 · 0faab49
1 parent caf5475
commit 0faab49
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 6 deletions.
diff --git a/src/client/api/init.c b/src/client/api/init.c
@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -270,6 +271,7 @@ daos_init(void)
 	if (rc != 0)
 		D_GOTO(out_obj, rc);
 #endif
+	daos_array_env_init();
 	module_initialized++;
 	D_GOTO(unlock, rc = 0);
 

diff --git a/src/client/array/dc_array.c b/src/client/array/dc_array.c
@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -69,6 +70,24 @@ struct io_params {
 	char			akey_val;
 };
 
+unsigned int array_list_io_limit;
+
+void
+daos_array_env_init()
+{
+	array_list_io_limit = DAOS_ARRAY_LIST_IO_LIMIT;
+	d_getenv_uint("DAOS_ARRAY_LIST_IO_LIMIT", &array_list_io_limit);
+	if (array_list_io_limit == 0) {
+		array_list_io_limit = UINT_MAX;
+	}
+	if (array_list_io_limit > DAOS_ARRAY_LIST_IO_LIMIT) {
+		D_WARN("Setting a high limit for list io descriptors (%u) is not recommended\n",
+		       array_list_io_limit);
+	} else {
+		D_DEBUG(DB_TRACE, "ARRAY List IO limit = %u\n", array_list_io_limit);
+	}
+}
+
 static void
 array_free(struct d_hlink *hlink)
 {
@@ -1436,6 +1455,38 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 		D_GOTO(err_task, rc = -DER_INVAL);
 	}
 
+	/*
+	 * If we are above the limit, check for small recx size. Just a best effort check for
+	 * extreme cases to reject.
+	 */
+	if (rg_iod->arr_nr > array_list_io_limit) {
+		daos_size_t i;
+		daos_size_t tiny_count = 0;
+
+		/* quick shortcut check */
+		for (i = 0; i < rg_iod->arr_nr; i = i * 2) {
+			if (rg_iod->arr_rgs[i].rg_len > DAOS_ARRAY_RG_LEN_THD)
+				break;
+			if (i == 0)
+				i++;
+		}
+
+		/** Full check if quick check fails */
+		if (i >= rg_iod->arr_nr) {
+			for (i = 0; i < rg_iod->arr_nr; i++) {
+				if (rg_iod->arr_rgs[i].rg_len <= DAOS_ARRAY_RG_LEN_THD)
+					tiny_count++;
+				if (tiny_count > array_list_io_limit)
+					break;
+			}
+			if (tiny_count > array_list_io_limit) {
+				D_ERROR("List io supports a max of %u offsets (using %zu)",
+					array_list_io_limit, rg_iod->arr_nr);
+				D_GOTO(err_task, rc = -DER_NOTSUPPORTED);
+			}
+		}
+	}
+
 	array = array_hdl2ptr(array_oh);
 	if (array == NULL) {
 		D_ERROR("Invalid array handle: "DF_RC"\n", DP_RC(-DER_NO_HDL));

diff --git a/src/include/daos/array.h b/src/include/daos/array.h
@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2017-2023 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -13,6 +14,11 @@
 #include <daos_types.h>
 #include <daos/tse.h>
 
+/** limits for list io write/read */
+extern unsigned int array_list_io_limit;
+void
+    daos_array_env_init();
+
 /* task functions for array operations */
 int dc_array_create(tse_task_t *task);
 int dc_array_open(tse_task_t *task);

diff --git a/src/include/daos/common.h b/src/include/daos/common.h
@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2015-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -605,6 +606,8 @@ daos_errno2der(int err)
 	case EINVAL:		return -DER_INVAL;
 	case ENOTDIR:		return -DER_NOTDIR;
 	case EIO:		return -DER_IO;
+	case ENOTSUP:
+		return -DER_NOTSUPPORTED;
 	case EFAULT:
 	case ENXIO:
 	case ENODEV:
@@ -661,6 +664,8 @@ daos_der2errno(int err)
 	case -DER_NOTDIR:	return ENOTDIR;
 	case -DER_STALE:	return ESTALE;
 	case -DER_TX_RESTART:	return ERESTART;
+	case -DER_NOTSUPPORTED:
+		return ENOTSUP;
 	default:		return EIO;
 	}
 };

diff --git a/src/include/daos_array.h b/src/include/daos_array.h
@@ -1,5 +1,6 @@
 /*
  * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -23,6 +24,11 @@ extern "C" {
 #include <daos_obj.h>
 #include <daos_obj_class.h>
 
+/** limit of arr_nr (list-io entries) for file offsets in a single update */
+#define DAOS_ARRAY_LIST_IO_LIMIT 16384
+/** Tiny recx limit (in bytes) in the array IODs where the list limit is high */
+#define DAOS_ARRAY_RG_LEN_THD    16
+
 /** Range of contiguous records */
 typedef struct {
 	/** Index of the first record in the range */
@@ -260,6 +266,8 @@ daos_array_close(daos_handle_t oh, daos_event_t *ev);
  * \param[in]	oh	Array object open handle.
  * \param[in]	th	Transaction handle.
  * \param[in]	iod	IO descriptor of ranges to read from the array.
+ *			There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if
+ *			the length on the ranges are under DAOS_ARRAY_RG_LEN_THD.
  * \param[in]	sgl	A scatter/gather list (sgl) to the store array data.
  *			Buffer sizes do not have to match the individual range
  *			sizes as long as the total size does. User allocates the
@@ -286,6 +294,8 @@ daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod,
  * \param[in]	oh	Array object open handle.
  * \param[in]	th	Transaction handle.
  * \param[in]	iod	IO descriptor of ranges to write to the array.
+ *			There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if
+ *			the length on the ranges are under DAOS_ARRAY_RG_LEN_THD.
  * \param[in]	sgl	A scatter/gather list (sgl) to the store array data.
  *			Buffer sizes do not have to match the individual range
  *			sizes as long as the total size does.

diff --git a/src/include/daos_fs.h b/src/include/daos_fs.h
@@ -1,5 +1,6 @@
 /*
  * (C) Copyright 2018-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -624,6 +625,8 @@ dfs_read(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off,
  * \param[in]	dfs	Pointer to the mounted file system.
  * \param[in]	obj	Opened file object.
  * \param[in]	iod	IO descriptor for list-io.
+ *			There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if
+ *			the length on the ranges are under DAOS_ARRAY_RG_LEN_THD.
  * \param[in]	sgl	Scatter/Gather list for data buffer.
  * \param[out]	read_size
  *			How much data is actually read.
@@ -657,7 +660,9 @@ dfs_write(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off,
  *
  * \param[in]	dfs	Pointer to the mounted file system.
  * \param[in]	obj	Opened file object.
- * \param[in]	iod	IO descriptor of file view.
+ * \param[in]	iod	IO descriptor for list-io.
+ *			There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if
+ *			the length on the ranges are under DAOS_ARRAY_RG_LEN_THD.
  * \param[in]	sgl	Scatter/Gather list for data buffer.
  * \param[in]	ev	Completion event, it is optional and can be NULL.
  *			Function will run in blocking mode if \a ev is NULL.

diff --git a/src/tests/suite/dfs_unit_test.c b/src/tests/suite/dfs_unit_test.c
@@ -1,12 +1,14 @@
 /**
  * (C) Copyright 2019-2024 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
 #define D_LOGFAC	DD_FAC(tests)
 
 #include "dfs_test.h"
 #include <daos/dfs_lib_int.h>
+#include <daos/array.h>
 #include <daos_types.h>
 #include <daos/placement.h>
 #include <pthread.h>
@@ -871,29 +873,50 @@ dfs_test_io_error_code(void **state)
 	test_arg_t	*arg = *state;
 	dfs_obj_t	*file;
 	daos_event_t	ev, *evp;
-	daos_range_t	iod_rgs;
+	daos_range_t     iod_rg;
+	daos_range_t    *iod_rgs;
 	dfs_iod_t	iod;
 	d_sg_list_t	sgl;
 	d_iov_t		iov;
-	char		buf[10];
+	char            *buf;
 	daos_size_t	read_size;
+	int              i;
 	int		rc;
 
 	if (arg->myrank != 0)
 		return;
 
+	D_ALLOC_ARRAY(iod_rgs, DAOS_ARRAY_LIST_IO_LIMIT + 1);
+	D_ALLOC_ARRAY(buf, DAOS_ARRAY_LIST_IO_LIMIT + 1);
+
 	rc = dfs_open(dfs_mt, NULL, "io_error", S_IFREG | S_IWUSR | S_IRUSR,
 		      O_RDWR | O_CREAT, 0, 0, NULL, &file);
 	assert_int_equal(rc, 0);
 
+	/** set an IOD with a large nr count that is not supported */
+	iod.iod_nr = DAOS_ARRAY_LIST_IO_LIMIT + 1;
+	for (i = 0; i < DAOS_ARRAY_LIST_IO_LIMIT + 1; i++) {
+		iod_rgs[i].rg_idx = i + 2;
+		iod_rgs[i].rg_len = 1;
+	}
+	iod.iod_rgs = iod_rgs;
+	d_iov_set(&iov, buf, DAOS_ARRAY_LIST_IO_LIMIT + 1);
+	sgl.sg_nr     = 1;
+	sgl.sg_nr_out = 1;
+	sgl.sg_iovs   = &iov;
+	rc            = dfs_writex(dfs_mt, file, &iod, &sgl, NULL);
+	assert_int_equal(rc, ENOTSUP);
+	rc = dfs_readx(dfs_mt, file, &iod, &sgl, &read_size, NULL);
+	assert_int_equal(rc, ENOTSUP);
+
 	/*
 	 * set an IOD that has writes more data than sgl to trigger error in
 	 * array layer.
 	 */
 	iod.iod_nr = 1;
-	iod_rgs.rg_idx = 0;
-	iod_rgs.rg_len = 10;
-	iod.iod_rgs = &iod_rgs;
+	iod_rg.rg_idx = 0;
+	iod_rg.rg_len = 10;
+	iod.iod_rgs   = &iod_rg;
 	d_iov_set(&iov, buf, 5);
 	sgl.sg_nr = 1;
 	sgl.sg_nr_out = 1;
@@ -942,6 +965,8 @@ dfs_test_io_error_code(void **state)
 	assert_int_equal(rc, 0);
 	rc = dfs_remove(dfs_mt, NULL, "io_error", 0, NULL);
 	assert_int_equal(rc, 0);
+	D_FREE(buf);
+	D_FREE(iod_rgs);
 }
 
 int dfs_test_rc[DFS_TEST_MAX_THREAD_NR];