From 0faab490470f76745db51d4c03abc68c4dd44054 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 6 Feb 2025 18:29:29 +0000 Subject: [PATCH] DAOS-17055 client: add a soft limit of 4k to nr ranges for list-io For dfs_readx/writex and array_read/write operations, add a limit for the number of IODs being passed to DAOS of 16k if the range lengths are under 16 bytes (best effort checking). Features: dfs Signed-off-by: Mohamad Chaarawi --- src/client/api/init.c | 2 ++ src/client/array/dc_array.c | 51 +++++++++++++++++++++++++++++++++ src/include/daos/array.h | 6 ++++ src/include/daos/common.h | 5 ++++ src/include/daos_array.h | 10 +++++++ src/include/daos_fs.h | 7 ++++- src/tests/suite/dfs_unit_test.c | 35 ++++++++++++++++++---- 7 files changed, 110 insertions(+), 6 deletions(-) diff --git a/src/client/api/init.c b/src/client/api/init.c index 4c6ab9dd186..de45e51b4c5 100644 --- a/src/client/api/init.c +++ b/src/client/api/init.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -270,6 +271,7 @@ daos_init(void) if (rc != 0) D_GOTO(out_obj, rc); #endif + daos_array_env_init(); module_initialized++; D_GOTO(unlock, rc = 0); diff --git a/src/client/array/dc_array.c b/src/client/array/dc_array.c index 06201d267db..f089579b9bf 100644 --- a/src/client/array/dc_array.c +++ b/src/client/array/dc_array.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -69,6 +70,24 @@ struct io_params { char akey_val; }; +unsigned int array_list_io_limit; + +void +daos_array_env_init() +{ + array_list_io_limit = DAOS_ARRAY_LIST_IO_LIMIT; + d_getenv_uint("DAOS_ARRAY_LIST_IO_LIMIT", &array_list_io_limit); + if (array_list_io_limit == 0) { + array_list_io_limit = UINT_MAX; + } + if (array_list_io_limit > DAOS_ARRAY_LIST_IO_LIMIT) { + D_WARN("Setting a high limit for list io descriptors (%u) is not recommended\n", + array_list_io_limit); + } else { + D_DEBUG(DB_TRACE, "ARRAY List IO limit = %u\n", array_list_io_limit); + } +} + static void array_free(struct d_hlink *hlink) { @@ -1436,6 +1455,38 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th, D_GOTO(err_task, rc = -DER_INVAL); } + /* + * If we are above the limit, check for small recx size. Just a best effort check for + * extreme cases to reject. + */ + if (rg_iod->arr_nr > array_list_io_limit) { + daos_size_t i; + daos_size_t tiny_count = 0; + + /* quick shortcut check */ + for (i = 0; i < rg_iod->arr_nr; i = i * 2) { + if (rg_iod->arr_rgs[i].rg_len > DAOS_ARRAY_RG_LEN_THD) + break; + if (i == 0) + i++; + } + + /** Full check if quick check fails */ + if (i >= rg_iod->arr_nr) { + for (i = 0; i < rg_iod->arr_nr; i++) { + if (rg_iod->arr_rgs[i].rg_len <= DAOS_ARRAY_RG_LEN_THD) + tiny_count++; + if (tiny_count > array_list_io_limit) + break; + } + if (tiny_count > array_list_io_limit) { + D_ERROR("List io supports a max of %u offsets (using %zu)", + array_list_io_limit, rg_iod->arr_nr); + D_GOTO(err_task, rc = -DER_NOTSUPPORTED); + } + } + } + array = array_hdl2ptr(array_oh); if (array == NULL) { D_ERROR("Invalid array handle: "DF_RC"\n", DP_RC(-DER_NO_HDL)); diff --git a/src/include/daos/array.h b/src/include/daos/array.h index d52e6c90e3c..a1c4fa4d067 100644 --- a/src/include/daos/array.h +++ b/src/include/daos/array.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2017-2023 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -13,6 +14,11 @@ #include #include +/** limits for list io write/read */ +extern unsigned int array_list_io_limit; +void + daos_array_env_init(); + /* task functions for array operations */ int dc_array_create(tse_task_t *task); int dc_array_open(tse_task_t *task); diff --git a/src/include/daos/common.h b/src/include/daos/common.h index 64ab66c04e4..2225a417c47 100644 --- a/src/include/daos/common.h +++ b/src/include/daos/common.h @@ -1,5 +1,6 @@ /** * (C) Copyright 2015-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -605,6 +606,8 @@ daos_errno2der(int err) case EINVAL: return -DER_INVAL; case ENOTDIR: return -DER_NOTDIR; case EIO: return -DER_IO; + case ENOTSUP: + return -DER_NOTSUPPORTED; case EFAULT: case ENXIO: case ENODEV: @@ -661,6 +664,8 @@ daos_der2errno(int err) case -DER_NOTDIR: return ENOTDIR; case -DER_STALE: return ESTALE; case -DER_TX_RESTART: return ERESTART; + case -DER_NOTSUPPORTED: + return ENOTSUP; default: return EIO; } }; diff --git a/src/include/daos_array.h b/src/include/daos_array.h index 6cc69a4b152..ace0735b3a7 100644 --- a/src/include/daos_array.h +++ b/src/include/daos_array.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2016-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -23,6 +24,11 @@ extern "C" { #include #include +/** limit of arr_nr (list-io entries) for file offsets in a single update */ +#define DAOS_ARRAY_LIST_IO_LIMIT 16384 +/** Tiny recx limit (in bytes) in the array IODs where the list limit is high */ +#define DAOS_ARRAY_RG_LEN_THD 16 + /** Range of contiguous records */ typedef struct { /** Index of the first record in the range */ @@ -260,6 +266,8 @@ daos_array_close(daos_handle_t oh, daos_event_t *ev); * \param[in] oh Array object open handle. * \param[in] th Transaction handle. * \param[in] iod IO descriptor of ranges to read from the array. + * There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if + * the length on the ranges are under DAOS_ARRAY_RG_LEN_THD. * \param[in] sgl A scatter/gather list (sgl) to the store array data. * Buffer sizes do not have to match the individual range * sizes as long as the total size does. User allocates the @@ -286,6 +294,8 @@ daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, * \param[in] oh Array object open handle. * \param[in] th Transaction handle. * \param[in] iod IO descriptor of ranges to write to the array. + * There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if + * the length on the ranges are under DAOS_ARRAY_RG_LEN_THD. * \param[in] sgl A scatter/gather list (sgl) to the store array data. * Buffer sizes do not have to match the individual range * sizes as long as the total size does. diff --git a/src/include/daos_fs.h b/src/include/daos_fs.h index 7c5ac13076d..ec597ad220c 100644 --- a/src/include/daos_fs.h +++ b/src/include/daos_fs.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2018-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -624,6 +625,8 @@ dfs_read(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off, * \param[in] dfs Pointer to the mounted file system. * \param[in] obj Opened file object. * \param[in] iod IO descriptor for list-io. + * There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if + * the length on the ranges are under DAOS_ARRAY_RG_LEN_THD. * \param[in] sgl Scatter/Gather list for data buffer. * \param[out] read_size * How much data is actually read. @@ -657,7 +660,9 @@ dfs_write(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off, * * \param[in] dfs Pointer to the mounted file system. * \param[in] obj Opened file object. - * \param[in] iod IO descriptor of file view. + * \param[in] iod IO descriptor for list-io. + * There is a limit on the number of descriptors (DAOS_ARRAY_LIST_IO_LIMIT) if + * the length on the ranges are under DAOS_ARRAY_RG_LEN_THD. * \param[in] sgl Scatter/Gather list for data buffer. * \param[in] ev Completion event, it is optional and can be NULL. * Function will run in blocking mode if \a ev is NULL. diff --git a/src/tests/suite/dfs_unit_test.c b/src/tests/suite/dfs_unit_test.c index a5feb6c3ca5..2f1388bfc05 100644 --- a/src/tests/suite/dfs_unit_test.c +++ b/src/tests/suite/dfs_unit_test.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2019-2024 Intel Corporation. + * (C) Copyright 2025 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -7,6 +8,7 @@ #include "dfs_test.h" #include +#include #include #include #include @@ -871,29 +873,50 @@ dfs_test_io_error_code(void **state) test_arg_t *arg = *state; dfs_obj_t *file; daos_event_t ev, *evp; - daos_range_t iod_rgs; + daos_range_t iod_rg; + daos_range_t *iod_rgs; dfs_iod_t iod; d_sg_list_t sgl; d_iov_t iov; - char buf[10]; + char *buf; daos_size_t read_size; + int i; int rc; if (arg->myrank != 0) return; + D_ALLOC_ARRAY(iod_rgs, DAOS_ARRAY_LIST_IO_LIMIT + 1); + D_ALLOC_ARRAY(buf, DAOS_ARRAY_LIST_IO_LIMIT + 1); + rc = dfs_open(dfs_mt, NULL, "io_error", S_IFREG | S_IWUSR | S_IRUSR, O_RDWR | O_CREAT, 0, 0, NULL, &file); assert_int_equal(rc, 0); + /** set an IOD with a large nr count that is not supported */ + iod.iod_nr = DAOS_ARRAY_LIST_IO_LIMIT + 1; + for (i = 0; i < DAOS_ARRAY_LIST_IO_LIMIT + 1; i++) { + iod_rgs[i].rg_idx = i + 2; + iod_rgs[i].rg_len = 1; + } + iod.iod_rgs = iod_rgs; + d_iov_set(&iov, buf, DAOS_ARRAY_LIST_IO_LIMIT + 1); + sgl.sg_nr = 1; + sgl.sg_nr_out = 1; + sgl.sg_iovs = &iov; + rc = dfs_writex(dfs_mt, file, &iod, &sgl, NULL); + assert_int_equal(rc, ENOTSUP); + rc = dfs_readx(dfs_mt, file, &iod, &sgl, &read_size, NULL); + assert_int_equal(rc, ENOTSUP); + /* * set an IOD that has writes more data than sgl to trigger error in * array layer. */ iod.iod_nr = 1; - iod_rgs.rg_idx = 0; - iod_rgs.rg_len = 10; - iod.iod_rgs = &iod_rgs; + iod_rg.rg_idx = 0; + iod_rg.rg_len = 10; + iod.iod_rgs = &iod_rg; d_iov_set(&iov, buf, 5); sgl.sg_nr = 1; sgl.sg_nr_out = 1; @@ -942,6 +965,8 @@ dfs_test_io_error_code(void **state) assert_int_equal(rc, 0); rc = dfs_remove(dfs_mt, NULL, "io_error", 0, NULL); assert_int_equal(rc, 0); + D_FREE(buf); + D_FREE(iod_rgs); } int dfs_test_rc[DFS_TEST_MAX_THREAD_NR];