From cf10b98fcddc30b8278b837cc9c258333512da45 Mon Sep 17 00:00:00 2001
From: Dalton Bohning <dalton.bohning@intel.com>
Date: Thu, 18 Jan 2024 12:24:30 -0800
Subject: [PATCH 01/11] DAOS-14831 test: use scm and nvme for ior/hard_rebuild
 (#13551)

- Use both SCM and NVMe when creating a pool to accomodate MD on SSD.
- Use GX so space is spread across targets.

Signed-off-by: Dalton Bohning <dalton.bohning@intel.com>
---
 src/tests/ftest/ior/hard_rebuild.yaml | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/src/tests/ftest/ior/hard_rebuild.yaml b/src/tests/ftest/ior/hard_rebuild.yaml
index a3a5f5f5444..97137c97b46 100644
--- a/src/tests/ftest/ior/hard_rebuild.yaml
+++ b/src/tests/ftest/ior/hard_rebuild.yaml
@@ -31,34 +31,28 @@ server_config:
       log_file: daos_server1.log
       log_mask: ERR
       storage: auto
-create_pool_max_size:
-  scm: true
-  percentage: 90
 pool:
-  control_method: dmg
+  size: 90%
 container:
   type: POSIX
   control_method: daos
-  properties: dedup:memcmp
 ior:
   api: "DFS"
   client_processes:
     np: 32
-  dfs_destroy: false
   iorflags:
     flags: "-C -k -e -w -g -G 27 -D 120 -Q 1 -vv"
     read_flags: "-C -k -e -r -R -g -G 27 -D 120 -Q 1 -vv"
   test_file: daos:testFile
   segment_count: 2000000
-  repetitions: 1
   chunk_block_transfer_sizes:
     # [ChunkSize, BlocksSize, TransferSize]
     - [47008, 47008, 47008]
   objectclass:
     dfs_oclass_list:
       # - [EC_Object_Class, Minimum number of servers]
-      - ["EC_2P2G1", 6]
-      - ["EC_4P2G1", 8]
-      - ["EC_8P2G1", 12]
+      - ["EC_2P2GX", 6]
+      - ["EC_4P2GX", 8]
+      - ["EC_8P2GX", 12]
   sw_wearout: 1
   sw_status_file: "/var/tmp/daos_testing/stoneWallingStatusFile"

From 91b93c84e8159b275baa42b0a4507229bb3a91a6 Mon Sep 17 00:00:00 2001
From: Liu Xuezhao <xuezhao.liu@intel.com>
Date: Fri, 19 Jan 2024 07:22:11 +0800
Subject: [PATCH 02/11] DAOS-13252 tests: set svcn for multiple_failure test
 (#13619)

original set 1 cannot work if the killed rank happened to be the
only server replica.

Signed-off-by: Xuezhao Liu <xuezhao.liu@intel.com>
---
 src/tests/ftest/erasurecode/multiple_failure.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tests/ftest/erasurecode/multiple_failure.yaml b/src/tests/ftest/erasurecode/multiple_failure.yaml
index dbd63f69bbe..73ceb3bfdc0 100644
--- a/src/tests/ftest/erasurecode/multiple_failure.yaml
+++ b/src/tests/ftest/erasurecode/multiple_failure.yaml
@@ -25,7 +25,7 @@ server_config:
       storage: auto
 pool:
   size: 93%
-  svcn: 1
+  svcn: 5
   control_method: dmg
 container:
   type: POSIX

From 1ed8f1da82d0dabddbfa171da634d4a89ae752e1 Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <94527853+knard-intel@users.noreply.github.com>
Date: Fri, 19 Jan 2024 14:06:46 +0100
Subject: [PATCH 03/11] DAOS-14981 gurt: restore d_getenv_int undefined symbol
 (#13622)

* DAOS-14981 gurt: restore d_getenv_int undefined symbol

Restore missing plain function d_getenv_int() to fix missing symbol with
libdaos.

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 src/gurt/misc.c           | 15 +++++++++++++++
 src/include/gurt/common.h | 10 +++-------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/gurt/misc.c b/src/gurt/misc.c
index ffb1a85bb0d..3b287ca73ff 100644
--- a/src/gurt/misc.c
+++ b/src/gurt/misc.c
@@ -1302,6 +1302,21 @@ d_getenv_uint(const char *name, unsigned *uint_val)
 	return -DER_SUCCESS;
 }
 
+/**
+ * get an unsigned integer type environment variables.
+ *
+ * \param[in]		name		name of the environment variable.
+ * \param[in,out]	uint_val	returned value of the ENV. Will not change the original
+ *					value if ENV is not set or set as a non-integer value.
+ * \return				0 on success, a negative value on error.
+ * \deprecated				d_getenv_int() is deprecated, please use d_getenv_uint().
+ */
+int
+d_getenv_int(const char *name, unsigned *uint_val)
+{
+	return d_getenv_uint(name, uint_val);
+}
+
 /**
  * get a 32bits unsigned integer type environment variables
  *
diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h
index 779a547768b..1cf40fc3292 100644
--- a/src/include/gurt/common.h
+++ b/src/include/gurt/common.h
@@ -587,6 +587,9 @@ d_getenv_bool(const char *name, bool *bool_val);
 int
 d_getenv_char(const char *name, char *char_val);
 int
+d_getenv_int(const char *name, unsigned int *uint_val)
+    __attribute__((deprecated("use d_getenv_uint")));
+int
 d_getenv_uint(const char *name, unsigned int *uint_val);
 int
 d_getenv_uint32_t(const char *name, uint32_t *uint32_val);
@@ -601,13 +604,6 @@ d_unsetenv(const char *name);
 int
 d_clearenv(void);
 
-static inline int
-d_getenv_int(const char *name, unsigned int *uint_val)
-{
-	D_WARN("d_getenv_int() is deprecated, please use d_getenv_uint()");
-	return d_getenv_uint(name, uint_val);
-}
-
 int
 d_write_string_buffer(struct d_string_buffer_t *buf, const char *fmt, ...);
 void

From be4402b995e27a488295e7ab322a1a35ec19be59 Mon Sep 17 00:00:00 2001
From: wangdi <di.wang@intel.com>
Date: Fri, 19 Jan 2024 07:33:30 -0800
Subject: [PATCH 04/11] DAOS-14884 pool: set the pool server handle before
 update (#13618)

Set the pool server handler before IV update, to make sure
IV server checking accurate on the PS leader once step up finish.

Signed-off-by: Di Wang <di.wang@intel.com>
---
 src/pool/srv_internal.h |  4 ---
 src/pool/srv_iv.c       | 57 -----------------------------------------
 src/pool/srv_pool.c     | 10 +++++---
 3 files changed, 7 insertions(+), 64 deletions(-)

diff --git a/src/pool/srv_internal.h b/src/pool/srv_internal.h
index c95f15cc715..f997d299f80 100644
--- a/src/pool/srv_internal.h
+++ b/src/pool/srv_internal.h
@@ -255,10 +255,6 @@ int ds_pool_iv_srv_hdl_invalidate(struct ds_pool *pool);
 int ds_pool_iv_conn_hdl_fetch(struct ds_pool *pool);
 int ds_pool_iv_conn_hdl_invalidate(struct ds_pool *pool, uuid_t hdl_uuid);
 
-int ds_pool_iv_srv_hdl_fetch_non_sys(struct ds_pool *pool,
-				     uuid_t *srv_cont_hdl,
-				     uuid_t *srv_pool_hdl);
-
 /*
  * srv_metrics.c
  */
diff --git a/src/pool/srv_iv.c b/src/pool/srv_iv.c
index 92970ff3d5f..a1969c67bd6 100644
--- a/src/pool/srv_iv.c
+++ b/src/pool/srv_iv.c
@@ -1496,63 +1496,6 @@ ds_pool_iv_srv_hdl_fetch(struct ds_pool *pool, uuid_t *pool_hdl_uuid,
 	return rc;
 }
 
-struct srv_hdl_ult_arg {
-	struct ds_pool	*pool;
-	ABT_eventual	eventual;
-};
-
-static void
-pool_iv_srv_hdl_fetch_ult(void *data)
-{
-	struct srv_hdl_ult_arg *arg = data;
-	int rc;
-
-	rc = ds_pool_iv_srv_hdl_fetch(arg->pool, NULL, NULL);
-
-	ABT_eventual_set(arg->eventual, (void *)&rc, sizeof(rc));
-}
-
-int
-ds_pool_iv_srv_hdl_fetch_non_sys(struct ds_pool *pool, uuid_t *srv_cont_hdl,
-				 uuid_t *srv_pool_hdl)
-{
-	struct srv_hdl_ult_arg	arg;
-	ABT_eventual		eventual;
-	int			*status;
-	int			rc;
-
-	/* Fetch the capability from the leader. To avoid extra locks,
-	 * all metadatas are maintained by xstream 0, so let's create
-	 * an ULT on xstream 0 to let xstream 0 to handle capa fetch
-	 * and update.
-	 */
-	rc = ABT_eventual_create(sizeof(*status), &eventual);
-	if (rc != ABT_SUCCESS)
-		return dss_abterr2der(rc);
-
-	arg.pool = pool;
-	arg.eventual = eventual;
-	rc = dss_ult_create(pool_iv_srv_hdl_fetch_ult, &arg, DSS_XS_SYS,
-			    0, 0, NULL);
-	if (rc)
-		D_GOTO(out_eventual, rc);
-
-	rc = ABT_eventual_wait(eventual, (void **)&status);
-	if (rc != ABT_SUCCESS)
-		D_GOTO(out_eventual, rc = dss_abterr2der(rc));
-	if (*status != 0)
-		D_GOTO(out_eventual, rc = *status);
-
-	if (srv_cont_hdl)
-		uuid_copy(*srv_cont_hdl, pool->sp_srv_cont_hdl);
-	if (srv_pool_hdl)
-		uuid_copy(*srv_pool_hdl, pool->sp_srv_pool_hdl);
-
-out_eventual:
-	ABT_eventual_free(&eventual);
-	return rc;
-}
-
 int
 ds_pool_iv_prop_update(struct ds_pool *pool, daos_prop_t *prop)
 {
diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c
index 6396e6bea59..8f857ea1e12 100644
--- a/src/pool/srv_pool.c
+++ b/src/pool/srv_pool.c
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1810,6 +1810,11 @@ pool_svc_step_up_cb(struct ds_rsvc *rsvc)
 	} else {
 		uuid_generate(pool_hdl_uuid);
 		uuid_generate(cont_hdl_uuid);
+		/* Only copy server handle to make is_from_srv() check correctly, and
+		 * container server handle will not be copied here, otherwise
+		 * ds_pool_iv_refresh_hdl will not open the server container handle.
+		 */
+		uuid_copy(svc->ps_pool->sp_srv_pool_hdl, pool_hdl_uuid);
 	}
 
 	rc = ds_pool_iv_srv_hdl_update(svc->ps_pool, pool_hdl_uuid,
@@ -4296,8 +4301,7 @@ ds_pool_query_handler(crt_rpc_t *rpc, int handler_version)
 	metrics = svc->ps_pool->sp_metrics[DAOS_POOL_MODULE];
 
 	/* See comment above, rebuild doesn't connect the pool */
-	if ((query_bits & DAOS_PO_QUERY_SPACE) &&
-	    !is_pool_from_srv(in->pqi_op.pi_uuid, in->pqi_op.pi_hdl)) {
+	if (query_bits & DAOS_PO_QUERY_SPACE) {
 		rc = pool_space_query_bcast(rpc->cr_ctx, svc, in->pqi_op.pi_hdl,
 					    &out->pqo_space);
 		if (unlikely(rc))

From 1946ef3cd4a09488747fd79d47b32fdcd58ac534 Mon Sep 17 00:00:00 2001
From: wangdi <di.wang@intel.com>
Date: Sun, 21 Jan 2024 08:21:39 -0800
Subject: [PATCH 05/11] DAOS-14969 container: retry IV might cause deadlock
 (#13632)

OID IV entry lock might be required again for retry
case.

Signed-off-by: Di Wang <di.wang@intel.com>
---
 src/cart/crt_iv.c      |  4 ++--
 src/container/oid_iv.c | 10 +++++++++-
 src/engine/server_iv.c |  6 +++---
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/cart/crt_iv.c b/src/cart/crt_iv.c
index af3226facd8..59ad504993f 100644
--- a/src/cart/crt_iv.c
+++ b/src/cart/crt_iv.c
@@ -3508,8 +3508,8 @@ crt_iv_update_internal(crt_iv_namespace_t ivns, uint32_t class_id,
 
 		D_GOTO(exit, rc);
 	} else {
-		DL_CDEBUG(rc == -DER_NONEXIST || rc == -DER_NOTLEADER, DLOG_INFO, DLOG_ERR, rc,
-			  "ivo_on_update failed");
+		DL_CDEBUG(rc == -DER_NONEXIST || rc == -DER_NOTLEADER || rc == -DER_BUSY,
+			  DLOG_INFO, DLOG_ERR, rc, "ivo_on_update failed");
 
 		update_comp_cb(ivns, class_id, iv_key, NULL,
 			       iv_value, rc, cb_arg);
diff --git a/src/container/oid_iv.c b/src/container/oid_iv.c
index d1041184006..f10f5d34f7e 100644
--- a/src/container/oid_iv.c
+++ b/src/container/oid_iv.c
@@ -31,6 +31,7 @@ struct oid_iv_entry {
 	struct oid_iv_range	rg;
 	/** protect the entry */
 	ABT_mutex		lock;
+	void			*current_req;
 };
 
 /** Priv data in the iv layer */
@@ -130,7 +131,14 @@ oid_iv_ent_update(struct ds_iv_entry *ns_entry, struct ds_iv_key *iv_key,
 	D_ASSERT(priv != NULL);
 
 	entry = ns_entry->iv_value.sg_iovs[0].iov_buf;
-	ABT_mutex_lock(entry->lock);
+	rc = ABT_mutex_trylock(entry->lock);
+	/* For retry requests, from _iv_op(), the lock may not be released
+	 * in some cases.
+	 */
+	if (rc == ABT_ERR_MUTEX_LOCKED && entry->current_req != src)
+		return -DER_BUSY;
+
+	entry->current_req = src;
 	avail = &entry->rg;
 
 	oids = src->sg_iovs[0].iov_buf;
diff --git a/src/engine/server_iv.c b/src/engine/server_iv.c
index a7d258705a3..5f5d00722cc 100644
--- a/src/engine/server_iv.c
+++ b/src/engine/server_iv.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2017-2023 Intel Corporation.
+ * (C) Copyright 2017-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1053,7 +1053,7 @@ _iv_op(struct ds_iv_ns *ns, struct ds_iv_key *key, d_sg_list_t *value,
 retry:
 	rc = iv_op_internal(ns, key, value, sync, shortcut, opc);
 	if (retry && !ns->iv_stop &&
-	    (daos_rpc_retryable_rc(rc) || rc == -DER_NOTLEADER)) {
+	    (daos_rpc_retryable_rc(rc) || rc == -DER_NOTLEADER || rc == -DER_BUSY)) {
 		if (rc == -DER_NOTLEADER && key->rank != (d_rank_t)(-1) &&
 		    sync && (sync->ivs_mode == CRT_IV_SYNC_LAZY ||
 			     sync->ivs_mode == CRT_IV_SYNC_EAGER)) {
@@ -1070,7 +1070,7 @@ _iv_op(struct ds_iv_ns *ns, struct ds_iv_key *key, d_sg_list_t *value,
 		 * but in-flight fetch request return IVCB_FORWARD, then queued RPC will
 		 * reply IVCB_FORWARD.
 		 */
-		D_WARN("ns %u retry for class %d opc %d rank %u/%u: " DF_RC "\n", ns->iv_ns_id,
+		D_INFO("ns %u retry for class %d opc %d rank %u/%u: " DF_RC "\n", ns->iv_ns_id,
 		       key->class_id, opc, key->rank, ns->iv_master_rank, DP_RC(rc));
 		/* sleep 1sec and retry */
 		dss_sleep(1000);

From 22687b767a8705213aa765673c2944b32e589119 Mon Sep 17 00:00:00 2001
From: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
Date: Mon, 22 Jan 2024 09:44:34 -0600
Subject: [PATCH 06/11] DAOS-14474 dfs: remove unused --evict option from
 checker (#13616)

The dfs fs check command already allows evicting containers using the
--flags=evict option. the --evict option was added by mistake and does
not do anything today, so remove it.

Signed-off-by: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
---
 src/control/cmd/daos/filesystem.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/control/cmd/daos/filesystem.go b/src/control/cmd/daos/filesystem.go
index e7fcf828a45..8baa1c93535 100644
--- a/src/control/cmd/daos/filesystem.go
+++ b/src/control/cmd/daos/filesystem.go
@@ -291,7 +291,6 @@ type fsCheckCmd struct {
 
 	FsckFlags FsCheckFlag `long:"flags" short:"f" description:"comma-separated flags: print, remove, relink, verify, evict"`
 	DirName   string      `long:"dir-name" short:"n" description:"directory name under lost+found to store leaked oids (a timestamp dir would be created if this is not specified)"`
-	Evict     bool        `long:"evict" short:"e" description:"evict all open handles on the container"`
 }
 
 func (cmd *fsCheckCmd) Execute(_ []string) error {

From 9152ed02ad0726543e9238fb494f8e8ebb9453cd Mon Sep 17 00:00:00 2001
From: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
Date: Mon, 22 Jan 2024 09:45:51 -0600
Subject: [PATCH 07/11] DAOS-14219 dfs: checker should not follow symlinks
 (#13625)

The DFS checker should mark the symlink oid and not dereference the
symlink value. the value can be invalid anyway and if a valid path in
the container, it would be reachable from the hardlink path.

Signed-off-by: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
---
 src/client/dfs/dfs.c            | 2 +-
 src/tests/suite/dfs_unit_test.c | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/client/dfs/dfs.c b/src/client/dfs/dfs.c
index a95acd9d31b..fe0b226cb90 100644
--- a/src/client/dfs/dfs.c
+++ b/src/client/dfs/dfs.c
@@ -6686,7 +6686,7 @@ oit_mark_cb(dfs_t *dfs, dfs_obj_t *parent, const char name[], void *args)
 	}
 
 	/** open the entry name and get the oid */
-	rc = dfs_lookup_rel(dfs, parent, name, O_RDONLY, &obj, NULL, NULL);
+	rc = dfs_lookup_rel(dfs, parent, name, O_RDONLY | O_NOFOLLOW, &obj, NULL, NULL);
 	if (rc) {
 		D_ERROR("dfs_lookup_rel() of %s failed: %d\n", name, rc);
 		return rc;
diff --git a/src/tests/suite/dfs_unit_test.c b/src/tests/suite/dfs_unit_test.c
index 6c2bf8fe1bb..34d49220f58 100644
--- a/src/tests/suite/dfs_unit_test.c
+++ b/src/tests/suite/dfs_unit_test.c
@@ -2503,7 +2503,7 @@ dfs_test_checker(void **state)
 	test_arg_t		*arg = *state;
 	dfs_t			*dfs;
 	int			nr = 100, i;
-	dfs_obj_t		*root, *lf;
+	dfs_obj_t               *root, *lf, *sym;
 	daos_obj_id_t		root_oid;
 	daos_handle_t		root_oh;
 	daos_handle_t		coh;
@@ -2574,6 +2574,12 @@ dfs_test_checker(void **state)
 		assert_int_equal(rc, 0);
 	}
 
+	/** create a symlink with a non-existent target in the container */
+	rc = dfs_open(dfs, NULL, "SL1", S_IFLNK | S_IWUSR | S_IRUSR, O_RDWR | O_CREAT | O_EXCL, 0,
+		      0, "/usr/local", &sym);
+	assert_int_equal(rc, 0);
+	rc = dfs_release(sym);
+
 	rc = dfs_disconnect(dfs);
 	assert_int_equal(rc, 0);
 	/** have to call fini to release the cached container handle for the checker to work */

From 20ae230f19d756cdb219c3ac02d86e1e87e778bb Mon Sep 17 00:00:00 2001
From: Cedric Koch-Hofer <94527853+knard-intel@users.noreply.github.com>
Date: Mon, 22 Jan 2024 16:47:19 +0100
Subject: [PATCH 08/11] DAOS-15036 control: Fix error messages (#13627)

Fix error messages with invlaid number of parameters.

Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
---
 src/control/lib/control/pool.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/control/lib/control/pool.go b/src/control/lib/control/pool.go
index 79b1efaf1e2..24327914275 100644
--- a/src/control/lib/control/pool.go
+++ b/src/control/lib/control/pool.go
@@ -1365,7 +1365,7 @@ func processNVMeSpaceStats(log debugLogger, filterRank filterRankFn, nvmeControl
 		for _, smdDevice := range controller.SmdDevices {
 			if !smdDevice.Roles.IsEmpty() && (smdDevice.Roles.OptionBits&storage.BdevRoleData) == 0 {
 				log.Debugf("Skipping SMD device %s (rank %d, ctrlr %s) not used for storing data",
-					smdDevice.UUID, smdDevice.Rank, controller.PciAddr, smdDevice.Rank)
+					smdDevice.UUID, smdDevice.Rank, controller.PciAddr)
 				continue
 			}
 
@@ -1377,7 +1377,7 @@ func processNVMeSpaceStats(log debugLogger, filterRank filterRankFn, nvmeControl
 
 			if !filterRank(smdDevice.Rank) {
 				log.Debugf("Skipping SMD device %s (rank %d, ctrlr %s) not in ranklist",
-					smdDevice.UUID, smdDevice.Rank, controller.PciAddr, smdDevice.Rank)
+					smdDevice.UUID, smdDevice.Rank, controller.PciAddr)
 				continue
 			}
 

From a2356d889ec09d1fc76d43ecdf5963e12925c97d Mon Sep 17 00:00:00 2001
From: Li Wei <wei.g.li@intel.com>
Date: Tue, 23 Jan 2024 00:48:14 +0900
Subject: [PATCH 09/11] DAOS-14443 rdb: Improve rdb_campaign error (#13180)

The recent unplanned raft update has introduced a new raft error,
RAFT_ERR_MIGHT_VIOLATE_LEASE, which is mapped to the default -DER_MISC.
This patch maps the new raft error to the slightly more meaningful
-DER_NO_PERM, to complete the raft update.

Signed-off-by: Li Wei <wei.g.li@intel.com>
---
 src/rdb/rdb.c      | 2 +-
 src/rdb/rdb_raft.c | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/rdb/rdb.c b/src/rdb/rdb.c
index f5c3eb47629..a1d39c507ef 100644
--- a/src/rdb/rdb.c
+++ b/src/rdb/rdb.c
@@ -684,7 +684,7 @@ rdb_resign(struct rdb *db, uint64_t term)
  *
  * \param[in]	db	database
  *
- * \retval -DER_INVAL	not a voting replica
+ * \retval -DER_NO_PERM	not a voting replica or might violate a lease
  */
 int
 rdb_campaign(struct rdb *db)
diff --git a/src/rdb/rdb_raft.c b/src/rdb/rdb_raft.c
index 8be8dc6ed99..886a873729c 100644
--- a/src/rdb/rdb_raft.c
+++ b/src/rdb/rdb_raft.c
@@ -61,6 +61,7 @@ rdb_raft_rc(int raft_rc)
 	case RAFT_ERR_NOMEM:			return -DER_NOMEM;
 	case RAFT_ERR_SNAPSHOT_ALREADY_LOADED:	return -DER_ALREADY;
 	case RAFT_ERR_INVALID_CFG_CHANGE:	return -DER_INVAL;
+	case RAFT_ERR_MIGHT_VIOLATE_LEASE:	return -DER_NO_PERM;
 	default:				return -DER_MISC;
 	}
 }
@@ -2854,7 +2855,7 @@ rdb_raft_campaign(struct rdb *db)
 	node = raft_get_my_node(db->d_raft);
 	if (node == NULL || !raft_node_is_voting(node)) {
 		D_DEBUG(DB_MD, DF_DB": must be voting node\n", DP_DB(db));
-		rc = -DER_INVAL;
+		rc = -DER_NO_PERM;
 		goto out_mutex;
 	}
 

From 89f9182c5e49898c37412b531d96366ee1567444 Mon Sep 17 00:00:00 2001
From: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
Date: Mon, 22 Jan 2024 10:32:02 -0600
Subject: [PATCH 10/11] DAOS-14476 dfs: update root stbuf on lookup of root
 (#13597)

- In order to avoid stale root stat data, update the root stbuf on lookup.
- add mtime/ctime proper querying for dfs_lookup()
- destroy the uns container after the link is remove since dfuse looks
  up the root if the container of being destroyed.

Signed-off-by: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
---
 src/client/dfs/dfs.c            | 92 ++++++++++++++++++++++++++-------
 src/client/dfs/duns.c           | 16 +++---
 src/tests/suite/dfs_unit_test.c | 32 +++++++++++-
 utils/node_local_test.py        |  1 +
 4 files changed, 112 insertions(+), 29 deletions(-)

diff --git a/src/client/dfs/dfs.c b/src/client/dfs/dfs.c
index fe0b226cb90..93f1c81e2a1 100644
--- a/src/client/dfs/dfs.c
+++ b/src/client/dfs/dfs.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2018-2023 Intel Corporation.
+ * (C) Copyright 2018-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -3416,13 +3416,10 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags,
 lookup_rel_path_loop:
 
 		/*
-		 * Open the directory object one level up.
-		 * Since fetch_entry does not support ".",
-		 * we can't support ".." as the last entry,
-		 * nor can we support "../.." because we don't
-		 * have parent.parent_oid and parent.mode.
-		 * For now, represent this partial state with
-		 * parent_fully_valid.
+		 * Open the directory object one level up.  Since fetch_entry does not support ".",
+		 * we can't support ".." as the last entry, nor can we support "../.." because we
+		 * don't have parent.parent_oid and parent.mode.  For now, represent this partial
+		 * state with parent_fully_valid.
 		 */
 		parent_fully_valid = true;
 		if (strcmp(token, "..") == 0) {
@@ -3508,15 +3505,23 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags,
 			}
 
 			if (stbuf) {
-				daos_size_t size;
+				daos_array_stbuf_t array_stbuf = {0};
 
-				rc = daos_array_get_size(obj->oh, DAOS_TX_NONE, &size, NULL);
+				rc = daos_array_stat(obj->oh, DAOS_TX_NONE, &array_stbuf, NULL);
 				if (rc) {
 					daos_array_close(obj->oh, NULL);
 					D_GOTO(err_obj, rc = daos_der2errno(rc));
 				}
-				stbuf->st_size = size;
+
+				stbuf->st_size   = array_stbuf.st_size;
 				stbuf->st_blocks = (stbuf->st_size + (1 << 9) - 1) >> 9;
+
+				rc = update_stbuf_times(entry, array_stbuf.st_max_epoch, stbuf,
+							NULL);
+				if (rc) {
+					daos_array_close(obj->oh, NULL);
+					D_GOTO(err_obj, rc);
+				}
 			}
 			break;
 		}
@@ -3617,14 +3622,28 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags,
 		}
 
 		obj->d.chunk_size = entry.chunk_size;
-		obj->d.oclass = entry.oclass;
-		if (stbuf)
-			stbuf->st_size = sizeof(entry);
-
+		obj->d.oclass     = entry.oclass;
 		oid_cp(&parent.oid, obj->oid);
 		oid_cp(&parent.parent_oid, obj->parent_oid);
 		parent.oh = obj->oh;
 		parent.mode = entry.mode;
+
+		if (stbuf) {
+			daos_epoch_t ep;
+
+			rc = daos_obj_query_max_epoch(obj->oh, DAOS_TX_NONE, &ep, NULL);
+			if (rc) {
+				daos_obj_close(obj->oh, NULL);
+				D_GOTO(err_obj, rc = daos_der2errno(rc));
+			}
+
+			rc = update_stbuf_times(entry, ep, stbuf, NULL);
+			if (rc) {
+				daos_obj_close(obj->oh, NULL);
+				D_GOTO(err_obj, rc = daos_der2errno(rc));
+			}
+			stbuf->st_size = sizeof(entry);
+		}
 	}
 
 	if (mode)
@@ -3632,16 +3651,49 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags,
 
 	if (stbuf) {
 		if (is_root) {
+			daos_epoch_t ep;
+
+			/** refresh possibly stale root stbuf */
+			rc = fetch_entry(dfs->layout_v, dfs->super_oh, DAOS_TX_NONE, "/", 1, false,
+					 &exists, &entry, 0, NULL, NULL, NULL);
+			if (rc) {
+				D_ERROR("fetch_entry() failed: %d (%s)\n", rc, strerror(rc));
+				D_GOTO(err_obj, rc);
+			}
+
+			if (!exists || !S_ISDIR(entry.mode)) {
+				/** something really bad happened! */
+				D_ERROR("Root object corrupted!");
+				D_GOTO(err_obj, rc = EIO);
+			}
+
+			if (mode)
+				*mode = entry.mode;
+			dfs->root_stbuf.st_mode = entry.mode;
+			dfs->root_stbuf.st_uid  = entry.uid;
+			dfs->root_stbuf.st_gid  = entry.gid;
+
+			rc = daos_obj_query_max_epoch(dfs->root.oh, DAOS_TX_NONE, &ep, NULL);
+			if (rc)
+				D_GOTO(err_obj, rc = daos_der2errno(rc));
+
+			/** object was updated since creation */
+			rc = update_stbuf_times(entry, ep, &dfs->root_stbuf, NULL);
+			if (rc)
+				D_GOTO(err_obj, rc);
+			if (tspec_gt(dfs->root_stbuf.st_ctim, dfs->root_stbuf.st_mtim)) {
+				dfs->root_stbuf.st_atim.tv_sec  = entry.ctime;
+				dfs->root_stbuf.st_atim.tv_nsec = entry.ctime_nano;
+			} else {
+				dfs->root_stbuf.st_atim.tv_sec  = entry.mtime;
+				dfs->root_stbuf.st_atim.tv_nsec = entry.mtime_nano;
+			}
 			memcpy(stbuf, &dfs->root_stbuf, sizeof(struct stat));
 		} else {
 			stbuf->st_nlink = 1;
 			stbuf->st_mode = obj->mode;
 			stbuf->st_uid = entry.uid;
-			stbuf->st_gid = entry.gid;
-			stbuf->st_mtim.tv_sec = entry.mtime;
-			stbuf->st_mtim.tv_nsec = entry.mtime_nano;
-			stbuf->st_ctim.tv_sec = entry.ctime;
-			stbuf->st_ctim.tv_nsec = entry.ctime_nano;
+			stbuf->st_gid   = entry.gid;
 			if (tspec_gt(stbuf->st_ctim, stbuf->st_mtim)) {
 				stbuf->st_atim.tv_sec = entry.ctime;
 				stbuf->st_atim.tv_nsec = entry.ctime_nano;
diff --git a/src/client/dfs/duns.c b/src/client/dfs/duns.c
index fadd2f1f769..9f262857e6f 100644
--- a/src/client/dfs/duns.c
+++ b/src/client/dfs/duns.c
@@ -1331,14 +1331,6 @@ duns_destroy_path(daos_handle_t poh, const char *path)
 		return rc;
 	}
 
-	/** Destroy the container */
-	rc = daos_cont_destroy(poh, dattr.da_cont, 1, NULL);
-	if (rc) {
-		D_ERROR("Failed to destroy container (%d)\n", rc);
-		/** recreate the link ? */
-		return daos_der2errno(rc);
-	}
-
 	if (dattr.da_type == DAOS_PROP_CO_LAYOUT_POSIX) {
 #ifdef LUSTRE_INCLUDE
 		if (dattr.da_on_lustre)
@@ -1369,6 +1361,14 @@ duns_destroy_path(daos_handle_t poh, const char *path)
 		}
 	}
 
+	/** Destroy the container */
+	rc = daos_cont_destroy(poh, dattr.da_cont, 1, NULL);
+	if (rc) {
+		D_ERROR("Failed to destroy container (%d)\n", rc);
+		/** recreate the link ? */
+		return daos_der2errno(rc);
+	}
+
 	return 0;
 }
 
diff --git a/src/tests/suite/dfs_unit_test.c b/src/tests/suite/dfs_unit_test.c
index 34d49220f58..af4a0294314 100644
--- a/src/tests/suite/dfs_unit_test.c
+++ b/src/tests/suite/dfs_unit_test.c
@@ -1415,15 +1415,45 @@ dfs_test_chown(void **state)
 	char		*filename_file2 = "open_stat2";
 	mode_t		create_mode = S_IWUSR | S_IRUSR;
 	int		create_flags = O_RDWR | O_CREAT | O_EXCL;
+	struct timespec  ctime_orig, mtime_orig;
+	mode_t           orig_mode;
 	int		rc;
 
 	if (arg->myrank != 0)
 		return;
 
-	rc = dfs_lookup(dfs_mt, "/", O_RDWR, &dir, NULL, &stbuf);
+	rc = dfs_lookup(dfs_mt, "/", O_RDWR, &dir, &orig_mode, &stbuf);
 	assert_int_equal(rc, 0);
 	assert_int_equal(stbuf.st_uid, geteuid());
 	assert_int_equal(stbuf.st_gid, getegid());
+	mtime_orig.tv_sec  = stbuf.st_mtim.tv_sec;
+	mtime_orig.tv_nsec = stbuf.st_mtim.tv_nsec;
+	ctime_orig.tv_sec  = stbuf.st_ctim.tv_sec;
+	ctime_orig.tv_nsec = stbuf.st_ctim.tv_nsec;
+
+	/** chown of root and see if visible */
+	print_message("Running chown tests on root object...\n");
+	memset(&stbuf, 0, sizeof(stbuf));
+	stbuf.st_uid          = 3;
+	stbuf.st_gid          = 4;
+	stbuf.st_mtim.tv_sec  = mtime_orig.tv_sec + 10;
+	stbuf.st_mtim.tv_nsec = mtime_orig.tv_nsec;
+	stbuf.st_mode         = orig_mode | S_IROTH | S_IWOTH | S_IXOTH;
+	rc                    = dfs_osetattr(dfs_mt, dir, &stbuf,
+					     DFS_SET_ATTR_UID | DFS_SET_ATTR_GID | DFS_SET_ATTR_MTIME |
+						 DFS_SET_ATTR_MODE);
+	assert_int_equal(rc, 0);
+	rc = dfs_release(dir);
+	assert_int_equal(rc, 0);
+
+	memset(&stbuf, 0, sizeof(stbuf));
+	rc = dfs_lookup(dfs_mt, "/", O_RDWR, &dir, NULL, &stbuf);
+	assert_int_equal(rc, 0);
+	assert_int_equal(stbuf.st_mode, orig_mode | S_IROTH | S_IWOTH | S_IXOTH);
+	assert_int_equal(stbuf.st_uid, 3);
+	assert_int_equal(stbuf.st_gid, 4);
+	assert_true(check_ts(ctime_orig, stbuf.st_ctim));
+	assert_int_equal(mtime_orig.tv_sec + 10, stbuf.st_mtim.tv_sec);
 	rc = dfs_release(dir);
 	assert_int_equal(rc, 0);
 
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 61d254ef9b7..97d48821c40 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -2838,6 +2838,7 @@ def test_uns_link(self):
         print(os.listdir(path))
         cmd = ['cont', 'destroy', '--path', path]
         rc = run_daos_cmd(self.conf, cmd)
+        assert rc.returncode == 0
 
         path = join(self.dfuse.dir, 'uns_link2')
         cmd = ['cont', 'link', self.pool.id(), container2.id(), '--path', path]

From 16ecc93b1826f47c79efd7ef88a779319488e998 Mon Sep 17 00:00:00 2001
From: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
Date: Mon, 22 Jan 2024 10:34:04 -0600
Subject: [PATCH 11/11] DAOS-15049 dfs: fix bug in setattr when mtime and size
 are being set (#13640)

- in dfs_osetattr(), if user sets both mtime and size, the hlc for the
  mtime is not being update on storage. To fix that, use the hlc
  reported from the array_stat and update that when updating the mtime in
  the entry.

Signed-off-by: Mohamad Chaarawi <mohamad.chaarawi@intel.com>
---
 src/client/dfs/dfs.c            | 56 +++++++++++++++++++--------------
 src/tests/suite/dfs_unit_test.c | 39 ++++++++++++++++++-----
 2 files changed, 64 insertions(+), 31 deletions(-)

diff --git a/src/client/dfs/dfs.c b/src/client/dfs/dfs.c
index 93f1c81e2a1..e05ed2e50bc 100644
--- a/src/client/dfs/dfs.c
+++ b/src/client/dfs/dfs.c
@@ -5460,11 +5460,12 @@ dfs_osetattr(dfs_t *dfs, dfs_obj_t *obj, struct stat *stbuf, int flags)
 	bool			set_size = false;
 	bool			set_mtime = false;
 	bool			set_ctime = false;
-	int			i = 0;
-	size_t			len;
-	int			rc;
+	int                     i = 0, hlc_recx_idx = 0;
+	size_t                  len;
 	uint64_t		obj_hlc = 0;
 	struct stat		rstat = {};
+	daos_array_stbuf_t      array_stbuf = {0};
+	int                     rc;
 
 	if (dfs == NULL || !dfs->mounted)
 		return EINVAL;
@@ -5561,6 +5562,10 @@ dfs_osetattr(dfs_t *dfs, dfs_obj_t *obj, struct stat *stbuf, int flags)
 		d_iov_set(&sg_iovs[i], &obj_hlc, sizeof(uint64_t));
 		recxs[i].rx_idx = HLC_IDX;
 		recxs[i].rx_nr = sizeof(uint64_t);
+		if (flags & DFS_SET_ATTR_SIZE) {
+			/** we need to update this again after the set size */
+			hlc_recx_idx = i;
+		}
 		i++;
 
 		set_mtime = true;
@@ -5610,38 +5615,41 @@ dfs_osetattr(dfs_t *dfs, dfs_obj_t *obj, struct stat *stbuf, int flags)
 		rstat.st_blocks = (stbuf->st_size + (1 << 9) - 1) >> 9;
 		rstat.st_size = stbuf->st_size;
 
-		/* mtime and ctime need to be updated too only if not set earlier */
-		if (!set_mtime || !set_ctime) {
-			daos_array_stbuf_t	array_stbuf = {0};
+		/**
+		 * if mtime is set, we need to to just update the hlc on the entry. if mtime and/or
+		 * ctime were not set, we need to update the stat buf returned. both cases require
+		 * an array stat for the hlc.
+		 */
+		/** TODO - need an array API to just stat the max epoch without size */
+		rc = daos_array_stat(obj->oh, th, &array_stbuf, NULL);
+		if (rc)
+			D_GOTO(out_obj, rc = daos_der2errno(rc));
 
-			/** TODO - need an array API to just stat the max epoch without size */
-			rc = daos_array_stat(obj->oh, th, &array_stbuf, NULL);
-			if (rc)
+		if (!set_mtime) {
+			rc = d_hlc2timespec(array_stbuf.st_max_epoch, &rstat.st_mtim);
+			if (rc) {
+				D_ERROR("d_hlc2timespec() failed " DF_RC "\n", DP_RC(rc));
 				D_GOTO(out_obj, rc = daos_der2errno(rc));
-
-			if (!set_mtime) {
-				rc = d_hlc2timespec(array_stbuf.st_max_epoch, &rstat.st_mtim);
-				if (rc) {
-					D_ERROR("d_hlc2timespec() failed "DF_RC"\n", DP_RC(rc));
-					D_GOTO(out_obj, rc = daos_der2errno(rc));
-				}
 			}
+		} else {
+			D_ASSERT(hlc_recx_idx > 0);
+			D_ASSERT(recxs[hlc_recx_idx].rx_idx == HLC_IDX);
+			d_iov_set(&sg_iovs[hlc_recx_idx], &array_stbuf.st_max_epoch,
+				  sizeof(uint64_t));
+		}
 
-			if (!set_ctime) {
-				rc = d_hlc2timespec(array_stbuf.st_max_epoch, &rstat.st_ctim);
-				if (rc) {
-					D_ERROR("d_hlc2timespec() failed "DF_RC"\n", DP_RC(rc));
-					D_GOTO(out_obj, rc = daos_der2errno(rc));
-				}
+		if (!set_ctime) {
+			rc = d_hlc2timespec(array_stbuf.st_max_epoch, &rstat.st_ctim);
+			if (rc) {
+				D_ERROR("d_hlc2timespec() failed " DF_RC "\n", DP_RC(rc));
+				D_GOTO(out_obj, rc = daos_der2errno(rc));
 			}
 		}
 	}
 
 	iod.iod_nr = i;
-
 	if (i == 0)
 		D_GOTO(out_stat, rc = 0);
-
 	sgl.sg_nr	= i;
 	sgl.sg_nr_out	= 0;
 	sgl.sg_iovs	= &sg_iovs[0];
diff --git a/src/tests/suite/dfs_unit_test.c b/src/tests/suite/dfs_unit_test.c
index af4a0294314..cc3091c3fe3 100644
--- a/src/tests/suite/dfs_unit_test.c
+++ b/src/tests/suite/dfs_unit_test.c
@@ -1525,6 +1525,11 @@ run_time_tests(dfs_obj_t *obj, char *name, int mode)
 	struct timespec		prev_ts, first_ts;
 	daos_size_t		size;
 	dfs_obj_t		*tmp_obj;
+	struct tm                tm = {0};
+	time_t                   ts;
+	char                    *p;
+	struct tm               *timeptr;
+	char                     time_str[64];
 	int			rc;
 
 	rc = dfs_stat(dfs_mt, NULL, name, &stbuf);
@@ -1612,8 +1617,34 @@ run_time_tests(dfs_obj_t *obj, char *name, int mode)
 	prev_ts.tv_sec = stbuf.st_mtim.tv_sec;
 	prev_ts.tv_nsec = stbuf.st_mtim.tv_nsec;
 
-	/** set size on file with dfs_osetattr and stat at same time */
 	if (S_ISREG(mode)) {
+		/** set mtime and size at the same time; mtime should be what we set */
+		memset(&stbuf, 0, sizeof(stbuf));
+		stbuf.st_size = 1000;
+		p             = strptime("2023-12-31", "%Y-%m-%d", &tm);
+		assert_non_null(p);
+		ts                    = mktime(&tm);
+		stbuf.st_mtim.tv_sec  = ts;
+		stbuf.st_mtim.tv_nsec = 0;
+		rc = dfs_osetattr(dfs_mt, obj, &stbuf, DFS_SET_ATTR_SIZE | DFS_SET_ATTR_MTIME);
+		assert_int_equal(rc, 0);
+		assert_int_equal(stbuf.st_size, 1000);
+		/** check the mtime was updated with the setattr */
+		assert_int_equal(ts, stbuf.st_mtim.tv_sec);
+		timeptr = localtime(&stbuf.st_mtim.tv_sec);
+		strftime(time_str, sizeof(time_str), "%Y-%m-%d", timeptr);
+		print_message("mtime = %s\n", time_str);
+		assert_true(strncmp("2023", time_str, 4) == 0);
+
+		memset(&stbuf, 0, sizeof(stbuf));
+		rc = dfs_ostat(dfs_mt, obj, &stbuf);
+		assert_int_equal(rc, 0);
+		assert_int_equal(stbuf.st_size, 1000);
+		timeptr = localtime(&stbuf.st_mtim.tv_sec);
+		strftime(time_str, sizeof(time_str), "%Y-%m-%d", timeptr);
+		assert_int_equal(ts, stbuf.st_mtim.tv_sec);
+		assert_true(strncmp("2023", time_str, 4) == 0);
+
 		memset(&stbuf, 0, sizeof(stbuf));
 		stbuf.st_size = 1024;
 		rc = dfs_osetattr(dfs_mt, obj, &stbuf, DFS_SET_ATTR_SIZE);
@@ -1623,12 +1654,6 @@ run_time_tests(dfs_obj_t *obj, char *name, int mode)
 		assert_true(check_ts(prev_ts, stbuf.st_mtim));
 	}
 
-	struct tm	tm = {0};
-	time_t		ts;
-	char		*p;
-	struct tm       *timeptr;
-	char		time_str[64];
-
 	/** set the mtime to 2020 */
 	p = strptime("2020-12-31", "%Y-%m-%d", &tm);
 	assert_non_null(p);