DAOS-14969 container: retry IV might cause deadlock

OID IV entry lock might be required again for retry case. Test-repeat: 10 Test-tag: test_daos_oid_allocator test_daos_management Required-githooks: true Signed-off-by: Di Wang <[email protected]>
daos-stack · Jan 18, 2024 · be67921 · be67921
1 parent da658ad
commit be67921
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/src/container/oid_iv.c b/src/container/oid_iv.c
@@ -31,6 +31,7 @@ struct oid_iv_entry {
 	struct oid_iv_range	rg;
 	/** protect the entry */
 	ABT_mutex		lock;
+	void			*current_req;
 };
 
 /** Priv data in the iv layer */
@@ -130,7 +131,14 @@ oid_iv_ent_update(struct ds_iv_entry *ns_entry, struct ds_iv_key *iv_key,
 	D_ASSERT(priv != NULL);
 
 	entry = ns_entry->iv_value.sg_iovs[0].iov_buf;
-	ABT_mutex_lock(entry->lock);
+	rc = ABT_mutex_trylock(entry->lock);
+	/* For retry requests, from _iv_op(), the lock may not be released
+	 * in some cases.
+	 */
+	if (rc == ABT_ERR_MUTEX_LOCKED && entry->current_req != src)
+		return -DER_BUSY;
+
+	entry->current_req = src;
 	avail = &entry->rg;
 
 	oids = src->sg_iovs[0].iov_buf;

diff --git a/src/engine/server_iv.c b/src/engine/server_iv.c
@@ -1053,7 +1053,7 @@ _iv_op(struct ds_iv_ns *ns, struct ds_iv_key *key, d_sg_list_t *value,
 retry:
 	rc = iv_op_internal(ns, key, value, sync, shortcut, opc);
 	if (retry && !ns->iv_stop &&
-	    (daos_rpc_retryable_rc(rc) || rc == -DER_NOTLEADER)) {
+	    (daos_rpc_retryable_rc(rc) || rc == -DER_NOTLEADER || rc == -DER_BUSY)) {
 		if (rc == -DER_NOTLEADER && key->rank != (d_rank_t)(-1) &&
 		    sync && (sync->ivs_mode == CRT_IV_SYNC_LAZY ||
 			     sync->ivs_mode == CRT_IV_SYNC_EAGER)) {