From 05122bdf29556e5c0d8d159cc64ac3ac935425dd Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Mon, 4 Mar 2024 17:43:09 +0000
Subject: [PATCH 01/47] DAOS-15338 dfuse: add a test for read caching.

Required-githooks: true

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse.h         |  9 ++++----
 src/client/dfuse/dfuse_core.c    | 12 ++++++++++-
 src/client/dfuse/dfuse_fuseops.c |  7 +++++++
 src/client/dfuse/inval.c         | 28 +++++++++++++++++++++----
 src/client/dfuse/ops/create.c    |  4 ++++
 src/client/dfuse/ops/open.c      |  1 +
 utils/node_local_test.py         | 36 +++++++++++++++++++++++++++++---
 7 files changed, 85 insertions(+), 12 deletions(-)

diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h
index 350323dbecb..5c6f6cf780d 100644
--- a/src/client/dfuse/dfuse.h
+++ b/src/client/dfuse/dfuse.h
@@ -524,6 +524,9 @@ struct dfuse_cont {
 	double                  dfc_dentry_dir_timeout;
 	double                  dfc_ndentry_timeout;
 	double                  dfc_data_timeout;
+
+	double                        dfc_dentry_inval_time;
+
 	bool                    dfc_data_otoc;
 	bool                    dfc_direct_io_disable;
 
@@ -539,9 +542,6 @@ struct dfuse_cont {
 #define DFUSE_IE_STAT_ADD(_ie, _stat)                                                              \
 	atomic_fetch_add_relaxed(&(_ie)->ie_dfs->dfs_stat_value[(_stat)], 1)
 
-void
-dfuse_set_default_cont_cache_values(struct dfuse_cont *dfc);
-
 /* Connect to a container via a label
  * Called either for labels on the command line or via dfuse_cont_get_handle() if opening via uuid
  *
@@ -838,7 +838,8 @@ dfuse_loop(struct dfuse_info *dfuse_info);
 #define DFUSE_REPLY_CREATE(inode, req, entry, fi)                                                  \
 	do {                                                                                       \
 		int __rc;                                                                          \
-		DFUSE_TRA_DEBUG(inode, "Returning create");                                        \
+		DFUSE_TRA_DEBUG(inode, "Returning create keep_cache %d direct %d",                 \
+				(fi)->keep_cache, (fi)->direct_io);                                \
 		ival_update_inode(inode, (entry).entry_timeout);                                   \
 		(inode) = NULL;                                                                    \
 		__rc    = fuse_reply_create(req, &entry, fi);                                      \
diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c
index 1d1bbd037e7..167f67400b1 100644
--- a/src/client/dfuse/dfuse_core.c
+++ b/src/client/dfuse/dfuse_core.c
@@ -749,7 +749,7 @@ dfuse_cont_get_cache(struct dfuse_cont *dfc)
  * dentries which represent directories and are therefore referenced much
  * more often during path-walk activities are set to five seconds.
  */
-void
+static void
 dfuse_set_default_cont_cache_values(struct dfuse_cont *dfc)
 {
 	dfc->dfc_attr_timeout       = 1;
@@ -873,6 +873,8 @@ dfuse_cont_open(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp, const cha
 		uuid_copy(dfc->dfc_uuid, c_info.ci_uuid);
 
 		if (dfuse_info->di_caching) {
+			double timeout;
+
 			rc = dfuse_cont_get_cache(dfc);
 			if (rc == ENODATA) {
 				DFUSE_TRA_INFO(dfc, "Using default caching values");
@@ -880,6 +882,14 @@ dfuse_cont_open(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp, const cha
 			} else if (rc != 0) {
 				D_GOTO(err_umount, rc);
 			}
+
+			timeout = max(dfc->dfc_attr_timeout, dfc->dfc_data_timeout);
+
+			timeout = min(timeout, 10 * 60);
+
+			timeout = max(timeout, dfc->dfc_dentry_timeout);
+
+			dfc->dfc_dentry_inval_time = timeout + 3;
 		}
 
 		rc = ival_add_cont_buckets(dfc);
diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c
index dcf7fadfa85..be9085451ae 100644
--- a/src/client/dfuse/dfuse_fuseops.c
+++ b/src/client/dfuse/dfuse_fuseops.c
@@ -44,6 +44,7 @@ dfuse_show_flags(void *handle, unsigned int cap, unsigned int want)
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_PARALLEL_DIROPS);
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_POSIX_ACL);
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_HANDLE_KILLPRIV);
+	SHOW_FLAG(handle, cap, want, FUSE_CAP_EXPIRE_ONLY);
 
 #ifdef FUSE_CAP_CACHE_SYMLINKS
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_CACHE_SYMLINKS);
@@ -96,6 +97,12 @@ dfuse_fuse_init(void *arg, struct fuse_conn_info *conn)
 	if (dfuse_info->di_wb_cache)
 		conn->want |= FUSE_CAP_WRITEBACK_CACHE;
 
+#if 1
+	conn->want |= FUSE_CAP_EXPLICIT_INVAL_DATA;
+
+	conn->want &= ~FUSE_CAP_AUTO_INVAL_DATA;
+#endif
+
 #ifdef FUSE_CAP_CACHE_SYMLINKS
 	conn->want |= FUSE_CAP_CACHE_SYMLINKS;
 #endif
diff --git a/src/client/dfuse/inval.c b/src/client/dfuse/inval.c
index 104bc21a591..fb1e1a45d55 100644
--- a/src/client/dfuse/inval.c
+++ b/src/client/dfuse/inval.c
@@ -344,7 +344,7 @@ ival_update_inode(struct dfuse_inode_entry *inode, double timeout)
 	if (S_ISDIR(inode->ie_stat.st_mode))
 		timeout += INVAL_DIRECTORY_GRACE;
 	else
-		timeout += INVAL_FILE_GRACE;
+		timeout = inode->ie_dfs->dfc_dentry_inval_time;
 
 	clock_gettime(CLOCK_MONOTONIC_COARSE, &now);
 
@@ -443,8 +443,28 @@ ival_bucket_dec_value(double timeout)
 	DFUSE_TRA_ERROR(&ival_data, "Unable to find ref for %.1lf", timeout);
 }
 
+#if 0
+/* How long to keep file inodes in cache before invalidating them
+ */
+static double
+ival_dentry_timeout(struct dfuse_cont *dfc)
+{
+	double timeout;
+
+	timeout = max(dfc->dfc_attr_timeout, dfc->dfc_data_timeout);
+
+	timeout = min(timeout, 10 * 60);
+
+	timeout = max(timeout, dfc->dfc_dentry_timeout);
+
+	return timeout + INVAL_FILE_GRACE;
+}
+#endif
+
 /* Ensure the correct buckets exist for a attached container.  Pools have a zero dentry timeout
- * so skip zero values
+ * so skip zero values.
+ *
+ * TODO: Make the dentry timeout be more than the data cache timeout.
  */
 int
 ival_add_cont_buckets(struct dfuse_cont *dfc)
@@ -457,7 +477,7 @@ ival_add_cont_buckets(struct dfuse_cont *dfc)
 	if (rc != 0)
 		goto out;
 	if (dfc->dfc_dentry_timeout != 0) {
-		rc = ival_bucket_add_value(dfc->dfc_dentry_timeout + INVAL_FILE_GRACE);
+		rc = ival_bucket_add_value(dfc->dfc_dentry_inval_time);
 		if (rc != 0)
 			ival_bucket_dec_value(dfc->dfc_dentry_dir_timeout + INVAL_DIRECTORY_GRACE);
 	}
@@ -473,7 +493,7 @@ ival_dec_cont_buckets(struct dfuse_cont *dfc)
 {
 	D_MUTEX_LOCK(&ival_lock);
 	if (dfc->dfc_dentry_timeout != 0)
-		ival_bucket_dec_value(dfc->dfc_dentry_timeout + INVAL_FILE_GRACE);
+		ival_bucket_dec_value(dfc->dfc_dentry_inval_time);
 	ival_bucket_dec_value(dfc->dfc_dentry_dir_timeout + INVAL_DIRECTORY_GRACE);
 	D_MUTEX_UNLOCK(&ival_lock);
 }
diff --git a/src/client/dfuse/ops/create.c b/src/client/dfuse/ops/create.c
index a389c253795..2c9062168ac 100644
--- a/src/client/dfuse/ops/create.c
+++ b/src/client/dfuse/ops/create.c
@@ -197,6 +197,10 @@ dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, const char *na
 	if (dfs->dfc_data_timeout != 0 || ie->ie_dfs->dfc_data_otoc) {
 		if (fi->flags & O_DIRECT)
 			fi_out.direct_io = 1;
+#if 1
+		else
+			fi_out.keep_cache = 1;
+#endif
 	} else {
 		fi_out.direct_io = 1;
 	}
diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 7365bb26ee6..51df2994cf6 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -229,6 +229,7 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		dfuse_inode_decref(dfuse_info, oh->doh_parent_dir);
 	}
 	if (ie) {
+		DFUSE_TRA_DEBUG(ie, "Evict on close, forgetting dentry");
 		rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, ie->ie_parent,
 						      ie->ie_name, strnlen(ie->ie_name, NAME_MAX));
 
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 72478aca49b..90d15c5dfa5 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -783,8 +783,8 @@ def _start(self):
                      '--runtime_dir', self.agent_dir,
                      '--logfile', self.agent_log.name]
 
-        if not self.conf.args.server_debug and not self.conf.args.client_debug:
-            agent_cmd.append('--debug')
+        #if not self.conf.args.server_debug and not self.conf.args.client_debug:
+        #     agent_cmd.append('--debug')
 
         self._agent = subprocess.Popen(agent_cmd)
         self.conf.agent_dir = self.agent_dir
@@ -1288,7 +1288,7 @@ class DFuse():
 
     # pylint: disable-next=too-many-arguments
     def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_path=None,
-                 caching=True, wbcache=True, multi_user=False, ro=False):
+                 caching=True, wbcache=False, multi_user=False, ro=False):
         if mount_path:
             self.dir = mount_path
         else:
@@ -1953,6 +1953,7 @@ def _helper(obj):
             if not self.dfuse_inval:
                 assert self.caching is True
                 cont_attrs = {'dfuse-attr-time': '5m',
+                              'dfuse-data-cache': '5m',
                               'dfuse-dentry-time': '5m',
                               'dfuse-dentry-dir-time': '5m',
                               'dfuse-ndentry-time': '5m'}
@@ -2276,6 +2277,35 @@ def test_pre_read(self):
         assert len(data5) == 0
         assert raw_data1 == data6
 
+    @needs_dfuse_with_opt(caching=True, wbcache=False, dfuse_inval=False)
+    def test_read_from_cache(self):
+        """Test a basic read.
+
+        Write to a file, then read from it.  With caching on dfuse won't see the read, with caching
+        off dfuse will see one truncated read, then another at EOF which will return zero bytes.
+        """
+        file_name = join(self.dfuse.dir, 'file')
+        with open(file_name, 'w') as fd:
+            fd.write('test')
+
+        with open(file_name, 'r') as fd:
+            data = fd.read(16)  # Pass in a buffer size here or python will only read file size.
+        print(data)
+        assert data == 'test'
+
+        sd = self.dfuse.check_usage()
+        sd_read = sd["statistics"].get("read", 0)
+        print(f'Number of reads {sd_read}')
+
+        with open(file_name, 'r') as fd:
+            data = fd.read(16)  # Pass in a buffer size here or python will only read file size.
+        print(data)
+        assert data == 'test'
+        pd = self.dfuse.check_usage()
+        pd_read = pd["statistics"].get("read", 0)
+        print(f'Number of reads {pd_read}')
+        assert pd_read == sd_read
+
     def test_two_mounts(self):
         """Create two mounts, and check that a file created in one can be read from the other"""
         dfuse0 = DFuse(self.server,

From e8c52a1b13b5a7e8f037de49213570e3610fe278 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Mon, 4 Mar 2024 17:58:12 +0000
Subject: [PATCH 02/47] Check for flag before logging it.

Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse_fuseops.c | 4 +++-
 utils/node_local_test.py         | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c
index be9085451ae..955f4ab4542 100644
--- a/src/client/dfuse/dfuse_fuseops.c
+++ b/src/client/dfuse/dfuse_fuseops.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -44,7 +44,9 @@ dfuse_show_flags(void *handle, unsigned int cap, unsigned int want)
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_PARALLEL_DIROPS);
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_POSIX_ACL);
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_HANDLE_KILLPRIV);
+#ifdef FUSE_CAP_EXPIRE_ONLY
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_EXPIRE_ONLY);
+#endif
 
 #ifdef FUSE_CAP_CACHE_SYMLINKS
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_CACHE_SYMLINKS);
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 90d15c5dfa5..7089f2a76a0 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -783,8 +783,8 @@ def _start(self):
                      '--runtime_dir', self.agent_dir,
                      '--logfile', self.agent_log.name]
 
-        #if not self.conf.args.server_debug and not self.conf.args.client_debug:
-        #     agent_cmd.append('--debug')
+        if not self.conf.args.server_debug and not self.conf.args.client_debug:
+            agent_cmd.append('--debug')
 
         self._agent = subprocess.Popen(agent_cmd)
         self.conf.agent_dir = self.agent_dir

From 21449e7fcab68f10d76503d3f2d07f06a547cf52 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 11:19:14 +0000
Subject: [PATCH 03/47] Add a ftest for this.

Test-tag: DFuseReadTest
Skip-unit-tests: true
Skip-build-el8-rpm: true
Skip-func-test-el9: false

Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py   | 68 +++++++++++++++++++++++++++++++++
 src/tests/ftest/dfuse/read.yaml | 20 ++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 src/tests/ftest/dfuse/read.py
 create mode 100644 src/tests/ftest/dfuse/read.yaml

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
new file mode 100644
index 00000000000..e84f6841bd4
--- /dev/null
+++ b/src/tests/ftest/dfuse/read.py
@@ -0,0 +1,68 @@
+"""
+  (C) Copyright 2024 Intel Corporation.
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+"""
+
+
+from dfuse_test_base import DfuseTestBase
+from run_utils import run_remote
+
+
+class DFuseReadTest(DfuseTestBase):
+    """Base ReadTest test class.
+
+    :avocado: recursive
+    """
+
+    def test_bashfd(self):
+        """
+
+        Test Description:
+            Run a simple Write/Read test to check for read caching.
+
+        :avocado: tags=all,full_regression
+        :avocado: tags=vm
+        :avocado: tags=dfuse,dfs
+        :avocado: tags=DFuseReadTest,test_dfuse_read
+        """
+
+        pool = self.get_pool(connect=False)
+        container = self.get_container(pool)
+
+        self.dfuse.disable_wb_cache.value = True
+
+        cont_attrs = {}
+
+        cont_attrs['dfuse-data-cache'] = '1h'
+        cont_attrs['dfuse-attr-time'] = '1h'
+        cont_attrs['dfuse-dentry-time'] = '1h'
+        cont_attrs['dfuse-ndentry-time'] = '1h'
+
+        self.container.set_attr(attrs=cont_attrs)
+
+        self.start_dfuse(self.hostlist_clients, pool, container)
+
+        fuse_root_dir = self.dfuse.mount_dir.value
+
+        cmd = f"dd if=/dev/zero of={fuse_root_dir}/test_file count=10 bs=1"
+
+        result = run_remote(self.log, self.hostlist_clients, cmd)
+        if not result.passed:
+            self.fail(f'"{cmd}" failed on {result.failed_hosts}')
+
+        cmd = f"daos filesystem query {fuse_root_dir}"
+        result = run_remote(self.log, self.hostlist_clients, cmd)
+        if not result.passed:
+            self.fail(f'"{cmd}" failed on {result.failed_hosts}')
+
+        cmd = f"dd if={fuse_root_dir}/test_file of=/dev/zero count=10 bs=1"
+
+        result = run_remote(self.log, self.hostlist_clients, cmd)
+        if not result.passed:
+            self.fail(f'"{cmd}" failed on {result.failed_hosts}')
+
+        cmd = f"daos filesystem query {fuse_root_dir}"
+        result = run_remote(self.log, self.hostlist_clients, cmd)
+        if not result.passed:
+            self.fail(f'"{cmd}" failed on {result.failed_hosts}')
diff --git a/src/tests/ftest/dfuse/read.yaml b/src/tests/ftest/dfuse/read.yaml
new file mode 100644
index 00000000000..cba38655bc1
--- /dev/null
+++ b/src/tests/ftest/dfuse/read.yaml
@@ -0,0 +1,20 @@
+hosts:
+  test_servers: 1
+timeout: 900
+server_config:
+  name: daos_server
+  engines_per_host: 1
+  engines:
+    0:
+      targets: 4
+      nr_xs_helpers: 0
+      storage:
+        0:
+          class: ram
+          scm_mount: /mnt/daos
+  system_ram_reserved: 1
+pool:
+  scm_size: 1G
+container:
+  type: POSIX
+  control_method: daos

From 5257102499c5a0fff85b4176323436e6975eb0ca Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 11:40:12 +0000
Subject: [PATCH 04/47] Fix the test tags.

Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: true
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 2 +-
 src/tests/ftest/tags.py       | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index e84f6841bd4..5c682075d1b 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -15,7 +15,7 @@ class DFuseReadTest(DfuseTestBase):
     :avocado: recursive
     """
 
-    def test_bashfd(self):
+    def test_dfuse_read(self):
         """
 
         Test Description:
diff --git a/src/tests/ftest/tags.py b/src/tests/ftest/tags.py
index d8c516c9275..4ef9109436c 100755
--- a/src/tests/ftest/tags.py
+++ b/src/tests/ftest/tags.py
@@ -314,6 +314,7 @@ def run_linter(paths=None):
                 if class_name not in tags:
                     tests_wo_class_as_tag.append(method_name)
                 if method_name not in tags:
+                    print(file_path)
                     tests_wo_method_as_tag.append(method_name)
                 if not set(tags).intersection(set(['vm', 'hw', 'manual'])):
                     tests_wo_hw_vm_manual.append(method_name)

From 10dae0090bdb2d1014fa445a131fadca21ed07d4 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 11:48:25 +0000
Subject: [PATCH 05/47] Try loading dfuse before access.

Skip-build-ubuntu20-rpm: true

Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: true
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index 5c682075d1b..ed8caa828d3 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -30,6 +30,8 @@ def test_dfuse_read(self):
         pool = self.get_pool(connect=False)
         container = self.get_container(pool)
 
+        self.load_dfuse(self.hostlist_clients, None)
+
         self.dfuse.disable_wb_cache.value = True
 
         cont_attrs = {}

From 20a5f62da7ebfdbbe66c2032984a5643090198f2 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 12:56:14 +0000
Subject: [PATCH 06/47] Setup a container.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: true
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 1 +
 src/tests/ftest/tags.py       | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index ed8caa828d3..7e4949c8433 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -28,6 +28,7 @@ def test_dfuse_read(self):
         """
 
         pool = self.get_pool(connect=False)
+        self.add_container(self.pool)
         container = self.get_container(pool)
 
         self.load_dfuse(self.hostlist_clients, None)
diff --git a/src/tests/ftest/tags.py b/src/tests/ftest/tags.py
index 4ef9109436c..d8c516c9275 100755
--- a/src/tests/ftest/tags.py
+++ b/src/tests/ftest/tags.py
@@ -314,7 +314,6 @@ def run_linter(paths=None):
                 if class_name not in tags:
                     tests_wo_class_as_tag.append(method_name)
                 if method_name not in tags:
-                    print(file_path)
                     tests_wo_method_as_tag.append(method_name)
                 if not set(tags).intersection(set(['vm', 'hw', 'manual'])):
                     tests_wo_hw_vm_manual.append(method_name)

From 1ae23d5c12888e1ad0a7640895e9321d7a8e488d Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 13:34:09 +0000
Subject: [PATCH 07/47] Change the pool setup.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: true
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index 7e4949c8433..8b6dbac965d 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -28,7 +28,6 @@ def test_dfuse_read(self):
         """
 
         pool = self.get_pool(connect=False)
-        self.add_container(self.pool)
         container = self.get_container(pool)
 
         self.load_dfuse(self.hostlist_clients, None)
@@ -42,7 +41,7 @@ def test_dfuse_read(self):
         cont_attrs['dfuse-dentry-time'] = '1h'
         cont_attrs['dfuse-ndentry-time'] = '1h'
 
-        self.container.set_attr(attrs=cont_attrs)
+        container.set_attr(attrs=cont_attrs)
 
         self.start_dfuse(self.hostlist_clients, pool, container)
 

From f9a75f4c2ef3a1c457e3522091927b4476d9de11 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 14:17:00 +0000
Subject: [PATCH 08/47] Turn on dfuse debugging.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: true
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index 8b6dbac965d..fa12eeb8f67 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -34,6 +34,10 @@ def test_dfuse_read(self):
 
         self.dfuse.disable_wb_cache.value = True
 
+        self.dfuse.env["D_LOG_MASK"] = "INFO,DFUSE=DEBUG"
+        self.dfuse.env["DD_MASK"] = "ALL"
+        self.dfuse.env["DD_SUBSYS"] = "ALL"
+
         cont_attrs = {}
 
         cont_attrs['dfuse-data-cache'] = '1h'

From 82b2d5ec73ec0b6f627bfc7834b1a9d89beed7ae Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 16:01:07 +0000
Subject: [PATCH 09/47] Use a larger write size.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: true
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index fa12eeb8f67..53ce156085d 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -51,7 +51,7 @@ def test_dfuse_read(self):
 
         fuse_root_dir = self.dfuse.mount_dir.value
 
-        cmd = f"dd if=/dev/zero of={fuse_root_dir}/test_file count=10 bs=1"
+        cmd = f"dd if=/dev/zero of={fuse_root_dir}/test_file count=16 bs=1M"
 
         result = run_remote(self.log, self.hostlist_clients, cmd)
         if not result.passed:
@@ -62,13 +62,16 @@ def test_dfuse_read(self):
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
 
-        cmd = f"dd if={fuse_root_dir}/test_file of=/dev/zero count=10 bs=1"
+        cmd = f"dd if={fuse_root_dir}/test_file of=/dev/zero count=16 bs=1M"
 
         result = run_remote(self.log, self.hostlist_clients, cmd)
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
 
-        cmd = f"daos filesystem query {fuse_root_dir}"
+        cmd = f"daos filesystem query --json {fuse_root_dir}"
         result = run_remote(self.log, self.hostlist_clients, cmd)
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
+
+        print(result)
+        print(result.output)

From 30f73a3f44ee06003b9f80a167c949ad3483fe10 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 17:02:19 +0000
Subject: [PATCH 10/47] Skip-build-ubuntu20-rpm: true Test-tag: DFuseReadTest
 Skip-fault-injection-test: true Skip-scan-el8-rpms: true Skip-unit-tests:
 true Skip-build-el8-rpm: false Skip-func-test-el9: false Required-githooks:
 true Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>

---
 src/client/dfuse/dfuse_fuseops.c | 4 ++--
 src/tests/ftest/dfuse/read.py    | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c
index 955f4ab4542..e19dc138119 100644
--- a/src/client/dfuse/dfuse_fuseops.c
+++ b/src/client/dfuse/dfuse_fuseops.c
@@ -99,9 +99,9 @@ dfuse_fuse_init(void *arg, struct fuse_conn_info *conn)
 	if (dfuse_info->di_wb_cache)
 		conn->want |= FUSE_CAP_WRITEBACK_CACHE;
 
-#if 1
+#ifdef FUSE_CAP_EXPLICIT_INVAL_DATA
+	/* DAOS-15338 Not not let the kernel evict data on mtime changes */
 	conn->want |= FUSE_CAP_EXPLICIT_INVAL_DATA;
-
 	conn->want &= ~FUSE_CAP_AUTO_INVAL_DATA;
 #endif
 
diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index 53ce156085d..422f57dc513 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -5,6 +5,8 @@
 """
 
 
+import json
+
 from dfuse_test_base import DfuseTestBase
 from run_utils import run_remote
 
@@ -75,3 +77,6 @@ def test_dfuse_read(self):
 
         print(result)
         print(result.output)
+
+        data = json.loads(result.output[0].stdout)
+        print(data)

From 8fe67e8351625cbf3692c660e853b7507df8b830 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 17:32:32 +0000
Subject: [PATCH 11/47] Debug the test.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py |  2 ++
 utils/node_local_test.py      | 20 ++++++++------------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index 422f57dc513..98e8284d738 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -77,6 +77,8 @@ def test_dfuse_read(self):
 
         print(result)
         print(result.output)
+        print(result.output[0].stdout)
+        print(result.output[0].stdout[0])
 
         data = json.loads(result.output[0].stdout)
         print(data)
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 7089f2a76a0..ae8235b6c45 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -2281,30 +2281,26 @@ def test_pre_read(self):
     def test_read_from_cache(self):
         """Test a basic read.
 
-        Write to a file, then read from it.  With caching on dfuse won't see the read, with caching
-        off dfuse will see one truncated read, then another at EOF which will return zero bytes.
+        Write to a file, then read from it.  With write-through caching on then the read should come
+        from the page cache.  Due to the way this is implement the cache will be truncated down
+        to a pagesize so this test only works for whole pages.
         """
         file_name = join(self.dfuse.dir, 'file')
-        with open(file_name, 'w') as fd:
-            fd.write('test')
 
-        with open(file_name, 'r') as fd:
-            data = fd.read(16)  # Pass in a buffer size here or python will only read file size.
-        print(data)
-        assert data == 'test'
+        subprocess.run(["dd", "if=/dev/zero", f"of={file_name}", 'count=1', 'bs=4k'], check=True)
 
+        subprocess.run(["dd", "of=/dev/zero", f"if={file_name}", 'count=4k', 'bs=1'], check=True)
         sd = self.dfuse.check_usage()
         sd_read = sd["statistics"].get("read", 0)
         print(f'Number of reads {sd_read}')
 
-        with open(file_name, 'r') as fd:
-            data = fd.read(16)  # Pass in a buffer size here or python will only read file size.
-        print(data)
-        assert data == 'test'
+        subprocess.run(["dd", "of=/dev/zero", f"if={file_name}", 'count=1', 'bs=4k'], check=True)
         pd = self.dfuse.check_usage()
         pd_read = pd["statistics"].get("read", 0)
         print(f'Number of reads {pd_read}')
+
         assert pd_read == sd_read
+        assert pd_read == 0
 
     def test_two_mounts(self):
         """Create two mounts, and check that a file created in one can be read from the other"""

From 6e657e0a2594f59541e69f78008f8f6b1b867b7f Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 17:48:48 +0000
Subject: [PATCH 12/47] Fix spelling.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/inval.c | 18 ------------------
 utils/node_local_test.py |  2 +-
 2 files changed, 1 insertion(+), 19 deletions(-)

diff --git a/src/client/dfuse/inval.c b/src/client/dfuse/inval.c
index fb1e1a45d55..58bdae265de 100644
--- a/src/client/dfuse/inval.c
+++ b/src/client/dfuse/inval.c
@@ -443,24 +443,6 @@ ival_bucket_dec_value(double timeout)
 	DFUSE_TRA_ERROR(&ival_data, "Unable to find ref for %.1lf", timeout);
 }
 
-#if 0
-/* How long to keep file inodes in cache before invalidating them
- */
-static double
-ival_dentry_timeout(struct dfuse_cont *dfc)
-{
-	double timeout;
-
-	timeout = max(dfc->dfc_attr_timeout, dfc->dfc_data_timeout);
-
-	timeout = min(timeout, 10 * 60);
-
-	timeout = max(timeout, dfc->dfc_dentry_timeout);
-
-	return timeout + INVAL_FILE_GRACE;
-}
-#endif
-
 /* Ensure the correct buckets exist for a attached container.  Pools have a zero dentry timeout
  * so skip zero values.
  *
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index ae8235b6c45..45c65bc9a9d 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -2283,7 +2283,7 @@ def test_read_from_cache(self):
 
         Write to a file, then read from it.  With write-through caching on then the read should come
         from the page cache.  Due to the way this is implement the cache will be truncated down
-        to a pagesize so this test only works for whole pages.
+        to a page size so this test only works for whole pages.
         """
         file_name = join(self.dfuse.dir, 'file')
 

From ffc647e6488227e29060a7cf71cd78620341ce95 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 5 Mar 2024 18:31:27 +0000
Subject: [PATCH 13/47] Re-attach the data before parsing it.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index 98e8284d738..a5cefc82d20 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -80,5 +80,5 @@ def test_dfuse_read(self):
         print(result.output[0].stdout)
         print(result.output[0].stdout[0])
 
-        data = json.loads(result.output[0].stdout)
+        data = json.loads("\n".join(result.output[0].stdout))
         print(data)

From 43bad1f48068ec803d45006b104500ee45f2d438 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Fri, 8 Mar 2024 08:26:15 +0000
Subject: [PATCH 14/47] check the number of reads.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index a5cefc82d20..c9a27edb922 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -75,10 +75,6 @@ def test_dfuse_read(self):
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
 
-        print(result)
-        print(result.output)
-        print(result.output[0].stdout)
-        print(result.output[0].stdout[0])
-
         data = json.loads("\n".join(result.output[0].stdout))
         print(data)
+        assert data["responce"]["statistics"].get("read", 0) == 0

From 1f9f67665a47600ace0704fc0ca3c6b0093d3edb Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Fri, 8 Mar 2024 08:53:27 +0000
Subject: [PATCH 15/47] Fix json key.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index c9a27edb922..b644a161021 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -77,4 +77,4 @@ def test_dfuse_read(self):
 
         data = json.loads("\n".join(result.output[0].stdout))
         print(data)
-        assert data["responce"]["statistics"].get("read", 0) == 0
+        assert data["response"]["statistics"].get("read", 0) == 0

From 16af3f0732c05cebc60b0dd68e823e27eecbb59a Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Fri, 8 Mar 2024 09:22:26 +0000
Subject: [PATCH 16/47] Make new code into a dfuse method.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py       | 28 ++++++++++++----------------
 src/tests/ftest/util/dfuse_utils.py | 15 +++++++++++++++
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index b644a161021..c3410d6c213 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -4,9 +4,6 @@
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
 
-
-import json
-
 from dfuse_test_base import DfuseTestBase
 from run_utils import run_remote
 
@@ -42,10 +39,10 @@ def test_dfuse_read(self):
 
         cont_attrs = {}
 
-        cont_attrs['dfuse-data-cache'] = '1h'
-        cont_attrs['dfuse-attr-time'] = '1h'
-        cont_attrs['dfuse-dentry-time'] = '1h'
-        cont_attrs['dfuse-ndentry-time'] = '1h'
+        cont_attrs["dfuse-data-cache"] = "1h"
+        cont_attrs["dfuse-attr-time"] = "1h"
+        cont_attrs["dfuse-dentry-time"] = "1h"
+        cont_attrs["dfuse-ndentry-time"] = "1h"
 
         container.set_attr(attrs=cont_attrs)
 
@@ -54,12 +51,6 @@ def test_dfuse_read(self):
         fuse_root_dir = self.dfuse.mount_dir.value
 
         cmd = f"dd if=/dev/zero of={fuse_root_dir}/test_file count=16 bs=1M"
-
-        result = run_remote(self.log, self.hostlist_clients, cmd)
-        if not result.passed:
-            self.fail(f'"{cmd}" failed on {result.failed_hosts}')
-
-        cmd = f"daos filesystem query {fuse_root_dir}"
         result = run_remote(self.log, self.hostlist_clients, cmd)
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
@@ -75,6 +66,11 @@ def test_dfuse_read(self):
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
 
-        data = json.loads("\n".join(result.output[0].stdout))
-        print(data)
-        assert data["response"]["statistics"].get("read", 0) == 0
+        data = self.dfuse.get_stats()
+
+        read_calls = data["statistics"].get("read", 0)
+        write_calls = data["statistics"].get("write")
+
+        print(f"Test caused {write_calls} write and {read_calls} reads.")
+
+        assert read_calls == 0, data
diff --git a/src/tests/ftest/util/dfuse_utils.py b/src/tests/ftest/util/dfuse_utils.py
index 058ea614965..5a54453510f 100644
--- a/src/tests/ftest/util/dfuse_utils.py
+++ b/src/tests/ftest/util/dfuse_utils.py
@@ -4,6 +4,7 @@
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
 
+import json
 import os
 import time
 
@@ -380,6 +381,20 @@ def stop(self):
         # Only assume clean if nothing above failed
         self.__need_cleanup = False
 
+    def get_stats(self):
+        """Return the I/O stats for the filesystem"""
+
+        cmd = f"daos filesystem query --json {self.mount_dir.value}"
+        result = run_remote(self.log, self.hosts, cmd)
+        if not result.passed:
+            raise CommandFailure(f'"{cmd}" failed on {result.failed_hosts}')
+
+        data = json.loads("\n".join(result.output[0].stdout))
+        print(data)
+        assert data["status"] == 0
+        assert data["error"] is None
+        return data["response"]
+
 
 def get_dfuse(test, hosts, namespace=None):
     """Get a new Dfuse instance.

From 28a0d4594d99874663dd6fcc644c82e487b60f32 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Fri, 8 Mar 2024 09:44:20 +0000
Subject: [PATCH 17/47] Tidy up.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse.h      | 3 +--
 src/client/dfuse/ops/create.c | 6 +-----
 src/client/dfuse/ops/open.c   | 3 +--
 src/tests/ftest/dfuse/read.py | 2 +-
 4 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h
index 5c6f6cf780d..eab2b08a23c 100644
--- a/src/client/dfuse/dfuse.h
+++ b/src/client/dfuse/dfuse.h
@@ -838,8 +838,7 @@ dfuse_loop(struct dfuse_info *dfuse_info);
 #define DFUSE_REPLY_CREATE(inode, req, entry, fi)                                                  \
 	do {                                                                                       \
 		int __rc;                                                                          \
-		DFUSE_TRA_DEBUG(inode, "Returning create keep_cache %d direct %d",                 \
-				(fi)->keep_cache, (fi)->direct_io);                                \
+		DFUSE_TRA_DEBUG(inode, "Returning create");                                        \
 		ival_update_inode(inode, (entry).entry_timeout);                                   \
 		(inode) = NULL;                                                                    \
 		__rc    = fuse_reply_create(req, &entry, fi);                                      \
diff --git a/src/client/dfuse/ops/create.c b/src/client/dfuse/ops/create.c
index 2c9062168ac..c164c653111 100644
--- a/src/client/dfuse/ops/create.c
+++ b/src/client/dfuse/ops/create.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -197,10 +197,6 @@ dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, const char *na
 	if (dfs->dfc_data_timeout != 0 || ie->ie_dfs->dfc_data_otoc) {
 		if (fi->flags & O_DIRECT)
 			fi_out.direct_io = 1;
-#if 1
-		else
-			fi_out.keep_cache = 1;
-#endif
 	} else {
 		fi_out.direct_io = 1;
 	}
diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 51df2994cf6..16d2d98ced2 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -229,7 +229,6 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		dfuse_inode_decref(dfuse_info, oh->doh_parent_dir);
 	}
 	if (ie) {
-		DFUSE_TRA_DEBUG(ie, "Evict on close, forgetting dentry");
 		rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, ie->ie_parent,
 						      ie->ie_name, strnlen(ie->ie_name, NAME_MAX));
 
diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index c3410d6c213..02e1583169d 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -71,6 +71,6 @@ def test_dfuse_read(self):
         read_calls = data["statistics"].get("read", 0)
         write_calls = data["statistics"].get("write")
 
-        print(f"Test caused {write_calls} write and {read_calls} reads.")
+        print(f"Test caused {write_calls} write and {read_calls} reads calls")
 
         assert read_calls == 0, data

From bf7628ad3ca328e97f1f8a3be04d370e88d5ccaa Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Mon, 18 Mar 2024 11:11:54 +0000
Subject: [PATCH 18/47] Remove date checks.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Required-githooks: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/ops/create.c | 2 +-
 src/client/dfuse/ops/open.c   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/client/dfuse/ops/create.c b/src/client/dfuse/ops/create.c
index c164c653111..a389c253795 100644
--- a/src/client/dfuse/ops/create.c
+++ b/src/client/dfuse/ops/create.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2016-2023 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 16d2d98ced2..7365bb26ee6 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2024 Intel Corporation.
+ * (C) Copyright 2016-2023 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */

From 20a352ccd0737421762038ee3727f7c35055473d Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 21 Mar 2024 16:28:12 +0000
Subject: [PATCH 19/47] Start adding back in the eviction code.

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse.h    |  6 ++++++
 src/client/dfuse/ops/open.c | 19 ++++++++++++++-----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h
index eab2b08a23c..c9a0b8b3132 100644
--- a/src/client/dfuse/dfuse.h
+++ b/src/client/dfuse/dfuse.h
@@ -980,6 +980,12 @@ struct dfuse_inode_entry {
 	/** File has been unlinked from daos */
 	bool                      ie_unlinked;
 
+	/* Data cache metdata, list known size/mtime for file, if these have been updated then
+	 * the data cache should be dropped.
+	 */
+	size_t                    ie_cache_size;
+	struct timespec           ie_cache_time;
+
 	/** Last file closed in this directory was read linearly.  Directories only.
 	 *
 	 * Set on close() of a file in the directory to the value of linear_read from the fh.
diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 7365bb26ee6..c356b6b122e 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -138,6 +138,8 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 	struct dfuse_inode_entry *ie         = NULL;
 	int                       rc;
 	uint32_t                  il_calls;
+	bool                      update_data_cache = false;
+	int                       oc;
 
 	/* Perform the opposite of what the ioctl call does, always change the open handle count
 	 * but the inode only tracks number of open handles with non-zero ioctl counts
@@ -180,7 +182,7 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 			if (il_calls == 0) {
 				DFUSE_TRA_DEBUG(oh, "Evicting metadata cache, setting data cache");
 				dfuse_mcache_evict(oh->doh_ie);
-				dfuse_dcache_set_time(oh->doh_ie);
+				update_data_cache = true;
 			} else {
 				DFUSE_TRA_DEBUG(oh, "Evicting cache");
 				dfuse_cache_evict(oh->doh_ie);
@@ -191,7 +193,8 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		if (oh->doh_caching) {
 			if (il_calls == 0) {
 				DFUSE_TRA_DEBUG(oh, "Saving data cache");
-				dfuse_dcache_set_time(oh->doh_ie);
+				update_data_cache = true;
+
 			} else {
 				DFUSE_TRA_DEBUG(oh, "Evicting cache");
 				dfuse_cache_evict(oh->doh_ie);
@@ -199,16 +202,22 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		}
 	}
 	DFUSE_TRA_DEBUG(oh, "il_calls %d, caching %d,", il_calls, oh->doh_caching);
-	if (il_calls != 0) {
+	if (il_calls != 0)
 		atomic_fetch_sub_relaxed(&oh->doh_ie->ie_il_count, 1);
-	}
-	atomic_fetch_sub_relaxed(&oh->doh_ie->ie_open_count, 1);
+	oc = atomic_fetch_sub_relaxed(&oh->doh_ie->ie_open_count, 1);
 
 	if (oh->doh_evict_on_close) {
 		ie = oh->doh_ie;
 		atomic_fetch_add_relaxed(&ie->ie_ref, 1);
 	}
 
+	if (update_data_cache) {
+		dfuse_dcache_set_time(oh->doh_ie);
+		if (oc == 1) {
+			/* Fire off a ostatx call to fetch the new mtime */
+		}
+	}
+
 	rc = dfs_release(oh->doh_obj);
 	if (rc == 0) {
 		DFUSE_REPLY_ZERO_OH(oh, req);

From 836953c5581f3628fc4226824f8cc38b01e17354 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Mon, 25 Mar 2024 10:17:48 +0000
Subject: [PATCH 20/47] Further tweak build

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 site_scons/prereq_tools/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py
index efd2fd3f9a4..534994b43f2 100644
--- a/site_scons/prereq_tools/base.py
+++ b/site_scons/prereq_tools/base.py
@@ -1265,11 +1265,11 @@ def set_environment(self, env, needed_libs):
                 # will adjust this to be a relative rpath later
                 env.AppendUnique(RPATH_FULL=[full_path])
                 # For binaries run during build
-                env.AppendENVPath("LD_LIBRARY_PATH", full_path)
+                # env.AppendENVPath("LD_LIBRARY_PATH", full_path)
 
             # Ensure RUNPATH is used rather than RPATH.  RPATH is deprecated
             # and this allows LD_LIBRARY_PATH to override RPATH
-            env.AppendUnique(LINKFLAGS=["-Wl,--enable-new-dtags"])
+            # env.AppendUnique(LINKFLAGS=["-Wl,--enable-new-dtags"])
         if self.component_prefix == "/usr" and self.package is None:
             # hack until we have everything installed in lib64
             env.AppendUnique(RPATH=["/usr/lib"])

From 5632d8dce421fc262ee2479e2e22159261600990 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 26 Mar 2024 14:37:58 +0000
Subject: [PATCH 21/47] Save my changes.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 site_scons/prereq_tools/base.py  |   4 +-
 src/client/dfuse/dfuse.h         |  41 ++++++++-
 src/client/dfuse/dfuse_core.c    |  52 +++++++++++
 src/client/dfuse/dfuse_fuseops.c |  13 ++-
 src/client/dfuse/inval.c         |   2 -
 src/client/dfuse/ops/open.c      | 153 +++++++++++++++++++++++++------
 utils/node_local_test.py         |  14 ++-
 7 files changed, 242 insertions(+), 37 deletions(-)

diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py
index 534994b43f2..efd2fd3f9a4 100644
--- a/site_scons/prereq_tools/base.py
+++ b/site_scons/prereq_tools/base.py
@@ -1265,11 +1265,11 @@ def set_environment(self, env, needed_libs):
                 # will adjust this to be a relative rpath later
                 env.AppendUnique(RPATH_FULL=[full_path])
                 # For binaries run during build
-                # env.AppendENVPath("LD_LIBRARY_PATH", full_path)
+                env.AppendENVPath("LD_LIBRARY_PATH", full_path)
 
             # Ensure RUNPATH is used rather than RPATH.  RPATH is deprecated
             # and this allows LD_LIBRARY_PATH to override RPATH
-            # env.AppendUnique(LINKFLAGS=["-Wl,--enable-new-dtags"])
+            env.AppendUnique(LINKFLAGS=["-Wl,--enable-new-dtags"])
         if self.component_prefix == "/usr" and self.package is None:
             # hack until we have everything installed in lib64
             env.AppendUnique(RPATH=["/usr/lib"])
diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h
index c9a0b8b3132..773f3030159 100644
--- a/src/client/dfuse/dfuse.h
+++ b/src/client/dfuse/dfuse.h
@@ -401,6 +401,7 @@ struct dfuse_event {
 	union {
 		size_t de_req_len;
 		size_t de_readahead_len;
+		struct dfuse_info *de_di;
 	};
 	void (*de_complete_cb)(struct dfuse_event *ev);
 	struct stat de_attr;
@@ -447,6 +448,7 @@ struct dfuse_pool {
 	ACTION(CREATE)                                                                             \
 	ACTION(MKNOD)                                                                              \
 	ACTION(FGETATTR)                                                                           \
+	ACTION(PRE_GETATTR)                                                                        \
 	ACTION(GETATTR)                                                                            \
 	ACTION(FSETATTR)                                                                           \
 	ACTION(SETATTR)                                                                            \
@@ -980,11 +982,36 @@ struct dfuse_inode_entry {
 	/** File has been unlinked from daos */
 	bool                      ie_unlinked;
 
-	/* Data cache metdata, list known size/mtime for file, if these have been updated then
+	/* Data cache metadata, list known size/mtime for file, if these have been updated then
 	 * the data cache should be dropped.
+	 *
+	 * When the last fd on a file is closed and all writes are completed then dfuse will launch
+	 * a stat to get the update size/mtime for the inode.  Future opens should block on this
+	 * stat in order to know if the file has been updated.
+	 *
+	 * Access is controlled via atomics and semaphore, when a decision to make the stat is taken
+	 * then active in increased, and the sem is posted.
+	 *
+	 * Future accesses of the inode should check active, if the value is 0 then there is nothing
+	 * to do.
+	 * If active is positive then it should increate active, wait on the semaphore, decrease
+	 * active and then post the semaphore if active != 0;
+	 *
+	 * After active is 0, (or the semaphore has been waited on) then the new stat structure is
+	 * valid.
+	 *
+	 * The release() code to initialize stat is atomic as it's only triggered by the last
+	 * release on a inode.  It could race with open() where the inode is known in advance
+	 * or create() where it is not.  Open will flush the stat before setting keep_cache.
 	 */
-	size_t                    ie_cache_size;
-	struct timespec           ie_cache_time;
+	struct {
+		struct stat     stat;
+		bool            valid;
+		ATOMIC uint32_t active;
+		struct timespec last_update;
+
+		sem_t           sem;
+	} ie_dc;
 
 	/** Last file closed in this directory was read linearly.  Directories only.
 	 *
@@ -997,6 +1024,9 @@ struct dfuse_inode_entry {
 	d_list_t                  ie_evict_entry;
 };
 
+void
+dfuse_ie_cs_flush(struct dfuse_inode_entry *ie);
+
 /* Lookup an inode and take a ref on it. */
 static inline struct dfuse_inode_entry *
 dfuse_inode_lookup(struct dfuse_info *dfuse_info, fuse_ino_t ino)
@@ -1097,6 +1127,11 @@ dfuse_mcache_get_valid(struct dfuse_inode_entry *ie, double max_age, double *tim
 bool
 dfuse_dentry_get_valid(struct dfuse_inode_entry *ie, double max_age, double *timeout);
 
+void
+dfuse_dc_cache_set_time(struct dfuse_inode_entry *ie);
+bool
+dfuse_dc_cache_get_valid(struct dfuse_inode_entry *ie, double max_age, double *timeout);
+
 /* inval.c */
 
 int
diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c
index 167f67400b1..c54265a1f9c 100644
--- a/src/client/dfuse/dfuse_core.c
+++ b/src/client/dfuse/dfuse_core.c
@@ -883,6 +883,12 @@ dfuse_cont_open(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp, const cha
 				D_GOTO(err_umount, rc);
 			}
 
+			/* Set the timeout for invalidate.  This is the duration after which inodes
+			 * will be evicted from the kernel.  Data timeout is considered here as
+			 * well as attrs are used to decide if keeping the data in cache is
+			 * correct.  Data timeout can be set to True/-1 so cap this duration at
+			 * ten minutes or nothing would ever get evicted.
+			 */
 			timeout = max(dfc->dfc_attr_timeout, dfc->dfc_data_timeout);
 
 			timeout = min(timeout, 10 * 60);
@@ -1055,6 +1061,51 @@ dfuse_mcache_get_valid(struct dfuse_inode_entry *ie, double max_age, double *tim
 	return use;
 }
 
+/* Set a timer to mark cache entry as valid */
+void
+dfuse_dc_cache_set_time(struct dfuse_inode_entry *ie)
+{
+	struct timespec now;
+
+	clock_gettime(CLOCK_MONOTONIC_COARSE, &now);
+	ie->ie_dc.last_update = now;
+}
+
+bool
+dfuse_dc_cache_get_valid(struct dfuse_inode_entry *ie, double max_age, double *timeout)
+{
+	bool            use = false;
+	struct timespec now;
+	struct timespec left;
+	double          time_left;
+
+	D_ASSERT(max_age != -1);
+	D_ASSERT(max_age >= 0);
+
+	if (ie->ie_mcache_last_update.tv_sec == 0)
+		return false;
+
+	clock_gettime(CLOCK_MONOTONIC_COARSE, &now);
+
+	left.tv_sec  = now.tv_sec - ie->ie_dc.last_update.tv_sec;
+	left.tv_nsec = now.tv_nsec - ie->ie_dc.last_update.tv_nsec;
+	if (left.tv_nsec < 0) {
+		left.tv_sec--;
+		left.tv_nsec += 1000000000;
+	}
+	time_left = max_age - (left.tv_sec + ((double)left.tv_nsec / 1000000000));
+	if (time_left > 0) {
+		use = true;
+
+		DFUSE_TRA_DEBUG(ie, "Allowing cache use, time remaining: %.1lf", time_left);
+
+		if (timeout)
+			*timeout = time_left;
+	}
+
+	return use;
+}
+
 bool
 dfuse_dentry_get_valid(struct dfuse_inode_entry *ie, double max_age, double *timeout)
 {
@@ -1258,6 +1309,7 @@ dfuse_ie_init(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie)
 	atomic_init(&ie->ie_open_count, 0);
 	atomic_init(&ie->ie_open_write_count, 0);
 	atomic_init(&ie->ie_il_count, 0);
+	sem_init(&ie->ie_dc.sem, 0, 0);
 	atomic_fetch_add_relaxed(&dfuse_info->di_inode_count, 1);
 	D_INIT_LIST_HEAD(&ie->ie_evict_entry);
 }
diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c
index e19dc138119..44ef298909b 100644
--- a/src/client/dfuse/dfuse_fuseops.c
+++ b/src/client/dfuse/dfuse_fuseops.c
@@ -100,7 +100,7 @@ dfuse_fuse_init(void *arg, struct fuse_conn_info *conn)
 		conn->want |= FUSE_CAP_WRITEBACK_CACHE;
 
 #ifdef FUSE_CAP_EXPLICIT_INVAL_DATA
-	/* DAOS-15338 Not not let the kernel evict data on mtime changes */
+	/* DAOS-15338 Do not let the kernel evict data on mtime changes */
 	conn->want |= FUSE_CAP_EXPLICIT_INVAL_DATA;
 	conn->want &= ~FUSE_CAP_AUTO_INVAL_DATA;
 #endif
@@ -179,6 +179,17 @@ df_ll_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		DFUSE_IE_STAT_ADD(inode, DS_GETATTR);
 	}
 
+	if (inode->ie_dfs->dfc_attr_timeout) {
+		double timeout;
+		dfuse_ie_cs_flush(inode);
+
+		if (dfuse_dc_cache_get_valid(inode, inode->ie_dfs->dfc_attr_timeout, &timeout)) {
+			DFUSE_IE_STAT_ADD(inode, DS_PRE_GETATTR);
+			DFUSE_REPLY_ATTR_FORCE(inode, req, timeout);
+			return;
+		}
+	}
+
 	if (inode->ie_dfs->dfc_attr_timeout &&
 	    (atomic_load_relaxed(&inode->ie_open_write_count) == 0) &&
 	    (atomic_load_relaxed(&inode->ie_il_count) == 0)) {
diff --git a/src/client/dfuse/inval.c b/src/client/dfuse/inval.c
index 58bdae265de..c0f5675dd2a 100644
--- a/src/client/dfuse/inval.c
+++ b/src/client/dfuse/inval.c
@@ -445,8 +445,6 @@ ival_bucket_dec_value(double timeout)
 
 /* Ensure the correct buckets exist for a attached container.  Pools have a zero dentry timeout
  * so skip zero values.
- *
- * TODO: Make the dentry timeout be more than the data cache timeout.
  */
 int
 ival_add_cont_buckets(struct dfuse_cont *dfc)
diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index c356b6b122e..78ad5212cd6 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -54,23 +54,41 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 	if (rc)
 		D_GOTO(err, rc);
 
+	/*
+	 * dfs_dup() just locally duplicates the file handle. If we have
+	 * O_TRUNC flag, we need to truncate the file manually.
+	 */
+	if (fi->flags & O_TRUNC) {
+		rc = dfs_punch(ie->ie_dfs->dfs_ns, ie->ie_obj, 0, DFS_MAX_FSIZE);
+		if (rc)
+			D_GOTO(err, rc);
+		dfuse_dcache_evict(oh->doh_ie);
+	}
+
+	/* There are no failure points between here and the kernel reply */
+
+	atomic_fetch_add_relaxed(&ie->ie_open_count, 1);
+
 	if ((fi->flags & O_ACCMODE) != O_RDONLY)
 		oh->doh_writeable = true;
 
 	if (ie->ie_dfs->dfc_data_timeout != 0) {
-		if (fi->flags & O_DIRECT)
+		if (fi->flags & O_DIRECT) {
 			fi_out.direct_io = 1;
-
-		/* If the file is already open or (potentially) in cache then allow any existing
-		 * kernel cache to be used.  If not then use pre-read.
-		 * This should mean that pre-read is only used on the first read, and on files
-		 * which pre-existed in the container.
-		 */
-		if (atomic_load_relaxed(&ie->ie_open_count) > 0 ||
-		    dfuse_dcache_get_valid(ie, ie->ie_dfs->dfc_data_timeout)) {
-			fi_out.keep_cache = 1;
 		} else {
-			prefetch = true;
+			dfuse_ie_cs_flush(ie);
+
+			/* If the file is already open or (potentially) in cache then allow any
+			 * existing kernel cache to be used.  If not then use pre-read. This should
+			 * mean that pre-read is only used on the first read, and on files which
+			 * pre-existed in the container.
+			 */
+			if (atomic_load_relaxed(&ie->ie_open_count) > 0 ||
+			    dfuse_dcache_get_valid(ie, ie->ie_dfs->dfc_data_timeout)) {
+				fi_out.keep_cache = 1;
+			} else {
+				prefetch = true;
+			}
 		}
 	} else if (ie->ie_dfs->dfc_data_otoc) {
 		/* Open to close caching, this allows the use of shared mmap */
@@ -91,19 +109,6 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 
 	fi_out.fh = (uint64_t)oh;
 
-	/*
-	 * dfs_dup() just locally duplicates the file handle. If we have
-	 * O_TRUNC flag, we need to truncate the file manually.
-	 */
-	if (fi->flags & O_TRUNC) {
-		rc = dfs_punch(ie->ie_dfs->dfs_ns, ie->ie_obj, 0, DFS_MAX_FSIZE);
-		if (rc)
-			D_GOTO(err, rc);
-		dfuse_dcache_evict(oh->doh_ie);
-	}
-
-	atomic_fetch_add_relaxed(&ie->ie_open_count, 1);
-
 	/* Enable this for files up to the max read size. */
 	if (prefetch && oh->doh_parent_dir &&
 	    atomic_load_relaxed(&oh->doh_parent_dir->ie_linear_read) && ie->ie_stat.st_size > 0 &&
@@ -130,6 +135,100 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 	DFUSE_REPLY_ERR_RAW(ie, req, rc);
 }
 
+void
+dfuse_ie_cs_flush(struct dfuse_inode_entry *ie)
+{
+	uint32_t new = 2;
+	uint32_t old = 1;
+
+restart:
+	if (atomic_compare_exchange(&ie->ie_dc.active, old, new)) {
+		/* ie->ie_ref was 1, now it's 2 */
+		sem_wait(&ie->ie_dc.sem);
+
+	} else if (old == 0) {
+		/* ie->ie_ref was and is 0 */
+		return;
+	} else {
+		new = old + 1;
+
+		/* ie->ie_ref was > 1 and is unchanged */
+		if (!atomic_compare_exchange(&ie->ie_dc.active, old, new))
+			goto restart;
+
+		/* ie->ie_ref was > 1 and is unchanged */
+
+		sem_wait(&ie->ie_dc.sem);
+	}
+
+	old = atomic_fetch_sub(&ie->ie_dc.active, 1);
+
+	if (old > 1)
+		sem_post(&ie->ie_dc.sem);
+}
+
+static void
+getattr_cb(struct dfuse_event *ev)
+{
+	if (ev->de_ev.ev_error != 0) {
+		ev->de_ie->ie_dc.valid = false;
+		goto free;
+	}
+
+	ev->de_ie->ie_dc.valid = true;
+	dfuse_dc_cache_set_time(ev->de_ie);
+
+	/* Data will have been read directory into ie->ie_dc.stat */
+
+free:
+	atomic_fetch_sub(&ev->de_ie->ie_dc.active, 1);
+	sem_post(&ev->de_ie->ie_dc.sem);
+	dfuse_inode_decref(ev->de_di, ev->de_ie);
+	D_FREE(ev);
+}
+
+void
+size_resample(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie)
+{
+	struct dfuse_event *ev;
+	uint64_t            eqt_idx;
+	struct dfuse_eq    *eqt;
+	int                 rc;
+
+	if (ie->ie_unlinked)
+		return;
+
+	eqt_idx = atomic_fetch_add_relaxed(&dfuse_info->di_eqt_idx, 1);
+	eqt     = &dfuse_info->di_eqt[eqt_idx % dfuse_info->di_eq_count];
+	D_ALLOC_PTR(ev);
+	if (ev == NULL)
+		return;
+
+	ev->de_complete_cb = getattr_cb;
+	ev->de_ie          = ie;
+	ev->de_di          = dfuse_info;
+
+	atomic_fetch_add_relaxed(&ie->ie_ref, 1);
+
+	rc = daos_event_init(&ev->de_ev, eqt->de_eq, NULL);
+	if (rc != -DER_SUCCESS)
+		goto err;
+
+	rc = dfs_ostatx(ie->ie_dfs->dfs_ns, ie->ie_obj, &ie->ie_dc.stat, &ev->de_ev);
+	if (rc != 0)
+		goto err;
+
+	atomic_fetch_add(&ie->ie_dc.active, 1);
+
+	sem_post(&eqt->de_sem);
+
+	return;
+err:
+	dfuse_inode_decref(ev->de_di, ev->de_ie);
+	ie->ie_dc.valid = false;
+	D_FREE(ev);
+}
+
 void
 dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 {
@@ -213,9 +312,9 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 
 	if (update_data_cache) {
 		dfuse_dcache_set_time(oh->doh_ie);
-		if (oc == 1) {
+		if (oc == 1)
 			/* Fire off a ostatx call to fetch the new mtime */
-		}
+			size_resample(dfuse_info, oh->doh_ie);
 	}
 
 	rc = dfs_release(oh->doh_obj);
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 45c65bc9a9d..3afbbc6f0b1 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -2284,6 +2284,16 @@ def test_read_from_cache(self):
         Write to a file, then read from it.  With write-through caching on then the read should come
         from the page cache.  Due to the way this is implement the cache will be truncated down
         to a page size so this test only works for whole pages.
+
+        The I/O that dfuse should see are:
+        create
+        write
+        release
+        stat
+        open
+        release
+        open
+        release
         """
         file_name = join(self.dfuse.dir, 'file')
 
@@ -5036,7 +5046,7 @@ def run_in_fg(server, conf, args):
 
         # Only set the container cache attributes when the container is initially created so they
         # can be modified later.
-        cont_attrs = {'dfuse-data-cache': False,
+        cont_attrs = {'dfuse-data-cache': 120,
                       'dfuse-attr-time': 67,
                       'dfuse-dentry-time': 19,
                       'dfuse-dentry-dir-time': 31,
@@ -5046,7 +5056,7 @@ def run_in_fg(server, conf, args):
         container = container.uuid
 
     dargs = {"caching": True,
-             "wbcache": True,
+             "wbcache": False,
              "multi_user": args.multi_user}
 
     if pool_on_cmd_line:

From 8737908d2b502271f4105b1133192f9ce793aa8d Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 4 Apr 2024 13:03:52 +0000
Subject: [PATCH 22/47] Hotpatch Jenkinsfile.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
PR-repos: fuse3@PR-11

Skip-build-leap15-rpm: true
Skip-build-el9-rpm: true
Skip-func-test-el9: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 Jenkinsfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1ec0c595e70..7e046f96b9a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -782,7 +782,6 @@ pipeline {
                                          valgrind_stash: 'el8-gcc-nlt-memcheck'
                             recordIssues enabledForFailure: true,
                                          failOnError: false,
-                                         ignoreFailedBuilds: true,
                                          ignoreQualityGate: true,
                                          name: 'NLT server leaks',
                                          qualityGates: [[threshold: 1, type: 'TOTAL', unstable: true]],
@@ -1020,7 +1019,6 @@ pipeline {
                                                       scm: 'daos-stack/daos'
                             recordIssues enabledForFailure: true,
                                          failOnError: false,
-                                         ignoreFailedBuilds: true,
                                          ignoreQualityGate: true,
                                          qualityGates: [[threshold: 1, type: 'TOTAL_ERROR'],
                                                         [threshold: 1, type: 'TOTAL_HIGH'],

From d5f92ef6fd0ce672794fe99cd05df0a00a324853 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Fri, 5 Apr 2024 17:24:24 +0000
Subject: [PATCH 23/47] Enpty commit for testing.

Features: dfuse
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
PR-repos: fuse3@PR-11
Skip-build-leap15-rpm: true
Skip-build-el9-rpm: true
Skip-func-test-el9: true
Skip-build-ubuntu20-rpm: true

From a46c40409cc8643924a8b8b7f1857b4335e78dcd Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 11 Apr 2024 09:37:20 +0000
Subject: [PATCH 24/47] Drop the data cache if metadata has changesd.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-tests: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse_core.c    |  2 +-
 src/client/dfuse/dfuse_fuseops.c | 12 ++++--
 src/client/dfuse/ops/fgetattr.c  | 20 ++++++++--
 src/client/dfuse/ops/open.c      | 10 ++---
 utils/node_local_test.py         | 63 +++++++++++++++++++++++++++-----
 5 files changed, 83 insertions(+), 24 deletions(-)

diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c
index c54265a1f9c..5936297e661 100644
--- a/src/client/dfuse/dfuse_core.c
+++ b/src/client/dfuse/dfuse_core.c
@@ -1082,7 +1082,7 @@ dfuse_dc_cache_get_valid(struct dfuse_inode_entry *ie, double max_age, double *t
 	D_ASSERT(max_age != -1);
 	D_ASSERT(max_age >= 0);
 
-	if (ie->ie_mcache_last_update.tv_sec == 0)
+	if (ie->ie_dc.last_update.tv_sec == 0)
 		return false;
 
 	clock_gettime(CLOCK_MONOTONIC_COARSE, &now);
diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c
index 44ef298909b..005bd38e388 100644
--- a/src/client/dfuse/dfuse_fuseops.c
+++ b/src/client/dfuse/dfuse_fuseops.c
@@ -44,9 +44,6 @@ dfuse_show_flags(void *handle, unsigned int cap, unsigned int want)
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_PARALLEL_DIROPS);
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_POSIX_ACL);
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_HANDLE_KILLPRIV);
-#ifdef FUSE_CAP_EXPIRE_ONLY
-	SHOW_FLAG(handle, cap, want, FUSE_CAP_EXPIRE_ONLY);
-#endif
 
 #ifdef FUSE_CAP_CACHE_SYMLINKS
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_CACHE_SYMLINKS);
@@ -57,6 +54,12 @@ dfuse_show_flags(void *handle, unsigned int cap, unsigned int want)
 #ifdef FUSE_CAP_EXPLICIT_INVAL_DATA
 	SHOW_FLAG(handle, cap, want, FUSE_CAP_EXPLICIT_INVAL_DATA);
 #endif
+#ifdef FUSE_CAP_EXPIRE_ONLY
+	SHOW_FLAG(handle, cap, want, FUSE_CAP_EXPIRE_ONLY);
+#endif
+#ifdef FUSE_CAP_SETXATTR_EXT
+	SHOW_FLAG(handle, cap, want, FUSE_CAP_SETXATTR_EXT);
+#endif
 
 	if (cap)
 		DFUSE_TRA_WARNING(handle, "Unknown capability flags %#x", cap);
@@ -179,6 +182,9 @@ df_ll_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		DFUSE_IE_STAT_ADD(inode, DS_GETATTR);
 	}
 
+	/* Check for stat-after-write/close.  On close a stat is performed so the first getattr
+	 * call can use the result of that.
+	 */
 	if (inode->ie_dfs->dfc_attr_timeout) {
 		double timeout;
 		dfuse_ie_cs_flush(inode);
diff --git a/src/client/dfuse/ops/fgetattr.c b/src/client/dfuse/ops/fgetattr.c
index 6fdee73515c..960d9ee652b 100644
--- a/src/client/dfuse/ops/fgetattr.c
+++ b/src/client/dfuse/ops/fgetattr.c
@@ -10,14 +10,28 @@
 static void
 dfuse_cb_getattr_cb(struct dfuse_event *ev)
 {
+	struct dfuse_inode_entry *ie = ev->de_ie;
+
 	if (ev->de_ev.ev_error != 0) {
-		DFUSE_REPLY_ERR_RAW(ev->de_ie, ev->de_req, ev->de_ev.ev_error);
+		DFUSE_REPLY_ERR_RAW(ie, ev->de_req, ev->de_ev.ev_error);
 		D_GOTO(release, 0);
 	}
 
-	ev->de_attr.st_ino = ev->de_ie->ie_stat.st_ino;
+	ev->de_attr.st_ino = ie->ie_stat.st_ino;
+
+	ie->ie_stat = ev->de_attr;
 
-	ev->de_ie->ie_stat = ev->de_attr;
+	if ((ie->ie_dfs->dfc_data_timeout != 0) &&
+	    (dfuse_dcache_get_valid(ie, ie->ie_dfs->dfc_data_timeout))) {
+		/* This tries to match the code in fuse_change_attributes in fs/fuse/inode.c,
+		 * if the mtime or the size has changed then drop the data cache.
+		 */
+		if ((ie->ie_stat.st_size != ie->ie_dc.stat.st_size) ||
+		    (d_timediff_ns(&ie->ie_stat.st_mtim, &ie->ie_dc.stat.st_mtim) != 0)) {
+			DFUSE_TRA_INFO(ie, "Invalidating data cache");
+			dfuse_dcache_evict(ie);
+		}
+	}
 
 	DFUSE_REPLY_ATTR(ev->de_ie, ev->de_req, &ev->de_attr);
 release:
diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 78ad5212cd6..b64f3e1d7ac 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -67,8 +67,6 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 
 	/* There are no failure points between here and the kernel reply */
 
-	atomic_fetch_add_relaxed(&ie->ie_open_count, 1);
-
 	if ((fi->flags & O_ACCMODE) != O_RDONLY)
 		oh->doh_writeable = true;
 
@@ -101,6 +99,8 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		fi_out.direct_io = 1;
 	}
 
+	atomic_fetch_add_relaxed(&ie->ie_open_count, 1);
+
 	if (ie->ie_dfs->dfc_direct_io_disable)
 		fi_out.direct_io = 0;
 
@@ -290,11 +290,7 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		atomic_fetch_sub_relaxed(&oh->doh_ie->ie_open_write_count, 1);
 	} else {
 		if (oh->doh_caching) {
-			if (il_calls == 0) {
-				DFUSE_TRA_DEBUG(oh, "Saving data cache");
-				update_data_cache = true;
-
-			} else {
+			if (il_calls > 0) {
 				DFUSE_TRA_DEBUG(oh, "Evicting cache");
 				dfuse_cache_evict(oh->doh_ie);
 			}
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 9696e1498e8..c355f150d53 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -2274,7 +2274,6 @@ def test_pre_read(self):
         assert len(data5) == 0
         assert raw_data1 == data6
 
-    @needs_dfuse_with_opt(caching=True, wbcache=False, dfuse_inval=False)
     def test_read_from_cache(self):
         """Test a basic read.
 
@@ -2291,23 +2290,67 @@ def test_read_from_cache(self):
         release
         open
         release
+
+        After that update the file time from a different path and re-read the file, this should
+        trigger an actual read due to the file size changing triggering a cache invalidation.
         """
-        file_name = join(self.dfuse.dir, 'file')
 
+        # The value of attr-time.  Too slow and the test won't run fast enough and will fail,
+        # too long and the wall-clock time will be affected.
+        attr_time = 20
+
+        self.container.set_attrs({'dfuse-attr-time': str(attr_time),
+                                  'dfuse-data-cache': '5m',
+                                  'dfuse-dentry-time': '5m'})
+
+        dfuse0 = DFuse(self.server,
+                       self.conf,
+                       caching=True,
+                       wbcache=False,
+                       container=self.container)
+        dfuse0.start(v_hint='rfc')
+
+        file_name = join(dfuse0.dir, 'file')
+
+        # Create the file.
         subprocess.run(["dd", "if=/dev/zero", f"of={file_name}", 'count=1', 'bs=4k'], check=True)
 
+        start = time.time()
+
+        # Read it after write, this should come from cache.
         subprocess.run(["dd", "of=/dev/zero", f"if={file_name}", 'count=4k', 'bs=1'], check=True)
-        sd = self.dfuse.check_usage()
-        sd_read = sd["statistics"].get("read", 0)
-        print(f'Number of reads {sd_read}')
+        sd = dfuse0.check_usage()
 
+        assert sd["statistics"].get("read", 0) == 0, sd
+
+        # Read it after read, this should also come from cache.
         subprocess.run(["dd", "of=/dev/zero", f"if={file_name}", 'count=1', 'bs=4k'], check=True)
-        pd = self.dfuse.check_usage()
-        pd_read = pd["statistics"].get("read", 0)
-        print(f'Number of reads {pd_read}')
+        sd = dfuse0.check_usage()
+        assert sd["statistics"].get("read", 0) == 0, sd
+
+        elapsed = time.time() - start
+
+        to_sleep = attr_time + 5 - elapsed
+
+        if to_sleep < 5:
+            raise NLTestFail("attr_time not high enough")
+
+        print(f"Sleeping for {to_sleep} seconds")
+        time.sleep(to_sleep)
+
+        # Now the attr time has expired but the dentry and data caches are still valid.  At this
+        # point change the file time using touch and then re-read the file which should perform
+        # an actual read this time.
+        self.server.run_daos_client_cmd_pil4dfs(['touch', 'file'], container=self.container)
+
+        # Read it after attr timeout, this should not come from cache due to the times changing.
+        subprocess.run(["dd", "of=/dev/zero", f"if={file_name}", 'count=1', 'bs=4k'], check=True)
+        sd = dfuse0.check_usage()
+        assert sd["statistics"].get("read", 0) > 0, sd
+
+        if dfuse0.stop():
+            self.fatal_errors = True
 
-        assert pd_read == sd_read
-        assert pd_read == 0
 
     def test_two_mounts(self):
         """Create two mounts, and check that a file created in one can be read from the other"""

From 6b32a7e6d0737ee7bb2628767b590a2aef2d2652 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 11 Apr 2024 10:33:34 +0000
Subject: [PATCH 25/47] Fix lint.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 utils/cq/words.dict      | 1 +
 utils/node_local_test.py | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/cq/words.dict b/utils/cq/words.dict
index c5ca4dd5d41..68563261646 100644
--- a/utils/cq/words.dict
+++ b/utils/cq/words.dict
@@ -138,6 +138,7 @@ debian
 debuginfo
 defusedxml
 del
+dentry
 deps
 dereference
 dereferencing
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index c355f150d53..3e0d7ebd65d 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -2351,7 +2351,6 @@ def test_read_from_cache(self):
         if dfuse0.stop():
             self.fatal_errors = True
 
-
     def test_two_mounts(self):
         """Create two mounts, and check that a file created in one can be read from the other"""
         dfuse0 = DFuse(self.server,

From 99c6a12bf455fd5fe48529715ef7bd28aecf8668 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 11 Apr 2024 11:43:21 +0000
Subject: [PATCH 26/47] Silence a new warning.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest DaosBuild
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h
index 25b40efb0ce..42295a956ed 100644
--- a/src/client/dfuse/dfuse.h
+++ b/src/client/dfuse/dfuse.h
@@ -739,7 +739,7 @@ dfuse_loop(struct dfuse_info *dfuse_info);
 		_Static_assert(IS_OH(_oh), "Param is not open handle");                            \
 		(_oh)->doh_ie = NULL;                                                              \
 		__rc          = fuse_reply_err(req, 0);                                            \
-		if (__rc != 0)                                                                     \
+		if ((__rc != 0) && (__rc != -ENOENT))                                              \
 			DS_ERROR(-__rc, "fuse_reply_err() error");                                 \
 	} while (0)
 

From faecce6f248a50923c071cd7f880a26a677d5123 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 11 Apr 2024 16:32:02 +0000
Subject: [PATCH 27/47] Back out NLT metadata changes.

Skip-build-ubuntu20-rpm: true
Test-tag: DFuseReadTest DaosBuild
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 utils/node_local_test.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 3e0d7ebd65d..9fe57e3e05c 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -1288,7 +1288,7 @@ class DFuse():
 
     # pylint: disable-next=too-many-arguments
     def __init__(self, daos, conf, pool=None, container=None, mount_path=None, uns_path=None,
-                 caching=True, wbcache=False, multi_user=False, ro=False):
+                 caching=True, wbcache=True, multi_user=False, ro=False):
         if mount_path:
             self.dir = mount_path
         else:
@@ -1950,7 +1950,6 @@ def _helper(obj):
             if not self.dfuse_inval:
                 assert self.caching is True
                 cont_attrs = {'dfuse-attr-time': '5m',
-                              'dfuse-data-cache': '5m',
                               'dfuse-dentry-time': '5m',
                               'dfuse-dentry-dir-time': '5m',
                               'dfuse-ndentry-time': '5m'}
@@ -5085,7 +5084,7 @@ def run_in_fg(server, conf, args):
 
         # Only set the container cache attributes when the container is initially created so they
         # can be modified later.
-        cont_attrs = {'dfuse-data-cache': 120,
+        cont_attrs = {'dfuse-data-cache': False,
                       'dfuse-attr-time': 67,
                       'dfuse-dentry-time': 19,
                       'dfuse-dentry-dir-time': 31,
@@ -5095,7 +5094,7 @@ def run_in_fg(server, conf, args):
         container = container.uuid
 
     dargs = {"caching": True,
-             "wbcache": False,
+             "wbcache": True,
              "multi_user": args.multi_user}
 
     if pool_on_cmd_line:

From 47368a1b51398e73fc195ab3120720bb1ce42d83 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 16 Apr 2024 14:48:19 +0000
Subject: [PATCH 28/47] Reply with the new inode data.

Test-tag: dfuseF

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse_fuseops.c | 1 +
 src/client/dfuse/ops/open.c      | 1 +
 src/client/dfuse/ops/setattr.c   | 2 ++
 utils/node_local_test.py         | 2 +-
 4 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c
index 4d3379cfe56..738a4ddabab 100644
--- a/src/client/dfuse/dfuse_fuseops.c
+++ b/src/client/dfuse/dfuse_fuseops.c
@@ -186,6 +186,7 @@ df_ll_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 
 		if (dfuse_dc_cache_get_valid(inode, inode->ie_dfs->dfc_attr_timeout, &timeout)) {
 			DFUSE_IE_STAT_ADD(inode, DS_PRE_GETATTR);
+			inode->ie_stat = inode->ie_dc.stat;
 			DFUSE_REPLY_ATTR_FORCE(inode, req, timeout);
 			return;
 		}
diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index f92ee0852e5..4a13b36980d 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -175,6 +175,7 @@ getattr_cb(struct dfuse_event *ev)
 		goto free;
 	}
 
+	ev->de_ie->ie_dc.stat.st_ino = ev->de_ie->ie_stat.st_ino;
 	ev->de_ie->ie_dc.valid = true;
 	dfuse_dc_cache_set_time(ev->de_ie);
 
diff --git a/src/client/dfuse/ops/setattr.c b/src/client/dfuse/ops/setattr.c
index 7ba3e0e1de3..12217891cf3 100644
--- a/src/client/dfuse/ops/setattr.c
+++ b/src/client/dfuse/ops/setattr.c
@@ -15,6 +15,8 @@ dfuse_cb_setattr(fuse_req_t req, struct dfuse_inode_entry *ie, struct stat *attr
 
 	DFUSE_TRA_DEBUG(ie, "flags %#x", to_set);
 
+	dfuse_ie_cs_flush(ie);
+
 	if (ie->ie_unlinked) {
 		DFUSE_TRA_DEBUG(ie, "File is unlinked, returning most recent data");
 
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index b7ba5f53f61..3bd49fab187 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -5087,7 +5087,7 @@ def run_in_fg(server, conf, args):
 
         # Only set the container cache attributes when the container is initially created so they
         # can be modified later.
-        cont_attrs = {'dfuse-data-cache': False,
+        cont_attrs = {'dfuse-data-cache': 120,
                       'dfuse-attr-time': 67,
                       'dfuse-dentry-time': 19,
                       'dfuse-dentry-dir-time': 31,

From 579c8c13cbfcb3926a9b1777bab45a92a929c990 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 16 Apr 2024 15:11:14 +0000
Subject: [PATCH 29/47] Only use the cached value once.

Test-tag: dfuse
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse_fuseops.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c
index 738a4ddabab..9779b009d7e 100644
--- a/src/client/dfuse/dfuse_fuseops.c
+++ b/src/client/dfuse/dfuse_fuseops.c
@@ -185,10 +185,12 @@ df_ll_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		dfuse_ie_cs_flush(inode);
 
 		if (dfuse_dc_cache_get_valid(inode, inode->ie_dfs->dfc_attr_timeout, &timeout)) {
-			DFUSE_IE_STAT_ADD(inode, DS_PRE_GETATTR);
-			inode->ie_stat = inode->ie_dc.stat;
-			DFUSE_REPLY_ATTR_FORCE(inode, req, timeout);
-			return;
+			if (inode->ie_dc.valid) {
+				DFUSE_IE_STAT_ADD(inode, DS_PRE_GETATTR);
+				inode->ie_dc.valid = false;
+				DFUSE_REPLY_ATTR_FORCE(inode, req, timeout);
+				return;
+			}
 		}
 	}
 	DFUSE_IE_WFLUSH(inode);

From 6fd51c0ee1f0bcb92b197fbd1d7968b0f1856512 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 16 Apr 2024 17:57:01 +0000
Subject: [PATCH 30/47] Fir clang-format and run tests.

Skip-build-ubuntu20-rpm: true
Test-tag: dfuse
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/ops/open.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 4a13b36980d..2b7ca99ffd9 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -176,7 +176,7 @@ getattr_cb(struct dfuse_event *ev)
 	}
 
 	ev->de_ie->ie_dc.stat.st_ino = ev->de_ie->ie_stat.st_ino;
-	ev->de_ie->ie_dc.valid = true;
+	ev->de_ie->ie_dc.valid       = true;
 	dfuse_dc_cache_set_time(ev->de_ie);
 
 	/* Data will have been read directory into ie->ie_dc.stat */

From a2130c21c1ccfa79efee529c2296822fff8018e0 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 18 Apr 2024 10:37:00 +0000
Subject: [PATCH 31/47] Add debugging to test.

Skip-build-ubuntu20-rpm: true
Test-tag: test_dfuse_caching_check
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-tests: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/caching_check.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/tests/ftest/dfuse/caching_check.py b/src/tests/ftest/dfuse/caching_check.py
index 852a24f0dfd..e468c3358fd 100644
--- a/src/tests/ftest/dfuse/caching_check.py
+++ b/src/tests/ftest/dfuse/caching_check.py
@@ -45,14 +45,18 @@ def test_dfuse_caching_check(self):
 
         self.log_step('Write to the dfuse mount point')
         self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
+        print(self.dfuse.get_stats())
 
         self.log_step('Get baseline read performance from dfuse with caching disabled')
         self.ior_cmd.update_params(flags=flags[1])
         base_read_arr = []
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
         base_read_arr.append(IorCommand.get_ior_metrics(out))
+        print(self.dfuse.get_stats())
+
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
         base_read_arr.append(IorCommand.get_ior_metrics(out))
+        print(self.dfuse.get_stats())
 
         # the index of max_mib
         max_mib = IorMetrics.MAX_MIB
@@ -66,10 +70,14 @@ def test_dfuse_caching_check(self):
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
         base_read_arr.append(IorCommand.get_ior_metrics(out))
 
+        print(self.dfuse.get_stats())
+
         self.log_step('Get cached read performance')
         out = self.run_ior_with_pool(fail_on_warning=False)
         with_caching = IorCommand.get_ior_metrics(out)
 
+        print(self.dfuse.get_stats())
+
         self.log_step('Verify cached read performance is greater than first read')
         # Log all the values first, then do the assert so that failures can be checked easily.
         for base_read in base_read_arr:

From 767bb4344b47569de10de3463ca2b210836320eb Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 18 Apr 2024 11:39:15 +0000
Subject: [PATCH 32/47] Fix caching check test.

Skip-build-ubuntu20-rpm: true
Test-tag: test_dfuse_caching_check
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-tests: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/caching_check.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/tests/ftest/dfuse/caching_check.py b/src/tests/ftest/dfuse/caching_check.py
index e468c3358fd..b765da808ee 100644
--- a/src/tests/ftest/dfuse/caching_check.py
+++ b/src/tests/ftest/dfuse/caching_check.py
@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2019-2023 Intel Corporation.
+  (C) Copyright 2019-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -73,10 +73,11 @@ def test_dfuse_caching_check(self):
         print(self.dfuse.get_stats())
 
         self.log_step('Get cached read performance')
-        out = self.run_ior_with_pool(fail_on_warning=False)
+        out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
         with_caching = IorCommand.get_ior_metrics(out)
 
         print(self.dfuse.get_stats())
+        self.dfuse.unmount(tries=1)
 
         self.log_step('Verify cached read performance is greater than first read')
         # Log all the values first, then do the assert so that failures can be checked easily.

From 200ca13096b503c7c343f39998a784c8875e0ca4 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 18 Apr 2024 13:33:25 +0000
Subject: [PATCH 33/47] Use the new fuse version.

PR-repos: fuse3@PR-11
Test-tag: test_dfuse_caching_check
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-tests: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-build-leap15-rpm: true
Skip-build-el9-rpm: true
Skip-func-test-el9: false
Skip-build-ubuntu20-rpm: true

Skip-func-hw-test: true
Skip-func-test: true
Quick-Functional: true
Test-tag: dfuse

From bc21d4fd540bf56f75acd3c2876fa753acb9ea48 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 18 Apr 2024 16:54:03 +0000
Subject: [PATCH 34/47] Bump timeouts.

Skip-build-ubuntu20-rpm: true
Test-tag: test_dfuse_caching_check
Skip-fault-injection-test: true
Skip-scan-el8-rpms: true
Skip-unit-test: true
Skip-unit-tests: true
Skip-unit-test-memcheck: true
Skip-build-el8-rpm: false
Skip-func-test-el9: false

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse_core.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c
index 89282d9332a..c4eca2a8b02 100644
--- a/src/client/dfuse/dfuse_core.c
+++ b/src/client/dfuse/dfuse_core.c
@@ -755,10 +755,10 @@ dfuse_cont_get_cache(struct dfuse_info *dfuse_info, struct dfuse_cont *dfc)
 static void
 dfuse_set_default_cont_cache_values(struct dfuse_cont *dfc)
 {
-	dfc->dfc_attr_timeout       = 1;
-	dfc->dfc_dentry_timeout     = 1;
-	dfc->dfc_dentry_dir_timeout = 5;
-	dfc->dfc_ndentry_timeout    = 1;
+	dfc->dfc_attr_timeout       = 300;
+	dfc->dfc_dentry_timeout     = 300;
+	dfc->dfc_dentry_dir_timeout = 300;
+	dfc->dfc_ndentry_timeout    = 300;
 	dfc->dfc_data_timeout       = 60 * 10;
 	dfc->dfc_direct_io_disable  = false;
 }

From 2a9a8a116d143fcc690d2c0f70baefaeddb78f28 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Mon, 22 Apr 2024 20:24:10 +0000
Subject: [PATCH 35/47] Do not run the complex rename test with caching.

Features: dfuse
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 utils/node_local_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 3bd49fab187..b34b1485d3e 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -3521,11 +3521,13 @@ def test_cont_chown(self):
         if dfuse.stop():
             self.fatal_errors = True
 
-    @needs_dfuse
+    @needs_dfuse_with_opt(caching=False)
     def test_complex_rename(self):
         """Test for rename semantics
 
-        Check that that rename is correctly updating the dfuse data for the moved file.
+        Check that that rename is correctly updating the dfuse data for the moved file.  For this
+        test to work correctly with caching then the attr-timeout needs to be shorter than the
+        time it takes to spin up the second DFuse instance, so do not this with caching on.
 
         # Create a file, read/write to it.
         # Check fstat works.

From 845fc1fec522da8ae7feb335c25dd14f5e3a776d Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 23 Apr 2024 18:09:10 +0000
Subject: [PATCH 36/47] Fix flake.

Features: dfuse
Priority: 2
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/caching_check.py | 13 ++-----------
 src/tests/ftest/dfuse/read.py          |  1 +
 src/tests/ftest/dfuse/read.yaml        |  2 +-
 3 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/tests/ftest/dfuse/caching_check.py b/src/tests/ftest/dfuse/caching_check.py
index b765da808ee..852a24f0dfd 100644
--- a/src/tests/ftest/dfuse/caching_check.py
+++ b/src/tests/ftest/dfuse/caching_check.py
@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2019-2024 Intel Corporation.
+  (C) Copyright 2019-2023 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -45,18 +45,14 @@ def test_dfuse_caching_check(self):
 
         self.log_step('Write to the dfuse mount point')
         self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
-        print(self.dfuse.get_stats())
 
         self.log_step('Get baseline read performance from dfuse with caching disabled')
         self.ior_cmd.update_params(flags=flags[1])
         base_read_arr = []
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
         base_read_arr.append(IorCommand.get_ior_metrics(out))
-        print(self.dfuse.get_stats())
-
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
         base_read_arr.append(IorCommand.get_ior_metrics(out))
-        print(self.dfuse.get_stats())
 
         # the index of max_mib
         max_mib = IorMetrics.MAX_MIB
@@ -70,15 +66,10 @@ def test_dfuse_caching_check(self):
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
         base_read_arr.append(IorCommand.get_ior_metrics(out))
 
-        print(self.dfuse.get_stats())
-
         self.log_step('Get cached read performance')
-        out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
+        out = self.run_ior_with_pool(fail_on_warning=False)
         with_caching = IorCommand.get_ior_metrics(out)
 
-        print(self.dfuse.get_stats())
-        self.dfuse.unmount(tries=1)
-
         self.log_step('Verify cached read performance is greater than first read')
         # Log all the values first, then do the assert so that failures can be checked easily.
         for base_read in base_read_arr:
diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index cc9712667d5..ab7ebe58ad9 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -97,6 +97,7 @@ def test_dfuse_pre_read(self):
 
         self.assertEqual(data["inodes"], 3, "expected 3 inodes in cache")
 
+
 class DFuseReadTest(DfuseTestBase):
     """Base ReadTest test class.
     
diff --git a/src/tests/ftest/dfuse/read.yaml b/src/tests/ftest/dfuse/read.yaml
index d8f5a0086f7..0bf858fe33f 100644
--- a/src/tests/ftest/dfuse/read.yaml
+++ b/src/tests/ftest/dfuse/read.yaml
@@ -15,7 +15,7 @@ server_config:
           scm_mount: /mnt/daos
   system_ram_reserved: 1
 pool:
-  scm_size: 1G
+  size: 1G
 container:
   type: POSIX
   control_method: daos

From a48cc2834c5c6608024d84280944744bc23dff1b Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 23 Apr 2024 18:11:45 +0000
Subject: [PATCH 37/47] Minor changes.

Features: dfuse
Priority: 2
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/ops/open.c            |  4 ++--
 src/tests/ftest/dfuse/caching_check.py |  9 +++++++--
 src/tests/ftest/dfuse/read.py          |  2 +-
 src/tests/ftest/util/dfuse_utils.py    | 10 ++++++++--
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 4867a315b07..b09b9e0fb79 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -82,9 +82,9 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 			 * pre-existed in the container.
 			 */
 			if (atomic_load_relaxed(&ie->ie_open_count) > 0 ||
-		    ((ie->ie_dcache_last_update.tv_sec != 0) &&
+			    ((ie->ie_dcache_last_update.tv_sec != 0) &&
 
-			    dfuse_dcache_get_valid(ie, ie->ie_dfs->dfc_data_timeout))) {
+			     dfuse_dcache_get_valid(ie, ie->ie_dfs->dfc_data_timeout))) {
 				fi_out.keep_cache = 1;
 			} else {
 				prefetch = true;
diff --git a/src/tests/ftest/dfuse/caching_check.py b/src/tests/ftest/dfuse/caching_check.py
index 852a24f0dfd..2d9332a6dc1 100644
--- a/src/tests/ftest/dfuse/caching_check.py
+++ b/src/tests/ftest/dfuse/caching_check.py
@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2019-2023 Intel Corporation.
+  (C) Copyright 2019-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -45,13 +45,16 @@ def test_dfuse_caching_check(self):
 
         self.log_step('Write to the dfuse mount point')
         self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
+        self.dfuse.get_stats()
 
         self.log_step('Get baseline read performance from dfuse with caching disabled')
         self.ior_cmd.update_params(flags=flags[1])
         base_read_arr = []
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
+        self.dfuse.get_stats()
         base_read_arr.append(IorCommand.get_ior_metrics(out))
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
+        self.dfuse.get_stats()
         base_read_arr.append(IorCommand.get_ior_metrics(out))
 
         # the index of max_mib
@@ -64,10 +67,12 @@ def test_dfuse_caching_check(self):
 
         self.log_step('Get first read performance with caching enabled')
         out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
+        self.dfuse.get_stats()
         base_read_arr.append(IorCommand.get_ior_metrics(out))
 
         self.log_step('Get cached read performance')
-        out = self.run_ior_with_pool(fail_on_warning=False)
+        out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
+        self.dfuse.get_stats()
         with_caching = IorCommand.get_ior_metrics(out)
 
         self.log_step('Verify cached read performance is greater than first read')
diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index ab7ebe58ad9..ccc718df218 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -100,7 +100,7 @@ def test_dfuse_pre_read(self):
 
 class DFuseReadTest(DfuseTestBase):
     """Base ReadTest test class.
-    
+
     :avocado: recursive
     """
 
diff --git a/src/tests/ftest/util/dfuse_utils.py b/src/tests/ftest/util/dfuse_utils.py
index f24fd1b07ce..8a5a083e7e5 100644
--- a/src/tests/ftest/util/dfuse_utils.py
+++ b/src/tests/ftest/util/dfuse_utils.py
@@ -385,6 +385,12 @@ def get_stats(self):
         """Return the I/O stats for the filesystem
 
         Only works if there is one entry in the client list.
+
+        Raises:
+            CommandFailure: if the command fails
+
+        Returns:
+            dict: the json response
         """
 
         if len(self.hosts) != 1:
@@ -393,11 +399,11 @@ def get_stats(self):
         cmd = f"daos filesystem query --json {self.mount_dir.value}"
         result = run_remote(self.log, self.hosts, cmd)
         if not result.passed:
-            raise CommandFailure(f'"fs query failed on {result.failed_hosts}')
+            raise CommandFailure(f"fs query failed on {result.failed_hosts}")
 
         data = json.loads("\n".join(result.output[0].stdout))
         if data["status"] != 0 or data["error"] is not None:
-            raise CommandFailure("fs query returned bad data.")
+            raise CommandFailure("fs query returned bad data")
         return data["response"]
 
 

From 085b30a8965ad2cd1d62f226eebf8eab5f9cb7ed Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 25 Apr 2024 16:57:25 +0000
Subject: [PATCH 38/47] Reset default cache settings.

Test-tag: dfuse
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse_core.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c
index f711f9d5b83..9e9a28651b5 100644
--- a/src/client/dfuse/dfuse_core.c
+++ b/src/client/dfuse/dfuse_core.c
@@ -755,10 +755,10 @@ dfuse_cont_get_cache(struct dfuse_info *dfuse_info, struct dfuse_cont *dfc)
 static void
 dfuse_set_default_cont_cache_values(struct dfuse_cont *dfc)
 {
-	dfc->dfc_attr_timeout       = 300;
-	dfc->dfc_dentry_timeout     = 300;
-	dfc->dfc_dentry_dir_timeout = 300;
-	dfc->dfc_ndentry_timeout    = 300;
+	dfc->dfc_attr_timeout       = 1;
+	dfc->dfc_dentry_timeout     = 1;
+	dfc->dfc_dentry_dir_timeout = 5;
+	dfc->dfc_ndentry_timeout    = 1;
 	dfc->dfc_data_timeout       = 60 * 10;
 	dfc->dfc_direct_io_disable  = false;
 }

From e45d17929bcc515bfd31afa770663f6c3a8f2b1b Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Fri, 26 Apr 2024 09:46:45 +0000
Subject: [PATCH 39/47] DAOS-15745 dfuse: Add the pre_read metrics whilst
 holding reference.

Increase the pre-read statistics before replying to the read,
otherwise the oh might not be valid which can lead to unexpected
behaviour.

Test-tag: dfuse

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/ops/read.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/client/dfuse/ops/read.c b/src/client/dfuse/ops/read.c
index 991755d461b..26c97204fbb 100644
--- a/src/client/dfuse/ops/read.c
+++ b/src/client/dfuse/ops/read.c
@@ -97,6 +97,7 @@ dfuse_readahead_reply(fuse_req_t req, size_t len, off_t position, struct dfuse_o
 				position + reply_len - 1, position + reply_len, position + len - 1);
 	}
 
+	DFUSE_IE_STAT_ADD(oh->doh_ie, DS_PRE_READ);
 	DFUSE_REPLY_BUFQ(oh, req, oh->doh_readahead->dra_ev->de_iov.iov_buf + position, reply_len);
 	return true;
 }
@@ -143,10 +144,8 @@ dfuse_cb_read(fuse_req_t req, fuse_ino_t ino, size_t len, off_t position, struct
 		replied = dfuse_readahead_reply(req, len, position, oh);
 		D_MUTEX_UNLOCK(&oh->doh_readahead->dra_lock);
 
-		if (replied) {
-			DFUSE_IE_STAT_ADD(oh->doh_ie, DS_PRE_READ);
+		if (replied)
 			return;
-		}
 	}
 
 	eqt_idx = atomic_fetch_add_relaxed(&dfuse_info->di_eqt_idx, 1);

From 8e9a03f119a8023b1dfe40711f9625c1bbc1070c Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 30 Apr 2024 07:54:31 +0000
Subject: [PATCH 40/47] Tune down a debug message.

Test-tag: dfuse,-test_dfuse_find
Skip-fault-injection-test: true
Skip-unit-tests: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/ops/fgetattr.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/client/dfuse/ops/fgetattr.c b/src/client/dfuse/ops/fgetattr.c
index 960d9ee652b..62645290f0e 100644
--- a/src/client/dfuse/ops/fgetattr.c
+++ b/src/client/dfuse/ops/fgetattr.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -28,7 +28,7 @@ dfuse_cb_getattr_cb(struct dfuse_event *ev)
 		 */
 		if ((ie->ie_stat.st_size != ie->ie_dc.stat.st_size) ||
 		    (d_timediff_ns(&ie->ie_stat.st_mtim, &ie->ie_dc.stat.st_mtim) != 0)) {
-			DFUSE_TRA_INFO(ie, "Invalidating data cache");
+			DFUSE_TRA_DEBUG(ie, "Invalidating data cache");
 			dfuse_dcache_evict(ie);
 		}
 	}

From bd84ad4b4fc83d9c563970e43a934882cfd5acd8 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 30 Apr 2024 10:49:42 +0000
Subject: [PATCH 41/47] Fix a bug where reads weren't going into cache.

Improve test to catch this.

zzTest-tag: dfuse,-test_dfuse_find
Test-tag: test_dfuse_read
Skip-fault-injection-test: true
Skip-unit-tests: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/ops/open.c   | 10 ++++++----
 src/tests/ftest/dfuse/read.py | 34 +++++++++++++++++++++++++++++-----
 2 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index b09b9e0fb79..286f8e2af6d 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -82,9 +82,7 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 			 * pre-existed in the container.
 			 */
 			if (atomic_load_relaxed(&ie->ie_open_count) > 0 ||
-			    ((ie->ie_dcache_last_update.tv_sec != 0) &&
-
-			     dfuse_dcache_get_valid(ie, ie->ie_dfs->dfc_data_timeout))) {
+			    dfuse_dcache_get_valid(ie, ie->ie_dfs->dfc_data_timeout)) {
 				fi_out.keep_cache = 1;
 			} else {
 				prefetch = true;
@@ -295,7 +293,11 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 		atomic_fetch_sub_relaxed(&oh->doh_ie->ie_open_write_count, 1);
 	} else {
 		if (oh->doh_caching) {
-			if (il_calls > 0) {
+			if (il_calls == 0) {
+				DFUSE_TRA_DEBUG(oh, "Saving data cache");
+				/* Set the time here but do not re-sample as there are no writes. */
+				dfuse_dcache_set_time(oh->doh_ie);
+			} else {
 				DFUSE_TRA_DEBUG(oh, "Evicting cache");
 				dfuse_cache_evict(oh->doh_ie);
 			}
diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index ccc718df218..41898733cb7 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -108,6 +108,11 @@ def test_dfuse_read(self):
         """
         Test Description:
             Run a simple Write/Read test to check for read caching.
+
+        Write a file, then read from it and verify that there were no reads at the dfuse level.
+
+        Evict the file, read from it twice and verify the second read comes from cache.
+
         :avocado: tags=all,full_regression
         :avocado: tags=vm
         :avocado: tags=dfuse,dfs
@@ -144,21 +149,40 @@ def test_dfuse_read(self):
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
 
         cmd = f"dd if={fuse_root_dir}/test_file of=/dev/zero count=16 bs=1M"
+        result = run_remote(self.log, self.hostlist_clients, cmd)
+        if not result.passed:
+            self.fail(f'"{cmd}" failed on {result.failed_hosts}')
+
+        data = self.dfuse.get_stats()
+
+        read_calls = data["statistics"].get("read", 0)
+        write_calls = data["statistics"].get("write")
+
+        print(f"Test caused {write_calls} write and {read_calls} reads calls")
 
+        self.assertEqual(data["statistics"].get("read", 0), 0, "Did not expect any read calls")
+
+        cmd = f"daos filesystem evict {fuse_root_dir}/test_file"
         result = run_remote(self.log, self.hostlist_clients, cmd)
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
 
-        cmd = f"daos filesystem query --json {fuse_root_dir}"
+        cmd = f"dd if={fuse_root_dir}/test_file of=/dev/zero count=16 bs=1M"
         result = run_remote(self.log, self.hostlist_clients, cmd)
         if not result.passed:
             self.fail(f'"{cmd}" failed on {result.failed_hosts}')
 
         data = self.dfuse.get_stats()
 
-        read_calls = data["statistics"].get("read", 0)
-        write_calls = data["statistics"].get("write")
+        self.assertGreater(
+            data["statistics"].get("read", 0), 0, "expected non-zero pre read"
+        )
 
-        print(f"Test caused {write_calls} write and {read_calls} reads calls")
+        result = run_remote(self.log, self.hostlist_clients, cmd)
+        if not result.passed:
+            self.fail(f'"{cmd}" failed on {result.failed_hosts}')
+
+        data2 = self.dfuse.get_stats()
+
+        self.assertEqual(data["statistics"].get("read", 0), data2["statistics"].get("read", 0), "Did not expect more read calls")
 
-        assert read_calls == 0, data

From f40a391b7942628034a7dfc35205b32b634602fa Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Tue, 30 Apr 2024 11:20:25 +0000
Subject: [PATCH 42/47] Fix flake and run more tests.

Test-tag: dfuse,-test_dfuse_find
Skip-fault-injection-test: true
Skip-unit-tests: true
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/tests/ftest/dfuse/read.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index 41898733cb7..4ba88cc875e 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -160,7 +160,9 @@ def test_dfuse_read(self):
 
         print(f"Test caused {write_calls} write and {read_calls} reads calls")
 
-        self.assertEqual(data["statistics"].get("read", 0), 0, "Did not expect any read calls")
+        self.assertEqual(
+            data["statistics"].get("read", 0), 0, "Did not expect any read calls"
+        )
 
         cmd = f"daos filesystem evict {fuse_root_dir}/test_file"
         result = run_remote(self.log, self.hostlist_clients, cmd)
@@ -184,5 +186,8 @@ def test_dfuse_read(self):
 
         data2 = self.dfuse.get_stats()
 
-        self.assertEqual(data["statistics"].get("read", 0), data2["statistics"].get("read", 0), "Did not expect more read calls")
-
+        self.assertEqual(
+            data["statistics"].get("read", 0),
+            data2["statistics"].get("read", 0),
+            "Did not expect more read calls",
+        )

From 1a48745a81758e5ee02ff44735fb98db986fa514 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 2 May 2024 07:54:46 +0000
Subject: [PATCH 43/47] Tidy up test code.

Test-tag: test_dfuse_caching_check
Skip-fault-injection-test: true
Skip-unit-tests: true

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse.h      |  1 -
 src/tests/ftest/dfuse/read.py | 11 -----------
 utils/node_local_test.py      |  2 +-
 3 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h
index e3c5b172cbe..d8c5fba8c2b 100644
--- a/src/client/dfuse/dfuse.h
+++ b/src/client/dfuse/dfuse.h
@@ -1027,7 +1027,6 @@ struct dfuse_inode_entry {
 		bool            valid;
 		ATOMIC uint32_t active;
 		struct timespec last_update;
-
 		sem_t           sem;
 	} ie_dc;
 
diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py
index cb7d1a096f6..5706c740176 100644
--- a/src/tests/ftest/dfuse/read.py
+++ b/src/tests/ftest/dfuse/read.py
@@ -153,12 +153,6 @@ def test_dfuse_read(self):
 
         self.load_dfuse(self.hostlist_clients, None)
 
-        self.dfuse.disable_wb_cache.value = True
-
-        self.dfuse.env["D_LOG_MASK"] = "INFO,DFUSE=DEBUG"
-        self.dfuse.env["DD_MASK"] = "ALL"
-        self.dfuse.env["DD_SUBSYS"] = "ALL"
-
         cont_attrs = {}
 
         cont_attrs["dfuse-data-cache"] = "1h"
@@ -184,11 +178,6 @@ def test_dfuse_read(self):
 
         data = self.dfuse.get_stats()
 
-        read_calls = data["statistics"].get("read", 0)
-        write_calls = data["statistics"].get("write")
-
-        print(f"Test caused {write_calls} write and {read_calls} reads calls")
-
         self.assertEqual(
             data["statistics"].get("read", 0), 0, "Did not expect any read calls"
         )
diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 44261a31603..d8845944b2e 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -5089,7 +5089,7 @@ def run_in_fg(server, conf, args):
 
         # Only set the container cache attributes when the container is initially created so they
         # can be modified later.
-        cont_attrs = {'dfuse-data-cache': 120,
+        cont_attrs = {'dfuse-data-cache': False,
                       'dfuse-attr-time': 67,
                       'dfuse-dentry-time': 19,
                       'dfuse-dentry-dir-time': 31,

From 7039849cb586dd2c175312a5c4feccc67e6297b7 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 2 May 2024 08:57:41 +0000
Subject: [PATCH 44/47] Run with features.

Features: dfuse,-test_dfuse_daos_build_wt_pil4dfs

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>

From bce5f4ea94bf0d2b9d23aa99c3e66e8e82626ddd Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Thu, 26 Sep 2024 14:55:49 +0000
Subject: [PATCH 45/47] Fix merge.

Test-tag: dfuse
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/ops/open.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c
index 268ad8cdfc8..bfc8fbb58d0 100644
--- a/src/client/dfuse/ops/open.c
+++ b/src/client/dfuse/ops/open.c
@@ -240,7 +240,6 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 	uint32_t                  oc;
 	uint32_t                  il_calls;
 	bool                      update_data_cache = false;
-	int                       oc;
 
 	/* Perform the opposite of what the ioctl call does, always change the open handle count
 	 * but the inode only tracks number of open handles with non-zero ioctl counts
@@ -307,7 +306,7 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
 	DFUSE_TRA_DEBUG(oh, "il_calls %d, caching %d,", il_calls, oh->doh_caching);
 	if (il_calls != 0)
 		atomic_fetch_sub_relaxed(&oh->doh_ie->ie_il_count, 1);
-	}
+
 	oc = atomic_fetch_sub_relaxed(&oh->doh_ie->ie_open_count, 1);
 	if (oc == 1) {
 		if (read_chunk_close(oh->doh_ie))

From fc82b2113f4f54a56f3aee4d10edc103866de2d0 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Mon, 30 Sep 2024 09:45:17 +0000
Subject: [PATCH 46/47] Bump test runtime.

Test-tag: dfuse
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 utils/node_local_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/node_local_test.py b/utils/node_local_test.py
index 80255c7b9f3..5db5cb2fed1 100755
--- a/utils/node_local_test.py
+++ b/utils/node_local_test.py
@@ -2318,7 +2318,7 @@ def test_read_from_cache(self):
 
         # The value of attr-time.  Too slow and the test won't run fast enough and will fail,
         # too long and the wall-clock time will be affected.
-        attr_time = 20
+        attr_time = 30
 
         self.container.set_attrs({'dfuse-attr-time': str(attr_time),
                                   'dfuse-data-cache': '5m',

From 288e2f6b88812cd918ca2d3ad587e1ed11deaa92 Mon Sep 17 00:00:00 2001
From: Ashley Pittman <ashley.m.pittman@intel.com>
Date: Mon, 30 Sep 2024 10:02:51 +0000
Subject: [PATCH 47/47] Update spelling.

Test-tag: dfuse
Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
---
 src/client/dfuse/dfuse.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h
index 4acdc44ccd6..6481ccd8efb 100644
--- a/src/client/dfuse/dfuse.h
+++ b/src/client/dfuse/dfuse.h
@@ -1013,7 +1013,7 @@ struct dfuse_inode_entry {
 	 *
 	 * Future accesses of the inode should check active, if the value is 0 then there is nothing
 	 * to do.
-	 * If active is positive then it should increate active, wait on the semaphore, decrease
+	 * If active is positive then it should increase active, wait on the semaphore, decrease
 	 * active and then post the semaphore if active != 0;
 	 *
 	 * After active is 0, (or the semaphore has been waited on) then the new stat structure is