Add some more coverage for salt/fileclient.py

salt/fileclient.py

@@ -307,19 +307,19 @@ def file_list(self, saltenv="base", prefix=""):
         """
         This function must be overwritten
         """
-        return []
+        return []  # pragma: no cover
 
     def dir_list(self, saltenv="base", prefix=""):
         """
         This function must be overwritten
         """
-        return []
+        return []  # pragma: no cover
 
     def symlink_list(self, saltenv="base", prefix=""):
         """
         This function must be overwritten
         """
-        return {}
+        return {}  # pragma: no cover
 
     def is_cached(self, path, saltenv="base", cachedir=None):
         """

tests/pytests/unit/fileclient/test_fileclient.py

@@ -223,3 +223,13 @@ def test_get_file_client(file_client):
     with patch("salt.fileclient.RemoteClient", MagicMock(return_value="remote_client")):
         ret = fileclient.get_file_client(minion_opts)
         assert "remote_client" == ret
+
+
+def test_getstate(file_client, mocked_opts):
+    assert file_client.__getstate__() == {"opts": mocked_opts}
+
+
+def test_setstate(file_client, mocked_opts):
+    mocked_opts["fake_opt"] = "fake"
+    file_client.__setstate__({"opts": mocked_opts})
+    assert file_client.opts == mocked_opts

tests/pytests/unit/fileclient/test_fileclient_cache.py

@@ -101,7 +101,8 @@ def _new_dir(path):
     _new_dir(cache_root)
 
 
-def test_cache_dir(mocked_opts, minion_opts):
+@pytest.mark.parametrize("subdir", [SUBDIR, f"{SUBDIR}{os.sep}"])
+def test_cache_dir(mocked_opts, minion_opts, subdir):
     """
     Ensure entire directory is cached to correct location
     """
@@ -111,7 +112,58 @@ def test_cache_dir(mocked_opts, minion_opts):
     with patch.dict(fileclient.__opts__, patched_opts):
         client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
         for saltenv in _saltenvs():
-            assert client.cache_dir(f"salt://{SUBDIR}", saltenv, cachedir=None)
+            assert client.cache_dir(
+                "salt://{}".format(subdir),
+                saltenv,
+                exclude_pat="*foo.txt",
+                cachedir=None,
+            )
+            for subdir_file in _subdir_files():
+                cache_loc = os.path.join(
+                    fileclient.__opts__["cachedir"],
+                    "files",
+                    saltenv,
+                    subdir,
+                    subdir_file,
+                )
+                if not subdir_file.endswith("foo.txt"):
+                    with salt.utils.files.fopen(cache_loc) as fp_:
+                        content = fp_.read()
+                    log.debug("cache_loc = %s", cache_loc)
+                    log.debug("content = %s", content)
+                    assert subdir_file in content
+                    assert SUBDIR in content
+                    assert saltenv in content
+                else:
+                    assert not os.path.exists(cache_loc)
+
+
+def test_cache_dir_include_empty(mocked_opts, minion_opts, fs_root):
+    """
+    Ensure entire directory is cached to correct location, including empty ones
+    """
+    patched_opts = minion_opts.copy()
+    patched_opts.update(mocked_opts)
+
+    with patch.dict(fileclient.__opts__, patched_opts):
+        client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
+        for saltenv in _saltenvs():
+            # Make an empty directory
+            os.makedirs(os.path.join(fs_root, saltenv, SUBDIR, "emptydir"))
+            os.makedirs(os.path.join(fs_root, saltenv, SUBDIR, "emptydir2"))
+            os.makedirs(
+                os.path.join(
+                    fileclient.__opts__["cachedir"],
+                    "files",
+                    saltenv,
+                    SUBDIR,
+                    "emptydir2",
+                )
+            )
+            os.makedirs(os.path.join(fs_root, saltenv, "emptydir"))
+            assert client.cache_dir(
+                "salt://{}".format(SUBDIR), saltenv, cachedir=None, include_empty=True
+            )
             for subdir_file in _subdir_files():
                 cache_loc = os.path.join(
                     fileclient.__opts__["cachedir"],
@@ -120,20 +172,27 @@ def test_cache_dir(mocked_opts, minion_opts):
                     SUBDIR,
                     subdir_file,
                 )
-                # Double check that the content of the cached file
-                # identifies it as being from the correct saltenv. The
-                # setUp function creates the file with the name of the
-                # saltenv mentioned in the file, so a simple 'in' check is
-                # sufficient here. If opening the file raises an exception,
-                # this is a problem, so we are not catching the exception
-                # and letting it be raised so that the test fails.
                 with salt.utils.files.fopen(cache_loc) as fp_:
                     content = fp_.read()
                 log.debug("cache_loc = %s", cache_loc)
                 log.debug("content = %s", content)
                 assert subdir_file in content
                 assert SUBDIR in content
                 assert saltenv in content
+            assert os.path.exists(
+                os.path.join(
+                    fileclient.__opts__["cachedir"],
+                    "files",
+                    saltenv,
+                    SUBDIR,
+                    "emptydir",
+                )
+            )
+            assert not os.path.exists(
+                os.path.join(
+                    fileclient.__opts__["cachedir"], "files", saltenv, "emptydir"
+                )
+            )
 
 
 def test_cache_dir_with_alternate_cachedir_and_absolute_path(
@@ -237,6 +296,81 @@ def test_cache_file(mocked_opts, minion_opts):
             assert saltenv in content
 
 
+def test_cache_files(mocked_opts, minion_opts):
+    """
+    Test caching multiple files
+    """
+    patched_opts = minion_opts.copy()
+    patched_opts.update(mocked_opts)
+
+    with patch.dict(fileclient.__opts__, patched_opts):
+        client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
+        files = ["salt://foo.txt", f"salt://{SUBDIR}/bar.txt"]
+        for saltenv in _saltenvs():
+            assert client.cache_files(files, saltenv, cachedir=None)
+            for fn in files:
+                fn = fn[7:]
+                cache_loc = os.path.join(
+                    fileclient.__opts__["cachedir"], "files", saltenv, fn
+                )
+                with salt.utils.files.fopen(cache_loc) as fp_:
+                    content = fp_.read()
+                log.debug("cache_loc = %s", cache_loc)
+                log.debug("content = %s", content)
+                assert saltenv in content
+
+
+def test_cache_files_string(mocked_opts, minion_opts):
+    """
+    Test caching multiple files
+    """
+    patched_opts = minion_opts.copy()
+    patched_opts.update(mocked_opts)
+
+    with patch.dict(fileclient.__opts__, patched_opts):
+        client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
+        files = ["salt://foo.txt", f"salt://{SUBDIR}/bar.txt"]
+        for saltenv in _saltenvs():
+            assert client.cache_files(",".join(files), saltenv, cachedir=None)
+            for fn in files:
+                fn = fn[7:]
+                cache_loc = os.path.join(
+                    fileclient.__opts__["cachedir"], "files", saltenv, fn
+                )
+                with salt.utils.files.fopen(cache_loc) as fp_:
+                    content = fp_.read()
+                log.debug("cache_loc = %s", cache_loc)
+                log.debug("content = %s", content)
+                assert saltenv in content
+
+
+def test_cache_master(mocked_opts, minion_opts, fs_root):
+    """
+    Test caching multiple files
+    """
+    patched_opts = minion_opts.copy()
+    patched_opts.update(mocked_opts)
+
+    with patch.dict(fileclient.__opts__, patched_opts):
+        client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
+        for saltenv in _saltenvs():
+            assert client.cache_master(saltenv=saltenv)
+            for dirpath, _, filenames in os.walk(os.path.join(fs_root, saltenv)):
+                for fn in filenames:
+                    fileserver_path = os.path.relpath(
+                        os.path.join(dirpath, fn), fs_root
+                    )
+                    cachedir_path = os.path.join(
+                        fileclient.__opts__["cachedir"], "files", fileserver_path
+                    )
+                    assert os.path.exists(cachedir_path)
+                    with salt.utils.files.fopen(cachedir_path) as fp_:
+                        content = fp_.read()
+                    log.debug("cache_loc = %s", cachedir_path)
+                    log.debug("content = %s", content)
+                    assert saltenv in content
+
+
 def test_cache_file_with_alternate_cachedir_and_absolute_path(
     mocked_opts, minion_opts, tmp_path
 ):
@@ -335,4 +469,36 @@ def _check(ret, expected):
 
         _check(client.cache_dest(f"salt://{relpath}?saltenv=dev"), _salt("dev"))
 
-        _check("/foo/bar", "/foo/bar")
+        _check(client.cache_dest("/foo/bar"), "/foo/bar")
+
+
+def test_is_cached_salt_proto_saltenv(mocked_opts, minion_opts):
+    """
+    Test that is_cached works with a salt protocol filepath and a saltenv query param
+    """
+    patched_opts = minion_opts.copy()
+    patched_opts.update(mocked_opts)
+
+    with patch.dict(fileclient.__opts__, patched_opts):
+        client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
+        file_url = "salt://foo.txt?saltenv=dev"
+        client.cache_file(file_url)
+        assert client.is_cached(file_url)
+
+
+def test_is_cached_local_files(mocked_opts, minion_opts):
+    """
+    Test that is_cached works with local files
+    """
+    patched_opts = minion_opts.copy()
+    patched_opts.update(mocked_opts)
+
+    with patch.dict(fileclient.__opts__, patched_opts):
+        client = fileclient.get_file_client(fileclient.__opts__, pillar=False)
+        local_file = os.path.join(
+            patched_opts["cachedir"], "localfiles", "testfile.txt"
+        )
+        os.makedirs(os.path.join(patched_opts["cachedir"], "localfiles"), exist_ok=True)
+        with salt.utils.files.fopen(local_file, "w") as fp:
+            fp.write("test")
+        assert client.is_cached("testfile.txt")