Add tests for habcache

hab/cache.py

@@ -16,7 +16,7 @@ class Cache:
 
     The caches are stored per-site file as file next to the site file using the
     same stem name. (Ie by default studio.json would have a cache file called
-    studio.cache).
+    studio.habcache).
 
     If this cache file exists it is used unless enabled is set to False. Cache
     files are useful when you have some sort of CI setup to ensure the cache is
@@ -25,7 +25,8 @@ class Cache:
 
     Properties:
         cache_template (dict): The str.format template used to find the cache files.
-            This template requires the kwarg `stem`.
+            This template requires the kwarg `stem`. This template can be overridden
+            by the first `site_cache_file_template` property in site.
         enabled (bool): Used to disable using of the cached data forcing a full
             glob and parse of files described by all site files.
     """
@@ -36,7 +37,7 @@ def __init__(self, site):
         self.enabled = True
 
         # Get the template filename used to find the cache files on disk
-        self.cache_template = self.site.get("site_cache_file", "{stem}.cache")
+        self.cache_template = self.site["site_cache_file_template"][0]
 
     @property
     def cached_keys(self):
@@ -84,6 +85,12 @@ def cache(self, force=False):
 
         return self._cache
 
+    def clear(self):
+        """Reset the cache forcing it to reload the next time its used."""
+        if self._cache:
+            logger.debug("Site cache contents cleared")
+        self._cache = None
+
     def config_paths(self, flat=False):
         if flat:
             return self.cache().get("flat", {}).get("config_paths", {})
@@ -108,13 +115,15 @@ def generate_cache(self, resolver, site_file, version=1):
 
         # read the site file to get paths to process
         temp_site = Site([site_file])
+        # Use this to convert platform specific paths to generic variables
+        platform_path_key = resolver.site.platform_path_key
 
         for key, stats in self.cached_keys.items():
             glob_str, cls = stats
             # Process each glob dir defined for this site
             for dirname in temp_site.get(key, []):
                 cfg_paths = output.setdefault(key, {}).setdefault(
-                    dirname.as_posix(), {}
+                    platform_path_key(dirname).as_posix(), {}
                 )
 
                 # Add each found hab config to the cache
@@ -131,7 +140,7 @@ def generate_cache(self, resolver, site_file, version=1):
                     ) as error:
                         logger.debug(str(error))
                     else:
-                        cfg_paths[path.as_posix()] = data
+                        cfg_paths[platform_path_key(path).as_posix()] = data
 
         return output
 
@@ -163,15 +172,37 @@ def iter_cache_paths(cls, name, paths, cache, glob_str=None, include_path=True):
                 for path in paths:
                     yield dirname, path, cached
 
-    def load_cache(self, filename):
+    def load_cache(self, filename, platform=None):
         """For each glob dir add or replace the contents. If a previous cache
         has the same glob dir, it's cache is ignored. This expects that
         load_cache is called from right to left for each path in `self.site.path`.
         """
+
+        def cache_to_platform(cache, mappings):
+            """Restore the cross-platform variables to current platform paths."""
+            ret = {}
+            for glob_str, files in cache.items():
+                new_glob = glob_str.format(**mappings)
+                new_glob = Path(new_glob).as_posix()
+                new_files = {}
+                for key, value in files.items():
+                    new_key = key.format(**mappings)
+                    new_key = Path(new_key).as_posix()
+                    new_files[new_key] = value
+                ret[new_glob] = new_files
+
+            return ret
+
+        if platform is None:
+            platform = utils.Platform.name()
+
         contents = utils.load_json_file(filename)
+        mappings = self.site.get("platform_path_maps", {})
+        mappings = {key: value[platform] for key, value in mappings.items()}
         for key in self.cached_keys:
             if key in contents:
-                self._cache.setdefault(key, {}).update(contents[key])
+                cache = cache_to_platform(contents[key], mappings)
+                self._cache.setdefault(key, {}).update(cache)
 
     def save_cache(self, resolver, site_file, version=1):
         cache_file = self.site_cache_path(site_file)

hab/resolver.py

@@ -74,6 +74,7 @@ def clear_caches(self):
         logger.debug("Resolver cache cleared.")
         self._configs = None
         self._distros = None
+        self.site.cache.clear()
 
     def closest_config(self, path, default=False):
         """Returns the most specific leaf or the tree root matching path. Ignoring any

hab/site.py

@@ -31,6 +31,7 @@ class Site(UserDict):
             "distro_paths": [],
             "ignored_distros": ["release", "pre"],
             "platforms": ["windows", "osx", "linux"],
+            "site_cache_file_template": ["{{stem}}.habcache"],
         }
     }
 
@@ -48,20 +49,22 @@ def __init__(self, paths=None, platform=None):
             paths = os.getenv("HAB_PATHS", "").split(os.pathsep)
         self.paths = [Path(os.path.expandvars(p)).expanduser() for p in paths if p]
 
-        self.cache = Cache(self)
         self.load()
+        self.cache = Cache(self)
 
     @property
     def data(self):
         return self.frozen_data.get(self.platform)
 
-    def dump(self, verbosity=0, color=None):
+    def dump(self, verbosity=0, color=None, width=80):
         """Return a string of the properties and their values.
 
         Args:
             verbosity (int, optional): More information is shown with higher values.
             color (bool, optional): Add console colorization to output. If None,
                 respect the site property "colorize" defaulting to True.
+            width (int, optional): The desired width for wrapping. The output may
+                exceed this value, but it will attempt to respect it.
 
         Returns:
             str: The configuration converted to a string
@@ -85,7 +88,7 @@ def cached_fmt(path, cached):
                 cache_file = self.cache.site_cache_path(path)
                 path = cached_fmt(path, cache_file.is_file())
             hab_paths.append(str(path))
-        site_ret = utils.dump_object({"HAB_PATHS": hab_paths}, color=color)
+        site_ret = utils.dump_object({"HAB_PATHS": hab_paths}, color=color, width=width)
         # Include all of the resolved site configurations
         ret = []
         for prop, value in self.items():
@@ -96,15 +99,22 @@ def cached_fmt(path, cached):
                     prop, value, cache, include_path=False
                 ):
                     paths.append(cached_fmt(dirname, cached))
-                txt = utils.dump_object(paths, label=f"{prop}:  ", color=color)
+                txt = utils.dump_object(
+                    paths, label=f"{prop}:  ", color=color, width=width
+                )
             elif verbosity < 1 and isinstance(value, dict):
                 # This is too complex for most site dumps, hide the details behind
                 # a higher verbosity setting.
                 txt = utils.dump_object(
-                    f"Dictionary keys: {len(value)}", label=f"{prop}:  ", color=color
+                    f"Dictionary keys: {len(value)}",
+                    label=f"{prop}:  ",
+                    color=color,
+                    width=width,
                 )
             else:
-                txt = utils.dump_object(value, label=f"{prop}:  ", color=color)
+                txt = utils.dump_object(
+                    value, label=f"{prop}:  ", color=color, width=width
+                )
 
             ret.append(txt)
 
@@ -216,6 +226,34 @@ def paths(self):
     def paths(self, paths):
         self._paths = paths
 
+    def platform_path_key(self, path, platform=None):
+        """Converts the provided full path to a str.format style path.
+
+        Uses mappings defined in `site.get('platform_path_maps', {})` to convert
+        full file paths to the map key name.
+        """
+        if self.platform == "windows":
+            path = PureWindowsPath(path)
+        else:
+            path = PurePosixPath(path)
+
+        platform = utils.Platform.name()
+        mappings = self.get("platform_path_maps", {})
+
+        for key in mappings:
+            m = mappings[key][platform]
+            try:
+                relative = path.relative_to(m)
+            except ValueError:
+                relative = ""
+            is_relative = bool(relative)
+            if is_relative:
+                # platform_path_maps only replace the start of a file path so
+                # there is no need to continue checking other mappings
+                relative = Path(f"{{{key}}}").joinpath(relative)
+                return relative
+        return path
+
     def platform_path_map(self, path, platform=None):
         """Convert the provided path to one valid for the platform.
 

tests/conftest.py

@@ -1,3 +1,4 @@
+import json
 import os
 from contextlib import contextmanager
 from pathlib import Path, PurePath
@@ -7,19 +8,109 @@
 
 from hab import Resolver, Site
 
+# Testing both cached and uncached every time adds extra testing time. This env
+# var can be used to disable cached testing for local testing.
+if os.getenv("HAB_TEST_UNCACHED_ONLY", "0") == "1":
+    resolver_tests = ["uncached"]
+else:
+    resolver_tests = ["uncached", "cached"]
 
-@pytest.fixture
+
+@pytest.fixture(scope="session")
 def config_root():
     return Path(__file__).parent
 
 
+def generate_habcached_site_file(config_root, dest):
+    """Returns the path to a site config file generated from `site_main.json`
+    configured so it can have a .habcache file generated next to it. The
+    config_paths and distro_paths of the site file are hard coded to point to
+    the repo's tests directory so it uses the same configs/distros. It also adds
+    a `config-root` entry to `platform_path_maps`.
+    """
+    site_file = Path(dest) / "site.json"
+    site_src = config_root / "site_main.json"
+
+    # Load the site_main.json files contents so we can modify it before saving
+    # it into the dest for testing.
+    data = json.load(site_src.open())
+    append = data["append"]
+
+    # Hard code relative_root to the tests folder so it works from
+    # a random testing directory without copying all configs/distros.
+    for key in ("config_paths", "distro_paths"):
+        for i in range(len(append[key])):
+            append[key][i] = append[key][i].format(relative_root=site_src.parent)
+
+    # Add platform_path_maps for the pytest directory to simplify testing and
+    # test cross-platform support. We need to add all platforms, but this only
+    # needs to run on the current platform, so add the same path to all.
+    append["platform_path_maps"]["config-root"] = {
+        platform: str(config_root) for platform in data["set"]["platforms"]
+    }
+
+    with site_file.open("w") as fle:
+        json.dump(data, fle, indent=4)
+
+    return site_file
+
+
+@pytest.fixture(scope="session")
+def habcached_site_file(config_root, tmp_path_factory):
+    """Generates a site.json file and generates its habcache file.
+    This file is stored in a `_cache` directory in the pytest directory.
+    This persists for the entire testing session and can be used by other tests
+    that need to test hab when it is using a habcache.
+    """
+    # Create the site file
+    shared = tmp_path_factory.mktemp("_cache")
+    ret = generate_habcached_site_file(config_root, shared)
+
+    # Generate the habcache file
+    site = Site([ret])
+    resolver = Resolver(site)
+    site.cache.save_cache(resolver, ret)
+
+    return ret
+
+
+@pytest.fixture
+def habcached_resolver(habcached_site_file):
+    """Returns a Resolver using a habcache file that was generated for this session.
+
+    See the `habcached_site_file` fixture for details on how the cache is setup.
+    For ease of testing the path to the saved habcache file is stored in
+    `_test_cache_file` on the returned resolver.
+    """
+    site = Site([habcached_site_file])
+    resolver = Resolver(site)
+    # Generate the cache and provide easy access to the habcache file path
+    resolver._test_cache_file = site.cache.site_cache_path(habcached_site_file)
+
+    return resolver
+
+
 @pytest.fixture
-def resolver(config_root):
-    """Return a standard testing resolver"""
+def uncached_resolver(config_root):
+    """Return a standard testing resolver not using any habcache files."""
     site = Site([config_root / "site_main.json"])
     return Resolver(site=site)
 
 
+@pytest.fixture(params=resolver_tests)
+def resolver(request):
+    """Returns a hab.Resolver instance using the site_main.json site config file.
+
+    This is a parameterized fixture that returns both cached and uncached versions
+    of the `site_main.json` site configuration. Note the cached version uses a
+    copy of it stored in the `_cache0` directory of the pytest temp files. This
+    should be used for most tests to ensure that all features are tested, but if
+    the test is not affected by caching you can use `uncached_resolver` instead.
+    """
+    test_map = {"uncached": "uncached_resolver", "cached": "habcached_resolver"}
+    return request.getfixturevalue(test_map[request.param])
+
+
 class Helpers(object):
     """A collection of reusable functions that tests can use."""
 

tests/distros/the_dcc/pre/.hab.json

@@ -0,0 +1,14 @@
+{
+    "name": "the_dcc",
+    "environment": {},
+    "distros": [
+        "the_dcc_plugin_a>=1.0",
+        "the_dcc_plugin_b>=0.9",
+        "the_dcc_plugin_e"
+    ],
+    "aliases": {
+        "windows": [
+            ["dcc", "{relative_root}\\the_dcc.exe"]
+        ]
+    }
+}

tests/distros/the_dcc/pre/README.md

@@ -0,0 +1,3 @@
+This directory enables testing that hab ignores distro version folders defined
+in the `ignored_distros` key of the site configuration. It's version is pulled
+from the parent directory of the .hab.json file, which is named `pre`.

tests/site/site_cache_file.json

@@ -0,0 +1,5 @@
+{
+    "set": {
+        "site_cache_file_template": ".{{stem}}.hab_cache"
+    }
+}