Reorganize Path handling

This is a major incompatible change to the existing output of
Carthage, and to the API for organizing that output:

* New directory cache_dir, similar to output_dir, for things like
  stamps and processed mako files that can be recreated.

* state_dir is clearly documented for things that are hard to
  reproduce.

* New log_dir

* output_dir removed from config

* New PathMixin that provides stamp_path, state_path, and log_path
  based on stamp_subdir.

* SetupTaskMixin is a PathMixin

* Refactor to set stamp_subdir everywhere.

* ContainerVolumes and ImageVolumes no longer store their stamps
  within the volume. As a result, clear out stamps when volumes are
  created.

* To do that introduce clear_stamps_and_cache to PathMixin

carthage-resource-agent

@@ -89,7 +89,7 @@ async def start(config, machines):
     layout = await get_layout(config)
     ainjector = layout.ainjector
     config_layout = await ainjector(ConfigLayout)
-    async with file_locked(Path(config_layout.output_dir)/"generation_lock"):
+    async with file_locked(Path(config_layout.cache_dir)/"generation_lock"):
         await layout.generate()
     models = await get_machines(layout, machines)
     import carthage_base.hosted
@@ -118,7 +118,7 @@ async def stop(config, machines):
         return ocf.OCF_ERR_UNIMPLEMENTED
     ainjector = layout.ainjector
     config_layout = await ainjector(ConfigLayout)
-    async with file_locked(Path(config_layout.output_dir)/"generation_lock"):
+    async with file_locked(Path(config_layout.cache_dir)/"generation_lock"):
         await layout.resolve_networking()
     models = await get_machines(layout, machines)
     for m in reversed(models):

carthage/config/base.py

@@ -27,8 +27,9 @@ class BaseSchema(ConfigSchema, prefix=""):
 
     base_dir: ConfigPath = "~/.carthage"
     checkout_dir: ConfigPath = "{base_dir}/checkout"
-    output_dir: ConfigPath = "{base_dir}/output"
+    cache_dir:ConfigPath = '{base_dir}/cache'
     image_dir: ConfigPath = "{base_dir}"
+    log_dir:ConfigPath = '{base_dir}/log'
     vm_image_dir: ConfigPath = "{base_dir}/vm"
     state_dir: ConfigPath = "{base_dir}/state"
     #: Directory for local ephemeral state like ssh_agent sockets

carthage/container.py

@@ -80,11 +80,8 @@ async def is_machine_running(self):
         return self.running
 
     @memoproperty
-    def stamp_path(self):
-        if self.volume is None:
-            raise RuntimeError('Volume not yet created')
-        return self.volume.path
-
+    def stamp_subdir(self):
+        return f'nspawn/{self.name}'
     async def do_network_config(self, networking):
         if networking and self.network_links:
             namespace = carthage.network.NetworkNamespace(self.full_name, self.network_links)

carthage/image.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018, 2019, 2020, 2021, 2024, Hadron Industries, Inc.
+# Copyright (C) 2018, 2019, 2020, 2021, 2024, 2025, Hadron Industries, Inc.
 # Carthage is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License version 3
 # as published by the Free Software Foundation. It is distributed
@@ -126,17 +126,17 @@ def close(self, canceled_futures=None):
         super().close(canceled_futures)
 
 
-@inject(config_layout=ConfigLayout)
+
 class ContainerVolume(AsyncInjectable, SetupTaskMixin):
 
     def __init__(self, name, *,
                  clone_from=None,
                  implementation=None,
-                 config_layout,
                  **kwargs):
         super().__init__(**kwargs)
-        path = Path(config_layout.image_dir).joinpath(name)
+        path = Path(self.config_layout.image_dir).joinpath(name)
         os.makedirs(path.parent, exist_ok=True)
+        already_created = path.exists()
         if implementation is None:
             try:
                 sh.btrfs(
@@ -148,8 +148,10 @@ def __init__(self, name, *,
         self.impl = implementation(name=name,
                                    path=path,
                                    injector=self.injector,
-                                   config_layout=config_layout,
+                                   config_layout=self.config_layout,
                                    clone_from=clone_from)
+        if not already_created:
+            self.clear_stamps_and_cache()
 
     async def async_ready(self):
         await self.impl.async_ready()
@@ -169,15 +171,20 @@ def name(self):
         return self.impl.name
 
     @property
-    def config_layout(self): return self.impl.config_layout
+    def config_layout(self): 
+        return self._config_layout
 
     @config_layout.setter
     def config_layout(self, cfg):
-        self.impl.config_layout = cfg
+        self._config_layout = cfg
+        try:
+            self.impl.config_layout = cfg
+        except AttributeError: pass
         return cfg
 
     @property
-    def stamp_path(self): return self.impl.path
+    def stamp_subdir(self):
+        return f'container_volume/{str(self.impl.path).replace("/","_")}'
 
     def __repr__(self):
         return f"<Container {self.impl.__class__.__name__} path:{self.impl.path}>"
@@ -277,7 +284,7 @@ class ImageVolume(SetupTaskMixin, AsyncInjectable):
     :param base_image: Rather than creating a zero-filled image, create an image as a copy of this file or :class:`ImageVolume`.
 
     :param size: If the image is not at least this large (MiB), resize it to be that large.
-    
+
 
     '''
 
@@ -296,7 +303,6 @@ def __init__(self, name=None, directory=None, *,
         if name is None and not self.name:
             raise TypeError('name must be set on the constructor or subclass')
         super().__init__(**kwargs)
-        self.config_layout = self.injector(ConfigLayout)
         self.path = None
         if base_image:
             self.base_image = base_image
@@ -322,8 +328,8 @@ async def async_ready(self):
         await self.run_setup_tasks()
         if not self.path: await self.find()
         return await AsyncInjectable.async_ready(self)
-    
-        
+
+
     async def find(self):
         if self.path and self.path.exists():
             assert not self.creating_path.exists(), 'Within a single run find should not be called while do_create runs.'
@@ -371,6 +377,7 @@ async def do_create(self):
         assert self.path
         assert self.qemu_format
         self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.clear_stamps_and_cache()
         try:
             self.creating_path.touch()
             if base_image:= self.base_image:
@@ -485,23 +492,22 @@ async def find_or_create(self):
         if self.size:
             await self.resize(self.size)
 
-    @find_or_create.invalidator()
-    async def find_or_create(self, last_run):
+    @find_or_create.check_completed()
+    async def find_or_create(self):
         return await self.find()
-                            
-                
+
+
     def __repr__(self):
         if self.path:
             return f"<{self.__class__.__name__} path={self.path}>"
         return f"<{self.__class__.__name__} name={self.name}>"
-        
+
 
     def _delete_volume(self):
         try:
             os.unlink(self.path)
-            shutil.rmtree(self.stamp_path)
-        except FileNotFoundError:
-            pass
+        except FileNotFoundError: pass
+        self.clear_stamps_and_cache()
 
     async def delete(self):
         await self.find()
@@ -511,10 +517,10 @@ async def dynamic_dependencies(self):
         if self.base_image and isinstance(self.base_image, ImageVolume):
             return [self.base_image]
         return []
-    
+
     @property
-    def stamp_path(self):
-        return Path(str(self.path) + '.stamps')
+    def stamp_subdir(self):
+        return 'libvirt/'+str(self.path.relative_to('/'))
 
     def close(self, canceled_futures=None):
         if self.config_layout.delete_volumes:
@@ -577,13 +583,13 @@ def image_mounted(self):
                             else:
                                 raise
 
-                            
+
 
 
 class BlockVolume(ImageVolume):
 
     qemu_format = 'raw'
-    
+
     def __init__(self, path, **kwargs):
         if 'name' in kwargs:
             raise ValueError('BlockVolume does not take name even though ImageVolume does')
@@ -597,12 +603,6 @@ async def find(self):
 
     async def do_create(self):
         raise NotImplementedError('Cannot create block device')
-
-    @memoproperty
-    def stamp_path(self):
-        res = Path(self.config_layout.state_dir) / "block_volume_stamps" / str(self.path)[1:]
-        os.makedirs(res, exist_ok=True)
-        return res
 
     def qemu_config(self, disk_config):
         res = dict(

carthage/local.py

@@ -76,8 +76,8 @@ async def is_machine_running(self):
         return True
 
     @memoproperty
-    def stamp_path(self):
-        return Path(self.config_layout.state_dir + "/localhost")
+    def stamp_subdir(self):
+        return 'localhost_machine'
 
 
 def process_local_network_config(model):

carthage/machine.py

@@ -752,7 +752,7 @@ async def customization_context(self):
             pass
 
     @property
-    def stamp_path(self):
+    def stamp_subdir(self):
         return self.host.stamp_path
 
     def create_stamp(self, stamp, contents):
@@ -980,8 +980,8 @@ async def find(self):
         return True
 
     @memoproperty
-    def stamp_path(self):
-        return Path(f'{self.config_layout.state_dir}/machines/{self.name}')
+    def stamp_subdir(self):
+        return 'machines/'+self.name
 
 
 def disk_config_from_model(model, default_disk_config):

carthage/modeling/ansible.py

@@ -51,7 +51,7 @@ def __init__(self, **kwargs):
         self.injector.add_provider(
             InjectionKey(AnsibleInventory),
             when_needed(AnsibleInventory,
-                        destination=self.config_layout.output_dir + "/inventory.yml"))
+                        destination=self.config_layout.cache_dir + "/inventory.yml"))
         enable_modeling_ansible(self.injector)
 
     async def generate(self):

carthage/modeling/base.py

@@ -17,7 +17,7 @@
 from .decorators import *
 from carthage.dependency_injection import *  # type: ignore
 from carthage.utils import when_needed, memoproperty
-from carthage import ConfigLayout, SetupTaskMixin
+from carthage import ConfigLayout, SetupTaskMixin, PathMixin
 import carthage.kvstore
 import carthage.network
 import carthage.machine
@@ -348,7 +348,7 @@ class MachineModelMixin:
 @inject_autokwargs(
     config_layout=ConfigLayout,
                    )
-class MachineModel(ModelContainer, carthage.machine.AbstractMachineModel, metaclass=MachineModelType, template=True):
+class MachineModel(ModelContainer, carthage.machine.AbstractMachineModel, PathMixin, metaclass=MachineModelType, template=True):
 
     '''
 
@@ -469,10 +469,8 @@ def machine_type(self):
         return res
 
     @memoproperty
-    def stamp_path(self):
-        path = self.config_layout.output_dir + f"/hosts/{self.name}"
-        os.makedirs(path, exist_ok=True)
-        return Path(path)
+    def stamp_subdir(self):
+        return 'machine_model/'+self.name
 
     async def resolve_networking(self, *args, **kwargs):
         '''
@@ -651,9 +649,9 @@ def our_key(cls):
         return InjectionKey(ModelTasks, name=name)
 
     @memoproperty
-    def stamp_path(self):
+    def stamp_subdir(self):
         name = getattr(self.__class__, 'name', self.__class__.__name__)
-        return Path(self.config_layout.output_dir) / name
+        return name
 
 
 __all__ += ['ModelTasks']

carthage/pki.py

@@ -49,8 +49,8 @@ def ca_file(certificates, *,
     '''
     assert name, 'Currently support for name=None is unimplemented; probably the right approach is to hash the certificates.'
     config = injector(ConfigLayout)
-    output_dir = pathlib.Path(config.output_dir)
-    truststores = output_dir/'truststores'
+    cache_dir = pathlib.Path(config.cache_dir)
+    truststores = cache_dir/'truststores'
     truststores.mkdir(parents=True, exist_ok=True)
     name = name.replace('/', '_')
     ca_path = truststores/name

carthage/podman/base.py

@@ -486,12 +486,8 @@ def __repr__(self):
         return f'<{self.__class__.__name__} {self.name} on {host}>'
 
     @memoproperty
-    def stamp_path(self):
-        state_dir = Path(self.config_layout.state_dir)
-        result = state_dir.joinpath("podman", self.name)
-        result.mkdir(exist_ok=True, parents=True)
-        return result
-
+    def stamp_subdir(self):
+        return 'podman/'+self.name
 
     def check_stamp(self, stamp, raise_on_error=False):
         mtime, text = super().check_stamp(stamp, raise_on_error)
@@ -710,12 +706,8 @@ def podman(self):
 
 
     @memoproperty
-    def stamp_path(self):
-        config = self.injector(ConfigLayout)
-        path = Path(config.output_dir)/"podman_image"/self.oci_image_tag.replace('/','_')
-        path.mkdir(exist_ok=True, parents=True)
-        return path
-
+    def stamp_subdir(self):
+        return 'podman_image/'+self.oci_image_tag
 
 
 __all__ += ['PodmanImage']
@@ -819,21 +811,19 @@ def __init__(self, container_context=None, **kwargs):
         if len(self.setup_tasks) > 2:
             # More than just find_or_create and copy_context_if_needed
             self.setup_tasks.sort(key=lambda t: 1 if t.func == OciManaged.find_or_create.func else 0)
-            self.container_context = self.output_path
-        self.injector.add_provider(InjectionKey("podman_log"), self.stamp_path/'podman.log')
+            self.container_context = self.stamp_path
+        self.injector.add_provider(InjectionKey("podman_log"), self.log_path/'podman.log')
 
 
 
 
     @memoproperty
     def output_path(self):
-        path = Path(self.config_layout.output_dir)/'podman_image'
-        tag = self.oci_image_tag.replace('/', '_')
-        path /= tag
-        path.mkdir(exist_ok=True, parents=True)
-        return path
+        return self.stamp_path
 
-    stamp_path = output_path
+    @property
+    def stamp_subdir(self):
+        return 'podman_image/'+self.oci_image_tag
 
     @setup_task("Copy Context if Needed", order=10)
     async def copy_context_if_needed(self):
@@ -952,11 +942,8 @@ async def delete(self):
             self.name)
 
     @memoproperty
-    def stamp_path(self):
-        config = self.injector(ConfigLayout)
-        path =  Path(config.output_dir)/'volumes'/self.name
-        path.mkdir(exist_ok=True, parents=True)
-        return path
+    def stamp_subdir(self):
+        return 'podman_volume/'+self.name
 
     @property
     def podman(self):