Add Support for Config Inheritance

docs/Configuration.md

@@ -753,3 +753,85 @@ use_legacy_include_asm: True
 
 #### Default
 `False`
+
+# Project Organization Configuration
+
+## Inheritance
+
+For projects with several similar overlays, splat supports configuration inheritance. Shared options and segments can be
+defined in a configuration file and then extended in another configuration file using the top level `parent` key.
+Parents are resolved recursively, allowing projects to be organized using a flexible configuration tree.
+
+Using the parent key is similar to specifying multiple configuration files at the command line when running `split`, but
+can be modeled as configuration rather than in the build system.
+
+#### Usage
+psx.yaml:
+```yaml
+options:
+  platform:     psx
+  base_path:    ..
+  compiler:     GCC
+  symbol_addrs_path:
+    - config/symbols.txt
+  asm_jtbl_label_macro: jlabel
+  extensions_path: tools/splat_ext
+  section_order:
+    - ".data"
+    - ".rodata"
+    - ".text"
+    - ".bss"
+    - ".sbss"
+```
+overlay.yaml:
+```yaml
+parent: psx.yaml
+options:
+  basename:     overlay
+  target_path:  disks/OVERLAY.BIN
+  asm_path:     asm/overlay
+  asset_path:   assets/overlay
+  src_path:     src/overlay
+  ld_script_path: build/overlay.ld
+  symbol_addrs_path:
+    - config/symbols.overlay.txt
+segments:
+  - name: overlay
+    type: code
+    start: 0x00000000
+    vram:  0x80000000
+    align: 4
+    subalign: 4
+```
+
+This produces a merged config equivalent to:
+```yaml
+options:
+  platform:     psx
+  base_path:    ..
+  compiler:     GCC
+  symbol_addrs_path:
+    - config/symbols.txt
+    - config/symbols.overlay.txt
+  asm_jtbl_label_macro: jlabel
+  extensions_path: tools/splat_ext
+  section_order:
+    - ".data"
+    - ".rodata"
+    - ".text"
+    - ".bss"
+    - ".sbss"
+  basename:     overlay
+  target_path:  disks/OVERLAY.BIN
+  asm_path:     asm/overlay
+  asset_path:   assets/overlay
+  src_path:     src/overlay
+  ld_script_path: build/overlay.ld
+segments:
+  - name: overlay
+    type: code
+    start: 0x00000000
+    vram:  0x80000000
+    align: 4
+    subalign: 4
+```

src/splat/scripts/split.py

@@ -8,11 +8,7 @@
 
 from .. import __package_name__, __version__
 from ..disassembler import disassembler_instance
-from ..util import cache_handler, progress_bar, vram_classes, statistics
-
-# This unused import makes the yaml library faster. don't remove
-import pylibyaml  # pyright: ignore
-import yaml
+from ..util import cache_handler, conf, progress_bar, vram_classes, statistics
 
 from colorama import Fore, Style
 from intervaltree import Interval, IntervalTree
@@ -142,34 +138,6 @@ def assign_symbols_to_segments():
                     seg.add_symbol(symbol)
 
 
-def merge_configs(main_config, additional_config):
-    # Merge rules are simple
-    # For each key in the dictionary
-    # - If list then append to list
-    # - If a dictionary then repeat merge on sub dictionary entries
-    # - Else assume string or number and replace entry
-
-    for curkey in additional_config:
-        if curkey not in main_config:
-            main_config[curkey] = additional_config[curkey]
-        elif type(main_config[curkey]) != type(additional_config[curkey]):
-            log.error(f"Type for key {curkey} in configs does not match")
-        else:
-            # keys exist and match, see if a list to append
-            if type(main_config[curkey]) == list:
-                main_config[curkey] += additional_config[curkey]
-            elif type(main_config[curkey]) == dict:
-                # need to merge sub areas
-                main_config[curkey] = merge_configs(
-                    main_config[curkey], additional_config[curkey]
-                )
-            else:
-                # not a list or dictionary, must be a number or string, overwrite
-                main_config[curkey] = additional_config[curkey]
-
-    return main_config
-
-
 def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str:
     s = seg.name.strip()
     if len(s) > limit:
@@ -203,25 +171,6 @@ def calc_segment_dependences(
     return vram_class_to_follows_segments
 
 
-def initialize_config(
-    config_path: List[str],
-    modes: Optional[List[str]],
-    verbose: bool,
-    disassemble_all: bool = False,
-) -> Dict[str, Any]:
-    config: Dict[str, Any] = {}
-    for entry in config_path:
-        with open(entry) as f:
-            additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
-        config = merge_configs(config, additional_config)
-
-    vram_classes.initialize(config.get("vram_classes"))
-
-    options.initialize(config, config_path, modes, verbose, disassemble_all)
-
-    return config
-
-
 def read_target_binary() -> bytes:
     rom_bytes = options.opts.target_path.read_bytes()
 
@@ -500,13 +449,14 @@ def main(
     skip_version_check: bool = False,
     stdout_only: bool = False,
     disassemble_all: bool = False,
+    include_path: List[Path] = [],
 ):
     if stdout_only:
         progress_bar.out_file = sys.stdout
 
     # Load config
     global config
-    config = initialize_config(config_path, modes, verbose, disassemble_all)
+    config = conf.initialize(config_path, include_path, modes, verbose, disassemble_all)
 
     disassembler_instance.create_disassembler_instance(skip_version_check, __version__)
 
@@ -585,6 +535,13 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser):
         help="Disasemble matched functions and migrated data",
         action="store_true",
     )
+    parser.add_argument(
+        "-I",
+        "--include-directory",
+        help="Add the directory to the list of search directories when including other config",
+        action="append",
+        type=Path,
+    )
 
 
 def process_arguments(args: argparse.Namespace):
@@ -596,6 +553,7 @@ def process_arguments(args: argparse.Namespace):
         args.skip_version_check,
         args.stdout_only,
         args.disassemble_all,
+        args.include_directory,
     )
 
 

src/splat/segtypes/segment.py

@@ -255,7 +255,7 @@ def parse_ld_align_segment_start(yaml: Union[dict, list]) -> Optional[int]:
 
     @staticmethod
     def parse_suggestion_rodata_section_start(
-        yaml: Union[dict, list]
+        yaml: Union[dict, list],
     ) -> Optional[bool]:
         if isinstance(yaml, dict):
             suggestion_rodata_section_start = yaml.get(

src/splat/util/conf.py

@@ -0,0 +1,110 @@
+"""
+This module is used to load splat configuration from a YAML file.
+
+A config dict can be loaded using `initialize`.
+
+    config = conf.initialize("path/to/splat.yaml")
+"""
+
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from pathlib import Path
+
+# This unused import makes the yaml library faster. don't remove
+import pylibyaml  # pyright: ignore
+import yaml
+
+import sys
+
+from . import log, options, vram_classes
+
+
+def _merge_configs(main_config, additional_config):
+    # Merge rules are simple
+    # For each key in the dictionary
+    # - If list then append to list
+    # - If a dictionary then repeat merge on sub dictionary entries
+    # - Else assume string or number and replace entry
+
+    for curkey in additional_config:
+        if curkey not in main_config:
+            main_config[curkey] = additional_config[curkey]
+        elif type(main_config[curkey]) != type(additional_config[curkey]):
+            log.error(f"Type for key {curkey} in configs does not match")
+        else:
+            # keys exist and match, see if a list to append
+            if type(main_config[curkey]) == list:
+                main_config[curkey] += additional_config[curkey]
+            elif type(main_config[curkey]) == dict:
+                # need to merge sub areas
+                main_config[curkey] = _merge_configs(
+                    main_config[curkey], additional_config[curkey]
+                )
+            else:
+                # not a list or dictionary, must be a number or string, overwrite
+                main_config[curkey] = additional_config[curkey]
+
+    return main_config
+
+
+def _resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path:
+    if (base / rel).exists():
+        return base / rel
+
+    for path in include_paths:
+        candidate = path / rel
+        if candidate.exists():
+            return candidate
+    log.error(f'"{rel}" not found')
+
+
+def _load_config(config_path: Path, include_path: List[Path]) -> Dict[str, Any]:
+    base_path = Path(config_path).parent
+    with open(config_path) as f:
+        config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+    if "parent" in config:
+        parent_path = _resolve_path(base_path, Path(config["parent"]), include_path)
+        parent = _load_config(parent_path, include_path)
+        config = _merge_configs(parent, config)
+        del config["parent"]
+
+    return config
+
+
+def initialize(
+    config_path: List[str],
+    include_path: List[Path] = [],
+    modes: Optional[List[str]] = None,
+    verbose: bool = False,
+    disassemble_all: bool = False,
+) -> Dict[str, Any]:
+    """
+    Returns a `dict` with resolved splat config.
+
+    Multiple configuration files can be passed in ``config_path`` with each
+    subsequent file merged into the previous. `parent` keys are resolved
+    prior to merging multiple files.
+
+    `include_path` can include any additional paths which should be searched for relative parent config files. Paths are
+    relative to the file being evaluated (i.e. a child config file).
+
+    `modes` specifies which modes are active (all, code, img, gfx, vtx, etc.). The default is all.
+
+    `verbose` may be used to determine whether or not to display additional output.
+
+    `disassemble_all` determines whether functions which are already compiled will be disassembled.
+
+    After being merged, static validation is done on the configuration.
+
+    The returned `dict` represents the merged and validated YAML.
+    """
+
+    config: Dict[str, Any] = {}
+    for entry in config_path:
+        additional_config = _load_config(Path(entry), include_path)
+        config = _merge_configs(config, additional_config)
+
+    vram_classes.initialize(config.get("vram_classes"))
+
+    options.initialize(config, config_path, modes, verbose, disassemble_all)
+
+    return config

test/basic_app/n64.yaml

@@ -0,0 +1,17 @@
+options:
+  platform:     n64
+  compiler:     GCC
+  base_path:    .
+  build_path:   build
+  asm_path:     split/asm
+  src_path:     split/src
+  cache_path:   split/.splache
+  asset_path: split/assets
+  compiler: GCC
+  symbol_addrs_path:
+    - config/symbols.txt
+  o_as_suffix: True
+segments:
+  - name: header
+    type: header
+    start: 0x00

test/basic_app/splat.yaml

@@ -1,24 +1,14 @@
+parent: n64.yaml
 options:
-  platform:     n64
-  compiler:     GCC
   basename:     basic_app
-  base_path:    .
-  build_path:   build
   target_path:  build/basic_app.bin
-  asm_path:     split/asm
-  src_path:     split/src
   ld_script_path: split/basic_app.ld
-  cache_path:     split/.splache
   symbol_addrs_path:    split/generated.symbols.txt
   undefined_funcs_auto_path: split/undefined_funcs_auto.txt
   undefined_syms_auto_path:  split/undefined_syms_auto.txt
-  asset_path: split/assets
-  compiler: GCC
-  o_as_suffix: True
+  symbol_addrs_path:
+    - config/symbols.splat.txt
 segments:
-  - name: header
-    type: header
-    start: 0x00
   - name: dummy_ipl3
     type: code
     start: 0x40