From 98220e2d18f9e5711db35078762619768330db0f Mon Sep 17 00:00:00 2001 From: Jonathan Hohle Date: Wed, 5 Feb 2025 11:53:10 -0700 Subject: [PATCH 1/5] Add Support for Config Inheritance Adds support for a top-level `parent` key in configuration files which will merge the contents of that path with the contents of the current file. The merge rules are the same as specifying multiple configuration files at the command line with the "parent" config being loaded first. This also moves configuration loading out into the `splat.conf` module so configuration loading can be used as a library by other projects. --- docs/Configuration.md | 82 +++++++++++++++++++++++++++++++ src/splat/scripts/split.py | 63 ++++-------------------- src/splat/util/conf.py | 80 ++++++++++++++++++++++++++++++ test/basic_app/expected/.splache | Bin 847 -> 869 bytes test/basic_app/n64.yaml | 17 +++++++ test/basic_app/splat.yaml | 16 ++---- 6 files changed, 192 insertions(+), 66 deletions(-) create mode 100644 src/splat/util/conf.py create mode 100644 test/basic_app/n64.yaml diff --git a/docs/Configuration.md b/docs/Configuration.md index 19e4009b..4b64dab3 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -753,3 +753,85 @@ use_legacy_include_asm: True #### Default `False` + +# Project Organization Configuration + +## Inheritance + +For projects with several similar overlays, Splat supports configuration inheritance. Shared options and segments can be +defined in a configuration file and then extended in another configuration file using the top level `parent` key. +Parents are resolved recursively, allowing projects to be organized using a flexible configuration tree. + +Using the parent key is similar to specifying multiple configuration files at the command line when running `split`, but +can be modeled as configuration rather than in the build system. + +#### Usage +psx.yaml: +```yaml +options: + platform: psx + base_path: .. + compiler: GCC + symbol_addrs_path: + - config/symbols.txt + asm_jtbl_label_macro: jlabel + extensions_path: tools/splat_ext + section_order: + - ".data" + - ".rodata" + - ".text" + - ".bss" + - ".sbss" +``` +overlay.yaml: +```yaml +parent: psx.yaml +options: + basename: overlay + target_path: disks/OVERLAY.BIN + asm_path: asm/overlay + asset_path: assets/overlay + src_path: src/overlay + ld_script_path: build/overlay.ld + symbol_addrs_path: + - config/symbols.overlay.txt +segments: + - name: overlay + type: code + start: 0x00000000 + vram: 0x80000000 + align: 4 + subalign: 4 +``` + +This produces a merged config equivalent to: +```yaml +options: + platform: psx + base_path: .. + compiler: GCC + symbol_addrs_path: + - config/symbols.txt + - config/symbols.overlay.txt + asm_jtbl_label_macro: jlabel + extensions_path: tools/splat_ext + section_order: + - ".data" + - ".rodata" + - ".text" + - ".bss" + - ".sbss" + basename: overlay + target_path: disks/OVERLAY.BIN + asm_path: asm/overlay + asset_path: assets/overlay + src_path: src/overlay + ld_script_path: build/overlay.ld +segments: + - name: overlay + type: code + start: 0x00000000 + vram: 0x80000000 + align: 4 + subalign: 4 +``` diff --git a/src/splat/scripts/split.py b/src/splat/scripts/split.py index 33f7388c..bf21b778 100644 --- a/src/splat/scripts/split.py +++ b/src/splat/scripts/split.py @@ -8,11 +8,7 @@ from .. import __package_name__, __version__ from ..disassembler import disassembler_instance -from ..util import cache_handler, progress_bar, vram_classes, statistics - -# This unused import makes the yaml library faster. don't remove -import pylibyaml # pyright: ignore -import yaml +from ..util import cache_handler, conf, progress_bar, vram_classes, statistics from colorama import Fore, Style from intervaltree import Interval, IntervalTree @@ -142,34 +138,6 @@ def assign_symbols_to_segments(): seg.add_symbol(symbol) -def merge_configs(main_config, additional_config): - # Merge rules are simple - # For each key in the dictionary - # - If list then append to list - # - If a dictionary then repeat merge on sub dictionary entries - # - Else assume string or number and replace entry - - for curkey in additional_config: - if curkey not in main_config: - main_config[curkey] = additional_config[curkey] - elif type(main_config[curkey]) != type(additional_config[curkey]): - log.error(f"Type for key {curkey} in configs does not match") - else: - # keys exist and match, see if a list to append - if type(main_config[curkey]) == list: - main_config[curkey] += additional_config[curkey] - elif type(main_config[curkey]) == dict: - # need to merge sub areas - main_config[curkey] = merge_configs( - main_config[curkey], additional_config[curkey] - ) - else: - # not a list or dictionary, must be a number or string, overwrite - main_config[curkey] = additional_config[curkey] - - return main_config - - def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str: s = seg.name.strip() if len(s) > limit: @@ -203,25 +171,6 @@ def calc_segment_dependences( return vram_class_to_follows_segments -def initialize_config( - config_path: List[str], - modes: Optional[List[str]], - verbose: bool, - disassemble_all: bool = False, -) -> Dict[str, Any]: - config: Dict[str, Any] = {} - for entry in config_path: - with open(entry) as f: - additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader) - config = merge_configs(config, additional_config) - - vram_classes.initialize(config.get("vram_classes")) - - options.initialize(config, config_path, modes, verbose, disassemble_all) - - return config - - def read_target_binary() -> bytes: rom_bytes = options.opts.target_path.read_bytes() @@ -500,13 +449,14 @@ def main( skip_version_check: bool = False, stdout_only: bool = False, disassemble_all: bool = False, + include_path: List[str] = [] ): if stdout_only: progress_bar.out_file = sys.stdout # Load config global config - config = initialize_config(config_path, modes, verbose, disassemble_all) + config = conf.initialize(config_path, include_path, modes, verbose, disassemble_all) disassembler_instance.create_disassembler_instance(skip_version_check, __version__) @@ -585,6 +535,12 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser): help="Disasemble matched functions and migrated data", action="store_true", ) + parser.add_argument( + "-I", + help="Add the directory to the list of search directories when including other config", + action="append", + type=Path + ) def process_arguments(args: argparse.Namespace): @@ -596,6 +552,7 @@ def process_arguments(args: argparse.Namespace): args.skip_version_check, args.stdout_only, args.disassemble_all, + args.I ) diff --git a/src/splat/util/conf.py b/src/splat/util/conf.py new file mode 100644 index 00000000..9a431fd3 --- /dev/null +++ b/src/splat/util/conf.py @@ -0,0 +1,80 @@ +from typing import Any, Dict, List, Optional, Set, Tuple, Union +from pathlib import Path + +# This unused import makes the yaml library faster. don't remove +import pylibyaml # pyright: ignore +import yaml + +import sys + +from . import log, options, vram_classes + +def merge_configs(main_config, additional_config): + # Merge rules are simple + # For each key in the dictionary + # - If list then append to list + # - If a dictionary then repeat merge on sub dictionary entries + # - Else assume string or number and replace entry + + for curkey in additional_config: + if curkey not in main_config: + main_config[curkey] = additional_config[curkey] + elif type(main_config[curkey]) != type(additional_config[curkey]): + log.error(f"Type for key {curkey} in configs does not match") + else: + # keys exist and match, see if a list to append + if type(main_config[curkey]) == list: + main_config[curkey] += additional_config[curkey] + elif type(main_config[curkey]) == dict: + # need to merge sub areas + main_config[curkey] = merge_configs( + main_config[curkey], additional_config[curkey] + ) + else: + # not a list or dictionary, must be a number or string, overwrite + main_config[curkey] = additional_config[curkey] + + return main_config + +def resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path: + if (base / rel).exists(): + return (base / rel) + + for path in include_paths: + candidate = path / rel + if candidate.exists(): + return candidate + log.error(f"\"{rel}\" not found") + return None + + +def load_config(config_path: str, include_path: List[Path]) -> Dict[str, Any]: + base_path = Path(config_path).parent + with open(config_path) as f: + config = yaml.load(f.read(), Loader=yaml.SafeLoader) + if "parent" in config: + parent_path = resolve_path(base_path, Path(config["parent"]), include_path) + parent = load_config(parent_path, include_path) + config = merge_configs(parent, config) + del config["parent"] + + return config + +def initialize( + config_path: List[str], + include_path: List[Path], + modes: Optional[List[str]], + verbose: bool, + disassemble_all: bool = False, +) -> Dict[str, Any]: + config: Dict[str, Any] = {} + for entry in config_path: + additional_config = load_config(entry, include_path) + config = merge_configs(config, additional_config) + + vram_classes.initialize(config.get("vram_classes")) + + options.initialize(config, config_path, modes, verbose, disassemble_all) + + return config + diff --git a/test/basic_app/expected/.splache b/test/basic_app/expected/.splache index 4bf620c10a486ff18f91e8c8cda5c65579339312..c5fdf07917a3a55a77df3df40f7f9dde35937084 100644 GIT binary patch delta 130 zcmX@l_LPmKfn{pcL>9Y=k*X6H=}r75H+dhUOqgJCWo}Y_PJCiYN>OoqL1Ib9l-Ma6 zJwnO(d1;yH`ViS-y^@NODLoQsg2e?ni6vmE)E@5q_{8G);?lIV%!(-;6CEWc)~QUq Ppf=f!v3&Cj#&kvi3&Jyu delta 154 zcmaFLcAkx;fn}=8L>48W|CMTsS; pDS8k!#d;+bC6ikjD<^J}<>StePb`ivE=^0zteDcV`5$9CBLM%tI#&Py diff --git a/test/basic_app/n64.yaml b/test/basic_app/n64.yaml new file mode 100644 index 00000000..3f45ef36 --- /dev/null +++ b/test/basic_app/n64.yaml @@ -0,0 +1,17 @@ +options: + platform: n64 + compiler: GCC + base_path: . + build_path: build + asm_path: split/asm + src_path: split/src + cache_path: split/.splache + asset_path: split/assets + compiler: GCC + symbol_addrs_path: + - config/symbols.txt + o_as_suffix: True +segments: + - name: header + type: header + start: 0x00 diff --git a/test/basic_app/splat.yaml b/test/basic_app/splat.yaml index dcb164dd..131f0c8d 100644 --- a/test/basic_app/splat.yaml +++ b/test/basic_app/splat.yaml @@ -1,24 +1,14 @@ +parent: n64.yaml options: - platform: n64 - compiler: GCC basename: basic_app - base_path: . - build_path: build target_path: build/basic_app.bin - asm_path: split/asm - src_path: split/src ld_script_path: split/basic_app.ld - cache_path: split/.splache symbol_addrs_path: split/generated.symbols.txt undefined_funcs_auto_path: split/undefined_funcs_auto.txt undefined_syms_auto_path: split/undefined_syms_auto.txt - asset_path: split/assets - compiler: GCC - o_as_suffix: True + symbol_addrs_path: + - config/symbols.splat.txt segments: - - name: header - type: header - start: 0x00 - name: dummy_ipl3 type: code start: 0x40 From 1ac75c8f8f10a8b16b1927139d4bd970ea5c2490 Mon Sep 17 00:00:00 2001 From: Jonathan Hohle Date: Wed, 5 Feb 2025 14:17:20 -0700 Subject: [PATCH 2/5] type and format errors --- src/splat/scripts/split.py | 6 +++--- src/splat/util/conf.py | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/splat/scripts/split.py b/src/splat/scripts/split.py index bf21b778..0ec46513 100644 --- a/src/splat/scripts/split.py +++ b/src/splat/scripts/split.py @@ -449,7 +449,7 @@ def main( skip_version_check: bool = False, stdout_only: bool = False, disassemble_all: bool = False, - include_path: List[str] = [] + include_path: List[Path] = [], ): if stdout_only: progress_bar.out_file = sys.stdout @@ -539,7 +539,7 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser): "-I", help="Add the directory to the list of search directories when including other config", action="append", - type=Path + type=Path, ) @@ -552,7 +552,7 @@ def process_arguments(args: argparse.Namespace): args.skip_version_check, args.stdout_only, args.disassemble_all, - args.I + args.I, ) diff --git a/src/splat/util/conf.py b/src/splat/util/conf.py index 9a431fd3..af047d53 100644 --- a/src/splat/util/conf.py +++ b/src/splat/util/conf.py @@ -9,6 +9,7 @@ from . import log, options, vram_classes + def merge_configs(main_config, additional_config): # Merge rules are simple # For each key in the dictionary @@ -36,19 +37,20 @@ def merge_configs(main_config, additional_config): return main_config + def resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path: if (base / rel).exists(): - return (base / rel) + return base / rel for path in include_paths: candidate = path / rel if candidate.exists(): return candidate - log.error(f"\"{rel}\" not found") + log.error(f'"{rel}" not found') return None -def load_config(config_path: str, include_path: List[Path]) -> Dict[str, Any]: +def load_config(config_path: Path, include_path: List[Path]) -> Dict[str, Any]: base_path = Path(config_path).parent with open(config_path) as f: config = yaml.load(f.read(), Loader=yaml.SafeLoader) @@ -60,6 +62,7 @@ def load_config(config_path: str, include_path: List[Path]) -> Dict[str, Any]: return config + def initialize( config_path: List[str], include_path: List[Path], @@ -69,7 +72,7 @@ def initialize( ) -> Dict[str, Any]: config: Dict[str, Any] = {} for entry in config_path: - additional_config = load_config(entry, include_path) + additional_config = load_config(Path(entry), include_path) config = merge_configs(config, additional_config) vram_classes.initialize(config.get("vram_classes")) @@ -77,4 +80,3 @@ def initialize( options.initialize(config, config_path, modes, verbose, disassemble_all) return config - From c66a599a7d85d32ad92b6fff57b047e337863867 Mon Sep 17 00:00:00 2001 From: Jonathan Hohle Date: Wed, 5 Feb 2025 14:25:57 -0700 Subject: [PATCH 3/5] add longopt for include directories --- src/splat/scripts/split.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/splat/scripts/split.py b/src/splat/scripts/split.py index 0ec46513..657d93f6 100644 --- a/src/splat/scripts/split.py +++ b/src/splat/scripts/split.py @@ -536,7 +536,7 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser): action="store_true", ) parser.add_argument( - "-I", + "-I", "--include-directory", help="Add the directory to the list of search directories when including other config", action="append", type=Path, @@ -552,7 +552,7 @@ def process_arguments(args: argparse.Namespace): args.skip_version_check, args.stdout_only, args.disassemble_all, - args.I, + args.include_directory, ) From 20fa6c96ea5969a6593ee3f654687e80c0a3996b Mon Sep 17 00:00:00 2001 From: Jonathan Hohle Date: Wed, 5 Feb 2025 16:34:16 -0700 Subject: [PATCH 4/5] review feedback --- docs/Configuration.md | 2 +- src/splat/util/conf.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 4b64dab3..988c17b0 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -758,7 +758,7 @@ use_legacy_include_asm: True ## Inheritance -For projects with several similar overlays, Splat supports configuration inheritance. Shared options and segments can be +For projects with several similar overlays, splat supports configuration inheritance. Shared options and segments can be defined in a configuration file and then extended in another configuration file using the top level `parent` key. Parents are resolved recursively, allowing projects to be organized using a flexible configuration tree. diff --git a/src/splat/util/conf.py b/src/splat/util/conf.py index af047d53..caf77666 100644 --- a/src/splat/util/conf.py +++ b/src/splat/util/conf.py @@ -47,7 +47,6 @@ def resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path: if candidate.exists(): return candidate log.error(f'"{rel}" not found') - return None def load_config(config_path: Path, include_path: List[Path]) -> Dict[str, Any]: From 75de750721927a8f4f3ea4ae37451ff57c401de6 Mon Sep 17 00:00:00 2001 From: Jonathan Hohle Date: Wed, 5 Feb 2025 19:45:00 -0700 Subject: [PATCH 5/5] documentation and default values --- src/splat/scripts/split.py | 3 +- src/splat/segtypes/segment.py | 2 +- src/splat/util/conf.py | 53 +++++++++++++++++++++++++++-------- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/splat/scripts/split.py b/src/splat/scripts/split.py index 657d93f6..9825fde3 100644 --- a/src/splat/scripts/split.py +++ b/src/splat/scripts/split.py @@ -536,7 +536,8 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser): action="store_true", ) parser.add_argument( - "-I", "--include-directory", + "-I", + "--include-directory", help="Add the directory to the list of search directories when including other config", action="append", type=Path, diff --git a/src/splat/segtypes/segment.py b/src/splat/segtypes/segment.py index 1c9e407c..cc923312 100644 --- a/src/splat/segtypes/segment.py +++ b/src/splat/segtypes/segment.py @@ -255,7 +255,7 @@ def parse_ld_align_segment_start(yaml: Union[dict, list]) -> Optional[int]: @staticmethod def parse_suggestion_rodata_section_start( - yaml: Union[dict, list] + yaml: Union[dict, list], ) -> Optional[bool]: if isinstance(yaml, dict): suggestion_rodata_section_start = yaml.get( diff --git a/src/splat/util/conf.py b/src/splat/util/conf.py index caf77666..fedb5371 100644 --- a/src/splat/util/conf.py +++ b/src/splat/util/conf.py @@ -1,3 +1,11 @@ +""" +This module is used to load splat configuration from a YAML file. + +A config dict can be loaded using `initialize`. + + config = conf.initialize("path/to/splat.yaml") +""" + from typing import Any, Dict, List, Optional, Set, Tuple, Union from pathlib import Path @@ -10,7 +18,7 @@ from . import log, options, vram_classes -def merge_configs(main_config, additional_config): +def _merge_configs(main_config, additional_config): # Merge rules are simple # For each key in the dictionary # - If list then append to list @@ -28,7 +36,7 @@ def merge_configs(main_config, additional_config): main_config[curkey] += additional_config[curkey] elif type(main_config[curkey]) == dict: # need to merge sub areas - main_config[curkey] = merge_configs( + main_config[curkey] = _merge_configs( main_config[curkey], additional_config[curkey] ) else: @@ -38,7 +46,7 @@ def merge_configs(main_config, additional_config): return main_config -def resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path: +def _resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path: if (base / rel).exists(): return base / rel @@ -49,14 +57,14 @@ def resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path: log.error(f'"{rel}" not found') -def load_config(config_path: Path, include_path: List[Path]) -> Dict[str, Any]: +def _load_config(config_path: Path, include_path: List[Path]) -> Dict[str, Any]: base_path = Path(config_path).parent with open(config_path) as f: config = yaml.load(f.read(), Loader=yaml.SafeLoader) if "parent" in config: - parent_path = resolve_path(base_path, Path(config["parent"]), include_path) - parent = load_config(parent_path, include_path) - config = merge_configs(parent, config) + parent_path = _resolve_path(base_path, Path(config["parent"]), include_path) + parent = _load_config(parent_path, include_path) + config = _merge_configs(parent, config) del config["parent"] return config @@ -64,15 +72,36 @@ def load_config(config_path: Path, include_path: List[Path]) -> Dict[str, Any]: def initialize( config_path: List[str], - include_path: List[Path], - modes: Optional[List[str]], - verbose: bool, + include_path: List[Path] = [], + modes: Optional[List[str]] = None, + verbose: bool = False, disassemble_all: bool = False, ) -> Dict[str, Any]: + """ + Returns a `dict` with resolved splat config. + + Multiple configuration files can be passed in ``config_path`` with each + subsequent file merged into the previous. `parent` keys are resolved + prior to merging multiple files. + + `include_path` can include any additional paths which should be searched for relative parent config files. Paths are + relative to the file being evaluated (i.e. a child config file). + + `modes` specifies which modes are active (all, code, img, gfx, vtx, etc.). The default is all. + + `verbose` may be used to determine whether or not to display additional output. + + `disassemble_all` determines whether functions which are already compiled will be disassembled. + + After being merged, static validation is done on the configuration. + + The returned `dict` represents the merged and validated YAML. + """ + config: Dict[str, Any] = {} for entry in config_path: - additional_config = load_config(Path(entry), include_path) - config = merge_configs(config, additional_config) + additional_config = _load_config(Path(entry), include_path) + config = _merge_configs(config, additional_config) vram_classes.initialize(config.get("vram_classes"))