From db45bab7acbfd3b9a464deb5a9133e09e419c90e Mon Sep 17 00:00:00 2001 From: Senthil Kumaran S Date: Wed, 12 Jun 2024 14:00:45 +0530 Subject: [PATCH] overlay: Implement overlay option The overlay option allows to supply any number of overlays with the destination path, where the overlay has to be applied. If the destination path is not given the default will be "/". This avoids hard coding overlay parameters as part of the jinja templates. We are not removing the existing overlay parameters from the jinja templates in order to not break the backwards compatibility for users who are already relying on this. Signed-off-by: Senthil Kumaran S --- lava_test_plans/__main__.py | 27 +++++++++++++++++- lava_test_plans/include/fastboot.jinja2 | 14 +++++++++ lava_test_plans/nfs.jinja2 | 14 +++++++++ lava_test_plans/qemu.jinja2 | 14 +++++++++ lava_test_plans/utils.py | 38 +++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 1 deletion(-) diff --git a/lava_test_plans/__main__.py b/lava_test_plans/__main__.py index 8814619e..784d4d11 100644 --- a/lava_test_plans/__main__.py +++ b/lava_test_plans/__main__.py @@ -35,7 +35,12 @@ from ruamel.yaml.composer import ComposerError from lava_test_plans import __version__ -from lava_test_plans.utils import get_context, validate_variables +from lava_test_plans.utils import ( + compression, + get_context, + overlay_action, + validate_variables, +) FORMAT = "[%(funcName)16s() ] %(message)s" logging.basicConfig(level=logging.INFO, format=FORMAT) @@ -297,6 +302,19 @@ def main(): type=int, default=logging.INFO, ) + parser.add_argument( + "--overlay", + default=[], + metavar="URL/String", + type=str, + help=( + "Tarball with overlay and optionally PATH to extract the tarball," + " default PATH '/'. Overlay can be specified multiple times" + ), + action=overlay_action, + nargs="+", + dest="overlays", + ) args = parser.parse_args() logger.setLevel(args.verbose) @@ -342,6 +360,11 @@ def main(): logger.error("QA_REPORTS_TOKEN and LAVA_TOKEN are missing") return 1 + overlays = [] + if args.overlays: + for index, item in enumerate(args.overlays): + overlays.append((f"overlay-{index:02}", item[0], item[1])) + lava_jobs = [] template_dirs = [ @@ -359,8 +382,10 @@ def main(): else StrictUndefined ), ) + j2_env.globals["compression"] = compression context = get_context(script_dirname, args.variables, args.overwrite_variables) context.update({"device_type": args.device_type}) + context.update({"overlays": overlays}) test_list = [] if args.test_plan: for test_plan in args.test_plan: diff --git a/lava_test_plans/include/fastboot.jinja2 b/lava_test_plans/include/fastboot.jinja2 index 4e687fb5..d804f382 100644 --- a/lava_test_plans/include/fastboot.jinja2 +++ b/lava_test_plans/include/fastboot.jinja2 @@ -168,6 +168,20 @@ reboot_to_fastboot: {{ reboot_to_fastboot }} {% if apply_overlay == "overlays" %} overlays: {% endif %} +{% if overlays %} + overlays: +{% endif %} +{% for name, overlay, dst in overlays %} + {{ name }}: + url: "{{ overlay }}" + format: {{ compression(overlay)[0] }} +{% if compression(overlay)[1] is not none %} + compression: {{ compression(overlay)[1] }} + path: "{{ dst }}" +{% else %} + path: "{{ dst }}{{ overlay.split('/')[-1] }}" +{% endif %} +{% endfor %} {% if OVERLAY_URL is defined %} over: url: {{OVERLAY_URL}} diff --git a/lava_test_plans/nfs.jinja2 b/lava_test_plans/nfs.jinja2 index 9f053f6b..bcb4f1c6 100644 --- a/lava_test_plans/nfs.jinja2 +++ b/lava_test_plans/nfs.jinja2 @@ -96,6 +96,20 @@ {% if apply_overlay == "overlays" %} overlays: {% endif %} +{% if overlays %} + overlays: +{% endif %} +{% for name, overlay, dst in overlays %} + {{ name }}: + url: "{{ overlay }}" + format: {{ compression(overlay)[0] }} +{% if compression(overlay)[1] is not none %} + compression: {{ compression(overlay)[1] }} + path: "{{ dst }}" +{% else %} + path: "{{ dst }}{{ overlay.split('/')[-1] }}" +{% endif %} +{% endfor %} {% if OVERLAY_URL is defined %} over: url: {{OVERLAY_URL}} diff --git a/lava_test_plans/qemu.jinja2 b/lava_test_plans/qemu.jinja2 index 7fdfc36e..8c196188 100644 --- a/lava_test_plans/qemu.jinja2 +++ b/lava_test_plans/qemu.jinja2 @@ -100,6 +100,20 @@ {% block rootfs_extra_args %} {% endblock rootfs_extra_args %} {# Process the following block if any overlays provided #} +{% if overlays %} + overlays: +{% endif %} +{% for name, overlay, dst in overlays %} + {{ name }}: + url: "{{ overlay }}" + format: {{ compression(overlay)[0] }} +{% if compression(overlay)[1] is not none %} + compression: {{ compression(overlay)[1] }} + path: "{{ dst }}" +{% else %} + path: "{{ dst }}{{ overlay.split('/')[-1] }}" +{% endif %} +{% endfor %} {% if OVERLAY_MODULES_URL is defined or OVERLAY_KSELFTEST_URL is defined or OVERLAY_PERF_URL is defined or diff --git a/lava_test_plans/utils.py b/lava_test_plans/utils.py index c89e1688..1492a533 100644 --- a/lava_test_plans/utils.py +++ b/lava_test_plans/utils.py @@ -8,6 +8,7 @@ import os +import argparse import logging from configobj import ConfigObj, ConfigObjError from ruamel.yaml import YAML @@ -61,3 +62,40 @@ def validate_variables( logger.error(f"Mandatory variables missing: {var_diff}") return 1 return 0 + + +class overlay_action(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + entries = len(values) + + pairs = getattr(namespace, self.dest, []) + + if entries > 2: + parser.error( + f"More than 2 arguments passed for {self.dest} options. Please check help options" + ) + + if entries == 1: + pairs.append([values[0], "/"]) + else: + pairs.append([values[0], values[1]]) + setattr(namespace, self.dest, pairs) + + +COMPRESSIONS = { + ".tar.xz": ("tar", "xz"), + ".tar.gz": ("tar", "gz"), + ".tgz": ("tar", "gz"), + ".gz": (None, "gz"), + ".xz": (None, "xz"), + ".zst": (None, "zstd"), + ".py": ("file", None), + ".sh": ("file", None), +} + + +def compression(path): + for ext, ret in COMPRESSIONS.items(): + if path.endswith(ext): + return ret + return (None, None)