diff --git a/README.md b/README.md index 05c3a531..99be6459 100644 --- a/README.md +++ b/README.md @@ -1,106 +1,97 @@ ![Build Status](https://github.com/Linaro/lava-test-plans/actions/workflows/test-plans-pipeline.yml/badge.svg) ![REUSE Compliance Check](https://github.com/Linaro/lava-test-plans/actions/workflows/reuse.yml/badge.svg) -# lava-test-plans - -The lava-test-plans project makes it easier to generate LAVA job definition files. -It generates the LAVA job definition file from a set of templates. +# LAVA Test Plans +The `lava-test-plans` project makes it easier to generate [LAVA](https://www.lavasoftware.org/) job definition files from a set of templates. # Installation +Install `lava-test-plans` from pip: + +```shell +pip install lava-test-plans +```` +or with podman/docker: +```shell +docker run -i -t lavasoftware/lava-test-plans /bin/bash +``` -Install lava-test-plans from pip: - - lava-test-plans -h - -or - -via podman/docker: - - docker run -i -t lavasoftware/lava-test-plans /bin/bash - -lavasoftware/lava-test-plans:latest points to the latest released version. -lavasoftware/lava-test-plans:master points to the latest development. +## Versions +- `lavasoftware/lava-test-plans:latest` points to the latest released version. +- `lavasoftware/lava-test-plans:master` points to the latest development. If the above commands succeed, you can run to check that the program starts correctly +```shell +lava-test-plans -h +``` - lava-test-plans -h - -## Developing - +## Development To install the latest development version: +```shell +git clone https://github.com/Linaro/lava-test-plans.git +cd ./lava-test-plans +pip3 install flit +flit install --symlink +``` +or you can use Docker: +```shell +docker run --volume /path/to/lava-test-plans:/xyz -i -t lavasoftware/lava-test-plans /bin/bash +cd /xyz +``` +There will be a directory with `/lava-test-plans` from either a "released" version or directly from master. - git clone https://github.com/Linaro/lava-test-plans.git - cd ./lava-test-plans - - pip3 install flit - flit install --symlink - -or - -You need to do if you have docker installed: - - docker run --volume $HOME/path/to/lava-test-plans:/xyz -i -t lavasoftware/lava-test-plans /bin/bash - cd /xyz - -lavasoftware/lava-test-plans:latest points to the latest released version. -lavasoftware/lava-test-plans:master points to the latest development. - -There will be a directory with /lava-test-plans from either a "released" -version or directly from master. - -If the above commands succeed, you can run to check that the program starts correctly - - python3 -m lava_test_plans -h +If the above commands succeed, you can run to check that the program starts correctly: +```shell +python3 -m lava_test_plans --version +``` +output should be similar to: +``` +__main__.py, 7c588ece +``` # External variables - External variables are set in the *variables.ini* file. Each line in this file -is in the form -``` -key=value -``` -Lines starting with *#* are omited. Variables can also be set using -*--overwrite-variables* parameter. List of used variables: - - * *PROJECT_NAME*: used as the first part in the test job name. Can be set to - differentiate LAVA test jobs between different teams/projects - * *BUILD_NUMBER*: used as last part in the test job name. - * *KERNEL_BRANCH*: used in test job name - * *OS_INFO*: used in test job name - * *LAVA_JOB_PRIORITY*: priority of the LAVA job, used by LAVA scheduler - * *LAVA_JOB_VISIBILITY*: defaults to *public*. This block can be used to restrict job visibility to user or group. - * *LAVA_JOB_VISIBILITY_GROUPS*: variable should contain groups required by job. Formtatting is important and this variable should be - formatted comma separated list. Example: group1, group2. In case of using just one group, end string with comma. Example: - group1, - * *AUTO_LOGIN_*: default *PROMPT='login:', *USERNAME='root' and *PASSWORD=''. - * *BOOT_LABEL*: default BOOT_LABEL='boot'. - * *TAGS*: variable should contain tags required by job. Formtatting is important and this variable should be - formatted comma separated list. Example: tag1, tag2. In case of using just one tag, end string with comma. Example: - tag1, - * *UBOOT_VERSION_STRING*: string that is matched in the u-boot shell from output of command *version* - * *OVERLAY_MODULES_* *: overlays modules into the rootfs. - * *TEST_DEFINITIONS_REPOSITORY*: points to the test repository to use, default: https://github.com/Linaro/test-definitions.git - -Variables can also be stored in YAML file. Usual YAML syntax applies. - -## Timeouts - -Overall job timeout is a sum of action timeouts. There are 6 components: - * *deploy_timeout* - * *boot_timeout* - * *install_fastboot_timeout* - * *fastboot_deploy_timeout* - * *target_deploy_timeout* - * *TARGET_BOOT_TIMEOUT* - * *test_timeout* - -When LXC is not in use all *lxc_* timeouts are set to 0. *test_timeout* is defined for each test template. *target_* timeouts can be set separately for each device. - -# CI for docker multiarch builds -lava-test-plans gets mirrored to gitlab -https://gitlab.com/Linaro/lava-test-plans to build multiarch docker containers -and publish them to https://hub.docker.com/r/lavasoftware/lava-test-plans, that -is why there is a .gitlab-ci.yml in this repository. - -# Repository -Pull requests are welcome to https://github.com/linaro/lava-test-plans. +is in the `key=value` form. + +Lines starting with `#` will be omitted. + +Variables can also be set using `--overwrite-variables` CLI option. + +List of used variables: + + * `PROJECT_NAME`: used as the first part in the LAVA test job name. Can be set to differentiate test jobs between different teams/projects. + * `BUILD_NUMBER`: used as the last part in test job's name. + * `KERNEL_BRANCH`: used in test job name. + * `OS_INFO`: used in test job name. + * `LAVA_JOB_PRIORITY`: priority of a job, used by LAVA scheduler. + * `LAVA_JOB_VISIBILITY`: defaults to *public*. This variable can be used to restrict job visibility to user or group. + * `LAVA_JOB_VISIBILITY_GROUPS`: variable should contain groups required by the job. Formatting is important and this variable - must be formatted as comma separated list. For example: `group1, group2`. In case of using just one group, end string with comma - `group1,`. + * `AUTO_LOGIN_`: default *PROMPT='login:', *USERNAME='root' and *PASSWORD=''. + * `BOOT_LABEL`: default is `boot`. + * `TAGS`: variable should contain tags required by a job. Formatting is important and this variable must be formatted comma separated list. For example: `tag1, tag2`. In case of using just one tag, end string with comma - `tag1,`. + * `UBOOT_VERSION_STRING`: string that is matched in the U-Boot shell after command `version`. + * `OVERLAY_MODULES_`: overlays modules. + * `TEST_DEFINITIONS_REPOSITORY`: points to the test definitions repository, default: https://github.com/Linaro/test-definitions.git + +Variables can also be stored in YAML-formatted file - `variables.yaml`. + +# Timeouts +Overall job timeout is a sum of action timeouts. There are several components: + * `deploy_timeout` + * `boot_timeout` + * `install_fastboot_timeout` + * `fastboot_deploy_timeout` + * `target_deploy_timeout` + * `TARGET_BOOT_TIMEOUT` + * `test_timeout`, is defined for each test template. + +### Notes +1. When LXC is not in use all `lxc_*` timeouts are set to 0. +2. `target_*` timeouts can be set separately for each device. + +# CI for Docker multi-arch builds +`lava-test-plans` repo gets mirrored to [Gitlab](https://gitlab.com/Linaro/lava-test-plans) + to build multi-arch Docker containers +and publish them to [Docker Hub](https://hub.docker.com/r/lavasoftware/lava-test-plans). + +# Contributing +Pull requests are welcome to https://github.com/linaro/lava-test-plans! \ No newline at end of file diff --git a/lava_test_plans/__main__.py b/lava_test_plans/__main__.py index 41e5a64e..2a7b4fef 100644 --- a/lava_test_plans/__main__.py +++ b/lava_test_plans/__main__.py @@ -7,7 +7,6 @@ # SPDX-License-Identifier: MIT import argparse -from configobj import ConfigObj, ConfigObjError import fnmatch from io import StringIO import itertools @@ -16,6 +15,7 @@ import re import requests import sys +from urllib.parse import urlsplit from jinja2 import ( Environment, @@ -23,7 +23,7 @@ StrictUndefined, make_logging_undefined, ) -from jinja2.exceptions import UndefinedError, TemplateSyntaxError +from jinja2.exceptions import UndefinedError, TemplateSyntaxError, TemplateNotFound from ruamel.yaml import YAML from ruamel.yaml.constructor import ( DuplicateKeyError, @@ -32,8 +32,7 @@ ) from ruamel.yaml.scanner import ScannerError from ruamel.yaml.parser import ParserError -from ruamel.yaml.composer import ComposerError - +import docker from lava_test_plans import __version__ from lava_test_plans.utils import get_context, validate_variables @@ -42,12 +41,6 @@ logger = logging.getLogger(__name__) -try: - from urllib.parse import urlsplit -except ImportError: - from urlparse import urlsplit - - # Templates base path script_dirname = os.path.dirname(os.path.abspath(__file__)) template_base_path = script_dirname @@ -57,17 +50,14 @@ def parse_template(yaml_string): - """ - Round trip lava_job through ruamel to test parsing and - improve formatting. Comments are preserved. - - In: yaml-formatted string - Out: validated yaml-formatted string + """Test LAVA job parsing and enhance formatting via ruamel, while keeping comments intact. + :param yaml_string: YAML string + :returns: Valid YAML string """ logger.debug(yaml_string) yaml = YAML() - # ruamel does not provide a mechanism to dump to string, so use StringIO - # to catch it + # ruamel does not provide a mechanism to dump to string, + # so use StringIO to catch it output = StringIO() yaml.dump(yaml.load(yaml_string), output) # strip empty lines from output @@ -75,9 +65,9 @@ def parse_template(yaml_string): def get_job_name(lava_job_string): - """ - In: yaml-formatted string - Out: LAVA job's name + """Get job name from LAVA job definition + :param lava_job_string: YAML-formatted string + :returns: LAVA job's name """ yaml = YAML() lava_job = yaml.load(lava_job_string) @@ -103,9 +93,9 @@ def _load_template(template_name, template_path, device_type): def _get_test_plan_list(test_plan_path): - """ - Returns list of all .yaml files in the directory - specified as parameter + """Get list of all .yaml files in the directory + :param test_plan_path: path to directory containing test plan files + :returns: list of test plan files """ logger.debug("Checking for files in %s" % test_plan_path) ret_list = [] @@ -117,6 +107,13 @@ def _get_test_plan_list(test_plan_path): def _submit_to_squad(lava_job, lava_url_base, qa_server_api, qa_server_base, qa_token): + """Submit LAVA job to SQAUD server using qa-reports API + :param lava_job: LAVA job definition + :param lava_url_base: LAVA server URL + :param qa_server_api: qa-reports API URL + :param qa_server_base: qa-reports server URL + :param qa_token: qa-reports API token + """ headers = {"Auth-Token": qa_token} try: @@ -135,10 +132,10 @@ def _submit_to_squad(lava_job, lava_url_base, qa_server_api, qa_server_base, qa_ logger.info(response.status_code) logger.info(response.text) response.raise_for_status() - except requests.exceptions.RequestException as err: - logger.error("QA Reports submission failed") - logger.info("offending job definition:") - logger.info(lava_job) + except requests.exceptions.RequestException: + logger.error( + "QA Reports submission failed. Offending job definition: %s", lava_job + ) return 1 @@ -332,7 +329,9 @@ def main(): if args.qa_server_project: if "/" in args.qa_server_project: - logger.error("--qa-server-project can not contain of a slash in the name") + logger.error( + "--qa-server-project parameter cannot include a slash in its name." + ) return 1 if args.dryrun: @@ -353,13 +352,17 @@ def main(): # prevent creating templates when variables are missing j2_env = Environment( loader=FileSystemLoader(template_dirs, followlinks=True), - undefined=make_logging_undefined(logger=logger, base=StrictUndefined) - if args.dryrun - else StrictUndefined, + undefined=( + make_logging_undefined(logger=logger, base=StrictUndefined) + if args.dryrun + else StrictUndefined + ), ) + context = get_context(script_dirname, args.variables, args.overwrite_variables) context.update({"device_type": args.device_type}) test_list = [] + if args.test_plan: for test_plan in args.test_plan: test_plan_path = os.path.abspath( @@ -376,25 +379,26 @@ def main(): return 1 # convert test_list to set to remove potential duplicates + exit_code = 0 for test in set(test_list): - """Prepare lava jobs""" - lava_job = None + # Prepare LAVA jobs + try: + template = j2_env.get_template(test) + except TemplateNotFound as e: + logger.error("Template not found: %s", e) + return 1 + try: - lava_job = j2_env.get_template(test).render(context) + lava_job = template.render(context) lava_job = parse_template(lava_job) lava_jobs.append(lava_job) - logger.debug(lava_job) - except DuplicateKeyError as e: - logger.error(e) - exit_code = 1 - except ConstructorError as e: - logger.error(e) - exit_code = 1 - except DuplicateKeyFutureWarning as e: - logger.error(e) - exit_code = 1 - except ScannerError as e: + except ( + DuplicateKeyError, + ConstructorError, + DuplicateKeyFutureWarning, + ScannerError, + ) as e: logger.error(e) exit_code = 1 except ParserError as e: @@ -404,29 +408,30 @@ def main(): exit_code = 1 except TemplateSyntaxError as e: testpath = os.path.join(output_path, args.device_type, test) - logger.error("Trying to render: %s" % testpath) - logger.error("Error in file: %s" % e.name) - logger.error("\tline: %s" % e.lineno) - logger.error("\tissue: %s" % e.message) + logger.error( + f"Trying to render: {testpath}\nError in file: {e.name}\n\tline:" + f" {e.lineno}\n\tissue: {e.message}" + ) exit_code = 1 except UndefinedError as e: testpath = os.path.join(output_path, args.device_type, test) - logger.error("Trying to render: %s" % testpath) - logger.error("\tissue: %s" % e.message) + logger.error(f"Trying to render: {testpath}\n\tissue: {e.message}") exit_code = 1 - if args.dryrun and lava_job is not None: - testpath = os.path.join( - output_path, args.device_type, os.path.basename(test) - ) - logger.info(testpath) - if not os.path.exists(os.path.dirname(testpath)): - os.makedirs(os.path.dirname(testpath), exist_ok=True) - with open(os.path.join(testpath), "w") as f: - f.write(lava_job) - if args.test_lava_validity: - import docker + if args.dryrun: + if 'lava_job' in locals(): + testpath = os.path.join( + output_path, args.device_type, os.path.basename(test) + ) + logger.info(f'Writing LAVA job in: {os.path.join(testpath)}') + if not os.path.exists(os.path.dirname(testpath)): + os.makedirs(os.path.dirname(testpath), exist_ok=True) + with open(os.path.join(testpath), "w") as f: + f.write(lava_job) + else: + logger.error("LAVA job is not defined") + if args.test_lava_validity: client = docker.from_env(version="1.38") logger.debug("Checking for LAVA validity") for test in set(test_list): @@ -458,10 +463,7 @@ def main(): return exit_code qa_server_base = args.qa_server - if not ( - qa_server_base.startswith("http://") - or qa_server_base.startswith("https://") - ): + if not qa_server_base.startswith(("http://", "https://")): qa_server_base = "https://" + qa_server_base qa_server_team = args.qa_server_team qa_server_project = args.qa_server_project @@ -484,9 +486,7 @@ def main(): qa_server_env, ) lava_server = args.lava_server - if not ( - lava_server.startswith("http://") or lava_server.startswith("https://") - ): + if not lava_server.startswith(("http://", "https://")): lava_server = "https://" + lava_server lava_url_base = "%s://%s/" % ( urlsplit(lava_server).scheme, @@ -494,7 +494,7 @@ def main(): ) for lava_job in lava_jobs: - """Submit lava jobs""" + # Submit LAVA job to QA Reports if args.qa_token: _submit_to_squad( lava_job,