From 793d6967ec6931de5d99274051c6364a614905f0 Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Mon, 4 Dec 2023 15:27:40 +0100 Subject: [PATCH 1/4] basic selftests logging updates A few basic tests related to logging has been masked because of #5817. This is a fix for those tests to catch up with latest changes. Signed-off-by: Jan Richter --- selftests/functional/basic.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/selftests/functional/basic.py b/selftests/functional/basic.py index 788c01635f..1e7be77da9 100644 --- a/selftests/functional/basic.py +++ b/selftests/functional/basic.py @@ -507,7 +507,7 @@ def test_runner_timeout(self): expected_rc, f"Avocado did not return rc {expected_rc}:\n{result}", ) - self.assertIn("timeout", result_json["tests"][0]["fail_reason"]) + self.assertIn("Timeout reached", result_json["tests"][0]["fail_reason"]) # Ensure no test aborted error messages show up self.assertNotIn(b"TestAbortError: Test aborted unexpectedly", output) @@ -553,9 +553,7 @@ def test_show_user_stream(self): ) result = process.run(cmd_line, ignore_status=True) self.assertEqual(result.exit_status, exit_codes.AVOCADO_ALL_OK) - self.assertIn( - b"Plant.test_plant_organic: preparing soil on row 0", result.stdout - ) + self.assertIn(b"preparing soil on row 0", result.stdout) lines = result.stdout.split(b"\n") self.assertEqual( len(lines), len(set(lines)), "The --show option has duplicities" @@ -783,9 +781,7 @@ def test_store_logging_stream(self): with open(progress_info, encoding="utf-8") as file: stream_line = file.readline() self.assertIn( - "avocado.test.progress INFO | " - "1-examples/tests/logging_streams.py:Plant.test_plant_organic: " - "preparing soil on row 0", + "logging_streams L0017 INFO | preparing soil on row 0", stream_line, ) @@ -805,10 +801,10 @@ def test_full_log(self): self.assertTrue(os.path.exists(progress_info)) with open(progress_info, encoding="utf-8") as file: stream = file.read() - self.assertIn("avocado.job", stream) - self.assertIn("avocado.core", stream) - self.assertIn("avocado.test", stream) - self.assertIn("avocado.app", stream) + self.assertIn("INFO | Avocado config:", stream) + self.assertIn("requested -> triagin", stream) + self.assertIn("preparing soil on row 0", stream) + self.assertIn("INFO | RESULTS : PASS 1 |", stream) @unittest.skipUnless( os.getenv("CI"), @@ -891,7 +887,7 @@ def test_store_logging_stream_level(self): with open(progress_info, encoding="utf-8") as file: stream_line = file.readline() self.assertIn( - "avocado.test.progress ERROR| Avocados are Gone", + "logging_streams L0037 ERROR| Avocados are Gone", stream_line, ) progress_info = os.path.join( @@ -903,7 +899,7 @@ def test_store_logging_stream_level(self): with open(progress_info, encoding="utf-8") as file: stream_line = file.readline() self.assertIn( - "avocado.test.progress ERROR| 1-examples/tests/logging_streams.py:Plant.test_plant_organic: Avocados are Gone", + "logging_streams L0037 ERROR| Avocados are Gone", stream_line, ) From bc1f4cec7ead9f12e932882da0a5981fded74673 Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Fri, 8 Dec 2023 17:47:28 +0100 Subject: [PATCH 2/4] add logging stream name to test logs In #5665 we introduced the logging stream name in test logs, this added more possibilities for user how to post-process test logs. Unfortunately, this change also hided information about modules where the log has been created. This issue has been solved in #5732. This commit connects those two solutions by storing stream and module names. Referencei: #5665 #5732 Signed-off-by: Jan Richter --- avocado/core/job.py | 4 +--- avocado/core/utils/messages.py | 2 +- selftests/functional/basic.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/avocado/core/job.py b/avocado/core/job.py index abb4d82f28..4a07a66567 100644 --- a/avocado/core/job.py +++ b/avocado/core/job.py @@ -209,9 +209,7 @@ def __exit__(self, _exc_type, _exc_value, _traceback): def __start_job_logging(self): # Enable test logger full_log = os.path.join(self.logdir, "full.log") - fmt = ( - "%(asctime)s %(module)-16.16s L%(lineno)-.4d %(levelname)-5.5s| %(message)s" - ) + fmt = "%(asctime)s %(name)s %(module)-16.16s L%(lineno)-.4d %(levelname)-5.5s| %(message)s" output.add_log_handler( LOG_JOB, logging.FileHandler, diff --git a/avocado/core/utils/messages.py b/avocado/core/utils/messages.py index a705090942..ab76b68abd 100644 --- a/avocado/core/utils/messages.py +++ b/avocado/core/utils/messages.py @@ -249,7 +249,7 @@ def start_logging(config, queue): log_handler = RunnerLogHandler(queue, "log") stdout_handler = RunnerLogHandler(queue, "stdout") stderr_handler = RunnerLogHandler(queue, "stderr") - fmt = "%(asctime)s %(module)-16.16s L%(lineno)-.4d %(levelname)-5.5s| %(message)s" + fmt = "%(asctime)s %(name)s %(module)-16.16s L%(lineno)-.4d %(levelname)-5.5s| %(message)s" formatter = logging.Formatter(fmt=fmt) log_handler.setFormatter(formatter) stdout_handler.setFormatter(formatter) diff --git a/selftests/functional/basic.py b/selftests/functional/basic.py index 1e7be77da9..ad0e93fe67 100644 --- a/selftests/functional/basic.py +++ b/selftests/functional/basic.py @@ -820,7 +820,7 @@ def check_matplotlib_logs(file_path): self.assertTrue(os.path.exists(file_path)) with open(file_path, encoding="utf-8") as file: stream = file.read() - self.assertIn("matplotlib DEBUG|", stream) + self.assertIn("matplotlib __init__ L0337 DEBUG|", stream) log_dir = os.path.join(self.tmpdir.name, "latest") test_log_dir = os.path.join( From ed96ed5bed6d3c2faf1b1ac15340329c179795e5 Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Mon, 4 Dec 2023 10:51:29 +0100 Subject: [PATCH 3/4] Selftest refactoring to fix masked tests Some functional tests have been masked out due to a safeloader behavior from #5817. This commit fixes this problem by changing the tests classes from unittests to avocado instrumented, where it is needed. Also, it adds warning to avocado documentation about this behavior. Reference: #5817 Signed-off-by: Jan Richter Signed-off-by: Cleber Rosa --- docs/source/guides/writer/chapters/writing.rst | 7 +++++++ selftests/functional/basic.py | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/source/guides/writer/chapters/writing.rst b/docs/source/guides/writer/chapters/writing.rst index 7051367960..b070e8c58b 100644 --- a/docs/source/guides/writer/chapters/writing.rst +++ b/docs/source/guides/writer/chapters/writing.rst @@ -29,6 +29,13 @@ leveraging its API power. As can be seen in the example above, an Avocado test is a method that starts with ``test`` in a class that inherits from :mod:`avocado.Test`. +.. warning:: Note that combining unittests and avocado-instrumented tests within + the same file is not feasible. If a class inherits from :mod:`avocado.Test`, + and another class inherits from :class:`unittest.TestCase` in the same file, + the unittest class will be excluded from testing. In such instances, it is + advisable to segregate these tests into separate files. + + .. note:: Avocado also supports coroutines as tests. Simply declare your test method using the ``async def`` syntax, and Avocado will run it inside an asyncio loop. diff --git a/selftests/functional/basic.py b/selftests/functional/basic.py index ad0e93fe67..7a9679e556 100644 --- a/selftests/functional/basic.py +++ b/selftests/functional/basic.py @@ -8,7 +8,6 @@ import xml.dom.minidom import zipfile -from avocado import Test from avocado.core import exit_codes from avocado.utils import path as utils_path from avocado.utils import process, script @@ -904,7 +903,7 @@ def test_store_logging_stream_level(self): ) -class DryRunTest(Test): +class DryRunTest(unittest.TestCase): def test_dry_run(self): examples_path = os.path.join("examples", "tests") passtest = os.path.join(examples_path, "passtest.py") From e02cf0e96ebafd60e95c6c72e8b0755ab13aaead Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Mon, 4 Dec 2023 16:52:49 +0100 Subject: [PATCH 4/4] selftest size check This adds selftest size check to the avocado check.py. This checks the size of avocado selftests to avoid test issues from #5817. Signed-off-by: Jan Richter --- selftests/check.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/selftests/check.py b/selftests/check.py index 7d3b274c70..a8a620bd46 100755 --- a/selftests/check.py +++ b/selftests/check.py @@ -16,6 +16,29 @@ from avocado.utils import process from selftests.utils import python_module_available +TEST_SIZE = { + "static-checks": 7, + "job-api-1": 1, + "job-api-2": 1, + "job-api-3": 2, + "job-api-4": 9, + "job-api-5": 12, + "job-api-6": 4, + "job-api-7": 1, + "nrunner-interface": 70, + "nrunner-requirement": 12, + "unit": 667, + "jobs": 11, + "functional-parallel": 297, + "functional-serial": 4, + "optional-plugins": 0, + "optional-plugins-golang": 2, + "optional-plugins-html": 3, + "optional-plugins-robot": 3, + "optional-plugins-varianter_cit": 40, + "optional-plugins-varianter_yaml_to_mux": 50, +} + class JobAPIFeaturesTest(Test): def check_directory_exists(self, path=None): @@ -579,6 +602,7 @@ def create_suites(args): # pylint: disable=W0621 # ======================================================================== # Run nrunner interface checks for all available runners # ======================================================================== + nrunner_interface_size = 10 config_nrunner_interface = { "resolver.references": ["selftests/functional/nrunner_interface.py"], "run.dict_variants.variant_id_keys": ["runner"], @@ -616,6 +640,7 @@ def create_suites(args): # pylint: disable=W0621 "runner": "avocado-runner-golang", } ) + TEST_SIZE["nrunner-interface"] += nrunner_interface_size if ( python_module_available("avocado-framework-plugin-robot") @@ -626,6 +651,7 @@ def create_suites(args): # pylint: disable=W0621 "runner": "avocado-runner-robot", } ) + TEST_SIZE["nrunner-interface"] += nrunner_interface_size if ( python_module_available("avocado-framework-plugin-ansible") @@ -636,6 +662,7 @@ def create_suites(args): # pylint: disable=W0621 "runner": "avocado-runner-ansible-module", } ) + TEST_SIZE["nrunner-interface"] += nrunner_interface_size if args.dict_tests["nrunner-interface"]: suites.append( @@ -727,6 +754,19 @@ def main(args): # pylint: disable=W0621 "optional-plugins": False, } + if python_module_available("avocado-framework-plugin-golang"): + TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-golang"] + if python_module_available("avocado-framework-plugin-result-html"): + TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-html"] + if python_module_available("avocado-framework-plugin-robot"): + TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-robot"] + if python_module_available("avocado-framework-plugin-varianter-cit"): + TEST_SIZE["optional-plugins"] += TEST_SIZE["optional-plugins-varianter_cit"] + if python_module_available("avocado-framework-plugin-varianter-yaml-to-mux"): + TEST_SIZE["optional-plugins"] += TEST_SIZE[ + "optional-plugins-varianter_yaml_to_mux" + ] + # Make a list of strings instead of a list with a single string if len(args.disable_plugin_checks) > 0: args.disable_plugin_checks = args.disable_plugin_checks[0].split(",") @@ -809,6 +849,19 @@ def main(args): # pylint: disable=W0621 print("check.py didn't clean test results.") print("uncleaned directories:") print(post_job_test_result_dirs.difference(pre_job_test_result_dirs)) + for suite in j.test_suites: + if suite.size != TEST_SIZE[suite.name]: + if exit_code == 0: + exit_code = 1 + print( + f"suite {suite.name} doesn't have {TEST_SIZE[suite.name]} tests" + f" it has {suite.size}." + ) + print( + "If you made some changes into selftests please update `TEST_SIZE`" + " variable in `check.py`. If you haven't done any changes to" + " selftests this behavior is an ERROR, and it needs to be fixed." + ) # tmp dirs clean up check process.run(f"{sys.executable} selftests/check_tmp_dirs")