From 867663ba976655580d8b33ebf3e02530e52ae3a8 Mon Sep 17 00:00:00 2001 From: Johannes Ernst Date: Wed, 23 Oct 2024 09:01:00 -0700 Subject: [PATCH 1/2] Re-imagine TestPlan as a single TestPlanSessionTemplate that is run once or more for several Constellations. That makes it consistent with how we generate HTML matrix reports. TestPlanSession so now is more correctly called TestPlanSessionTemplate Generate a single HTML file for single-Constellation TestPlans, and N+1 HTML files for N-Constellation TestPlans Reorganize Serializers Reorganize jinja2 templates a bit Add missing type annotations Update unit tests --- .../cli/commands/convert_transcript.py | 36 ++- .../cli/commands/create_session_template.py | 1 - src/feditest/cli/commands/create_testplan.py | 6 +- src/feditest/cli/commands/run.py | 45 ++- src/feditest/cli/utils.py | 49 +--- .../default/partials/shared/footer.jinja2 | 4 - .../default/partials/shared/mobile.jinja2 | 6 - .../default/partials/shared/summary.jinja2 | 127 --------- .../partials/test_matrix/metadata.jinja2 | 29 -- .../default/partials/test_matrix/table.jinja2 | 54 ---- .../default/partials/test_matrix/title.jinja2 | 4 - .../partials/test_session/metadata.jinja2 | 29 -- .../partials/test_session/results.jinja2 | 67 ----- .../partials/test_session/title.jinja2 | 5 - .../templates/default/test_matrix.jinja2 | 39 --- .../templates/default/test_session.jinja2 | 38 --- src/feditest/testplan.py | 56 +--- src/feditest/testrun.py | 61 ++-- src/feditest/testruncontroller.py | 23 +- src/feditest/testruntranscript.py | 268 +----------------- .../testruntranscriptserializer/__init__.py | 43 +++ .../testruntranscriptserializer/html.py | 153 ++++++++++ .../testruntranscriptserializer/json.py | 12 + .../testruntranscriptserializer/summary.py | 21 ++ .../testruntranscriptserializer/tap.py | 69 +++++ .../testplantranscript_default/matrix.jinja2 | 36 +++ .../partials/matrix/matrix.jinja2 | 54 ++++ .../partials/matrix/metadata.jinja2 | 29 ++ .../partials/matrix}/testresult.jinja2 | 0 .../partials/shared/footer.jinja2 | 4 + .../partials/shared/head.jinja2 | 4 + .../partials/shared/mobile.jinja2 | 5 + .../partials/shared/summary.jinja2 | 136 +++++++++ .../partials/shared_session/metadata.jinja2 | 29 ++ .../partials/shared_session/results.jinja2 | 59 ++++ .../shared_session}/testresult.jinja2 | 0 .../session_single.jinja2 | 33 +++ .../session_with_matrix.jinja2 | 34 +++ .../static/feditest.css | 3 +- tests.unit/test_10_register_nodedrivers.py | 1 + tests.unit/test_10_register_tests.py | 8 +- ...st_20_create_test_plan_session_template.py | 52 +--- tests.unit/test_30_create_testplan.py | 34 +-- ...llback_fediverse_accounts_from_testplan.py | 12 +- .../test_40_report_node_driver_errors.py | 36 ++- ...unt_manager_fallback_fediverse_accounts.py | 2 - ...40_ubos_mastodon_accounts_from_testplan.py | 8 +- tests.unit/test_50_run_assertion_raises.py | 20 +- tests.unit/test_50_run_exceptions.py | 15 +- .../test_50_run_multistep_assertion_raises.py | 14 +- tests.unit/test_50_run_not_implemented.py | 15 +- tests.unit/test_50_run_passes.py | 15 +- tests.unit/test_50_run_skip.py | 6 +- tests.unit/test_50_run_testplan_sandbox.py | 24 +- 54 files changed, 957 insertions(+), 976 deletions(-) delete mode 100644 src/feditest/templates/default/partials/shared/footer.jinja2 delete mode 100644 src/feditest/templates/default/partials/shared/mobile.jinja2 delete mode 100644 src/feditest/templates/default/partials/shared/summary.jinja2 delete mode 100644 src/feditest/templates/default/partials/test_matrix/metadata.jinja2 delete mode 100644 src/feditest/templates/default/partials/test_matrix/table.jinja2 delete mode 100644 src/feditest/templates/default/partials/test_matrix/title.jinja2 delete mode 100644 src/feditest/templates/default/partials/test_session/metadata.jinja2 delete mode 100644 src/feditest/templates/default/partials/test_session/results.jinja2 delete mode 100644 src/feditest/templates/default/partials/test_session/title.jinja2 delete mode 100644 src/feditest/templates/default/test_matrix.jinja2 delete mode 100644 src/feditest/templates/default/test_session.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/__init__.py create mode 100644 src/feditest/testruntranscriptserializer/html.py create mode 100644 src/feditest/testruntranscriptserializer/json.py create mode 100644 src/feditest/testruntranscriptserializer/summary.py create mode 100644 src/feditest/testruntranscriptserializer/tap.py create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/matrix.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/metadata.jinja2 rename src/feditest/{templates/default/partials/test_matrix => testruntranscriptserializer/templates/testplantranscript_default/partials/matrix}/testresult.jinja2 (100%) create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/footer.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/head.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/mobile.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/metadata.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 rename src/feditest/{templates/default/partials/test_session => testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session}/testresult.jinja2 (100%) create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 create mode 100644 src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_with_matrix.jinja2 rename src/feditest/{templates/default => testruntranscriptserializer/templates/testplantranscript_default}/static/feditest.css (99%) diff --git a/src/feditest/cli/commands/convert_transcript.py b/src/feditest/cli/commands/convert_transcript.py index 2596a3e..df42523 100644 --- a/src/feditest/cli/commands/convert_transcript.py +++ b/src/feditest/cli/commands/convert_transcript.py @@ -5,17 +5,13 @@ from argparse import ArgumentParser, Namespace, _SubParsersAction from feditest.reporting import warning -from feditest.testruntranscript import ( - JsonTestRunTranscriptSerializer, - MultifileRunTranscriptSerializer, - SummaryTestRunTranscriptSerializer, - TapTestRunTranscriptSerializer, - TestRunTranscript -) +from feditest.testruntranscript import TestRunTranscript +from feditest.testruntranscriptserializer.json import JsonTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.html import HtmlRunTranscriptSerializer +from feditest.testruntranscriptserializer.summary import SummaryTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.tap import TapTestRunTranscriptSerializer from feditest.utils import FEDITEST_VERSION -DEFAULT_TEMPLATE_PATH = "default" - def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: """ Run this command. @@ -28,21 +24,21 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: if not transcript.has_compatible_version(): warning(f'Transcript was created by FediTest { transcript.feditest_version }, you are running FediTest { FEDITEST_VERSION }: incompatibilities may occur.') - if isinstance(args.tap, str) or args.tap: - tap_serializer = TapTestRunTranscriptSerializer(transcript) - tap_serializer.write(args.tap) + if isinstance(args.html, str): + HtmlRunTranscriptSerializer(args.template_path).write(transcript, args.html) + elif args.html: + warning('--html requires a filename: skipping') + elif args.template_path: + warning('--template-path only supported with --html. Ignoring.') - if isinstance(args.html, str) or args.html: - multifile_serializer = MultifileRunTranscriptSerializer(args.html, args.template_path) - multifile_serializer.write(transcript) + if isinstance(args.tap, str) or args.tap: + TapTestRunTranscriptSerializer().write(transcript, args.tap) if isinstance(args.json, str) or args.json: - json_serializer = JsonTestRunTranscriptSerializer(transcript) - json_serializer.write(args.json) + JsonTestRunTranscriptSerializer().write(transcript, args.json) if isinstance(args.summary, str) or args.summary: - summary_serializer = SummaryTestRunTranscriptSerializer(transcript) - summary_serializer.write(args.json) + SummaryTestRunTranscriptSerializer().write(transcript, args.summary) return 0 @@ -60,7 +56,7 @@ def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> None: html_group = parser.add_argument_group('html', 'HTML options') html_group.add_argument('--html', help="Write results in HTML format to the provided file.") - html_group.add_argument('--template-path', default=DEFAULT_TEMPLATE_PATH, + html_group.add_argument('--template-path', required=False, help="When specifying --html, use this template path override (comma separated directory names)") parser.add_argument('--json', nargs="?", const=True, default=False, help="Write results in JSON format to stdout, or to the provided file (if given).") diff --git a/src/feditest/cli/commands/create_session_template.py b/src/feditest/cli/commands/create_session_template.py index 61d6b6b..1326b3b 100644 --- a/src/feditest/cli/commands/create_session_template.py +++ b/src/feditest/cli/commands/create_session_template.py @@ -12,7 +12,6 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: """ Run this command. - > feditest generate-settion-template --testsdir tests --all --out full-partially-automated-template.json """ if len(remaining): parser.print_help() diff --git a/src/feditest/cli/commands/create_testplan.py b/src/feditest/cli/commands/create_testplan.py index 97bbe65..08d2af0 100644 --- a/src/feditest/cli/commands/create_testplan.py +++ b/src/feditest/cli/commands/create_testplan.py @@ -4,7 +4,7 @@ from argparse import ArgumentParser, Namespace, _SubParsersAction import feditest -from feditest.cli.utils import create_plan_from_session_templates_and_constellations +from feditest.cli.utils import create_plan_from_session_and_constellations from feditest.reporting import fatal def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: @@ -18,7 +18,7 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: feditest.load_default_tests() feditest.load_tests_from(args.testsdir) - test_plan = create_plan_from_session_templates_and_constellations(args) + test_plan = create_plan_from_session_and_constellations(args) if test_plan: if args.out: test_plan.save(args.out) @@ -43,7 +43,7 @@ def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> None: # test plan options parser.add_argument('--name', default=None, required=False, help='Name of the generated test plan') parser.add_argument('--constellation', action='append', help='File(s) each containing a JSON fragment defining a constellation') - parser.add_argument('--session', '--session-template', action='append', help='File(s) each containing a JSON fragment defining a test session') + parser.add_argument('--session', '--session-template', required=False, help='File containing a JSON fragment defining a test session') parser.add_argument('--node', action='append', help="Use role=file to specify that the node definition in 'file' is supposed to be used for constellation role 'role'") parser.add_argument('--filter-regex', default=None, help='Only include tests whose name matches this regular expression') diff --git a/src/feditest/cli/commands/run.py b/src/feditest/cli/commands/run.py index 32b8cf8..f59b6f5 100644 --- a/src/feditest/cli/commands/run.py +++ b/src/feditest/cli/commands/run.py @@ -3,10 +3,11 @@ """ from argparse import ArgumentParser, Namespace, _SubParsersAction +from typing import cast import feditest from feditest.cli.utils import ( - create_plan_from_session_templates_and_constellations, + create_plan_from_session_and_constellations, create_plan_from_testplan ) from feditest.registry import Registry, set_registry_singleton @@ -14,18 +15,13 @@ from feditest.testplan import TestPlan from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController, InteractiveTestRunController, TestRunController -from feditest.testruntranscript import ( - JsonTestRunTranscriptSerializer, - MultifileRunTranscriptSerializer, - SummaryTestRunTranscriptSerializer, - TapTestRunTranscriptSerializer, - TestRunTranscriptSerializer, -) +from feditest.testruntranscriptserializer.json import JsonTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.html import HtmlRunTranscriptSerializer +from feditest.testruntranscriptserializer.summary import SummaryTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.tap import TapTestRunTranscriptSerializer from feditest.utils import FEDITEST_VERSION, hostname_validate -DEFAULT_TEMPLATE = 'default' - def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: """ Run this command. @@ -49,7 +45,7 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: if args.testplan: plan = create_plan_from_testplan(args) else: - plan = create_plan_from_session_templates_and_constellations(args) + plan = create_plan_from_session_and_constellations(args) if not plan: fatal('Cannot find or create test plan ') @@ -71,22 +67,21 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: transcript : feditest.testruntranscript.TestRunTranscript = test_run.transcribe() - summary_serializer = SummaryTestRunTranscriptSerializer(transcript) - serializer : TestRunTranscriptSerializer | None = None if isinstance(args.tap, str) or args.tap: - serializer = TapTestRunTranscriptSerializer(transcript) - serializer.write(args.tap) + TapTestRunTranscriptSerializer().write(transcript, cast(str|None, args.tap)) - if isinstance(args.html, str) or args.html: - multifile_serializer = MultifileRunTranscriptSerializer(args.html, args.template) - multifile_serializer.write(transcript) + if isinstance(args.html, str): + HtmlRunTranscriptSerializer(args.template_path).write(transcript, args.html) + elif args.html: + warning('--html requires a filename: skipping') + elif args.template_path: + warning('--template-path only supported with --html. Ignoring.') if isinstance(args.json, str) or args.json: - serializer = JsonTestRunTranscriptSerializer(transcript) - serializer.write(args.json) + JsonTestRunTranscriptSerializer().write(transcript, args.json) - if isinstance(args.summary, str) or args.summary or serializer is None: - summary_serializer.write(args.summary) + if isinstance(args.summary, str) or args.summary: + SummaryTestRunTranscriptSerializer().write(transcript, args.summary) if transcript.build_summary().n_failed > 0: print('FAILED.') @@ -112,7 +107,7 @@ def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> None: parser.add_argument('--name', default=None, required=False, help='Name of the generated test plan') parser.add_argument('--testplan', help='Name of the file that contains the test plan to run') parser.add_argument('--constellation', action='append', help='File(s) each containing a JSON fragment defining a constellation') - parser.add_argument('--session', '--session-template', action='append', help='File(s) each containing a JSON fragment defining a test session') + parser.add_argument('--session', '--session-template', required=False, help='File(s) each containing a JSON fragment defining a test session') parser.add_argument('--node', action='append', help="Use role=file to specify that the node definition in 'file' is supposed to be used for constellation role 'role'") parser.add_argument('--filter-regex', default=None, help='Only include tests whose name matches this regular expression') @@ -124,8 +119,8 @@ def add_sub_parser(parent_parser: _SubParsersAction, cmd_name: str) -> None: html_group = parser.add_argument_group('html', 'HTML options') html_group.add_argument('--html', help="Write results in HTML format to the provided file.") - html_group.add_argument('--template', default=DEFAULT_TEMPLATE, - help=f"When specifying --html, use this template (defaults to '{ DEFAULT_TEMPLATE }').") + html_group.add_argument('--template-path', required=False, + help="When specifying --html, use this template path override (comma separated directory names)") parser.add_argument('--json', '--testresult', nargs="?", const=True, default=False, help="Write results in JSON format to stdout, or to the provided file (if given).") parser.add_argument('--summary', nargs="?", const=True, default=False, diff --git a/src/feditest/cli/utils.py b/src/feditest/cli/utils.py index a0093c4..2daf52f 100644 --- a/src/feditest/cli/utils.py +++ b/src/feditest/cli/utils.py @@ -4,19 +4,18 @@ from argparse import ArgumentError, Namespace import re -from typing import Any from msgspec import ValidationError import feditest from feditest.tests import Test -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec def create_plan_from_testplan(args: Namespace) -> TestPlan: if args.constellation: raise ArgumentError(None, '--testplan already defines --constellation. Do not provide both.') if args.session: - raise ArgumentError(None, '--testplan already defines --session-template. Do not provide both.') + raise ArgumentError(None, '--testplan already defines --session. Do not provide both.') if args.node: raise ArgumentError(None, '--testplan already defines --node via the contained constellation. Do not provide both.') if args.test: @@ -25,46 +24,33 @@ def create_plan_from_testplan(args: Namespace) -> TestPlan: return plan -def create_plan_from_session_templates_and_constellations(args: Namespace) -> TestPlan | None: - session_templates = create_session_templates(args) +def create_plan_from_session_and_constellations(args: Namespace) -> TestPlan | None: + session = create_session(args) constellations = create_constellations(args) - plan : TestPlan | None = None - sessions = [] - for session_template in session_templates: - for constellation in constellations: - session = session_template.instantiate_with_constellation(constellation, constellation.name) - sessions.append(session) - if sessions: - plan = TestPlan(sessions, args.name) - plan.simplify() + plan = TestPlan(session, constellations, args.name) + plan.simplify() return plan -def create_session_templates(args: Namespace) -> list[TestPlanSession]: +def create_session(args: Namespace) -> TestPlanSessionTemplate: if args.session: - session_templates = create_session_templates_from_files(args) + session_template = create_session_from_files(args) else: session_template = create_session_template_from_tests(args) - session_templates = [ session_template ] - return session_templates + return session_template -def create_session_templates_from_files(args: Namespace) -> list[TestPlanSession]: +def create_session_from_files(args: Namespace) -> TestPlanSessionTemplate: if args.filter_regex: raise ArgumentError(None, '--session already defines the tests, do not provide --filter-regex') if args.test: raise ArgumentError(None, '--session already defines --test. Do not provide both.') - session_templates = [] - for session_file in args.session: - session_templates.append(TestPlanSession.load(session_file)) - return session_templates + return TestPlanSessionTemplate.load(args.session) -def create_session_template_from_tests(args: Namespace) -> TestPlanSession: +def create_session_template_from_tests(args: Namespace) -> TestPlanSessionTemplate: test_plan_specs : list[TestPlanTestSpec]= [] - constellation_role_names : dict[str,Any] = {} - constellation_roles: dict[str,TestPlanConstellationNode | None] = {} tests : list[Test]= [] if args.test: @@ -101,16 +87,7 @@ def create_session_template_from_tests(args: Namespace) -> TestPlanSession: test_plan_spec = TestPlanTestSpec(name) test_plan_specs.append(test_plan_spec) - for role_name in test.needed_local_role_names(): - constellation_role_names[role_name] = 1 - if not test_plan_spec.rolemapping: - test_plan_spec.rolemapping = {} - test_plan_spec.rolemapping[role_name] = role_name - - for constellation_role_name in constellation_role_names: - constellation_roles[constellation_role_name] = None - - session = TestPlanSession(TestPlanConstellation(constellation_roles), test_plan_specs, args.name) + session = TestPlanSessionTemplate(test_plan_specs, args.name) return session diff --git a/src/feditest/templates/default/partials/shared/footer.jinja2 b/src/feditest/templates/default/partials/shared/footer.jinja2 deleted file mode 100644 index 3a43408..0000000 --- a/src/feditest/templates/default/partials/shared/footer.jinja2 +++ /dev/null @@ -1,4 +0,0 @@ - \ No newline at end of file diff --git a/src/feditest/templates/default/partials/shared/mobile.jinja2 b/src/feditest/templates/default/partials/shared/mobile.jinja2 deleted file mode 100644 index d346414..0000000 --- a/src/feditest/templates/default/partials/shared/mobile.jinja2 +++ /dev/null @@ -1,6 +0,0 @@ -
-

This FediTest report requires a desktop or laptop-class monitor.

-

It cannot be viewed on a mobile device with a small screen.

-

Sorry about that. But then, you weren't going to use a phone to fix - reported bugs, either, right?

-
\ No newline at end of file diff --git a/src/feditest/templates/default/partials/shared/summary.jinja2 b/src/feditest/templates/default/partials/shared/summary.jinja2 deleted file mode 100644 index bcfd868..0000000 --- a/src/feditest/templates/default/partials/shared/summary.jinja2 +++ /dev/null @@ -1,127 +0,0 @@ -{% if summary.n_total %} - - - - - - - - - - -{%- if summary.n_passed != 0 %} - -{%- else %} - -{%- endif %} - - - - - - - -{%- if summary.n_skipped != 0 %} - -{%- else %} - -{%- endif %} - - - -{%- if summary.n_errored != 0 %} - -{%- else %} - -{%- endif %} - - - - - - - - -
StatusCount
Passed -
{{ summary.n_passed }} ({{ '%.1f' % ( summary.n_passed * 100.0 / summary.n_total ) }}%)
-
-
0
-
Failed - - - - - - - - - - - - - - - - - - -{%- for interop_level in feditest.InteropLevel %} - -{%- endfor %} - - -{%- for spec_level in [ feditest.SpecLevel.SHOULD, feditest.SpecLevel.IMPLIED, feditest.SpecLevel.UNSPECIFIED ] %} - - -{%- for interop_level in feditest.InteropLevel %} - -{%- endfor %} - - -{%- endfor %} - - -{%- for interop_level in feditest.InteropLevel %} - -{%- endfor %} - - - -
Interoperability
CompromisedDegradedUnaffectedUnknownTotal
ConformanceMust -
{{ summary.count_failures_for(feditest.SpecLevel.MUST, interop_level) }}
-
-
- {{ summary.count_failures_for(feditest.SpecLevel.MUST, None) }} - ({{ '%.1f' % ( summary.count_failures_for(feditest.SpecLevel.MUST, None) * 100.0 / summary.n_total ) }}%) -
-
{{ spec_level.formatted_name() }} -
{{ summary.count_failures_for(spec_level, interop_level) }}
-
-
- {{ summary.count_failures_for(spec_level, None) }} - ({{ '%.1f' % ( summary.count_failures_for(spec_level, None) * 100.0 / summary.n_total ) }}%) -
-
Total -
- {{ summary.count_failures_for(None, interop_level) }} - ({{ '%.1f' % ( summary.count_failures_for(None, interop_level) * 100.0 / summary.n_total ) }}%) -
-
-
- {{ summary.n_failed }} ({{ '%.1f' % ( summary.n_failed * 100.0 / summary.n_total ) }}%) -
-
-
Skipped -
{{ summary.n_skipped }} ({{ '%.1f' % ( summary.n_skipped * 100.0 / summary.n_total ) }}%)
-
-
0
-
Errors -
{{ summary.n_errored }} ({{ '%.1f' % ( summary.n_errored * 100.0 / summary.n_total ) }}%)
-
-
0
-
Total -
{{ summary.n_total }}
-
-{%- else %} -

No tests were run.

-{%- endif %} \ No newline at end of file diff --git a/src/feditest/templates/default/partials/test_matrix/metadata.jinja2 b/src/feditest/templates/default/partials/test_matrix/metadata.jinja2 deleted file mode 100644 index 69d9ead..0000000 --- a/src/feditest/templates/default/partials/test_matrix/metadata.jinja2 +++ /dev/null @@ -1,29 +0,0 @@ -
-{%- set started = getattr(run, 'started') %} -{%- set ended = getattr(run, 'ended') %} -{%- set duration = ended - started %} - - - - - - - - - - -{%- for key in ['username', 'hostname', 'platform'] -%} -{%- if getattr(run,key) %} - - - - -{%- endif %} -{%- endfor %} - - - - - -
Started{{ format_timestamp(started) }}
Ended{{ format_timestamp(ended) }} (total: {{ format_duration(duration) }})
{{ key.capitalize() }}{{ getattr(run, key) }}
Feditest version{{ getattr(run, 'feditest_version') }}
-
\ No newline at end of file diff --git a/src/feditest/templates/default/partials/test_matrix/table.jinja2 b/src/feditest/templates/default/partials/test_matrix/table.jinja2 deleted file mode 100644 index fc7d322..0000000 --- a/src/feditest/templates/default/partials/test_matrix/table.jinja2 +++ /dev/null @@ -1,54 +0,0 @@ -{% macro column_headings(first) %} - - {{ first }} -{%- for run_session in run.sessions %} -{%- set plan_session = run.plan.sessions[run_session.plan_session_index] %} -{%- set constellation = plan_session.constellation %} - -
-{%- if session_links -%} - -{%- endif -%} - {{ constellation }} -{%- if session_links -%} - -{%- endif -%} -
- -{%- endfor %} - -{% endmacro %} - -
- - - -{%- for run_session in run.sessions %} - -{%- endfor %} - - - {{ column_headings("{0} tests in {1} sessions (alphabetical order)".format(len(run.test_meta), len(run.sessions))) }} - - -{%- for test_index, ( _, test_meta ) in enumerate(sorted(run.test_meta.items())) %} - - -{%- for session_index, run_session in enumerate(run.sessions) %} -{%- for result in get_results_for(run, run_session, test_meta) %} -{% include "partials/test_matrix/testresult.jinja2" %} -{%- endfor %} -{%- endfor %} - -{%- endfor %} - - - {{ column_headings("") }} - -
- {{ permit_line_breaks_in_identifier(test_meta.name) }} -{%- if test_meta.description %} - {{ test_meta.description }} -{%- endif %} -
-
diff --git a/src/feditest/templates/default/partials/test_matrix/title.jinja2 b/src/feditest/templates/default/partials/test_matrix/title.jinja2 deleted file mode 100644 index 2386cb7..0000000 --- a/src/feditest/templates/default/partials/test_matrix/title.jinja2 +++ /dev/null @@ -1,4 +0,0 @@ -
-

Feditest Summary Report: {{ run.plan.name }}

-

{{ run.id }}

-
diff --git a/src/feditest/templates/default/partials/test_session/metadata.jinja2 b/src/feditest/templates/default/partials/test_session/metadata.jinja2 deleted file mode 100644 index 1da69fc..0000000 --- a/src/feditest/templates/default/partials/test_session/metadata.jinja2 +++ /dev/null @@ -1,29 +0,0 @@ -
-{%- set started = run_session.started %} -{%- set ended = run_session.ended %} -{%- set duration = ended - started %} - - - - - - - - - - -{%- for key in ['username', 'hostname', 'platform'] -%} -{%- if getattr(run,key) %} - - - - -{%- endif %} -{%- endfor %} - - - - - -
Started{{ format_timestamp(started) }}
Ended{{ format_timestamp(ended) }} (total: {{ format_duration(duration) }})
{{ key.capitalize() }}{{ getattr(run, key) }}
Feditest version{{ getattr(run, 'feditest_version') }}
-
\ No newline at end of file diff --git a/src/feditest/templates/default/partials/test_session/results.jinja2 b/src/feditest/templates/default/partials/test_session/results.jinja2 deleted file mode 100644 index 1e365be..0000000 --- a/src/feditest/templates/default/partials/test_session/results.jinja2 +++ /dev/null @@ -1,67 +0,0 @@ -{%- set plan_session = run.plan.sessions[run_session.plan_session_index] %} -{%- set constellation = plan_session.constellation %} -

Constellation

-
- -{%- for role_name, node in constellation.roles.items() %} -{% if role_name in run_session.constellation.nodes %} -{%- set session_node = run_session.constellation.nodes[role_name] %} -
-

{{ role_name }}

-
{{ local_name_with_tooltip(node.nodedriver) }}
-
{{ session_node.appdata['app'] }}
-
{{ session_node.appdata['app_version'] or '?'}}
-{%- if node.parameters %} - - - - - - - -{%- for key, value in node.parameters.items() %} - - - - -{%- endfor %} - -
Parameters:
{{ key }}{{ value }}
-{%- endif %} -{%- endif %} -
-{%- else %} - (no roles) -{%- endfor %} -
- -

Test Results

-
-{%- for test_index, run_test in enumerate(run_session.run_tests) %} -{%- set plan_test_spec = plan_session.tests[run_test.plan_test_index] %} -{%- set test_meta = run.test_meta[plan_test_spec.name] %} -
-

Test: {{ test_meta.name }}

-{%- if test_meta.description %} -
{{ test_meta.description}}
-{%- endif %} -

Started {{ format_timestamp(run_test.started) }}, ended {{ format_timestamp(run_test.ended) }} (duration: {{ format_duration(run_test.ended - run_test.started) }})

- {%- with result=run_test.worst_result %} -{%- include "partials/test_session/testresult.jinja2" %} -{%- endwith %} -{%- for test_step_index, run_step in enumerate(run_test.run_steps or []) %} -
-{% set test_step_meta = test_meta.steps[run_step.plan_step_index] %} -
Test step: {{ test_step_meta.name }}
-{%- if test_step_meta.description %} -
{{ test_step_meta.description}}
-{%- endif %} -

Started {{ format_timestamp(run_test.started) }}, ended {{ format_timestamp(run_test.ended) }} (duration: {{ format_duration(run_test.ended - run_test.started) }})

-{%- with result=run_step.result, idmod='step' %} -{%- include "partials/test_session/testresult.jinja2" %} -{%- endwith %} -
-{%- endfor %} -
-{%- endfor %} -
diff --git a/src/feditest/templates/default/partials/test_session/title.jinja2 b/src/feditest/templates/default/partials/test_session/title.jinja2 deleted file mode 100644 index 47f233e..0000000 --- a/src/feditest/templates/default/partials/test_session/title.jinja2 +++ /dev/null @@ -1,5 +0,0 @@ -{% set plan_session = run.plan.sessions[run_session.plan_session_index] %} -
-

Feditest Session Report: {{ plan_session.name }}

-

{{ run.id }} [Summary]

-
\ No newline at end of file diff --git a/src/feditest/templates/default/test_matrix.jinja2 b/src/feditest/templates/default/test_matrix.jinja2 deleted file mode 100644 index 5229035..0000000 --- a/src/feditest/templates/default/test_matrix.jinja2 +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - Feditest Test Report - - - - -{% include "partials/test_matrix/title.jinja2" %} -{% include "partials/shared/mobile.jinja2" %} - -
-

Test Run Summary

-{% include "partials/shared/summary.jinja2" %} -
-
-{% with session_links=true %} -

Test Results

-{% include "partials/test_matrix/table.jinja2" %} -{% endwith %} -
-
-

Test Run Metadata

-{% include "partials/test_matrix/metadata.jinja2" %} -
-{% include "partials/shared/footer.jinja2" %} - - diff --git a/src/feditest/templates/default/test_session.jinja2 b/src/feditest/templates/default/test_session.jinja2 deleted file mode 100644 index 79639f1..0000000 --- a/src/feditest/templates/default/test_session.jinja2 +++ /dev/null @@ -1,38 +0,0 @@ - - - - - -{% set plan_session = run.plan.sessions[run_session.plan_session_index] %} - {{ plan_session.name }} | Feditest - - - - - -{% include "partials/test_session/title.jinja2" %} -{% include "partials/shared/mobile.jinja2" %} - -
-

Test Run Summary

-{% include "partials/shared/summary.jinja2" %} -
-
-{% include "partials/test_session/results.jinja2" %} -
-
-

Test Run Metadata

-{% include "partials/test_session/metadata.jinja2" %} -
-{% include "partials/shared/footer.jinja2" %} - - diff --git a/src/feditest/testplan.py b/src/feditest/testplan.py index 5b32028..e7c0d54 100644 --- a/src/feditest/testplan.py +++ b/src/feditest/testplan.py @@ -339,52 +339,36 @@ def simplify(self) -> None: self.rolemapping = new_rolemapping -class TestPlanSession(msgspec.Struct): +class TestPlanSessionTemplate(msgspec.Struct): """ - A TestPlanSession spins up and tears down a constellation of Nodes against which a sequence of tests - is run. The constellation has 1 or more roles, which are bound to nodes that communicate with - each other according to the to-be-tested protocol(s) during the test. - - This class is used in two ways: - 1. as part of a TestPlan, which means the roles in the constellation are bound to particular NodeDrivers - 2. as a template in TestPlan generation, which means the roles in the constellation have been defined - but aren't bound to particular NodeDrivers yet + A TestPlanSessionTemplate defines a list of tests that will be executed in the context of + a particular TestPlanConstellation. The session template and the constellation together make the session. """ - constellation : TestPlanConstellation tests : list[TestPlanTestSpec] name: str | None = None @staticmethod - def load(filename: str) -> 'TestPlanSession': + def load(filename: str) -> 'TestPlanSessionTemplate': """ Read a file, and instantiate a TestPlanSession from what we find. """ with open(filename, 'r', encoding='utf-8') as f: testplansession_json = json.load(f) - return msgspec.convert(testplansession_json, type=TestPlanSession) + return msgspec.convert(testplansession_json, type=TestPlanSessionTemplate) def __str__(self): return self.name if self.name else 'Unnamed' - def is_template(self): - """ - Returns true if the roles in the constellation have not all been bound to NodeDrivers. - """ - return self.constellation.is_template() - - - def check_can_be_executed(self, context_msg: str = "") -> None: - self.constellation.check_can_be_executed(context_msg) - + def check_can_be_executed(self, constellation: TestPlanConstellation, context_msg: str = "") -> None: if not self.tests: raise TestPlanError(context_msg + 'No tests have been defined.') for index, test_spec in enumerate(self.tests): - test_spec.check_can_be_executed(self.constellation, context_msg + f'Test (index {index}): ') + test_spec.check_can_be_executed(constellation, context_msg + f'Test (index {index}): ') def needed_constellation_role_names(self) -> set[str]: @@ -394,15 +378,6 @@ def needed_constellation_role_names(self) -> set[str]: return ret - def instantiate_with_constellation(self, constellation: TestPlanConstellation, name: str | None = None) -> 'TestPlanSession': - """ - Treat this session as a template. Create a new (non-template) session that's like this one - and that uses the provided constellation. - """ - constellation.check_defines_all_role_names(self.needed_constellation_role_names()) - return TestPlanSession(constellation=constellation,tests=self.tests, name=name) - - def simplify(self) -> None: """ If possible, simplify this test plan session. @@ -428,10 +403,10 @@ def print(self) -> None: class TestPlan(msgspec.Struct): """ - A TestPlan defines one or more TestPlanSessions. TestPlanSessions can be run sequentially, or - (in theory; no code yet) in parallel. + A TestPlan runs the same TestPlanSession with one or more TestPlanConstellations. """ - sessions : list[TestPlanSession] = [] + session_template : TestPlanSessionTemplate + constellations : list[TestPlanConstellation] name: str | None = None type: str = 'feditest-testplan' feditest_version: str = FEDITEST_VERSION @@ -441,8 +416,7 @@ def simplify(self) -> None: """ If possible, simplify this test plan. """ - for session in self.sessions: - session.simplify() + self.session_template.simplify() @staticmethod @@ -489,8 +463,6 @@ def check_can_be_executed(self, context_msg: str = "") -> None: """ Check that this TestPlan is ready for execution. If not, raise a TestPlanEerror that explains the problem. """ - if not self.sessions: - raise TestPlanError('No TestPlanSessions have been defined in TestPlan') - - for index, session in enumerate(self.sessions): - session.check_can_be_executed(context_msg + f'TestPlanSession {index}: ') + for constellation in self.constellations: + constellation.check_can_be_executed(context_msg + f'Constellation { constellation }: ') + self.session_template.check_can_be_executed(constellation, context_msg + f'TestPlanSession with Constellation { constellation }: ') diff --git a/src/feditest/testrun.py b/src/feditest/testrun.py index 5d83307..ef847d2 100644 --- a/src/feditest/testrun.py +++ b/src/feditest/testrun.py @@ -19,7 +19,8 @@ from feditest.testplan import ( TestPlan, TestPlanConstellation, - TestPlanSession, + TestPlanConstellationNode, + TestPlanSessionTemplate, TestPlanTestSpec, ) from feditest.testruntranscript import ( @@ -58,7 +59,7 @@ class TestRunConstellation: """ The instance of a TestPlanConstellation associated with a particular test run. """ - def __init__(self, plan_constellation: TestPlanConstellation ): + def __init__(self, plan_constellation: TestPlanConstellation): self._plan_constellation = plan_constellation self._nodes : dict[str, Node] = {} self._appdata : dict[str, dict[str, str | None]] = {} # Record what apps and versions are running here. Preserved beyond teardown. @@ -160,9 +161,10 @@ def __init__(self) -> None: class TestRunTest(HasStartEndResults): - def __init__(self, run_session: 'TestRunSession', plan_test_index: int): + def __init__(self, run_session: 'TestRunSession', run_constellation: TestRunConstellation, plan_test_index: int): super().__init__() self.run_session = run_session + self.run_constellation = run_constellation self.plan_test_index = plan_test_index @@ -179,8 +181,8 @@ def outcome(self) -> Exception | None: class TestRunFunction(TestRunTest): - def __init__(self, run_session: 'TestRunSession', test_from_test_function: feditest.TestFromTestFunction, plan_test_index: int): - super().__init__(run_session, plan_test_index) + def __init__(self, run_session: 'TestRunSession', run_constellation: TestRunConstellation, test_from_test_function: feditest.TestFromTestFunction, plan_test_index: int): + super().__init__(run_session, run_constellation, plan_test_index) self.test_from_test_function = test_from_test_function @@ -201,7 +203,7 @@ def run(self, controller: feditest.testruncontroller.TestRunController) -> None: constellation_role_name = local_role_name if self.plan_testspec.rolemapping and local_role_name in self.plan_testspec.rolemapping: constellation_role_name = self.plan_testspec.rolemapping[local_role_name] - args[local_role_name] = self.run_session.run_constellation.get_node(constellation_role_name) # type: ignore[union-attr] + args[local_role_name] = self.run_constellation.get_node(constellation_role_name) # type: ignore[union-attr] try: self.test_from_test_function.test_function(**args) @@ -250,8 +252,8 @@ def run(self, test_instance: object, controller: feditest.testruncontroller.Test class TestRunClass(TestRunTest): - def __init__(self, run_session: 'TestRunSession', test_from_test_class: feditest.TestFromTestClass, plan_test_index: int): - super().__init__(run_session, plan_test_index) + def __init__(self, run_session: 'TestRunSession', run_constellation: TestRunConstellation, test_from_test_class: feditest.TestFromTestClass, plan_test_index: int): + super().__init__(run_session, run_constellation, plan_test_index) self.run_steps : list[TestRunStepInClass] = [] self.test_from_test_class = test_from_test_class @@ -273,7 +275,7 @@ def run(self, controller: feditest.testruncontroller.TestRunController) -> None: constellation_role_name = local_role_name if self.plan_testspec.rolemapping and local_role_name in self.plan_testspec.rolemapping: constellation_role_name = self.plan_testspec.rolemapping[local_role_name] - args[local_role_name] = self.run_session.run_constellation.get_node(constellation_role_name) # type: ignore[union-attr] + args[local_role_name] = self.run_constellation.get_node(constellation_role_name) # type: ignore[union-attr] try: test_instance = self.test_from_test_class.clazz(**args) @@ -311,17 +313,17 @@ def run(self, controller: feditest.testruncontroller.TestRunController) -> None: class TestRunSession(HasStartEndResults): - def __init__(self, the_run: 'TestRun', plan_session_index: int): + def __init__(self, the_run: 'TestRun', plan_constellation_index: int): super().__init__() self.the_run = the_run - self.plan_session_index = plan_session_index + self.plan_constellation_index = plan_constellation_index self.run_tests : list[TestRunTest] = [] - self.run_constellation : TestRunConstellation | None = None + self.run_constellation : TestRunConstellation | None = None # keep around for transcript @property - def plan_session(self) -> TestPlanSession: - return self.the_run.plan.sessions[self.plan_session_index] + def plan_session(self) -> TestPlanSessionTemplate: + return self.the_run.plan.session_template def __str__(self): @@ -335,7 +337,6 @@ def run(self, controller: feditest.testruncontroller.TestRunController) -> None: return: the number of tests run, or a negative number to signal that not all tests were run or completed """ self.started = datetime.now(UTC) - info(f'Started TestRunSession for TestPlanSession { self }') try: plan_test_index = controller.determine_next_test_index() @@ -346,21 +347,21 @@ def run(self, controller: feditest.testruncontroller.TestRunController) -> None: if test_spec.skip: info('Skipping Test:', test_spec.skip) else: + if not self.run_constellation: + # only allocate the constellation if we actually want to run a test + self.run_constellation = TestRunConstellation(self.the_run.plan.constellations[self.plan_constellation_index]) + self.run_constellation.setup() + test = test_spec.get_test() run_test : TestRunTest | None = None if isinstance(test, feditest.TestFromTestFunction): - run_test = TestRunFunction(self, test, plan_test_index) + run_test = TestRunFunction(self, self.run_constellation, test, plan_test_index) elif isinstance(test, feditest.TestFromTestClass): - run_test = TestRunClass(self, test, plan_test_index) + run_test = TestRunClass(self, self.run_constellation, test, plan_test_index) else: fatal('What is this?', test) return # does not actually return, but makes lint happy - if not self.run_constellation: - # only allocate the constellation if we actually want to run a test - self.run_constellation = TestRunConstellation(self.plan_session.constellation) - self.run_constellation.setup() - self.run_tests.append(run_test) # constellation.setup() may raise, so don't add before that run_test.run(controller) @@ -431,14 +432,14 @@ def run(self, controller: feditest.testruncontroller.TestRunController) -> None: info(f'Started TestRun { self }') try: - plan_session_index = controller.determine_next_session_index() - while plan_session_index >=0 and plan_session_index=0 and plan_constellation_index TestRunTranscript: trans_test_metas = {} for run_session in self.run_sessions: nodes_transcript: dict[str, TestRunNodeTranscript] = {} - for node_role, appdata in cast(TestRunConstellation, run_session.run_constellation)._appdata.items(): - nodes_transcript[node_role] = TestRunNodeTranscript(appdata) + run_constellation = cast(TestRunConstellation, run_session.run_constellation) + for node_role, appdata in run_constellation._appdata.items(): + node = cast(TestPlanConstellationNode, run_constellation._plan_constellation.roles[node_role]) + nodes_transcript[node_role] = TestRunNodeTranscript(appdata, cast(str, node.nodedriver)) trans_constellation = TestRunConstellationTranscript(nodes_transcript) trans_tests : list[TestRunTestTranscript] = [] for run_test in run_session.run_tests: @@ -492,7 +495,7 @@ def transcribe(self) -> TestRunTranscript: trans_test_metas[test.name] = TestMetaTranscript(test.name, test.needed_local_role_names(), meta_steps, test.description) trans_sessions.append(TestRunSessionTranscript( - run_session.plan_session_index, + run_session.plan_constellation_index, cast(datetime, run_session.started), cast(datetime, run_session.ended), trans_constellation, diff --git a/src/feditest/testruncontroller.py b/src/feditest/testruncontroller.py index 7fbd722..cdba0f5 100644 --- a/src/feditest/testruncontroller.py +++ b/src/feditest/testruncontroller.py @@ -42,9 +42,9 @@ def __init__(self, run: 'feditest.testrun.TestRun' ): @abstractmethod - def determine_next_session_index(self, last_session_index: int = -1) -> int: + def determine_next_constellation_index(self, last_constellation_index: int = -1) -> int: """ - last_session_index: -1 means: we haven't started yet + last_constellation_index: -1 means: we haven't started yet """ ... @@ -66,8 +66,8 @@ def determine_next_test_step_index(self, last_test_step_index: int = -1) -> int: class AutomaticTestRunController(TestRunController): - def determine_next_session_index(self, last_session_index: int = -1) -> int: - return last_session_index+1 + def determine_next_constellation_index(self, last_constellation_index: int = -1) -> int: + return last_constellation_index+1 def determine_next_test_index(self, last_test_index: int = -1) -> int: @@ -79,21 +79,22 @@ def determine_next_test_step_index(self, last_test_step_index: int = -1) -> int: class InteractiveTestRunController(TestRunController): - def determine_next_session_index(self, last_session_index: int = -1) -> int: + def determine_next_constellation_index(self, last_constellation_index: int = -1) -> int: """ - A TestRunSession has just completed. Which TestRunSession should we run next? + A TestRunSession with a certain TestRunConstellation has just completed. Which TestRunConstellation should + we run it with next? """ - if last_session_index >= 0: - prompt = 'Which TestSession to run next? n(ext session), r(repeat just completed session), (session number), q(uit): ' + if last_constellation_index >= 0: + prompt = 'Which Constellation to run tests with next? n(ext constellation), r(repeat just completed constellation), (constellation number), q(uit): ' else: - prompt = 'Which TestSession to run first? n(ext/first session), (session number), q(uit): ' + prompt = 'Which Constellation to run first? n(ext/first constellation), (constellation number), q(uit): ' while True: answer = self._prompt_user(prompt) match answer: case 'n': - return last_session_index+1 + return last_constellation_index+1 case 'r': - return last_session_index + return last_constellation_index case 'q': raise AbortTestRunException() try: diff --git a/src/feditest/testruntranscript.py b/src/feditest/testruntranscript.py index 0699495..494b3b3 100644 --- a/src/feditest/testruntranscript.py +++ b/src/feditest/testruntranscript.py @@ -1,18 +1,10 @@ -import contextlib -import io import json import os import os.path -import re -import shutil -import sys import traceback -from abc import ABC, abstractmethod from datetime import datetime -import html -from typing import IO, Iterator, Optional +from typing import IO, Optional -import jinja2 import msgspec import feditest @@ -26,6 +18,7 @@ class TestRunNodeTranscript(msgspec.Struct): Information about a node in a constellation in a transcript. """ appdata: dict[str, str | None] + node_driver: str """ So far, contains: app: name of the app running at the node (required) @@ -39,6 +32,9 @@ class TestRunConstellationTranscript(msgspec.Struct): """ nodes: dict[str,TestRunNodeTranscript] + def __str__(self): + return ', '.join( [ f'{ role }: { node.node_driver }' for role, node in self.nodes.items() ] ) + _result_transcript_tracker : list['TestRunResultTranscript'] = [] """ @@ -299,7 +295,7 @@ class TestRunSessionTranscript(msgspec.Struct): """ Captures information about the run of a single session in a transcript. """ - plan_session_index: int + run_session_index: int started : datetime ended : datetime constellation: TestRunConstellationTranscript @@ -319,7 +315,7 @@ def build_summary(self, augment_this: TestRunTranscriptSummary | None = None ): def __str__(self): - return f"Session {self.plan_session_index}" + return f"Session {self.run_session_index}" class TestRunTranscript(msgspec.Struct): @@ -390,253 +386,3 @@ def __str__(self): return f'{ self.id } ({ self.plan.name })' return self.id - -class TestRunTranscriptSerializer(ABC): - """ - An object that knows how to serialize a TestRunTranscript into some output format. - """ - def __init__(self, transcript: TestRunTranscript ): - self.transcript = transcript - - - def write(self, dest: str | None = None): - """ - dest: name of the file to write to, or stdout - """ - if dest and isinstance(dest,str): - with open(dest, "w", encoding="utf8") as out: - self._write(out) - else: - self._write(sys.stdout) - - - def write_to_string(self): - """ - Return the written content as a string; this is for testing. - """ - string_io = io.StringIO() - self._write(string_io) - return string_io.getvalue() - - - @abstractmethod - def _write(self, fd: IO[str]): - ... - - -class SummaryTestRunTranscriptSerializer(TestRunTranscriptSerializer): - """ - Knows how to serialize a TestRunTranscript into a single-line summary. - """ - def _write(self, fd: IO[str]): - summary = self.transcript.build_summary() - - print(f'Test summary: total={ summary.n_total }' - + f', passed={ summary.n_passed }' - + f', failed={ summary.n_failed }' - + f', skipped={ summary.n_skipped }' - + f', errors={ summary.n_errored }.', - file=fd) - - -class TapTestRunTranscriptSerializer(TestRunTranscriptSerializer): - """ - Knows how to serialize a TestRunTranscript into a report in TAP format. - """ - def _write(self, fd: IO[str]): - plan = self.transcript.plan - summary = self.transcript.build_summary() - - fd.write("TAP version 14\n") - fd.write(f"# test plan: { plan }\n") - for key in ['started', 'ended', 'platform', 'username', 'hostname']: - value = getattr(self.transcript, key) - if value: - fd.write(f"# {key}: {value}\n") - - test_id = 0 - for session_transcript in self.transcript.sessions: - plan_session = plan.sessions[session_transcript.plan_session_index] - constellation = plan_session.constellation - - fd.write(f"# session: { plan_session }\n") - fd.write(f"# constellation: { constellation }\n") - fd.write("# roles:\n") - for role_name, node in plan_session.constellation.roles.items(): - if role_name in session_transcript.constellation.nodes: - transcript_role = session_transcript.constellation.nodes[role_name] - fd.write(f"# - name: {role_name}\n") - if node: - fd.write(f"# driver: {node.nodedriver}\n") - fd.write(f"# app: {transcript_role.appdata['app']}\n") - fd.write(f"# app_version: {transcript_role.appdata['app_version'] or '?'}\n") - else: - fd.write(f"# - name: {role_name} -- not instantiated\n") - - for test_index, run_test in enumerate(session_transcript.run_tests): - test_id += 1 - - plan_test_spec = plan_session.tests[run_test.plan_test_index] - test_meta = self.transcript.test_meta[plan_test_spec.name] - - result = run_test.worst_result - if result: - fd.write(f"not ok {test_id} - {test_meta.name}\n") - fd.write(" ---\n") - fd.write(f" problem: {result.type} ({ result.spec_level }, { result.interop_level })\n") - if result.msg: - fd.write(" message:\n") - fd.write("\n".join( [ f" { p }" for p in result.msg.strip().split("\n") ] ) + "\n") - fd.write(" where:\n") - for loc in result.stacktrace: - fd.write(f" {loc[0]} {loc[1]}\n") - fd.write(" ...\n") - else: - directives = "" # FIXME f" # SKIP {test.skip}" if test.skip else "" - fd.write(f"ok {test_id} - {test_meta.name}{directives}\n") - - fd.write(f"1..{test_id}\n") - fd.write("# test run summary:\n") - fd.write(f"# total: {summary.n_total}\n") - fd.write(f"# passed: {summary.n_passed}\n") - fd.write(f"# failed: {summary.n_failed}\n") - fd.write(f"# skipped: {summary.n_skipped}\n") - fd.write(f"# errors: {summary.n_errored}\n") - - -class MultifileRunTranscriptSerializer: - """Generates the Feditest reports into a test matrix and a linked, separate - file per session. It uses a template path (comma-delimited string) - so variants of reports can be generated while sharing common templates. The file_ext - can be specified to support generating other formats like MarkDown. - - The generation uses two primary templates. The 'test_matrix.jinja2' template is used for - the matrix generation and 'test_session.jinja2' is used for session file generation. - - Any files in the a directory called "static" in a template folder will be copied verbatim - to the output directory (useful for css, etc.). If a file exists in the static folder of more - than one directory in the template path, earlier path entries will overwrite later ones. - """ - - def __init__( - self, - matrix_file: str | os.PathLike, - template_path: str, - file_ext: str = "html" - ): - self.matrix_file = matrix_file - templates_base_dir = os.path.join(os.path.dirname(__file__), "templates") - self.template_path = [ - os.path.join(templates_base_dir, t) for t in template_path.split(",") - ] - self.file_ext = file_ext - - - def write(self, transcript: TestRunTranscript): - dir, base_matrix_file = os.path.split(self.matrix_file) - self._copy_static_files_to(dir) - - jinja2_env = self._init_jinja2_env() - - def session_file_path(plan_session): - last_dot = base_matrix_file.rfind('.') - if last_dot > 0: - ret = f'{ base_matrix_file[0:last_dot] }.{ plan_session.name }.{ self.file_ext }' - else: - ret = f'{ base_matrix_file }.{ plan_session.name }.{ self.file_ext }' - return ret - - context = dict( - feditest=feditest, - run=transcript, - summary=transcript.build_summary(), - getattr=getattr, - sorted=sorted, - enumerate=enumerate, - get_results_for=_get_results_for, - remove_white=lambda s: re.sub("[ \t\n\a]", "_", str(s)), - session_file_path=session_file_path, - matrix_file_path=base_matrix_file, - permit_line_breaks_in_identifier=lambda s: re.sub( - r"(\.|::)", r"\1", s - ), - local_name_with_tooltip=lambda n: f'{ n.split(".")[-1] }', - format_timestamp=lambda ts: ts.strftime("%Y:%m:%d-%H:%M:%S.%fZ") if ts else "", - format_duration=lambda s: str(s), # makes it easier to change in the future - len=len, - html_escape=lambda s: html.escape(str(s)) - ) - - try: - with open(self.matrix_file, "w") as fp: - matrix_template = jinja2_env.get_template("test_matrix.jinja2") - fp.write(matrix_template.render(**context)) - - session_template = jinja2_env.get_template("test_session.jinja2") - for run_session in transcript.sessions: - session_context = dict(context) - session_context.update( - run_session=run_session, - summary=run_session.build_summary(), - ) - plan_session = transcript.plan.sessions[run_session.plan_session_index] - with open(os.path.join(dir, session_file_path(plan_session)), "w" ) as fp: - fp.write(session_template.render(**session_context)) - except jinja2.exceptions.TemplateNotFound as ex: - with contextlib.redirect_stdout(sys.stderr): - print(f"ERROR: template '{ex}' not found") - print("Searched in the following directories:") - for entry in self.template_path: - print(f" {entry}") - sys.exit(1) - - - def _init_jinja2_env(self) -> jinja2.Environment: - templates = jinja2.Environment( - loader=jinja2.FileSystemLoader(self.template_path) - ) - templates.filters["regex_sub"] = lambda s, pattern, replacement: re.sub( - pattern, replacement, s - ) - return templates - - - def _copy_static_files_to(self, dir: str | os.PathLike): - for path in reversed(self.template_path): - static_dir = os.path.join(path, "static") - if os.path.exists(static_dir): - for dirpath, _, filenames in os.walk(static_dir): - if len(filenames) == 0: - continue - filedir_to = os.path.join( - dir, os.path.relpath(dirpath, static_dir) - ) - if not os.path.exists(filedir_to): - os.makedirs(filedir_to, exist_ok=True) - for filename in filenames: - filepath_from = os.path.join(dirpath, filename) - filepath_to = os.path.join(filedir_to, filename) - # Notusing copytree since it would copy static too - shutil.copyfile(filepath_from, filepath_to) - - -def _get_results_for(run_transcript: TestRunTranscript, session_transcript: TestRunSessionTranscript, test_meta: TestMetaTranscript) -> Iterator[TestRunResultTranscript | None]: - """ - Determine the set of test results running test_meta within session_transcript, and return it as an Iterator. - This is a set, not a single value, because we might run the same test multiple times (perhaps with differing role - assignments) in the same session. The run_transcript is passed in because session_transcript does not have a pointer "up". - """ - plan_session = run_transcript.plan.sessions[session_transcript.plan_session_index] - for test_transcript in session_transcript.run_tests: - plan_testspec = plan_session.tests[test_transcript.plan_test_index] - if plan_testspec.name == test_meta.name: - yield test_transcript.worst_result - return None - - -class JsonTestRunTranscriptSerializer(TestRunTranscriptSerializer): - """ - An object that knows how to serialize a TestRun into JSON format - """ - def _write(self, fd: IO[str]): - self.transcript.write(fd) diff --git a/src/feditest/testruntranscriptserializer/__init__.py b/src/feditest/testruntranscriptserializer/__init__.py new file mode 100644 index 0000000..40bc6ae --- /dev/null +++ b/src/feditest/testruntranscriptserializer/__init__.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod +import io +import sys +from typing import IO + +from feditest.testruntranscript import TestRunTranscript + + +class TestRunTranscriptSerializer(ABC): + """ + An object that knows how to serialize a TestRunTranscript into some output format. + """ + @abstractmethod + def write(self, transcript: TestRunTranscript, dest: str | None): + ... + + +class FileOrStdoutTestRunTranscriptSerializer(TestRunTranscriptSerializer): + def write(self, transcript: TestRunTranscript, dest: str | None) -> None: + """ + dest: name of the file to write to, or stdout + """ + if dest and isinstance(dest,str): + with open(dest, "w", encoding="utf8") as out: + self._write(transcript, out) + else: + self._write(transcript, sys.stdout) + + + def write_to_string(self, transcript: TestRunTranscript) -> str: + """ + Return the written content as a string; this is for testing. + """ + string_io = io.StringIO() + self._write(transcript, string_io) + return string_io.getvalue() + + + @abstractmethod + def _write(self, transcript: TestRunTranscript, fd: IO[str]) -> None: + ... + + diff --git a/src/feditest/testruntranscriptserializer/html.py b/src/feditest/testruntranscriptserializer/html.py new file mode 100644 index 0000000..16a2e06 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/html.py @@ -0,0 +1,153 @@ +# import contextlib +import html +import os.path +import re +import shutil +from typing import Any, Iterator + +import jinja2 + +import feditest +from feditest.reporting import fatal +from feditest.testruntranscript import TestMetaTranscript, TestRunResultTranscript, TestRunSessionTranscript, TestRunTranscript +from feditest.testruntranscriptserializer import TestRunTranscriptSerializer + + +def _get_results_for(run_transcript: TestRunTranscript, session_transcript: TestRunSessionTranscript, test_meta: TestMetaTranscript) -> Iterator[TestRunResultTranscript | None]: + """ + Determine the set of test results running test_meta within session_transcript, and return it as an Iterator. + This is a set, not a single value, because we might run the same test multiple times (perhaps with differing role + assignments) in the same session. The run_transcript is passed in because session_transcript does not have a pointer "up". + """ + plan_session_template = run_transcript.plan.session_template + for test_transcript in session_transcript.run_tests: + plan_testspec = plan_session_template.tests[test_transcript.plan_test_index] + if plan_testspec.name == test_meta.name: + yield test_transcript.worst_result + return None + + +def _derive_full_and_local_filename(base: str, suffix: str) -> tuple[str,str]: + """ + Given a base filename, derive another filename (e.g. generate a .css filename from an .html filename). + Return the full filename with path, and the local filename + """ + dir = os.path.dirname(base) + local = os.path.basename(base) + last_dot = local.rfind('.') + if last_dot > 0: + derived = f'{ local[0:last_dot] }{ suffix }' + else: + derived = f'{ local }.{ suffix }' + return (os.path.join(dir, derived), derived) + + +class HtmlRunTranscriptSerializer(TestRunTranscriptSerializer): + """ + Generates the Feditest reports as HTML. + If the transcript contains one session, it will generate one HTML file to the provided destination. + + If the transcript contains multiple sessions, it will generate one HTML file per session and + an overview test matrix. The test matrix will be at the provided destination, and the session + files will have longer file names starting with the filename of the destination. + + A CSS file will be written to the provided destination with an extra extension. + """ + + def __init__(self, template_path: str): + if template_path: + self.template_path = [ t.strip() for t in template_path.split(",") ] + else: + self.template_path = [ os.path.join(os.path.dirname(__file__), "templates/testplantranscript_default") ] + + self.jinja2_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(self.template_path) + ) + self.jinja2_env.filters["regex_sub"] = lambda s, pattern, replacement: re.sub( + pattern, replacement, s + ) + + + # Python 3.12 @override + def write(self, transcript: TestRunTranscript, dest: str | None): + if dest is None: + fatal('Cannot write --html to stdout.') + return # make linter happy + if len(transcript.sessions) == 0: + fatal('No session in transcript: cannot transcribe') + + ( cssfile, local_cssfile ) = _derive_full_and_local_filename(dest, '.css') + base_context = dict( + feditest=feditest, + cssfile = local_cssfile, + getattr=getattr, + sorted=sorted, + enumerate=enumerate, + get_results_for=_get_results_for, + remove_white=lambda s: re.sub("[ \t\n\a]", "_", str(s)), + permit_line_breaks_in_identifier=lambda s: re.sub( + r"(\.|::)", r"\1", s + ), + local_name_with_tooltip=lambda n: f'{ n.split(".")[-1] }', + format_timestamp=lambda ts: ts.strftime("%Y:%m:%d-%H:%M:%S.%fZ") if ts else "", + format_duration=lambda s: str(s), # makes it easier to change in the future + len=len, + html_escape=lambda s: html.escape(str(s)) + ) + + try: + if len(transcript.sessions) == 1: + self.write_single_session(transcript, base_context, dest) + else: + self.write_matrix_and_sessions(transcript, base_context, dest) + + except jinja2.exceptions.TemplateNotFound as ex: + msg = f"ERROR: template '{ex}' not found.\n" + msg += "Searched in the following directories:" + for entry in self.template_path: + msg += f"\n {entry}" + fatal(msg) + + # One this worked, we can add the CSS file + for path in self.template_path: + css_candidate = os.path.join(path, 'static', 'feditest.css') + if os.path.exists(css_candidate): + shutil.copyfile(css_candidate, cssfile) + break + + + def write_single_session(self, transcript: TestRunTranscript, context: dict[str, Any], dest: str): + run_session = transcript.sessions[0] + context.update( + transcript=transcript, + run_session=run_session, + summary=run_session.build_summary() # if we use 'summary', we can use shared/summary.jinja2 + ) + with open(dest, "w") as fp: + session_template = self.jinja2_env.get_template("session_single.jinja2") + fp.write(session_template.render(**context)) + + + def write_matrix_and_sessions(self, transcript: TestRunTranscript, context: dict[str, Any], dest: str): + matrix_context = dict(context) + matrix_context.update( + transcript=transcript, + session_file_path=lambda session: _derive_full_and_local_filename(dest, f'.{ session.run_session_index }.html')[1], + summary=transcript.build_summary() # if we use 'summary', we can use shared/summary.jinja2 + ) + with open(dest, "w") as fp: + matrix_template = self.jinja2_env.get_template("matrix.jinja2") + fp.write(matrix_template.render(**matrix_context)) + + for run_session in transcript.sessions: + session_context = dict(context) + session_context.update( + transcript=transcript, + run_session=run_session, + summary=run_session.build_summary(), # if we use 'summary', we can use shared/summary.jinja2 + matrix_file_path=os.path.basename(dest) + ) + session_dest = _derive_full_and_local_filename(dest, f'.{ run_session.run_session_index }.html')[0] + with open(session_dest, "w") as fp: + session_template = self.jinja2_env.get_template("session_with_matrix.jinja2") + fp.write(session_template.render(**session_context)) diff --git a/src/feditest/testruntranscriptserializer/json.py b/src/feditest/testruntranscriptserializer/json.py new file mode 100644 index 0000000..68206b9 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/json.py @@ -0,0 +1,12 @@ +from typing import IO + +from feditest.testruntranscript import TestRunTranscript +from feditest.testruntranscriptserializer import FileOrStdoutTestRunTranscriptSerializer + + +class JsonTestRunTranscriptSerializer(FileOrStdoutTestRunTranscriptSerializer): + """ + An object that knows how to serialize a TestRun into JSON format + """ + def _write(self, transcript: TestRunTranscript, fd: IO[str]): + transcript.write(fd) diff --git a/src/feditest/testruntranscriptserializer/summary.py b/src/feditest/testruntranscriptserializer/summary.py new file mode 100644 index 0000000..9951924 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/summary.py @@ -0,0 +1,21 @@ +from typing import IO + +from feditest.testruntranscript import TestRunTranscript +from feditest.testruntranscriptserializer import FileOrStdoutTestRunTranscriptSerializer + + +class SummaryTestRunTranscriptSerializer(FileOrStdoutTestRunTranscriptSerializer): + """ + Knows how to serialize a TestRunTranscript into a single-line summary. + """ + def _write(self, transcript: TestRunTranscript, fd: IO[str]): + summary = transcript.build_summary() + + print(f'Test summary: total={ summary.n_total }' + + f', passed={ summary.n_passed }' + + f', failed={ summary.n_failed }' + + f', skipped={ summary.n_skipped }' + + f', errors={ summary.n_errored }.', + file=fd) + + diff --git a/src/feditest/testruntranscriptserializer/tap.py b/src/feditest/testruntranscriptserializer/tap.py new file mode 100644 index 0000000..f68e3eb --- /dev/null +++ b/src/feditest/testruntranscriptserializer/tap.py @@ -0,0 +1,69 @@ +from typing import IO + +from feditest.testruntranscript import TestRunTranscript +from feditest.testruntranscriptserializer import FileOrStdoutTestRunTranscriptSerializer + + +class TapTestRunTranscriptSerializer(FileOrStdoutTestRunTranscriptSerializer): + """ + Knows how to serialize a TestRunTranscript into a report in TAP format. + """ + def _write(self, transcript: TestRunTranscript, fd: IO[str]): + plan = transcript.plan + summary = transcript.build_summary() + + fd.write("TAP version 14\n") + fd.write(f"# test plan: { plan }\n") + for key in ['started', 'ended', 'platform', 'username', 'hostname']: + value = getattr(transcript, key) + if value: + fd.write(f"# {key}: {value}\n") + + test_id = 0 + for session_transcript in transcript.sessions: + plan_session_template = plan.session_template + constellation = session_transcript.constellation + + fd.write(f"# session: { session_transcript }\n") + fd.write(f"# constellation: { constellation }\n") + fd.write("# roles:\n") + for role_name, node in constellation.nodes.items(): + if role_name in session_transcript.constellation.nodes: + transcript_role = session_transcript.constellation.nodes[role_name] + fd.write(f"# - name: {role_name}\n") + if node: + fd.write(f"# driver: {node.node_driver}\n") + fd.write(f"# app: {transcript_role.appdata['app']}\n") + fd.write(f"# app_version: {transcript_role.appdata['app_version'] or '?'}\n") + else: + fd.write(f"# - name: {role_name} -- not instantiated\n") + + for test_index, run_test in enumerate(session_transcript.run_tests): + test_id += 1 + + plan_test_spec = plan_session_template.tests[run_test.plan_test_index] + test_meta = transcript.test_meta[plan_test_spec.name] + + result = run_test.worst_result + if result: + fd.write(f"not ok {test_id} - {test_meta.name}\n") + fd.write(" ---\n") + fd.write(f" problem: {result.type} ({ result.spec_level }, { result.interop_level })\n") + if result.msg: + fd.write(" message:\n") + fd.write("\n".join( [ f" { p }" for p in result.msg.strip().split("\n") ] ) + "\n") + fd.write(" where:\n") + for loc in result.stacktrace: + fd.write(f" {loc[0]} {loc[1]}\n") + fd.write(" ...\n") + else: + directives = "" # FIXME f" # SKIP {test.skip}" if test.skip else "" + fd.write(f"ok {test_id} - {test_meta.name}{directives}\n") + + fd.write(f"1..{test_id}\n") + fd.write("# test run summary:\n") + fd.write(f"# total: {summary.n_total}\n") + fd.write(f"# passed: {summary.n_passed}\n") + fd.write(f"# failed: {summary.n_failed}\n") + fd.write(f"# skipped: {summary.n_skipped}\n") + fd.write(f"# errors: {summary.n_errored}\n") diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 new file mode 100644 index 0000000..7be8890 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 @@ -0,0 +1,36 @@ + + + +{% include "partials/shared/head.jinja2" %} + {{ transcript.name }} | Feditest + + +
+

Feditest Summary Report: {{ transcript.plan.name }}

+

{{ transcript.id }}

+
+{% include "partials/shared/mobile.jinja2" %} + +
+

Test Run Summary

+{% include "partials/shared/summary.jinja2" %} +
+
+{% with session_links=true %} +

Test Results

+{% include "partials/matrix/matrix.jinja2" %} +{% endwith %} +
+ +{% include "partials/shared/footer.jinja2" %} + + diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/matrix.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/matrix.jinja2 new file mode 100644 index 0000000..fbbd276 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/matrix.jinja2 @@ -0,0 +1,54 @@ +{% macro column_headings(first) %} + + {{ first }} +{%- for run_session in transcript.sessions %} +{%- set constellation = run_session.constellation %} + + + +{%- endfor %} + +{% endmacro %} + +
+ + + +{%- for run_session in transcript.sessions %} + +{%- endfor %} + + +{{ column_headings("{0} tests in {1} sessions (alphabetical order)".format(len(transcript.test_meta), len(transcript.sessions))) }} + + +{%- for test_index, ( _, test_meta ) in enumerate(sorted(transcript.test_meta.items())) %} + + +{%- for session_index, run_session in enumerate(transcript.sessions) %} +{%- for result in get_results_for(transcript, run_session, test_meta) %} +{% include "partials/matrix/testresult.jinja2" %} +{%- endfor %} +{%- endfor %} + +{%- endfor %} + + +{{ column_headings("") }} + +
+ {{ permit_line_breaks_in_identifier(test_meta.name) }} +{%- if test_meta.description %} + {{ test_meta.description }} +{%- endif %} +
+
diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/metadata.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/metadata.jinja2 new file mode 100644 index 0000000..fd76e83 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/metadata.jinja2 @@ -0,0 +1,29 @@ + \ No newline at end of file diff --git a/src/feditest/templates/default/partials/test_matrix/testresult.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/testresult.jinja2 similarity index 100% rename from src/feditest/templates/default/partials/test_matrix/testresult.jinja2 rename to src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/matrix/testresult.jinja2 diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/footer.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/footer.jinja2 new file mode 100644 index 0000000..3ac0a3b --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/footer.jinja2 @@ -0,0 +1,4 @@ + \ No newline at end of file diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/head.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/head.jinja2 new file mode 100644 index 0000000..9831643 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/head.jinja2 @@ -0,0 +1,4 @@ + + + + diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/mobile.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/mobile.jinja2 new file mode 100644 index 0000000..3fd529b --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/mobile.jinja2 @@ -0,0 +1,5 @@ +
+

This FediTest report requires a desktop or laptop-class monitor.

+

It cannot be viewed on a mobile device with a small screen.

+

Sorry about that. But then, you weren't going to use a phone to fix reported bugs, either, right?

+
\ No newline at end of file diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 new file mode 100644 index 0000000..b897e3a --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 @@ -0,0 +1,136 @@ +{% if summary.n_total %} + + + + + + + + + + +{%- if summary.n_passed != 0 %} + +{%- else %} + +{%- endif %} + + + + + + + +{%- if summary.n_skipped != 0 %} + +{%- else %} + +{%- endif %} + + + +{%- if summary.n_errored != 0 %} + +{%- else %} + +{%- endif %} + + + + + + + + +
StatusCount
Passed +
{{ summary.n_passed }} ({{ '%.1f' % ( summary.n_passed * 100.0 / summary.n_total ) }}%)
+
+
0
+
Failed + + + + + + + + + + + + + + + + + + +{%- for interop_level in feditest.InteropLevel %} + +{%- endfor %} + + +{%- for spec_level in [ feditest.SpecLevel.SHOULD, feditest.SpecLevel.IMPLIED, feditest.SpecLevel.UNSPECIFIED ] %} + + +{%- for interop_level in feditest.InteropLevel %} + +{%- endfor %} + + +{%- endfor %} + + +{%- for interop_level in feditest.InteropLevel %} + +{%- endfor %} + + + +
Interoperability
CompromisedDegradedUnaffectedUnknownTotal
ConformanceMust +
{{ summary.count_failures_for(feditest.SpecLevel.MUST, interop_level) }}
+
+
+{{ summary.count_failures_for(feditest.SpecLevel.MUST, None) }} + ({{ '%.1f' % ( summary.count_failures_for(feditest.SpecLevel.MUST, None) * 100.0 / summary.n_total ) }}%) +
+
{{ spec_level.formatted_name() }} +
+{{ summary.count_failures_for(spec_level, interop_level) }} +
+
+
+{{ summary.count_failures_for(spec_level, None) }} + ({{ '%.1f' % ( summary.count_failures_for(spec_level, None) * 100.0 / summary.n_total ) }}%) +
+
Total +
+{{ summary.count_failures_for(None, interop_level) }} + ({{ '%.1f' % ( summary.count_failures_for(None, interop_level) * 100.0 / summary.n_total ) }}%) +
+
+
+{{ summary.n_failed }} + ({{ '%.1f' % ( summary.n_failed * 100.0 / summary.n_total ) }}%) +
+
+
Skipped +
+{{ summary.n_skipped }} + ({{ '%.1f' % ( summary.n_skipped * 100.0 / summary.n_total ) }}%) +
+
+
0
+
Errors +
+{{ summary.n_errored }} + ({{ '%.1f' % ( summary.n_errored * 100.0 / summary.n_total ) }}%) +
+
+
0
+
Total +
{{ summary.n_total }}
+
+{%- else %} +

No tests were run.

+{%- endif %} \ No newline at end of file diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/metadata.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/metadata.jinja2 new file mode 100644 index 0000000..299284a --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/metadata.jinja2 @@ -0,0 +1,29 @@ + \ No newline at end of file diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 new file mode 100644 index 0000000..b8c6508 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 @@ -0,0 +1,59 @@ +

Constellation

+
+{%- for role_name, node in run_session.constellation.nodes.items() %} +
+

{{ role_name }}

+
{{ local_name_with_tooltip(node.node_driver) }}
+
{{ node.appdata['app'] }}
+
{{ node.appdata['app_version'] or '?'}}
+{%- if node.parameters %} + + + + + + + +{%- for key, value in node.parameters.items() %} + + + + +{%- endfor %} + +
Parameters:
{{ key }}{{ value }}
+{%- endif %} +
+{%- endfor %} +
+ +

Test Results

+
+{%- for test_index, run_test in enumerate(run_session.run_tests) %} +{%- set plan_test_spec = transcript.plan.session.tests[run_test.plan_test_index] %} +{%- set test_meta = transcript.test_meta[plan_test_spec.name] %} +
+

Test: {{ test_meta.name }}

+{%- if test_meta.description %} +
{{ test_meta.description}}
+{%- endif %} +

Started {{ format_timestamp(run_test.started) }}, ended {{ format_timestamp(run_test.ended) }} (duration: {{ format_duration(run_test.ended - run_test.started) }})

+{%- with result=run_test.worst_result %} +{%- include "partials/shared_session/testresult.jinja2" %} +{%- endwith %} +{%- for test_step_index, run_step in enumerate(run_test.run_steps or []) %} +
+{% set test_step_meta = test_meta.steps[run_step.plan_step_index] %} +
Test step: {{ test_step_meta.name }}
+{%- if test_step_meta.description %} +
{{ test_step_meta.description}}
+{%- endif %} +

Started {{ format_timestamp(run_test.started) }}, ended {{ format_timestamp(run_test.ended) }} (duration: {{ format_duration(run_test.ended - run_test.started) }})

+{%- with result=run_step.result, idmod='step' %} +{%- include "partials/shared_session/testresult.jinja2" %} +{%- endwith %} +
+{%- endfor %} +
+{%- endfor %} +
diff --git a/src/feditest/templates/default/partials/test_session/testresult.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/testresult.jinja2 similarity index 100% rename from src/feditest/templates/default/partials/test_session/testresult.jinja2 rename to src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/testresult.jinja2 diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 new file mode 100644 index 0000000..2ab578c --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 @@ -0,0 +1,33 @@ + + + +{% include "partials/shared/head.jinja2" %} + {{ transcript.name }} | Feditest + + +
+

Feditest Report: {{ transcript.plan.name }}

+

{{ transcript.id }}

+
+{% include "partials/shared/mobile.jinja2" %} + +
+

Test Run Summary

+{% include "partials/shared/summary.jinja2" %} +
+
+{% include "partials/shared_session/results.jinja2" %} +
+ +{% include "partials/shared/footer.jinja2" %} + + diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_with_matrix.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_with_matrix.jinja2 new file mode 100644 index 0000000..093ce30 --- /dev/null +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_with_matrix.jinja2 @@ -0,0 +1,34 @@ + + + +{% include "partials/shared/head.jinja2" %} + {{ run_session.name }} | Feditest + + +
+

Feditest Session Report: {{ run_session }}

+

{{ transcript.id }} [Summary]

+
+ + {% include "partials/shared/mobile.jinja2" %} + +
+

Test Run Summary

+{% include "partials/shared/summary.jinja2" %} +
+
+{% include "partials/shared_session/results.jinja2" %} +
+ +{% include "partials/shared/footer.jinja2" %} + + diff --git a/src/feditest/templates/default/static/feditest.css b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/static/feditest.css similarity index 99% rename from src/feditest/templates/default/static/feditest.css rename to src/feditest/testruntranscriptserializer/templates/testplantranscript_default/static/feditest.css index ec24139..1907417 100644 --- a/src/feditest/templates/default/static/feditest.css +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/static/feditest.css @@ -345,8 +345,9 @@ div.feditest.tests .namedesc > span.name { .feditest dl.roles { display: grid; grid-template-columns: auto auto; - width: fit-content; + justify-items: center; column-gap: 8px; + text-align: left; } .feditest dl.roles dt::after { diff --git a/tests.unit/test_10_register_nodedrivers.py b/tests.unit/test_10_register_nodedrivers.py index 386aa17..decd02d 100644 --- a/tests.unit/test_10_register_nodedrivers.py +++ b/tests.unit/test_10_register_nodedrivers.py @@ -11,6 +11,7 @@ @pytest.fixture(scope="module", autouse=True) def init(): + """ Keep these isolated to this module """ feditest.all_node_drivers = {} feditest._loading_node_drivers = True diff --git a/tests.unit/test_10_register_tests.py b/tests.unit/test_10_register_tests.py index 217e0d0..85c43d4 100644 --- a/tests.unit/test_10_register_tests.py +++ b/tests.unit/test_10_register_tests.py @@ -50,6 +50,9 @@ def test_tests_registered() -> None: def test_functions() -> None: functions = [ testInstance for testInstance in feditest.all_tests.values() if isinstance(testInstance, TestFromTestFunction) ] assert len(functions) == 3 + functions[0].name.endswith('test1') + functions[1].name.endswith('test2') + functions[2].name.endswith('test3') def test_classes() -> None: @@ -57,5 +60,8 @@ def test_classes() -> None: assert len(classes) == 1 singleClass = classes[0] - assert len(singleClass.steps) == 2 + assert singleClass.name.endswith('TestA') + assert len(singleClass.steps) == 2 + singleClass.steps[0].name.startswith('testa') + singleClass.steps[1].name.startswith('testa') diff --git a/tests.unit/test_20_create_test_plan_session_template.py b/tests.unit/test_20_create_test_plan_session_template.py index 2ccf20d..4f150f6 100644 --- a/tests.unit/test_20_create_test_plan_session_template.py +++ b/tests.unit/test_20_create_test_plan_session_template.py @@ -8,7 +8,7 @@ import feditest from feditest import test from feditest.nodedrivers import Node -from feditest.testplan import TestPlanConstellation, TestPlanConstellationNode, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec @pytest.fixture(scope="module", autouse=True) @@ -38,70 +38,36 @@ def test3(role_b: Node) -> None: SESSION_TEMPLATE_NAME = 'My Session' -def _session_template(session_name: str | None) -> TestPlanSession: +def _session_template(session_name: str | None) -> TestPlanSessionTemplate: test_plan_specs : list[TestPlanTestSpec]= [] - constellation_role_names : dict[str,Any] = {} for name in sorted(feditest.all_tests.keys()): test = feditest.all_tests.get(name) if test is None: # make linter happy continue - test_plan_spec = TestPlanTestSpec(name) test_plan_specs.append(test_plan_spec) - for role_name in test.needed_local_role_names(): - constellation_role_names[role_name] = 1 - if not test_plan_spec.rolemapping: - test_plan_spec.rolemapping = {} - test_plan_spec.rolemapping[role_name] = role_name - - constellation_roles: dict[str,TestPlanConstellationNode | None] = {} - for constellation_role_name in constellation_role_names: - constellation_roles[constellation_role_name] = None - - session = TestPlanSession(TestPlanConstellation(constellation_roles), test_plan_specs, session_name) + session = TestPlanSessionTemplate(test_plan_specs, session_name) return session @pytest.fixture() -def unnamed() -> TestPlanSession: +def unnamed() -> TestPlanSessionTemplate: return _session_template(None) @pytest.fixture() -def named() -> TestPlanSession: +def named() -> TestPlanSessionTemplate: return _session_template(SESSION_TEMPLATE_NAME) -def test_session_template_unnamed(unnamed: TestPlanSession) -> None: +def test_session_template_unnamed(unnamed: TestPlanSessionTemplate) -> None: assert unnamed.name is None assert str(unnamed) == 'Unnamed' assert len(unnamed.tests) == 3 - assert unnamed.constellation - assert unnamed.constellation.name is None - assert str(unnamed.constellation) - assert unnamed.constellation.is_template() - assert len(unnamed.constellation.roles) == 3 - assert 'role_a' in unnamed.constellation.roles - assert 'role_b' in unnamed.constellation.roles - assert 'role_c' in unnamed.constellation.roles - assert unnamed.constellation.roles['role_a'] is None - assert unnamed.constellation.roles['role_b'] is None - assert unnamed.constellation.roles['role_c'] is None - - -def test_session_template_named(named: TestPlanSession) -> None: + + +def test_session_template_named(named: TestPlanSessionTemplate) -> None: assert named.name == SESSION_TEMPLATE_NAME assert str(named) == SESSION_TEMPLATE_NAME assert len(named.tests) == 3 - assert named.constellation - assert named.constellation.name is None - assert str(named.constellation) - assert named.constellation.is_template() - assert len(named.constellation.roles) == 3 - assert 'role_a' in named.constellation.roles - assert 'role_b' in named.constellation.roles - assert 'role_c' in named.constellation.roles - assert named.constellation.roles['role_a'] is None - assert named.constellation.roles['role_b'] is None - assert named.constellation.roles['role_c'] is None diff --git a/tests.unit/test_30_create_testplan.py b/tests.unit/test_30_create_testplan.py index 5f06669..1c39ba3 100644 --- a/tests.unit/test_30_create_testplan.py +++ b/tests.unit/test_30_create_testplan.py @@ -7,7 +7,7 @@ import feditest from feditest import test from feditest.nodedrivers import Node -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec @pytest.fixture(scope="module", autouse=True) @@ -50,52 +50,44 @@ def unnamed_constellations() -> list[TestPlanConstellation]: @pytest.fixture -def unnamed_session_templates(test_specs: list[TestPlanTestSpec]) -> list[TestPlanSession]: - return [ - TestPlanSession(TestPlanConstellation({'role_a': None, 'role_b': None}, None), test_specs), - TestPlanSession(TestPlanConstellation({'role_a': None, 'role_b': None, 'role_c': None}, None), test_specs), - ] +def unnamed_session_template(test_specs: list[TestPlanTestSpec]) -> TestPlanSessionTemplate: + return TestPlanSessionTemplate(test_specs) -def construct_testplan(constellations: list[TestPlanConstellation], session_templates: list[TestPlanSession], testplan_name: str) -> TestPlan: +def construct_testplan(constellations: list[TestPlanConstellation], session_template: TestPlanSessionTemplate, testplan_name: str) -> TestPlan: """ Helper to put it together. """ - sessions = [] - for session_template in session_templates: - for constellation in constellations: - session = session_template.instantiate_with_constellation(constellation, constellation.name) - sessions.append(session) - - test_plan = TestPlan(sessions, testplan_name) + test_plan = TestPlan(session_template, constellations, testplan_name) test_plan.simplify() return test_plan -def test_structure(unnamed_constellations: list[TestPlanConstellation], unnamed_session_templates: list[TestPlanSession]) -> None: +def test_structure(unnamed_constellations: list[TestPlanConstellation], unnamed_session_template: TestPlanSessionTemplate) -> None: """ Test the structure of the TestPlan, ignore the naming. """ - test_plan = construct_testplan(unnamed_constellations, unnamed_session_templates, None) - assert len(test_plan.sessions) == 4 + test_plan = construct_testplan(unnamed_constellations, unnamed_session_template, None) + assert test_plan.session_template + assert len(test_plan.constellations) == 2 -def test_all_unnamed(unnamed_constellations: list[TestPlanConstellation], unnamed_session_templates: list[TestPlanSession]) -> None: +def test_all_unnamed(unnamed_constellations: list[TestPlanConstellation], unnamed_session_template: TestPlanSessionTemplate) -> None: """ Only test the naming. """ - test_plan = construct_testplan(unnamed_constellations, unnamed_session_templates, None) + test_plan = construct_testplan(unnamed_constellations, unnamed_session_template, None) assert test_plan.name is None assert str(test_plan) == "Unnamed" -def test_testplan_named(unnamed_constellations: list[TestPlanConstellation], unnamed_session_templates: list[TestPlanSession]) -> None: +def test_testplan_named(unnamed_constellations: list[TestPlanConstellation], unnamed_session_template: TestPlanSessionTemplate) -> None: """ Only test the naming. """ TESTPLAN_NAME = 'My test plan' - test_plan = construct_testplan(unnamed_constellations, unnamed_session_templates, TESTPLAN_NAME) + test_plan = construct_testplan(unnamed_constellations, unnamed_session_template, TESTPLAN_NAME) assert test_plan.name == TESTPLAN_NAME assert str(test_plan) == TESTPLAN_NAME diff --git a/tests.unit/test_40_fallback_fediverse_accounts_from_testplan.py b/tests.unit/test_40_fallback_fediverse_accounts_from_testplan.py index 10d54ef..c5a35bf 100644 --- a/tests.unit/test_40_fallback_fediverse_accounts_from_testplan.py +++ b/tests.unit/test_40_fallback_fediverse_accounts_from_testplan.py @@ -17,7 +17,7 @@ FediverseAccount, FediverseNonExistingAccount ) -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSession +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate HOSTNAME = 'localhost' @@ -37,7 +37,7 @@ def init(): @pytest.fixture(autouse=True) -def the_test_plan() -> TestPlan: +def test_plan_fixture() -> TestPlan: node_driver = FediverseSaasNodeDriver() parameters = { 'hostname' : 'example.com', # Avoid interactive question @@ -57,16 +57,16 @@ def the_test_plan() -> TestPlan: ] node1 = TestPlanConstellationNode(node_driver, parameters, plan_accounts, plan_non_existing_accounts) constellation = TestPlanConstellation( { NODE1_ROLE : node1 }) - session = TestPlanSession(constellation, []) - ret = TestPlan( [ session ] ) + session_template = TestPlanSessionTemplate([]) + ret = TestPlan( session_template, [ constellation ] ) return ret -def test_parse(the_test_plan: TestPlan) -> None: +def test_parse(test_plan_fixture: TestPlan) -> None: """ Tests parsing the TestPlan """ - node1 = the_test_plan.sessions[0].constellation.roles[NODE1_ROLE] + node1 = test_plan_fixture.constellations[0].roles[NODE1_ROLE] node_driver = node1.nodedriver node_config, account_manager = node_driver.create_configuration_account_manager(NODE1_ROLE, node1) diff --git a/tests.unit/test_40_report_node_driver_errors.py b/tests.unit/test_40_report_node_driver_errors.py index eadc031..e668505 100644 --- a/tests.unit/test_40_report_node_driver_errors.py +++ b/tests.unit/test_40_report_node_driver_errors.py @@ -10,16 +10,14 @@ TestPlan, TestPlanConstellation, TestPlanConstellationNode, - TestPlanSession, + TestPlanSessionTemplate, TestPlanTestSpec, ) from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController -from feditest.testruntranscript import ( - JsonTestRunTranscriptSerializer, - SummaryTestRunTranscriptSerializer, - TapTestRunTranscriptSerializer, -) +from feditest.testruntranscriptserializer.json import JsonTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.summary import SummaryTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.tap import TapTestRunTranscriptSerializer class NodeDriverTestException(Exception): @@ -62,20 +60,23 @@ def _provision_node( for t in feditest.all_tests: print( f'TEST: { t }') + def test_faulty_node_driver_reporting() -> None: - plan = TestPlan( [ - TestPlanSession( + plan = TestPlan( + TestPlanSessionTemplate( + [ + TestPlanTestSpec('test_40_report_node_driver_errors::init..dummy') + ] + ), + [ TestPlanConstellation( { 'node' : TestPlanConstellationNode( nodedriver = 'test_40_report_node_driver_errors.init..Faulty_NodeDriver', parameters = { 'app' : 'Dummy for test_faulty_node_driver_reporting'} ) }), - [ - TestPlanTestSpec('test_40_report_node_driver_errors::init..dummy') - ] - ) - ]) + ] + ) run = TestRun(plan) controller = AutomaticTestRunController(run) @@ -84,17 +85,14 @@ def test_faulty_node_driver_reporting() -> None: transcript : feditest.testruntranscript.TestRunTranscript = run.transcribe() # transcript.save('transcript.json') - summary_serializer = SummaryTestRunTranscriptSerializer(transcript) - summary : str = summary_serializer.write_to_string() + summary = SummaryTestRunTranscriptSerializer().write_to_string(transcript) # print(summary) assert 'errors=1' in summary - tap_serializer = TapTestRunTranscriptSerializer(transcript) - tap : str = tap_serializer.write_to_string() + tap = TapTestRunTranscriptSerializer().write_to_string(transcript) # print(tap) assert 'errors: 1' in tap - json_serializer = JsonTestRunTranscriptSerializer(transcript) - j : str = json_serializer.write_to_string() + j = JsonTestRunTranscriptSerializer().write_to_string(transcript) # print(j) assert f'"type": "{NodeDriverTestException.__name__}"' in j diff --git a/tests.unit/test_40_static_account_manager_fallback_fediverse_accounts.py b/tests.unit/test_40_static_account_manager_fallback_fediverse_accounts.py index 5a27338..3c736c3 100644 --- a/tests.unit/test_40_static_account_manager_fallback_fediverse_accounts.py +++ b/tests.unit/test_40_static_account_manager_fallback_fediverse_accounts.py @@ -35,14 +35,12 @@ def account_manager() -> StaticAccountManager: FediverseAccount(None, 'user-2-unallocated'), FediverseAccount('role3', 'user-3-role3'), ] - initial_non_existing_accounts : list[FediverseNonExistingAccount] = [ FediverseNonExistingAccount(None, 'nonuser-0-unallocated'), FediverseNonExistingAccount('role1', 'nonuser-1-role1'), FediverseNonExistingAccount(None, 'nonuser-2-unallocated'), FediverseNonExistingAccount('role3', 'nonuser-3-role3'), ] - ret = StaticAccountManager(initial_accounts, initial_non_existing_accounts) return ret diff --git a/tests.unit/test_40_ubos_mastodon_accounts_from_testplan.py b/tests.unit/test_40_ubos_mastodon_accounts_from_testplan.py index f6ca2c7..e8c91aa 100644 --- a/tests.unit/test_40_ubos_mastodon_accounts_from_testplan.py +++ b/tests.unit/test_40_ubos_mastodon_accounts_from_testplan.py @@ -22,7 +22,7 @@ ) from feditest.nodedrivers.mastodon.ubos import MastodonUbosNodeDriver from feditest.protocols.fediverse import FediverseNonExistingAccount -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSession +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate HOSTNAME = 'localhost' @@ -66,8 +66,8 @@ def the_test_plan() -> TestPlan: ] node1 = TestPlanConstellationNode(node_driver, parameters, plan_accounts, plan_non_existing_accounts) constellation = TestPlanConstellation( { NODE1_ROLE : node1 }) - session = TestPlanSession(constellation, []) - ret = TestPlan( [ session ] ) + session = TestPlanSessionTemplate([]) + ret = TestPlan( session, [ constellation ] ) return ret @@ -75,7 +75,7 @@ def test_parse(the_test_plan: TestPlan) -> None: """ Tests parsing the TestPlan """ - node1 = the_test_plan.sessions[0].constellation.roles[NODE1_ROLE] + node1 = the_test_plan.constellations[0].roles[NODE1_ROLE] node_driver = node1.nodedriver node_config, account_manager = node_driver.create_configuration_account_manager(NODE1_ROLE, node1) diff --git a/tests.unit/test_50_run_assertion_raises.py b/tests.unit/test_50_run_assertion_raises.py index b716270..4d957e4 100644 --- a/tests.unit/test_50_run_assertion_raises.py +++ b/tests.unit/test_50_run_assertion_raises.py @@ -8,10 +8,14 @@ import feditest from feditest import SpecLevel, InteropLevel, assert_that, test -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController -from feditest.testruntranscript import TapTestRunTranscriptSerializer, JsonTestRunTranscriptSerializer, MultifileRunTranscriptSerializer, SummaryTestRunTranscriptSerializer, TestRunTestTranscript +from feditest.testruntranscript import TestRunTestTranscript +from feditest.testruntranscriptserializer.json import JsonTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.html import HtmlRunTranscriptSerializer +from feditest.testruntranscriptserializer.summary import SummaryTestRunTranscriptSerializer +from feditest.testruntranscriptserializer.tap import TapTestRunTranscriptSerializer @pytest.fixture(scope="module", autouse=True) @@ -174,8 +178,8 @@ def the_test_plan() -> TestPlan: constellation = TestPlanConstellation({}, 'No nodes needed') tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] - session = TestPlanSession(constellation, tests, "Test tests that raise various AssertionFailures") - ret = TestPlan( [ session ] ) + session = TestPlanSessionTemplate(tests, "Test tests that raise various AssertionFailures") + ret = TestPlan( session, [ constellation ] ) return ret @@ -229,10 +233,10 @@ def test_run_testplan(the_test_plan: TestPlan): multi_assert( 23, transcript.sessions[0].run_tests, SpecLevel.UNSPECIFIED, InteropLevel.UNKNOWN ) if False: # Make linter happy with import - TapTestRunTranscriptSerializer(transcript).write(f'{ basename(__file__) }.transcript.tap') - MultifileRunTranscriptSerializer(f'{ basename(__file__) }.transcript.html', 'default').write(transcript) - JsonTestRunTranscriptSerializer(transcript).write(f'{ basename(__file__) }.transcript.json') - SummaryTestRunTranscriptSerializer(transcript).write(f'{ basename(__file__) }.transcript.summary.txt') + TapTestRunTranscriptSerializer().write(transcript, f'{ basename(__file__) }.transcript.tap') + HtmlRunTranscriptSerializer().write( transcript, f'{ basename(__file__) }.transcript.html') + JsonTestRunTranscriptSerializer().write(transcript, f'{ basename(__file__) }.transcript.json') + SummaryTestRunTranscriptSerializer().write(transcript, f'{ basename(__file__) }.transcript.summary.txt') def multi_assert(index: int, t: list[TestRunTestTranscript], spec_level: SpecLevel, interop_level: InteropLevel): diff --git a/tests.unit/test_50_run_exceptions.py b/tests.unit/test_50_run_exceptions.py index c807789..0ae0484 100644 --- a/tests.unit/test_50_run_exceptions.py +++ b/tests.unit/test_50_run_exceptions.py @@ -5,7 +5,7 @@ import pytest import feditest -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController from feditest import test @@ -78,22 +78,21 @@ def value_error() -> None: @pytest.fixture(autouse=True) -def the_test_plan() -> TestPlan: +def test_plan_fixture() -> TestPlan: """ The test plan tests all known tests. """ - constellation = TestPlanConstellation({}, 'No nodes needed') tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] - session = TestPlanSession(constellation, tests, "Tests buggy tests") - ret = TestPlan( [ session ] ) + session = TestPlanSessionTemplate(tests, "Tests buggy tests") + ret = TestPlan(session, [ constellation ]) return ret -def test_run_testplan(the_test_plan: TestPlan): - the_test_plan.check_can_be_executed() +def test_run_testplan(test_plan_fixture: TestPlan): + test_plan_fixture.check_can_be_executed() - test_run = TestRun(the_test_plan) + test_run = TestRun(test_plan_fixture) controller = AutomaticTestRunController(test_run) test_run.run(controller) diff --git a/tests.unit/test_50_run_multistep_assertion_raises.py b/tests.unit/test_50_run_multistep_assertion_raises.py index 3ccb493..ababba8 100644 --- a/tests.unit/test_50_run_multistep_assertion_raises.py +++ b/tests.unit/test_50_run_multistep_assertion_raises.py @@ -6,7 +6,7 @@ import feditest from feditest import assert_that, step, test, InteropLevel, SpecLevel -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController @@ -172,22 +172,22 @@ def step24_unspecified_unknown(self) -> None: @pytest.fixture(autouse=True) -def the_test_plan() -> TestPlan: +def test_plan_fixture() -> TestPlan: """ The test plan tests all known tests. """ constellation = TestPlanConstellation({}, 'No nodes needed') tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] - session = TestPlanSession(constellation, tests, "Test a test whose steps raises multiple AssertionFailures") - ret = TestPlan( [ session ] ) + session = TestPlanSessionTemplate(tests, "Test a test whose steps raises multiple AssertionFailures") + ret = TestPlan(session, [ constellation ]) return ret -def test_run_testplan(the_test_plan: TestPlan): - the_test_plan.check_can_be_executed() +def test_run_testplan(test_plan_fixture: TestPlan): + test_plan_fixture.check_can_be_executed() - test_run = TestRun(the_test_plan) + test_run = TestRun(test_plan_fixture) controller = AutomaticTestRunController(test_run) test_run.run(controller) diff --git a/tests.unit/test_50_run_not_implemented.py b/tests.unit/test_50_run_not_implemented.py index 2ea88d6..02de338 100644 --- a/tests.unit/test_50_run_not_implemented.py +++ b/tests.unit/test_50_run_not_implemented.py @@ -6,7 +6,7 @@ import feditest from feditest.nodedrivers import AccountManager, Node, NodeConfiguration, NodeDriver, NotImplementedByNodeError -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController from feditest import test @@ -59,7 +59,6 @@ def not_implemented_by_node_error() -> None: raise NotImplementedByNodeError(node, DummyNode.missing_method) - ## ## FediTest tests end here ## (Don't forget the next two lines) @@ -70,22 +69,22 @@ def not_implemented_by_node_error() -> None: @pytest.fixture(autouse=True) -def the_test_plan() -> TestPlan: +def test_plan_fixture() -> TestPlan: """ The test plan tests all known tests. """ constellation = TestPlanConstellation({}, 'No nodes needed') tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] - session = TestPlanSession(constellation, tests, "Test tests that throw NotImplemented errors") - ret = TestPlan( [ session ] ) + session = TestPlanSessionTemplate(tests, "Test tests that throw NotImplemented errors") + ret = TestPlan(session, [ constellation ]) return ret -def test_run_testplan(the_test_plan: TestPlan): - the_test_plan.check_can_be_executed() +def test_run_testplan(test_plan_fixture: TestPlan): + test_plan_fixture.check_can_be_executed() - test_run = TestRun(the_test_plan) + test_run = TestRun(test_plan_fixture) controller = AutomaticTestRunController(test_run) test_run.run(controller) diff --git a/tests.unit/test_50_run_passes.py b/tests.unit/test_50_run_passes.py index afb29ce..e61b171 100644 --- a/tests.unit/test_50_run_passes.py +++ b/tests.unit/test_50_run_passes.py @@ -5,7 +5,7 @@ import pytest import feditest -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController from feditest import test @@ -51,22 +51,21 @@ def passes() -> None: @pytest.fixture(autouse=True) -def the_test_plan() -> TestPlan: +def test_plan_fixture() -> TestPlan: """ The test plan tests all known tests. """ - constellation = TestPlanConstellation({}, 'No nodes needed') tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] - session = TestPlanSession(constellation, tests, "Test a test that passes") - ret = TestPlan( [ session ] ) + session = TestPlanSessionTemplate(tests, "Test a test that passes") + ret = TestPlan(session, [ constellation ]) return ret -def test_run_testplan(the_test_plan: TestPlan): - the_test_plan.check_can_be_executed() +def test_run_testplan(test_plan_fixture: TestPlan): + test_plan_fixture.check_can_be_executed() - test_run = TestRun(the_test_plan) + test_run = TestRun(test_plan_fixture) controller = AutomaticTestRunController(test_run) test_run.run(controller) diff --git a/tests.unit/test_50_run_skip.py b/tests.unit/test_50_run_skip.py index de04fac..9f49f33 100644 --- a/tests.unit/test_50_run_skip.py +++ b/tests.unit/test_50_run_skip.py @@ -6,7 +6,7 @@ import feditest from feditest.nodedrivers import SkipTestException -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanSessionTemplate, TestPlanTestSpec from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController from feditest import test @@ -59,8 +59,8 @@ def the_test_plan() -> TestPlan: constellation = TestPlanConstellation({}, 'No nodes needed') tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] - session = TestPlanSession(constellation, tests, "Test a test that wants to be skipped") - ret = TestPlan( [ session ] ) + session = TestPlanSessionTemplate(tests, "Test a test that wants to be skipped") + ret = TestPlan(session, [ constellation ]) return ret diff --git a/tests.unit/test_50_run_testplan_sandbox.py b/tests.unit/test_50_run_testplan_sandbox.py index d3be2cc..22683d0 100644 --- a/tests.unit/test_50_run_testplan_sandbox.py +++ b/tests.unit/test_50_run_testplan_sandbox.py @@ -10,7 +10,7 @@ import feditest from feditest import assert_that, step, test, SpecLevel from feditest.protocols.sandbox import SandboxLogEvent, SandboxMultClient, SandboxMultServer -from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSession, TestPlanTestSpec +from feditest.testplan import TestPlan, TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec from feditest.testrun import TestRun from feditest.testruncontroller import AutomaticTestRunController @@ -35,6 +35,10 @@ def init_tests(): feditest._registered_as_test_step = {} feditest._loading_tests = True + ## + ## FediTest tests start here + ## + @test class ExampleTest1: """ @@ -69,6 +73,7 @@ def step1(self): assert_that(log[0].b, equal_to(self.b)) assert_that(log[0].c, equal_to(self.c)) + @step def step2(self): @@ -134,12 +139,17 @@ def example_test3( assert_that(c, equal_to(a * b)) + ## + ## FediTest tests end here + ## (Don't forget the next two lines) + ## + feditest._loading_tests = False feditest._load_tests_pass2() @pytest.fixture(autouse=True) -def the_test_plan() -> TestPlan: +def test_plan_fixture() -> TestPlan: """ The test plan tests all known tests. """ @@ -149,15 +159,15 @@ def the_test_plan() -> TestPlan: } constellation = TestPlanConstellation(roles, 'clientA vs server1') tests = [ TestPlanTestSpec(name) for name in sorted(feditest.all_tests.keys()) if feditest.all_tests.get(name) is not None ] - session = TestPlanSession(constellation, tests, "clientA vs server") - ret = TestPlan( [ session ], "All sandbox tests running clientA against server1") + session = TestPlanSessionTemplate(tests, "clientA vs server") + ret = TestPlan(session, [ constellation ], "All sandbox tests running clientA against server1") return ret -def test_run_testplan(the_test_plan: TestPlan): - the_test_plan.check_can_be_executed() +def test_run_testplan(test_plan_fixture: TestPlan): + test_plan_fixture.check_can_be_executed() - test_run = TestRun(the_test_plan) + test_run = TestRun(test_plan_fixture) controller = AutomaticTestRunController(test_run) test_run.run(controller) From b8aa9058fff2f98da6d5a70bb0a80678a857e2a5 Mon Sep 17 00:00:00 2001 From: Johannes Ernst Date: Wed, 23 Oct 2024 09:03:19 -0700 Subject: [PATCH 2/2] Lint improvements --- tests.unit/test_20_create_test_plan_session_template.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests.unit/test_20_create_test_plan_session_template.py b/tests.unit/test_20_create_test_plan_session_template.py index 4f150f6..62729f6 100644 --- a/tests.unit/test_20_create_test_plan_session_template.py +++ b/tests.unit/test_20_create_test_plan_session_template.py @@ -1,14 +1,13 @@ """ Test the equivalent of `feditest create-session-template` """ -from typing import Any import pytest import feditest from feditest import test from feditest.nodedrivers import Node -from feditest.testplan import TestPlanConstellation, TestPlanConstellationNode, TestPlanSessionTemplate, TestPlanTestSpec +from feditest.testplan import TestPlanSessionTemplate, TestPlanTestSpec @pytest.fixture(scope="module", autouse=True)