From 8897a41f11bc4731bd8f0f3bacfd454e209f6f0a Mon Sep 17 00:00:00 2001 From: Hristiyan Ivanov Date: Sat, 20 Jun 2020 16:28:05 +0200 Subject: [PATCH 1/7] Add skipped parameter to steps - the skipped parameter can be used to track which steps have been skipped and adjust the behaviour appropriatelly (ex. change the output colour to yellow) --- src/pytest_bdd/parser.py | 2 ++ src/pytest_bdd/reporting.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/pytest_bdd/parser.py b/src/pytest_bdd/parser.py index aa7c4ec7..fe4a8292 100644 --- a/src/pytest_bdd/parser.py +++ b/src/pytest_bdd/parser.py @@ -296,6 +296,7 @@ class Step: indent: int keyword: str failed: bool = field(init=False, default=False) + skipped: bool = field(init=False, default=False) scenario: ScenarioTemplate | None = field(init=False, default=None) background: Background | None = field(init=False, default=None) lines: list[str] = field(init=False, default_factory=list) @@ -308,6 +309,7 @@ def __init__(self, name: str, type: str, indent: int, line_number: int, keyword: self.keyword = keyword self.failed = False + self.skipped = False self.scenario = None self.background = None self.lines = [] diff --git a/src/pytest_bdd/reporting.py b/src/pytest_bdd/reporting.py index 26e1cb0e..de8ecfe7 100644 --- a/src/pytest_bdd/reporting.py +++ b/src/pytest_bdd/reporting.py @@ -22,6 +22,7 @@ class StepReport: """Step execution report.""" + skipped = False failed = False stopped = None @@ -44,16 +45,20 @@ def serialize(self) -> dict[str, Any]: "type": self.step.type, "keyword": self.step.keyword, "line_number": self.step.line_number, + "skipped": self.skipped, "failed": self.failed, "duration": self.duration, } - def finalize(self, failed: bool) -> None: + + def finalize(self, failed: bool, skipped=False) -> None: """Stop collecting information and finalize the report. :param bool failed: Whether the step execution is failed. + :param bool skipped: Indicates if the step execution is skipped. """ self.stopped = time.perf_counter() + self.skipped = skipped self.failed = failed @property From a2e48b153186e3f53914bef702547ce88db22d7a Mon Sep 17 00:00:00 2001 From: Hristiyan Ivanov Date: Sat, 20 Jun 2020 16:33:33 +0200 Subject: [PATCH 2/7] Add hook and plugin to modify steps when skipped - the hook and the pluging work in a similar way to the fail hook and plugin where they set a flag wether the step is skipped or not --- src/pytest_bdd/hooks.py | 4 ++++ src/pytest_bdd/plugin.py | 13 +++++++++++++ src/pytest_bdd/reporting.py | 23 +++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/src/pytest_bdd/hooks.py b/src/pytest_bdd/hooks.py index 9351b2e3..b4475eee 100644 --- a/src/pytest_bdd/hooks.py +++ b/src/pytest_bdd/hooks.py @@ -25,6 +25,10 @@ def pytest_bdd_after_step(request, feature, scenario, step, step_func, step_func """Called after step function is successfully executed.""" +def pytest_bdd_step_skip(request, feature, scenario, step, step_func, step_func_args, exception): + """Called when step function is skipped.""" + + def pytest_bdd_step_error(request, feature, scenario, step, step_func, step_func_args, exception): """Called when step function failed to execute.""" diff --git a/src/pytest_bdd/plugin.py b/src/pytest_bdd/plugin.py index 486cdf87..dedf90a3 100644 --- a/src/pytest_bdd/plugin.py +++ b/src/pytest_bdd/plugin.py @@ -87,6 +87,19 @@ def pytest_bdd_before_scenario(request: FixtureRequest, feature: Feature, scenar reporting.before_scenario(request, feature, scenario) +@pytest.hookimpl(tryfirst=True) +def pytest_bdd_step_skip( + request: FixtureRequest, + feature: Feature, + scenario: Scenario, + step: Step, + step_func: Callable, + step_func_args: dict, + exception: Exception, +) -> None: + reporting.step_skip(request, feature, scenario, step, step_func, step_func_args, exception) + + @pytest.hookimpl(tryfirst=True) def pytest_bdd_step_error( request: FixtureRequest, diff --git a/src/pytest_bdd/reporting.py b/src/pytest_bdd/reporting.py index de8ecfe7..aa68d6f8 100644 --- a/src/pytest_bdd/reporting.py +++ b/src/pytest_bdd/reporting.py @@ -138,6 +138,17 @@ def fail(self) -> None: report.finalize(failed=True) self.add_step_report(report) + def skip(self): + """Stop collecting information and finalize the report as skipped.""" + self.current_step_report.finalize(failed=False, skipped=True) + remaining_steps = self.scenario.steps[len(self.step_reports) :] + + # Skip the rest of the steps and make reports. + for step in remaining_steps: + report = StepReport(step=step) + report.finalize(failed=False, skipped=True) + self.add_step_report(report) + def runtest_makereport(item: Item, call: CallInfo, rep: TestReport) -> None: """Store item in the report object.""" @@ -155,6 +166,18 @@ def before_scenario(request: FixtureRequest, feature: Feature, scenario: Scenari request.node.__scenario_report__ = ScenarioReport(scenario=scenario) +def step_skip( + request: FixtureRequest, + feature: Feature, + scenario: Scenario, + step: Step, + step_func: Callable, + step_func_args: dict, + exception: Exception, +) -> None: + """Finalize the step report as skipped.""" + request.node.__scenario_report__.skip() + def step_error( request: FixtureRequest, feature: Feature, From 0a8a4e300bac1cb38e48f1d1ac3a439d7ff62f16 Mon Sep 17 00:00:00 2001 From: Hristiyan Ivanov Date: Sat, 20 Jun 2020 16:38:30 +0200 Subject: [PATCH 3/7] Use the skip hook and plugin if the step is skipped - trigger the skip hook if a step is skipped --- src/pytest_bdd/scenario.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pytest_bdd/scenario.py b/src/pytest_bdd/scenario.py index df7c029c..889826f4 100644 --- a/src/pytest_bdd/scenario.py +++ b/src/pytest_bdd/scenario.py @@ -21,6 +21,7 @@ import pytest from _pytest.fixtures import FixtureDef, FixtureManager, FixtureRequest, call_fixture_func from _pytest.nodes import iterparentnodeids +from _pytest.outcomes import Skipped from . import exceptions from .feature import get_feature, get_features @@ -160,6 +161,9 @@ def _execute_step_function( except Exception as exception: request.config.hook.pytest_bdd_step_error(exception=exception, **kw) raise + except Skipped as exception: + request.config.hook.pytest_bdd_step_skip(exception=exception, **kw) + raise if context.target_fixture is not None: inject_fixture(request, context.target_fixture, return_value) From bf2057a0d53eb888d223d7b8c4ce8ce2884a9405 Mon Sep 17 00:00:00 2001 From: Hristiyan Ivanov Date: Sat, 20 Jun 2020 16:39:49 +0200 Subject: [PATCH 4/7] Color the steps independently of the scenario - use the output of the individual steps to decide the output colour --- src/pytest_bdd/gherkin_terminal_reporter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pytest_bdd/gherkin_terminal_reporter.py b/src/pytest_bdd/gherkin_terminal_reporter.py index 1e7d6ae7..fa3d466e 100644 --- a/src/pytest_bdd/gherkin_terminal_reporter.py +++ b/src/pytest_bdd/gherkin_terminal_reporter.py @@ -93,7 +93,10 @@ def pytest_runtest_logreport(self, report: TestReport) -> Any: self._tw.write(report.scenario["name"], **scenario_markup) self._tw.write("\n") for step in report.scenario["steps"]: - self._tw.write(f" {step['keyword']} {step['name']}\n", **scenario_markup) + step_markup = ( + {"red": True} if step["failed"] else ({"yellow": True} if step["skipped"] else {"green": True}) + ) + self._tw.write(f" {step['keyword']} {step['name']}\n", **step_markup) self._tw.write(" " + word, **word_markup) self._tw.write("\n\n") else: From 9529e4676ffa41ecb813f06c1d595b00c292975a Mon Sep 17 00:00:00 2001 From: Hristiyan Ivanov Date: Wed, 24 Jun 2020 20:13:05 +0200 Subject: [PATCH 5/7] Modify report test - the PR proposes to expand the step with an additional skipped bool, to track weather the step has been skipped or not, therefor the expected behaviour of the steps in the tests should be modified --- tests/feature/test_report.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/feature/test_report.py b/tests/feature/test_report.py index 727a7486..e53124b8 100644 --- a/tests/feature/test_report.py +++ b/tests/feature/test_report.py @@ -119,6 +119,7 @@ def _(cucumbers, left): "keyword": "Given", "line_number": 6, "name": "a passing step", + "skipped": False, "type": "given", }, { @@ -127,6 +128,7 @@ def _(cucumbers, left): "keyword": "And", "line_number": 7, "name": "some other passing step", + "skipped": False, "type": "given", }, ], @@ -154,6 +156,7 @@ def _(cucumbers, left): "keyword": "Given", "line_number": 11, "name": "a passing step", + "skipped": False, "type": "given", }, { @@ -162,6 +165,7 @@ def _(cucumbers, left): "keyword": "And", "line_number": 12, "name": "a failing step", + "skipped": False, "type": "given", }, ], @@ -188,6 +192,7 @@ def _(cucumbers, left): "keyword": "Given", "line_number": 15, "name": "there are 12 cucumbers", + "skipped": False, "type": "given", }, { @@ -196,6 +201,7 @@ def _(cucumbers, left): "keyword": "When", "line_number": 16, "name": "I eat 5 cucumbers", + "skipped": False, "type": "when", }, { @@ -204,6 +210,7 @@ def _(cucumbers, left): "keyword": "Then", "line_number": 17, "name": "I should have 7 cucumbers", + "skipped": False, "type": "then", }, ], @@ -230,6 +237,7 @@ def _(cucumbers, left): "keyword": "Given", "line_number": 15, "name": "there are 5 cucumbers", + "skipped": False, "type": "given", }, { @@ -238,6 +246,7 @@ def _(cucumbers, left): "keyword": "When", "line_number": 16, "name": "I eat 4 cucumbers", + "skipped": False, "type": "when", }, { @@ -246,6 +255,7 @@ def _(cucumbers, left): "keyword": "Then", "line_number": 17, "name": "I should have 1 cucumbers", + "skipped": False, "type": "then", }, ], From 6eca0a3fcf4cc644958e0b328dd7033a63ab702c Mon Sep 17 00:00:00 2001 From: Hristiyan Ivanov Date: Thu, 25 Jun 2020 08:46:33 +0200 Subject: [PATCH 6/7] Extend test_report with a skipping scenario - when a step is skipped the step should have skipped set to True --- tests/feature/test_report.py | 82 ++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/tests/feature/test_report.py b/tests/feature/test_report.py index e53124b8..18458f3e 100644 --- a/tests/feature/test_report.py +++ b/tests/feature/test_report.py @@ -25,6 +25,7 @@ def test_step_trace(pytester): feature-tag scenario-passing-tag scenario-failing-tag + scenario-skipping-tag """ ), ) @@ -33,7 +34,7 @@ def test_step_trace(pytester): test=textwrap.dedent( """ @feature-tag - Feature: One passing scenario, one failing scenario + Feature: One passing scenario, one failing scenario, one skipping scenario @scenario-passing-tag Scenario: Passing @@ -45,6 +46,12 @@ def test_step_trace(pytester): Given a passing step And a failing step + @scenario-skipping-tag + Scenario: Skipping + Given a passing step + And a skipping step + And a passing step + Scenario Outline: Outlined Given there are cucumbers When I eat cucumbers @@ -76,6 +83,10 @@ def _(): def _(): raise Exception('Error') + @given('a skipping step') + def _(): + pytest.skip() + @given(parsers.parse('there are {start:d} cucumbers'), target_fixture="cucumbers") def _(start): assert isinstance(start, int) @@ -106,7 +117,7 @@ def _(cucumbers, left): "description": "", "filename": str(feature), "line_number": 2, - "name": "One passing scenario, one failing scenario", + "name": "One passing scenario, one failing scenario, one skipping scenario", "rel_filename": str(relpath), "tags": ["feature-tag"], }, @@ -143,7 +154,7 @@ def _(cucumbers, left): "description": "", "filename": str(feature), "line_number": 2, - "name": "One passing scenario, one failing scenario", + "name": "One passing scenario, one failing scenario, one skipping scenario", "rel_filename": str(relpath), "tags": ["feature-tag"], }, @@ -173,24 +184,69 @@ def _(cucumbers, left): } assert report == expected + report = result.matchreport("test_skipping", when="call").scenario + expected = { + "feature": { + "description": "", + "filename": str(feature), + "line_number": 2, + "name": "One passing scenario, one failing scenario, one skipping scenario", + "rel_filename": str(relpath), + "tags": ["feature-tag"], + }, + "line_number": 15, + "name": "Skipping", + "steps": [ + { + "duration": OfType(float), + "failed": False, + "keyword": "Given", + "line_number": 16, + "name": "a passing step", + "skipped": False, + "type": "given", + }, + { + "duration": OfType(float), + "failed": False, + "keyword": "And", + "line_number": 17, + "name": "a skipping step", + "skipped": True, + "type": "given", + }, + { + "duration": OfType(float), + "failed": False, + "keyword": "And", + "line_number": 18, + "name": "a passing step", + "skipped": True, + "type": "given", + }, + ], + "tags": ["scenario-skipping-tag"], + } + assert report == expected + report = result.matchreport("test_outlined[12-5-7]", when="call").scenario expected = { "feature": { "description": "", "filename": str(feature), "line_number": 2, - "name": "One passing scenario, one failing scenario", + "name": "One passing scenario, one failing scenario, one skipping scenario", "rel_filename": str(relpath), "tags": ["feature-tag"], }, - "line_number": 14, + "line_number": 20, "name": "Outlined", "steps": [ { "duration": OfType(float), "failed": False, "keyword": "Given", - "line_number": 15, + "line_number": 21, "name": "there are 12 cucumbers", "skipped": False, "type": "given", @@ -199,7 +255,7 @@ def _(cucumbers, left): "duration": OfType(float), "failed": False, "keyword": "When", - "line_number": 16, + "line_number": 22, "name": "I eat 5 cucumbers", "skipped": False, "type": "when", @@ -208,7 +264,7 @@ def _(cucumbers, left): "duration": OfType(float), "failed": False, "keyword": "Then", - "line_number": 17, + "line_number": 23, "name": "I should have 7 cucumbers", "skipped": False, "type": "then", @@ -224,18 +280,18 @@ def _(cucumbers, left): "description": "", "filename": str(feature), "line_number": 2, - "name": "One passing scenario, one failing scenario", + "name": "One passing scenario, one failing scenario, one skipping scenario", "rel_filename": str(relpath), "tags": ["feature-tag"], }, - "line_number": 14, + "line_number": 20, "name": "Outlined", "steps": [ { "duration": OfType(float), "failed": False, "keyword": "Given", - "line_number": 15, + "line_number": 21, "name": "there are 5 cucumbers", "skipped": False, "type": "given", @@ -244,7 +300,7 @@ def _(cucumbers, left): "duration": OfType(float), "failed": False, "keyword": "When", - "line_number": 16, + "line_number": 22, "name": "I eat 4 cucumbers", "skipped": False, "type": "when", @@ -253,7 +309,7 @@ def _(cucumbers, left): "duration": OfType(float), "failed": False, "keyword": "Then", - "line_number": 17, + "line_number": 23, "name": "I should have 1 cucumbers", "skipped": False, "type": "then", From 45336779bdedc4d5f70743cc971ad3983e11b566 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 18 Jun 2023 11:00:15 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pytest_bdd/reporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_bdd/reporting.py b/src/pytest_bdd/reporting.py index aa68d6f8..e5a80e5d 100644 --- a/src/pytest_bdd/reporting.py +++ b/src/pytest_bdd/reporting.py @@ -50,7 +50,6 @@ def serialize(self) -> dict[str, Any]: "duration": self.duration, } - def finalize(self, failed: bool, skipped=False) -> None: """Stop collecting information and finalize the report. @@ -178,6 +177,7 @@ def step_skip( """Finalize the step report as skipped.""" request.node.__scenario_report__.skip() + def step_error( request: FixtureRequest, feature: Feature,