Fix strict mode (#2601)

Fixes: #2592

.github/workflows/tox.yml

@@ -166,7 +166,7 @@ jobs:
       WSLENV: FORCE_COLOR:PYTEST_REQPASS:TOXENV:TOX_PARALLEL_NO_SPINNER
       # Number of expected test passes, safety measure for accidental skip of
       # tests. Update value if you add/remove tests.
-      PYTEST_REQPASS: 705
+      PYTEST_REQPASS: 707
 
     steps:
       - name: Activate WSL1

examples/playbooks/strict-mode.yml

@@ -0,0 +1,7 @@
+---
+- name: Fixture for test_strict
+  hosts: localhost
+  tasks:
+    - ansible.builtin.debug: # <-- name should be first key (warning)
+        msg: "Hello World"
+      name: Display debug information

src/ansiblelint/__main__.py

@@ -214,8 +214,9 @@ def main(argv: list[str] | None = None) -> int:  # noqa: C901
     if options.write_list:
         _do_transform(result, options)
 
-    mark_as_success = False
+    mark_as_success = True
     if result.matches and options.progressive:
+        mark_as_success = False
         _logger.info(
             "Matches found, running again on previous revision in order to detect regressions"
         )
@@ -247,6 +248,9 @@ def main(argv: list[str] | None = None) -> int:  # noqa: C901
                     ignored,
                 )
 
+    if options.strict and result.matches:
+        mark_as_success = False
+
     app.render_matches(result.matches)
 
     _perform_mockings_cleanup()

src/ansiblelint/app.py

@@ -193,17 +193,23 @@ def report_outcome(self, result: LintResult, mark_as_success: bool = False) -> i
                 "because 'yaml' is in 'skip_list'."
             )
 
+        if summary.failures:
+            mark_as_success = False
+
         if not self.options.quiet:
             console_stderr.print(render_yaml(msg))
-            self.report_summary(summary, changed_files_count, files_count)
+            self.report_summary(
+                summary, changed_files_count, files_count, is_success=mark_as_success
+            )
 
-        if not self.options.strict and (mark_as_success or not summary.failures):
-            return SUCCESS_RC
-        return VIOLATIONS_FOUND_RC
+        return SUCCESS_RC if mark_as_success else VIOLATIONS_FOUND_RC
 
     @staticmethod
     def report_summary(  # pylint: disable=too-many-branches,too-many-locals
-        summary: SummarizedResults, changed_files_count: int, files_count: int
+        summary: SummarizedResults,
+        changed_files_count: int,
+        files_count: int,
+        is_success: bool,
     ) -> None:
         """Report match and file counts."""
         # sort the stats by profiles
@@ -269,10 +275,10 @@ def report_summary(  # pylint: disable=too-many-branches,too-many-locals
             console_stderr.print(table)
             console_stderr.print()
 
-        if summary.failures:
-            msg = "[red][bold]Failed[/][/] after "
-        else:
+        if is_success:
             msg = "[green]Passed[/] with "
+        else:
+            msg = "[red][bold]Failed[/][/] after "
 
         if summary.passed_profile:
             msg += f"[bold]{summary.passed_profile}[/] profile"

src/ansiblelint/rules/only_builtins.py

@@ -52,6 +52,7 @@ def test_only_builtin_fail() -> None:
         """Test rule matches."""
         result = run_ansible_lint(
             "--config-file=/dev/null",
+            "--strict",
             "--warn-list=",
             "--enable-list",
             "only-builtins",

test/test_strict.py

@@ -0,0 +1,23 @@
+"""Test strict mode."""
+import pytest
+
+from ansiblelint.testing import run_ansible_lint
+
+
+@pytest.mark.parametrize(
+    ("strict", "returncode", "message"),
+    (
+        pytest.param(True, 2, "Failed", id="on"),
+        pytest.param(False, 0, "Passed", id="off"),
+    ),
+)
+def test_strict(strict: bool, returncode: int, message: str) -> None:
+    """Test running from inside meta folder."""
+    args = ["examples/playbooks/strict-mode.yml"]
+    if strict:
+        args.insert(0, "--strict")
+    result = run_ansible_lint(*args)
+    assert result.returncode == returncode
+    assert "key-order[task]" in result.stdout
+    summary_line = result.stderr.splitlines()[-1]
+    assert message in summary_line