add TMT plans for all our notable use cases

Signed-off-by: Jiri Jaburek <[email protected]>

docs/TESTS.md

@@ -68,3 +68,47 @@ they don't install virtual machines.
 
 These are things like `grep`-ing for specific strings (not) present in the built
 content, syntax-checking Ansible playbooks, or verifying HTTP URLs.
+
+# Test tags
+
+These are some of the commonly-used tags amongst tests.
+
+Note that we use tags to indicate properties of tests, not to categorize them
+(think: "needs virtualization", not: "runs during release testing").
+
+## `needs-param`
+
+This indicates a test that is used as a "tool" in automation-assisted use
+cases. It should not run automatically in regular "all tests" runs, as it
+requires the user to give it input via environment variables (parameters).
+
+## `always-fails`
+
+This is a test that uses the `fail` status to indicate some unwanted findings,
+expecting the user to review the list manually. These `fail`s should not be
+waived automatically as they are specific to the configuration the user
+requested.
+
+A test like this is another form of a "tool" and should not be run regularly
+in use cases that expect `pass` to be the norm and `fail` to be a regression.
+
+## `broken`
+
+This is a perfectly valid working test, but the functionality it tests is
+either completely broken, or under very active development, creating interface
+incompatibilities, such as config directive changes, and frequently breaking
+the test.
+
+Despite this, we don't want to disable the test outright, as it is useful for
+debugging and stabilizing the tested functionality via manual use.
+
+However a test like this should not be run by automation, it is not useful
+for preventing regressions.
+
+## `destructive`
+
+A destructive tests modifies the OS it runs on to the point where it is
+unusable for further testing, typically by hardening it.
+
+A test that just installs extra RPMs from the package manager, or enables
+extra services, is not considered destructive.

hardening/image-builder/with-gui.fmf

@@ -5,6 +5,7 @@ duration: 2h
 tag:
   - NoProductization
   - NoStabilization
+  - broken
 
 /anssi_bp28_high:
     environment+:

per-rule/main.fmf

@@ -49,6 +49,7 @@ tag:
     tag:
       - NoProductization
       - NoStabilization
+      - needs-param
     /oscap:
         extra-summary: /CoreOS/scap-security-guide/per-rule/from-env/oscap
         extra-nitrate: TC#0617199

plans/daily.fmf

@@ -0,0 +1,23 @@
+summary: Regular daily "productization" testing
+discover:
+    how: fmf
+    filter:
+      - tag:-needs-param
+      - tag:-always-fails
+      - tag:-broken
+    test:
+      # every remediation method, but only the basic reference environment
+      # without GUI, UEFI or other versions
+      - /hardening/oscap/[^/]+$
+      - /hardening/anaconda/[^/]+$
+      - /hardening/ansible/[^/]+$
+      - /hardening/image-builder/[^/]+$
+      # run host-os as well - not because it would be very useful compared to
+      # the above, but because we use it for CaC/content TestingFarm and want to
+      # detect impact of waivers on RHEL early after an upstream-related change
+      - /hardening/host-os
+      # run /per-rule as oscap only - this almost halves the runtime (for now)
+      - /per-rule/[^/]+/oscap$
+      # the rest is cheap to run
+      - /scanning
+      - /static-checks

plans/default.fmf

@@ -1 +1,3 @@
-summary: Default plan (simple for now)
+summary: Default plan discovering all tests for ad-hoc use
+discover:
+    how: fmf

plans/errata.fmf

@@ -0,0 +1,14 @@
+summary: Testing builds added to errata
+discover:
+    how: fmf
+    filter:
+      - tag:-needs-param
+      - tag:-always-fails
+      - tag:-broken
+    test:
+      # just some basic smoke testing that should never fail
+      - /scanning/oscap-eval
+      - /static-checks
+    exclude:
+      # often fails on temporary retrieval issues
+      - /static-checks/html-links

plans/gating-ci.fmf

@@ -0,0 +1 @@
+errata.fmf

plans/main.fmf

@@ -2,9 +2,6 @@
 /:
     inherit: false
 
-discover:
-    how: fmf
-
 execute:
     how: tmt
 

plans/stabilization.fmf

@@ -0,0 +1,7 @@
+summary: Pre-release "stabilization" testing
+discover:
+    how: fmf
+    # all tests are included by default, except these
+    filter:
+      - tag:-needs-param
+      - tag:-broken

plans/weekly.fmf

@@ -0,0 +1,8 @@
+summary: Regular weekly "productization" testing
+discover:
+    how: fmf
+    # all tests are included by default, except these
+    filter:
+      - tag:-needs-param
+      - tag:-always-fails
+      - tag:-broken

static-checks/diff/main.fmf

@@ -10,31 +10,39 @@ adjust:
 /profiles:
     summary: Diff datastreams, output added/removed profiles
     test: python3 -m lib.runtest ./profiles.py
-    tag: [NoProductization]
+    tag:
+      - NoProductization
+      - always-fails
     extra-summary: /CoreOS/scap-security-guide/static-checks/diff/profiles
     extra-nitrate: TC#0617452
     id: 5f0d1df1-f2b5-4212-84e7-2c25ec5566c1
 
 /profile-titles:
     summary: Diff datastreams, output profile title differences
     test: python3 -m lib.runtest ./profile-titles.py
-    tag: [NoProductization]
+    tag:
+      - NoProductization
+      - always-fails
     extra-summary: /CoreOS/scap-security-guide/static-checks/diff/profile-titles
     extra-nitrate: TC#0617450
     id: 9e43b634-eaab-4e4f-81c2-bbab571f7db1
 
 /profile-rules:
     summary: Diff datastreams, output profile rule/variable differences
     test: python3 -m lib.runtest ./profile-rules.py
-    tag: [NoProductization]
+    tag:
+      - NoProductization
+      - always-fails
     extra-summary: /CoreOS/scap-security-guide/static-checks/diff/profile-rules
     extra-nitrate: TC#0617449
     id: d17358be-d702-4786-a9df-6716036c8428
 
 /profile-variables:
     summary: Diff datastreams, output profile variable refine differences
     test: python3 -m lib.runtest ./profile-variables.py
-    tag: [NoProductization]
+    tag:
+      - NoProductization
+      - always-fails
     extra-summary: /CoreOS/scap-security-guide/static-checks/diff/profile-variables
     extra-nitrate: TC#0617451
     id: dd504436-0d67-4f1c-96f4-faeff18a2c0c