add TMT plans for all our notable use cases

Signed-off-by: Jiri Jaburek <[email protected]>
RHSecurityCompliance · Apr 16, 2024 · 62a1421 · 62a1421
1 parent b2fcff5
commit 62a1421
Show file tree

Hide file tree

Showing 9 changed files with 69 additions and 4 deletions.
diff --git a/docs/TESTS.md b/docs/TESTS.md
@@ -68,3 +68,34 @@ they don't install virtual machines.
 
 These are things like `grep`-ing for specific strings (not) present in the built
 content, syntax-checking Ansible playbooks, or verifying HTTP URLs.
+
+# Test tags
+
+These are some of the commonly-used tags amongst tests.
+
+Note that we use tags to indicate properties of tests, not to categorize them
+(think: "needs virtualization", not: "runs during release testing").
+
+## `adhoc`
+
+This indicates a test that is used as a "tool", in automation-assisted use
+cases. It should not run automatically in regular "all tests" runs, as it may
+need configuring via environment variables.
+
+## `broken`
+
+This is a perfectly valid working test, but the functionality it tests is
+either completely broken, or under very active development, creating interface
+incompatibilities, such as config directive changes, and frequently breaking
+the test.
+
+Despite this, we don't want to disable the test outright, as it is useful for
+debugging and stabilizing the tested functionality.
+
+## `destructive`
+
+A destructive tests modifies the OS it runs on to the point where it is
+unusable for further testing, typically by hardening it.
+
+A test that just installs extra RPMs from the package manager, or enables
+extra services, is not considered destructive.
diff --git a/hardening/image-builder/with-gui.fmf b/hardening/image-builder/with-gui.fmf
@@ -5,6 +5,7 @@ duration: 2h
 tag:
   - NoProductization
   - NoStabilization
+  - broken
 
 /anssi_bp28_high:
     environment+:

diff --git a/per-rule/main.fmf b/per-rule/main.fmf
@@ -46,6 +46,7 @@ tag:
     tag:
       - NoProductization
       - NoStabilization
+      - adhoc
     extra-summary: /CoreOS/scap-security-guide/per-rule/from-env
     extra-nitrate: TC#0617199
     id: 237c1eaf-bf68-4eba-b097-6cc0222ca282

diff --git a/plans/default.fmf b/plans/default.fmf
@@ -1 +1,3 @@
-summary: Default plan (simple for now)
+summary: Default plan discovering all tests for ad-hoc use
+discover:
+    how: fmf
diff --git a/plans/errata.fmf b/plans/errata.fmf
@@ -0,0 +1,5 @@
+summary: Testing builds added to errata
+discover:
+    how: fmf
+    test:
+      - /scanning/oscap-eval
diff --git a/plans/gating-ci.fmf b/plans/gating-ci.fmf
@@ -0,0 +1,8 @@
+summary: Testing during OSCI Gating
+discover:
+    how: fmf
+    test:
+      - /scanning/oscap-eval
+      - /static-checks/ansible/allowed-modules
+      - /static-checks/fetch-remote-resources
+      - /static-checks/rpmbuild-ctest
diff --git a/plans/main.fmf b/plans/main.fmf
@@ -2,9 +2,6 @@
 /:
     inherit: false
 
-discover:
-    how: fmf
-
 execute:
     how: tmt
 

diff --git a/plans/productization.fmf b/plans/productization.fmf
@@ -0,0 +1,12 @@
+summary: Regular "productization" testing
+discover:
+    how: fmf
+    filter:
+      - tag:-adhoc
+      - tag:-broken
+    exclude:
+      # these intentionally fail on any difference, to show up for manual review
+      # during stabilization
+      - /static-checks/diff
+environment:
+    CONTEST_CONTENT_BRANCH: master
diff --git a/plans/stabilization.fmf b/plans/stabilization.fmf
@@ -0,0 +1,8 @@
+summary: Pre-release "stabilization" testing
+discover:
+    how: fmf
+    filter:
+      - tag:-adhoc
+      - tag:-broken
+environment:
+    CONTEST_CONTENT_BRANCH: master