diff --git a/docs/TESTS.md b/docs/TESTS.md index faa94c8c..b7afb09a 100644 --- a/docs/TESTS.md +++ b/docs/TESTS.md @@ -68,3 +68,34 @@ they don't install virtual machines. These are things like `grep`-ing for specific strings (not) present in the built content, syntax-checking Ansible playbooks, or verifying HTTP URLs. + +# Test tags + +These are some of the commonly-used tags amongst tests. + +Note that we use tags to indicate properties of tests, not to categorize them +(think: "needs virtualization", not: "runs during release testing"). + +## `adhoc` + +This indicates a test that is used as a "tool", in automation-assisted use +cases. It should not run automatically in regular "all tests" runs, as it may +need configuring via environment variables. + +## `broken` + +This is a perfectly valid working test, but the functionality it tests is +either completely broken, or under very active development, creating interface +incompatibilities, such as config directive changes, and frequently breaking +the test. + +Despite this, we don't want to disable the test outright, as it is useful for +debugging and stabilizing the tested functionality. + +## `destructive` + +A destructive tests modifies the OS it runs on to the point where it is +unusable for further testing, typically by hardening it. + +A test that just installs extra RPMs from the package manager, or enables +extra services, is not considered destructive. diff --git a/hardening/image-builder/with-gui.fmf b/hardening/image-builder/with-gui.fmf index a2e93c8b..5bd22cf5 100644 --- a/hardening/image-builder/with-gui.fmf +++ b/hardening/image-builder/with-gui.fmf @@ -5,6 +5,7 @@ duration: 2h tag: - NoProductization - NoStabilization + - broken /anssi_bp28_high: environment+: diff --git a/per-rule/main.fmf b/per-rule/main.fmf index e9da930e..ab8ff4aa 100644 --- a/per-rule/main.fmf +++ b/per-rule/main.fmf @@ -46,6 +46,7 @@ tag: tag: - NoProductization - NoStabilization + - adhoc extra-summary: /CoreOS/scap-security-guide/per-rule/from-env extra-nitrate: TC#0617199 id: 237c1eaf-bf68-4eba-b097-6cc0222ca282 diff --git a/plans/default.fmf b/plans/default.fmf index 6e1bfa42..b9792920 100644 --- a/plans/default.fmf +++ b/plans/default.fmf @@ -1 +1,3 @@ -summary: Default plan (simple for now) +summary: Default plan discovering all tests for ad-hoc use +discover: + how: fmf diff --git a/plans/errata.fmf b/plans/errata.fmf new file mode 100644 index 00000000..4c0ca2a0 --- /dev/null +++ b/plans/errata.fmf @@ -0,0 +1,5 @@ +summary: Testing builds added to errata +discover: + how: fmf + test: + - /scanning/oscap-eval diff --git a/plans/gating-ci.fmf b/plans/gating-ci.fmf new file mode 100644 index 00000000..a4985631 --- /dev/null +++ b/plans/gating-ci.fmf @@ -0,0 +1,8 @@ +summary: Testing during OSCI Gating +discover: + how: fmf + test: + - /scanning/oscap-eval + - /static-checks/ansible/allowed-modules + - /static-checks/fetch-remote-resources + - /static-checks/rpmbuild-ctest diff --git a/plans/main.fmf b/plans/main.fmf index b28d69c5..7d44fb25 100644 --- a/plans/main.fmf +++ b/plans/main.fmf @@ -2,9 +2,6 @@ /: inherit: false -discover: - how: fmf - execute: how: tmt diff --git a/plans/productization.fmf b/plans/productization.fmf new file mode 100644 index 00000000..f42820bc --- /dev/null +++ b/plans/productization.fmf @@ -0,0 +1,12 @@ +summary: Regular "productization" testing +discover: + how: fmf + filter: + - tag:-adhoc + - tag:-broken + exclude: + # these intentionally fail on any difference, to show up for manual review + # during stabilization + - /static-checks/diff +environment: + CONTEST_CONTENT_BRANCH: master diff --git a/plans/stabilization.fmf b/plans/stabilization.fmf new file mode 100644 index 00000000..82bdf497 --- /dev/null +++ b/plans/stabilization.fmf @@ -0,0 +1,8 @@ +summary: Pre-release "stabilization" testing +discover: + how: fmf + filter: + - tag:-adhoc + - tag:-broken +environment: + CONTEST_CONTENT_BRANCH: master