add /per-rule test for running automatus.py in rule mode

Signed-off-by: Jiri Jaburek <[email protected]>

conf/waivers-upstream

@@ -270,4 +270,26 @@
 /hardening/image-builder/.*/enable_fips_mode
     True
 
+# /per-rule (Automatus rule mode) waivers
+# - these intentionally don't apply to /per-rule/from-env (ad-hoc RULE run)
+#
+# TODO: all of these are unknown and need investigation
+/per-rule/[^/]+/accounts_password_set_max_life_root/correct.pass
+/per-rule/[^/]+/accounts_password_set_max_life_root/wrong.fail
+/per-rule/[^/]+/postfix_client_configure_mail_alias/correct.pass
+    True
+/per-rule/[^/]+/grub2_audit_backlog_limit_argument/correct_grubenv.pass
+/per-rule/[^/]+/grub2_password/invalid_username.fail
+/per-rule/[^/]+/harden_sshd_ciphers_openssh_conf_crypto_policy/stig_correct.pass
+/per-rule/[^/]+/harden_sshd_ciphers_openssh_conf_crypto_policy/stig_correct_followed_by_incorrect_commented.pass
+/per-rule/[^/]+/sudo_add_umask/0027_var_multiple_values.pass
+    rhel == 8
+/per-rule/[^/]+/directory_permissions_var_log_audit/correct_value_0700.pass
+/per-rule/[^/]+/directory_permissions_var_log_audit/incorrect_value_0700.fail
+/per-rule/[^/]+/dconf_gnome_lock_screen_on_smartcard_removal/wrong_value.fail
+/per-rule/[^/]+/file_ownership_var_log_audit_stig/correct_value_default_file.pass
+/per-rule/[^/]+/tftpd_uses_secure_mode/correct.pass
+/per-rule/[^/]+/tftpd_uses_secure_mode/wrong.fail
+    rhel == 9
+
 # vim: syntax=python

lib/oscap.py

@@ -142,3 +142,39 @@ def unselect_rules(orig_ds, new_ds, rules):
                     line = line.replace('selected="true"', 'selected="false"')
                     util.log(f'unselected {line.strip()}')
                 new_ds_f.write(line)
+
+
+def get_all_profiles():
+    """
+    Yield all profile names present in the datastream.
+    """
+    cmd = ['oscap', 'info', '--profiles', util.get_datastream()]
+    _, lines = util.subprocess_stream(cmd, check=True)
+    for line in lines:
+        # xccdf_org.ssgproject.content_profile_stig:DISA STIG for Red Hat Enterprise Linux 9
+        yield line.partition(':')[0]
+
+
+def get_all_rules(profile):
+    """
+    Yield all rules in a profile.
+    """
+    pattern = re.compile(r"# BEGIN fix .* for 'xccdf_org\.ssgproject\.content_rule_([^']+)'")
+    # oscap doesn't have any "list all rules" command
+    cmd = ['oscap', 'xccdf', 'generate', '--profile', profile, 'fix', util.get_datastream()]
+    _, lines = util.subprocess_stream(cmd, check=True)
+    for line in lines:
+        match = pattern.fullmatch(line)
+        if match:
+            yield match.group(1)
+
+
+def get_all_profiles_rules():
+    """
+    Return a deduplicated unified list of all rules from all profiles.
+    """
+    all_rules = set()
+    for profile in get_all_profiles():
+        for rule in get_all_rules(profile):
+            all_rules.add(rule)
+    return sorted(all_rules)

per-rule/main.fmf

@@ -0,0 +1,126 @@
+summary: Runs content unit tests for every rule in every supported RHEL profile
+description: |-
+    Downloads and builds a content source, unless provided via CONTENT_SOURCE
+    as a path to a directory. This can be either built (with 'build') or unbuilt
+    source code - the test will detect and build the content if necessary.
+    It then runs automatus.py in rule mode on every rule from every profile
+    built for the current platform.
+    The RULE variable (with space-separated one or more rule names) can be used
+    to override this and run tests for only specific rule(s).
+test: python3 -m lib.runtest ./test.py
+result: custom
+environment+:
+    PYTHONPATH: ..
+    TOTAL_SLICES: 15
+duration: 3h
+require+:
+  # virt library dependencies
+  - libvirt-daemon
+  - libvirt-daemon-driver-qemu
+  - libvirt-daemon-driver-storage-core
+  - libvirt-daemon-driver-network
+  - firewalld
+  - qemu-kvm
+  - libvirt-client
+  - virt-install
+  - rpm-build
+  - createrepo
+  # for builddep on scap-security-guide.spec
+  - python-srpm-macros
+  # automatus dependencies (oscap-ssh, etc.)
+  - openscap-utils
+extra-hardware: |
+    keyvalue = HVM=1
+    hostrequire = memory>=3720
+adjust:
+  - when: arch != x86_64
+    enabled: false
+    because: we want to run virtualization on x86_64 only
+  - when: distro == rhel-7
+    enabled: false
+    because: the code is not compatible with RHEL-7 yum/python
+tag:
+  - max1
+  - daily
+
+# for use with the RULE environment variable
+/from-env:
+    tag:
+      - NoProductization
+      - NoStabilization
+    extra-summary: /CoreOS/scap-security-guide/per-rule/from-env
+
+/1:
+    environment+:
+        SLICE: 1
+    extra-summary: /CoreOS/scap-security-guide/per-rule/1
+
+/2:
+    environment+:
+        SLICE: 2
+    extra-summary: /CoreOS/scap-security-guide/per-rule/2
+
+/3:
+    environment+:
+        SLICE: 3
+    extra-summary: /CoreOS/scap-security-guide/per-rule/3
+
+/4:
+    environment+:
+        SLICE: 4
+    extra-summary: /CoreOS/scap-security-guide/per-rule/4
+
+/5:
+    environment+:
+        SLICE: 5
+    extra-summary: /CoreOS/scap-security-guide/per-rule/5
+
+/6:
+    environment+:
+        SLICE: 6
+    extra-summary: /CoreOS/scap-security-guide/per-rule/6
+
+/7:
+    environment+:
+        SLICE: 7
+    extra-summary: /CoreOS/scap-security-guide/per-rule/7
+
+/8:
+    environment+:
+        SLICE: 8
+    extra-summary: /CoreOS/scap-security-guide/per-rule/8
+
+/9:
+    environment+:
+        SLICE: 9
+    extra-summary: /CoreOS/scap-security-guide/per-rule/9
+
+/10:
+    environment+:
+        SLICE: 10
+    extra-summary: /CoreOS/scap-security-guide/per-rule/10
+
+/11:
+    environment+:
+        SLICE: 11
+    extra-summary: /CoreOS/scap-security-guide/per-rule/11
+
+/12:
+    environment+:
+        SLICE: 12
+    extra-summary: /CoreOS/scap-security-guide/per-rule/12
+
+/13:
+    environment+:
+        SLICE: 13
+    extra-summary: /CoreOS/scap-security-guide/per-rule/13
+
+/14:
+    environment+:
+        SLICE: 14
+    extra-summary: /CoreOS/scap-security-guide/per-rule/14
+
+/15:
+    environment+:
+        SLICE: 15
+    extra-summary: /CoreOS/scap-security-guide/per-rule/15