overlay.d & tests: Add alternatives migration and test

- Add an overlay with the migration logic for alternatives
- Add a test for the migration script

This should make sure that the system is setup properly and that the
migration script will do the right thing on older systems.

See: coreos/fedora-coreos-tracker#1818
See: coreos/fedora-coreos-tracker#677
See: https://docs.fedoraproject.org/en-US/fedora-coreos/alternatives/

manifests/fedora-coreos-base.yaml

@@ -22,6 +22,7 @@ ostree-layers:
   - overlay/25azure-udev-rules
   - overlay/30lvmdevices
   - overlay/40grub
+  - overlay/50alternatives
 
 # Be minimal
 recommends: false

overlay.d/50alternatives/statoverride

@@ -0,0 +1,2 @@
+# Config file for overriding permission bits on overlay files/dirs
+# Format: =<file mode in decimal> <absolute path to a file or directory>

overlay.d/50alternatives/usr/libexec/coreos-alternatives-migration

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+# set -x
+
+main() {
+    # Should never happen as systemd checks this, but just in case
+    if [[ ! -d "/var/lib/alternatives" ]]; then
+        echo "Skipped /var/lib/alternatives as it is not a directory"
+        exit 0
+    fi
+
+    # We can safely directly try to remove the directory as rmdir will fail on
+    # a non-empty directory
+    rmdir "/var/lib/alternatives" || echo "Warning: /var/lib/alternatives is not empty"
+
+    # Do the migration, explicitely using the new configuration directory to
+    # ignore /var/lib/alternatives if it still exists
+    alternatives --admindir /etc/alternatives-admindir --set iptables /usr/sbin/iptables-nft
+    return $?
+}
+
+main "${@}"

overlay.d/50alternatives/usr/systemd/system/coreos-alternatives-migration.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Migrate systems to fixed alternatives configuration
+ConditionPathExists=/var/lib/alternatives
+ConditionPathIsDirectory=/var/lib/alternatives
+
+[Service]
+ExecStart=/usr/libexec/coreos-alternatives-migration
+Type=oneshot
+RemainAfterExit=yes
+
+[Install]
+WantedBy=basic.target

overlay.d/README.md

@@ -93,3 +93,8 @@ information.
 
 Add in static grub configs that will be leveraged by bootupd when
 managing bootloaders. See https://github.com/coreos/bootupd/pull/543
+
+50alternatives
+--------------
+
+Temporary overlay for the alternatives migration scripts.

tests/kola/files/alternatives

@@ -0,0 +1,79 @@
+#!/bin/bash
+## kola:
+##   description: Verify that the alternatives config is properly migrated and test the migration
+
+# See
+# - https://github.com/coreos/fedora-coreos-tracker/issues/1818
+
+set -xeuo pipefail
+
+# shellcheck disable=SC1091
+. "$KOLA_EXT_DATA/commonlib.sh"
+
+if test -e "/var/lib/alternatives"; then
+    ls -al "/var/lib/alternatives"
+    fatal "Error: Found '/var/lib/alternatives' which should not exists"
+fi
+if ! test -d "/etc/alternatives"; then
+    fatal "Error: '/etc/alternatives' is missing"
+fi
+if ! test -d "/etc/alternatives-admindir"; then
+    fatal "Error: '/etc/alternatives-admindir' is missing"
+fi
+
+# To test the migration we will re-create the setup from an older FCOS node
+
+# First, reset iptables to the legacy backend
+alternatives --set iptables /usr/sbin/iptables-legacy
+if [[ $(alternatives --display iptables | grep -c "link currently points to /usr/sbin/iptables-legacy") != "1" ]]; then
+    fatal "Could not set iptables to legacy backend for testing"
+fi
+if [[ $(iptables --version | grep -c "legacy") != "1" ]]; then
+    fatal "Could not set iptables to legacy backend for testing"
+fi
+
+# Then re-create the broken alternatives folder in /var
+install -dm0755 /var/lib/alternatives
+
+# Do the migration
+/usr/libexec/coreos-alternatives-migration
+
+if [[ $(alternatives --admindir /etc/alternatives-admindir --display iptables | grep -c "link currently points to /usr/sbin/iptables-nft") != "1" ]]; then
+    fatal "Error: migration did not set iptables to nft backend"
+fi
+if [[ $(iptables --version | grep -c "nf_tables") != "1" ]]; then
+    fatal "Error: iptables not reset to nftables backend"
+fi
+if [[ -d "/var/lib/alternatives" ]]; then
+    fatal "Error: /var/lib/alternatives should not exists anymore"
+fi
+
+# Second case, if an admin set some config up for alternatives
+
+# First, reset iptables to the legacy backend
+alternatives --set iptables /usr/sbin/iptables-legacy
+if [[ $(alternatives --display iptables | grep -c "link currently points to /usr/sbin/iptables-legacy") != "1" ]]; then
+    fatal "Could not set iptables to legacy backend for testing"
+fi
+if [[ $(iptables --version | grep -c "legacy") != "1" ]]; then
+    fatal "Could not set iptables to legacy backend for testing"
+fi
+
+# Then re-create the broken alternatives folder in /var
+install -dm0755 /var/lib/alternatives
+
+# And add some fake config
+touch /var/lib/alternatives/foo
+
+# Do the migration
+/usr/libexec/coreos-alternatives-migration
+
+if [[ $(alternatives --admindir /etc/alternatives-admindir --display iptables | grep -c "link currently points to /usr/sbin/iptables-nft") != "1" ]]; then
+    fatal "Error: migration did not set iptables to nft backend"
+fi
+if [[ $(iptables --version | grep -c "nf_tables") != "1" ]]; then
+    fatal "Error: iptables not reset to nftables backend"
+fi
+if [[ ! -d "/var/lib/alternatives" ]]; then
+    fatal "Error: /var/lib/alternatives should still exists"
+fi