add systemd services for configuration after start

this adds 4 small systemd services that:
- creates crc specific configurations for dnsmasq
- sets a new uuid as cluster id
- creates the pod for routes-controller
- tries to grow the disk and filesystem
- checks if the cluster operators are ready
- adds the pull secret to the cluster
- sets kubeadmin and developer user passwords

createdisk-library.sh

@@ -400,3 +400,21 @@ function remove_pull_secret_from_disk() {
     esac
 }
 
+function copy_systemd_units() {
+    ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts'
+    ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/
+    ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+
+    case "${BUNDLE_TYPE}" in
+      "snc"|"okd")
+        ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/
+        ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+	;;
+    esac
+
+    ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%'
+    ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable'
+    ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts'
+}

createdisk.sh

@@ -130,6 +130,8 @@ if [ "${ARCH}" == "aarch64" ] && [ ${BUNDLE_TYPE} != "okd" ]; then
    ${SSH} core@${VM_IP} -- "sudo rpm-ostree install https://kojipkgs.fedoraproject.org//packages/qemu/8.2.6/3.fc40/aarch64/qemu-user-static-x86-8.2.6-3.fc40.aarch64.rpm"
 fi
 
+copy_systemd_units
+
 cleanup_vm_image ${VM_NAME} ${VM_IP}
 
 # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state.

systemd/crc-dnsmasq.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=CRC Unit for configuring dnsmasq
+Requires=ovs-configuration.service
+After=ovs-configuration.service
+
+[Service]
+Type=oneshot
+ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0"
+ExecStart=/usr/local/bin/crc-dnsmasq.sh
+ExecStartPost=/usr/bin/systemctl start dnsmasq.service
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-dnsmasq.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -x
+
+hostName=$(hostname)
+ip=$(ip -4 addr show br-ex | grep -oP '(?<=inet\s)192+(\.\d+){3}')
+iip=$(hostname -i)
+
+cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf
+listen-address=$ip
+expand-hosts
+log-queries
+local=/crc.testing/
+domain=crc.testing
+address=/apps-crc.testing/$ip
+address=/api.crc.testing/$ip
+address=/api-int.crc.testing/$ip
+address=/$hostName.crc.testing/$iip
+EOF
+

systemd/crc-routes-controller.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=CRC Unit starting routes controller
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0"
+ExecStart=/usr/local/bin/crc-routes-controller.sh
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-routes-controller.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG=/opt/kubeconfig
+
+retry=0
+max_retry=20
+until `oc get pods > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+oc apply -f /opt/crc/routes-controller.yaml
+

systemd/ocp-cluster-status.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit checking if cluster is ready
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-cluster-status.sh
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-cluster-status.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG=/opt/kubeconfig
+
+function check_cluster_unhealthy() {
+    WAIT="authentication|console|etcd|ingress|openshift-apiserver"
+
+    until `oc get co > /dev/null 2>&1`
+    do
+        sleep 2
+    done
+
+    for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }')
+    do
+        if [[ $i == "False" ]]
+        then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# rm -rf /tmp/.crc-cluster-ready
+
+COUNTER=0
+CLUSTER_HEALTH_SLEEP=8
+CLUSTER_HEALTH_RETRIES=500
+
+while $(check_cluster_unhealthy)
+do
+    sleep $CLUSTER_HEALTH_SLEEP
+    if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]]
+    then
+        return 1
+    fi
+    ((COUNTER++))
+done
+
+# need to set a marker to let `crc` know the cluster is ready
+# touch /tmp/.crc-cluster-ready
+

systemd/ocp-clusterid.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit setting random cluster ID
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-clusterid.sh
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-clusterid.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG="/opt/kubeconfig"
+uuid=$(uuidgen)
+
+retry=0
+max_retry=20
+until `oc get clusterversion > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge

systemd/ocp-growfs.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=CRC Unit to grow the root filesystem
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-growfs.sh
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-growfs.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -x
+
+root_partition=$(/usr/sbin/blkid -t TYPE=xfs -o device)
+/usr/bin/growpart "${root_partition::-1}" "${root_partition#/dev/???}"
+
+rootFS="/sysroot"
+mount -o remount,rw "${rootFS}"
+xfs_growfs "${rootFS}"
+#mount -o remount,ro "${rootFS}"

systemd/ocp-pullsecret.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=CRC Unit for adding pull secret to cluster
+After=kubelet.service
+Requires=kubelet.service
+ConditionPathExists=/opt/crc/pull-secret
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-pullsecret.sh
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-pullsecret.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG="/opt/kubeconfig"
+
+retry=0
+max_retry=20
+until `oc get secret > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret
+existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}")
+existingPs=$(echo "${existingPsB64}" | base64 -d)
+
+echo "${existingPs}" | jq -e '.auths'
+
+if [[ $? != 0 ]]; then
+    pullSecretB64=$(cat /opt/crc/pull-secret)
+    oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}"
+fi
+
+rm -f /opt/crc/pull-secret

systemd/ocp-userpasswords.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=CRC Unit for adding pull secret to cluster
+After=kubelet.service
+Requires=kubelet.service
+ConditionPathExists=/opt/crc/pass_developer
+ConditionPathExists=/opt/crc/pass_kubeadmin
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-userpasswords.sh
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-userpasswords.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG="/opt/kubeconfig"
+
+retry=0
+max_retry=20
+until `oc get secret > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+PASS_DEVELOPER=$(cat /opt/crc/pass_developer)
+PASS_KUBEADMIN=$(cat /opt/crc/pass_kubeadmin)
+
+podman run --rm -ti xmartlabs/htpasswd developer $PASS_DEVELOPER > /tmp/htpasswd.developer
+podman run --rm -ti xmartlabs/htpasswd kubeadmin $PASS_KUBEADMIN > /tmp/htpasswd.kubeadmin
+
+cat /tmp/htpasswd.developer > /tmp/htpasswd.txt
+cat /tmp/htpasswd.kubeadmin >> /tmp/htpasswd.txt
+sed -i '/^\s*$/d' /tmp/htpasswd.txt
+
+oc create secret generic htpass-secret --from-file=htpasswd=/tmp/htpasswd.txt -n openshift-config --dry-run=client -o yaml > /tmp/htpass-secret.yaml
+oc replace -f /tmp/htpass-secret.yaml
+
+rm -f /opt/crc/pass_developer /opt/crc/pass_kubeadmin