add systemd services for configuration after start

this adds 4 small systemd services that:
- creates crc specific configurations for dnsmasq
- sets a new uuid as cluster id
- creates the pod for routes-controller
- tries to grow the disk and filesystem
- checks if the cluster operators are ready
- adds the pull secret to the cluster
- sets kubeadmin and developer user passwords
- sets a custom ca for authentication
- sets custom nip.io cluster domain

createdisk-library.sh

@@ -216,6 +216,7 @@ function prepare_hyperV() {
             echo 'CONST{virt}=="microsoft", RUN{builtin}+="kmod load hv_sock"' > /etc/udev/rules.d/90-crc-vsock.rules
 EOF
 }
+
 function prepare_qemu_guest_agent() {
     local vm_ip=$1
 
@@ -400,3 +401,23 @@ function remove_pull_secret_from_disk() {
     esac
 }
 
+function copy_systemd_units() {
+    ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts'
+    ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/
+    ${SCP} systemd/crc-*.path core@${VM_IP}:/home/core/systemd-units/
+    ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+
+    case "${BUNDLE_TYPE}" in
+      "snc"|"okd")
+        ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/
+        ${SCP} systemd/ocp-*.path core@${VM_IP}:/home/core/systemd-units/
+        ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+	;;
+    esac
+
+    ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%'
+    ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable'
+    ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts'
+}

createdisk.sh

@@ -130,6 +130,8 @@ if [ "${ARCH}" == "aarch64" ] && [ ${BUNDLE_TYPE} != "okd" ]; then
    ${SSH} core@${VM_IP} -- "sudo rpm-ostree install https://kojipkgs.fedoraproject.org//packages/qemu/8.2.6/3.fc40/aarch64/qemu-user-static-x86-8.2.6-3.fc40.aarch64.rpm"
 fi
 
+copy_systemd_units
+
 cleanup_vm_image ${VM_NAME} ${VM_IP}
 
 # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state.

systemd/crc-cluster-status.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit checking if cluster is ready
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/crc-cluster-status.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-cluster-status.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG=/opt/kubeconfig
+
+function check_cluster_unhealthy() {
+    WAIT="authentication|console|etcd|ingress|openshift-apiserver"
+
+    until `oc get co > /dev/null 2>&1`
+    do
+        sleep 2
+    done
+
+    for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }')
+    do
+        if [[ $i == "False" ]]
+        then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# rm -rf /tmp/.crc-cluster-ready
+
+COUNTER=0
+CLUSTER_HEALTH_SLEEP=8
+CLUSTER_HEALTH_RETRIES=500
+
+while $(check_cluster_unhealthy)
+do
+    sleep $CLUSTER_HEALTH_SLEEP
+    if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]]
+    then
+        return 1
+    fi
+    ((COUNTER++))
+done
+
+# need to set a marker to let `crc` know the cluster is ready
+# touch /tmp/.crc-cluster-ready
+

systemd/crc-dnsmasq.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=CRC Unit for configuring dnsmasq
+Requires=ovs-configuration.service
+After=ovs-configuration.service
+
+[Service]
+Type=oneshot
+ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0"
+ExecStart=/usr/local/bin/crc-dnsmasq.sh
+ExecStartPost=/usr/bin/systemctl start dnsmasq.service
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-dnsmasq.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -x
+
+hostName=$(hostname)
+ip=$(ip -4 addr show br-ex | grep -oP '(?<=inet\s)192+(\.\d+){3}')
+iip=$(hostname -i)
+
+cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf
+listen-address=$ip
+expand-hosts
+log-queries
+local=/crc.testing/
+domain=crc.testing
+address=/apps-crc.testing/$ip
+address=/api.crc.testing/$ip
+address=/api-int.crc.testing/$ip
+address=/$hostName.crc.testing/$iip
+EOF
+

systemd/crc-pullsecret.path

@@ -0,0 +1,9 @@
+[Unit]
+Description=CRC Unit for monitoring the pull secret path
+After=kubelet.service
+
+[Path]
+PathExists=/opt/crc/pull-secret
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit for adding pull secret to cluster
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/crc-pullsecret.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG="/opt/kubeconfig"
+
+retry=0
+max_retry=20
+until `oc get secret > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret
+existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}")
+existingPs=$(echo "${existingPsB64}" | base64 -d)
+
+echo "${existingPs}" | jq -e '.auths'
+
+if [[ $? != 0 ]]; then
+    pullSecretB64=$(cat /opt/crc/pull-secret)
+    oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}"
+fi
+

systemd/crc-routes-controller.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit starting routes controller
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0"
+ExecStart=/usr/local/bin/crc-routes-controller.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-routes-controller.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG=/opt/kubeconfig
+
+retry=0
+max_retry=20
+until `oc get pods > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+oc apply -f /opt/crc/routes-controller.yaml
+

systemd/ocp-cluster-ca.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit setting custom cluster ca
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/opt/crc/crc-cloud
+ExecStart=/usr/local/bin/ocp-cluster-ca.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-cluster-ca.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# To generate the custom-ca.crt
+# USER="system:admin"
+# GROUP="system:masters"
+# USER_SUBJ="/O=${GROUP}/CN=${USER}"
+# NAME="custom"
+# CA_SUBJ="/OU=openshift/CN=admin-kubeconfig-signer-custom"
+# VALIDITY=3650
+
+# openssl genrsa -out $NAME-ca.key 4096
+# openssl req -x509 -new -nodes -key $NAME-ca.key -sha256 -days $VALIDITY -out $NAME-ca.crt -subj "$CA_SUBJ"
+# openssl req -nodes -newkey rsa:2048 -keyout $USER.key -subj "$USER_SUBJ" -out $USER.csr
+# openssl x509 -extfile <(printf "extendedKeyUsage = clientAuth") -req -in $USER.csr \
+#    -CA $NAME-ca.crt -CAkey $NAME-ca.key -CAcreateserial -out $USER.crt -days $VALIDITY -sha256
+
+set -x
+
+if [ -z $CRC_CLOUD ]; then
+    exit 1
+fi
+
+export KUBECONFIG="/opt/kubeconfig"
+
+retry=0
+max_retry=20
+until `oc get configmap > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+custom_ca_path=/opt/crc/custom-ca.crt
+
+retry=0
+max_retry=20
+until `ls ${custom_ca_path} > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+oc create configmap client-ca-custom -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path}
+oc patch apiserver cluster --type=merge -p '{"spec": {"clientCA": {"name": "client-ca-custom"}}}'
+oc create configmap admin-kubeconfig-client-ca -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} \
+--dry-run -o yaml | oc replace -f -
+

systemd/ocp-clusterid.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit setting random cluster ID
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-clusterid.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-clusterid.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG="/opt/kubeconfig"
+uuid=$(uuidgen)
+
+retry=0
+max_retry=20
+until `oc get clusterversion > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge

systemd/ocp-custom-domain.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit setting nip.io domain for cluster
+After=kubelet.service
+Requires=kubelet.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/opt/crc/crc-cloud
+ExecStart=/usr/local/bin/ocp-custom-domain.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-custom-domain.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -x
+
+if [ -z $CRC_CLOUD ]; then
+    exit 1
+fi
+
+export KUBECONFIG="/opt/kubeconfig"
+export EIP=$(hostname -i)
+
+STEPS_SLEEP_TIME=30
+
+retry=0
+max_retry=20
+until `oc get secret > /dev/null 2>&1`
+do
+    [ $retry == $max_retry ] && exit 1
+    sleep 5
+    ((retry++))
+done
+
+# create cert and add as secret
+openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout nip.key -out nip.crt -subj "/CN=$EIP.nip.io" -addext "subjectAltName=DNS:apps.$EIP.nip.io,DNS:*.apps.$EIP.nip.io,DNS:api.$EIP.nip.io"
+oc create secret tls nip-secret --cert=nip.crt --key=nip.key -n openshift-config
+sleep $STEPS_SLEEP_TIME
+
+# patch ingress
+    cat <<EOF > ingress-patch.yaml
+spec:
+  appsDomain: apps.$EIP.nip.io
+  componentRoutes:
+  - hostname: console-openshift-console.apps.$EIP.nip.io
+    name: console
+    namespace: openshift-console
+    servingCertKeyPairSecret:
+      name: nip-secret
+  - hostname: oauth-openshift.apps.$EIP.nip.io
+    name: oauth-openshift
+    namespace: openshift-authentication
+    servingCertKeyPairSecret:
+      name: nip-secret
+EOF
+oc patch ingresses.config.openshift.io cluster --type=merge --patch-file=ingress-patch.yaml
+
+# patch API server to use new CA secret
+oc patch apiserver cluster --type=merge -p '{"spec":{"servingCerts": {"namedCertificates":[{"names":["api.'$EIP'.nip.io"],"servingCertificate": {"name": "nip-secret"}}]}}}'
+
+# patch image registry route
+oc patch -p '{"spec": {"host": "default-route-openshift-image-registry.'$EIP'.nip.io"}}' route default-route -n openshift-image-registry --type=merge
+
+#wait_cluster_become_healthy "authentication|console|etcd|ingress|openshift-apiserver"

systemd/ocp-growfs.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=CRC Unit to grow the root filesystem
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-growfs.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-growfs.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -x
+
+root_partition=$(/usr/sbin/blkid -t TYPE=xfs -o device)
+/usr/bin/growpart "${root_partition#?}" "${root_partition#/dev/???}"
+
+rootFS="/sysroot"
+mount -o remount,rw "${rootFS}"
+xfs_growfs "${rootFS}"
+#mount -o remount,ro "${rootFS}"