From 4343a2273830c78b94467c802d9e7faa29673a29 Mon Sep 17 00:00:00 2001 From: Anjan Nath Date: Wed, 9 Oct 2024 18:41:51 +0530 Subject: [PATCH] add systemd services for configuration after start this adds 4 small systemd services that: - creates crc specific configurations for dnsmasq - sets a new uuid as cluster id - creates the pod for routes-controller - tries to grow the disk and filesystem - checks if the cluster operators are ready - adds the pull secret to the cluster - sets kubeadmin and developer user passwords --- createdisk-library.sh | 18 +++++++++++ createdisk.sh | 2 ++ systemd/crc-cluster-status.service | 12 ++++++++ systemd/crc-cluster-status.sh | 43 +++++++++++++++++++++++++++ systemd/crc-dnsmasq.service | 14 +++++++++ systemd/crc-dnsmasq.sh | 20 +++++++++++++ systemd/crc-pullsecret.service | 12 ++++++++ systemd/crc-pullsecret.sh | 36 ++++++++++++++++++++++ systemd/crc-routes-controller.service | 13 ++++++++ systemd/crc-routes-controller.sh | 17 +++++++++++ systemd/ocp-clusterid.service | 12 ++++++++ systemd/ocp-clusterid.sh | 17 +++++++++++ systemd/ocp-growfs.service | 10 +++++++ systemd/ocp-growfs.sh | 11 +++++++ systemd/ocp-userpasswords.service | 12 ++++++++ systemd/ocp-userpasswords.sh | 39 ++++++++++++++++++++++++ 16 files changed, 288 insertions(+) create mode 100644 systemd/crc-cluster-status.service create mode 100644 systemd/crc-cluster-status.sh create mode 100644 systemd/crc-dnsmasq.service create mode 100644 systemd/crc-dnsmasq.sh create mode 100644 systemd/crc-pullsecret.service create mode 100644 systemd/crc-pullsecret.sh create mode 100644 systemd/crc-routes-controller.service create mode 100644 systemd/crc-routes-controller.sh create mode 100644 systemd/ocp-clusterid.service create mode 100644 systemd/ocp-clusterid.sh create mode 100644 systemd/ocp-growfs.service create mode 100644 systemd/ocp-growfs.sh create mode 100644 systemd/ocp-userpasswords.service create mode 100644 systemd/ocp-userpasswords.sh diff --git a/createdisk-library.sh b/createdisk-library.sh index 282b6e1c..651fd305 100755 --- a/createdisk-library.sh +++ b/createdisk-library.sh @@ -400,3 +400,21 @@ function remove_pull_secret_from_disk() { esac } +function copy_systemd_units() { + ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts' + ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + + case "${BUNDLE_TYPE}" in + "snc"|"okd") + ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + ;; + esac + + ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%' + ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable' + ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts' +} diff --git a/createdisk.sh b/createdisk.sh index 95601077..a54ae179 100755 --- a/createdisk.sh +++ b/createdisk.sh @@ -130,6 +130,8 @@ if [ "${ARCH}" == "aarch64" ] && [ ${BUNDLE_TYPE} != "okd" ]; then ${SSH} core@${VM_IP} -- "sudo rpm-ostree install https://kojipkgs.fedoraproject.org//packages/qemu/8.2.6/3.fc40/aarch64/qemu-user-static-x86-8.2.6-3.fc40.aarch64.rpm" fi +copy_systemd_units + cleanup_vm_image ${VM_NAME} ${VM_IP} # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state. diff --git a/systemd/crc-cluster-status.service b/systemd/crc-cluster-status.service new file mode 100644 index 00000000..564e659b --- /dev/null +++ b/systemd/crc-cluster-status.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit checking if cluster is ready +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-cluster-status.sh +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-cluster-status.sh b/systemd/crc-cluster-status.sh new file mode 100644 index 00000000..e8afc8a5 --- /dev/null +++ b/systemd/crc-cluster-status.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -x + +export KUBECONFIG=/opt/kubeconfig + +function check_cluster_unhealthy() { + WAIT="authentication|console|etcd|ingress|openshift-apiserver" + + until `oc get co > /dev/null 2>&1` + do + sleep 2 + done + + for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }') + do + if [[ $i == "False" ]] + then + return 0 + fi + done + return 1 +} + +# rm -rf /tmp/.crc-cluster-ready + +COUNTER=0 +CLUSTER_HEALTH_SLEEP=8 +CLUSTER_HEALTH_RETRIES=500 + +while $(check_cluster_unhealthy) +do + sleep $CLUSTER_HEALTH_SLEEP + if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]] + then + return 1 + fi + ((COUNTER++)) +done + +# need to set a marker to let `crc` know the cluster is ready +# touch /tmp/.crc-cluster-ready + diff --git a/systemd/crc-dnsmasq.service b/systemd/crc-dnsmasq.service new file mode 100644 index 00000000..9e5e164e --- /dev/null +++ b/systemd/crc-dnsmasq.service @@ -0,0 +1,14 @@ +[Unit] +Description=CRC Unit for configuring dnsmasq +Requires=ovs-configuration.service +After=ovs-configuration.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-dnsmasq.sh +ExecStartPost=/usr/bin/systemctl start dnsmasq.service +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-dnsmasq.sh b/systemd/crc-dnsmasq.sh new file mode 100644 index 00000000..908a10ce --- /dev/null +++ b/systemd/crc-dnsmasq.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -x + +hostName=$(hostname) +ip=$(ip -4 addr show br-ex | grep -oP '(?<=inet\s)192+(\.\d+){3}') +iip=$(hostname -i) + +cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf +listen-address=$ip +expand-hosts +log-queries +local=/crc.testing/ +domain=crc.testing +address=/apps-crc.testing/$ip +address=/api.crc.testing/$ip +address=/api-int.crc.testing/$ip +address=/$hostName.crc.testing/$iip +EOF + diff --git a/systemd/crc-pullsecret.service b/systemd/crc-pullsecret.service new file mode 100644 index 00000000..598b9ee0 --- /dev/null +++ b/systemd/crc-pullsecret.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit for adding pull secret to cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-pullsecret.sh +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-pullsecret.sh b/systemd/crc-pullsecret.sh new file mode 100644 index 00000000..83456011 --- /dev/null +++ b/systemd/crc-pullsecret.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -x + +export KUBECONFIG="/opt/kubeconfig" + +retry=0 +max_retry=20 +until `oc get secret > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret +existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}") +existingPs=$(echo "${existingPsB64}" | base64 -d) + +echo "${existingPs}" | jq -e '.' + +if [[ $? != 0 ]]; then + retry=0 + max_retry=20 + + until `ls /opt/crc/pull-secret > /dev/null 2>&1` + do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) + done + + pullSecretB64=$(cat /opt/crc/pull-secret) + oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}" +fi + diff --git a/systemd/crc-routes-controller.service b/systemd/crc-routes-controller.service new file mode 100644 index 00000000..647ab308 --- /dev/null +++ b/systemd/crc-routes-controller.service @@ -0,0 +1,13 @@ +[Unit] +Description=CRC Unit starting routes controller +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-routes-controller.sh +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-routes-controller.sh b/systemd/crc-routes-controller.sh new file mode 100644 index 00000000..b6a3378e --- /dev/null +++ b/systemd/crc-routes-controller.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -x + +export KUBECONFIG=/opt/kubeconfig + +retry=0 +max_retry=20 +until `oc get pods > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +oc apply -f /opt/crc/routes-controller.yaml + diff --git a/systemd/ocp-clusterid.service b/systemd/ocp-clusterid.service new file mode 100644 index 00000000..18fb04e0 --- /dev/null +++ b/systemd/ocp-clusterid.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit setting random cluster ID +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-clusterid.sh +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-clusterid.sh b/systemd/ocp-clusterid.sh new file mode 100644 index 00000000..072312bc --- /dev/null +++ b/systemd/ocp-clusterid.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -x + +export KUBECONFIG="/opt/kubeconfig" +uuid=$(uuidgen) + +retry=0 +max_retry=20 +until `oc get clusterversion > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge diff --git a/systemd/ocp-growfs.service b/systemd/ocp-growfs.service new file mode 100644 index 00000000..5d33babe --- /dev/null +++ b/systemd/ocp-growfs.service @@ -0,0 +1,10 @@ +[Unit] +Description=CRC Unit to grow the root filesystem + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-growfs.sh +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-growfs.sh b/systemd/ocp-growfs.sh new file mode 100644 index 00000000..4c657bb2 --- /dev/null +++ b/systemd/ocp-growfs.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +root_partition=$(/usr/sbin/blkid -t TYPE=xfs -o device) +/usr/bin/growpart "${root_partition#?}" "${root_partition#/dev/???}" + +rootFS="/sysroot" +mount -o remount,rw "${rootFS}" +xfs_growfs "${rootFS}" +#mount -o remount,ro "${rootFS}" diff --git a/systemd/ocp-userpasswords.service b/systemd/ocp-userpasswords.service new file mode 100644 index 00000000..07d30da1 --- /dev/null +++ b/systemd/ocp-userpasswords.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit for adding pull secret to cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-userpasswords.sh +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-userpasswords.sh b/systemd/ocp-userpasswords.sh new file mode 100644 index 00000000..cf717b0b --- /dev/null +++ b/systemd/ocp-userpasswords.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -x + +export KUBECONFIG="/opt/kubeconfig" + +retry=0 +max_retry=20 +until `oc get secret > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + + +retry=0 +max_retry=20 +until `ls /opt/crc/pass_developer /opt/crc/pass_kubeadmin > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +PASS_DEVELOPER=$(cat /opt/crc/pass_developer) +PASS_KUBEADMIN=$(cat /opt/crc/pass_kubeadmin) + +podman run --rm -ti xmartlabs/htpasswd developer $PASS_DEVELOPER > /tmp/htpasswd.developer +podman run --rm -ti xmartlabs/htpasswd kubeadmin $PASS_KUBEADMIN > /tmp/htpasswd.kubeadmin + +cat /tmp/htpasswd.developer > /tmp/htpasswd.txt +cat /tmp/htpasswd.kubeadmin >> /tmp/htpasswd.txt +sed -i '/^\s*$/d' /tmp/htpasswd.txt + +oc create secret generic htpass-secret --from-file=htpasswd=/tmp/htpasswd.txt -n openshift-config --dry-run=client -o yaml > /tmp/htpass-secret.yaml +oc replace -f /tmp/htpass-secret.yaml + +rm -rf /opt/crc/pass_developer /opt/crc/pass_kubeadmin