From 6a9aaf6c124646291e08349c5a4186d3663dd5a9 Mon Sep 17 00:00:00 2001 From: Anjan Nath Date: Wed, 9 Oct 2024 18:41:51 +0530 Subject: [PATCH] add systemd services for configuration after start this adds 4 small systemd services that: - creates crc specific configurations for dnsmasq - sets a new uuid as cluster id - creates the pod for routes-controller - tries to grow the disk and filesystem - checks if the cluster operators are ready - adds the pull secret to the cluster - sets kubeadmin and developer user passwords - sets a custom ca for authentication - sets custom nip.io cluster domain --- createdisk-library.sh | 21 +++++++++++ createdisk.sh | 2 ++ systemd/crc-cluster-status.service | 12 +++++++ systemd/crc-cluster-status.sh | 43 ++++++++++++++++++++++ systemd/crc-dnsmasq.service | 13 +++++++ systemd/crc-dnsmasq.sh | 20 +++++++++++ systemd/crc-pullsecret.path | 9 +++++ systemd/crc-pullsecret.service | 11 ++++++ systemd/crc-pullsecret.sh | 26 ++++++++++++++ systemd/crc-routes-controller.service | 12 +++++++ systemd/crc-routes-controller.sh | 17 +++++++++ systemd/ocp-cluster-ca.service | 12 +++++++ systemd/ocp-cluster-ca.sh | 49 +++++++++++++++++++++++++ systemd/ocp-clusterid.service | 11 ++++++ systemd/ocp-clusterid.sh | 17 +++++++++ systemd/ocp-custom-domain.service | 12 +++++++ systemd/ocp-custom-domain.sh | 52 +++++++++++++++++++++++++++ systemd/ocp-growfs.service | 9 +++++ systemd/ocp-growfs.sh | 11 ++++++ systemd/ocp-userpasswords.path | 10 ++++++ systemd/ocp-userpasswords.service | 11 ++++++ systemd/ocp-userpasswords.sh | 29 +++++++++++++++ 22 files changed, 409 insertions(+) create mode 100644 systemd/crc-cluster-status.service create mode 100644 systemd/crc-cluster-status.sh create mode 100644 systemd/crc-dnsmasq.service create mode 100644 systemd/crc-dnsmasq.sh create mode 100644 systemd/crc-pullsecret.path create mode 100644 systemd/crc-pullsecret.service create mode 100644 systemd/crc-pullsecret.sh create mode 100644 systemd/crc-routes-controller.service create mode 100644 systemd/crc-routes-controller.sh create mode 100644 systemd/ocp-cluster-ca.service create mode 100644 systemd/ocp-cluster-ca.sh create mode 100644 systemd/ocp-clusterid.service create mode 100644 systemd/ocp-clusterid.sh create mode 100644 systemd/ocp-custom-domain.service create mode 100644 systemd/ocp-custom-domain.sh create mode 100644 systemd/ocp-growfs.service create mode 100644 systemd/ocp-growfs.sh create mode 100644 systemd/ocp-userpasswords.path create mode 100644 systemd/ocp-userpasswords.service create mode 100644 systemd/ocp-userpasswords.sh diff --git a/createdisk-library.sh b/createdisk-library.sh index 282b6e1c..672088a2 100755 --- a/createdisk-library.sh +++ b/createdisk-library.sh @@ -216,6 +216,7 @@ function prepare_hyperV() { echo 'CONST{virt}=="microsoft", RUN{builtin}+="kmod load hv_sock"' > /etc/udev/rules.d/90-crc-vsock.rules EOF } + function prepare_qemu_guest_agent() { local vm_ip=$1 @@ -400,3 +401,23 @@ function remove_pull_secret_from_disk() { esac } +function copy_systemd_units() { + ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts' + ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/crc-*.path core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + + case "${BUNDLE_TYPE}" in + "snc"|"okd") + ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/ocp-*.path core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + ;; + esac + + ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%' + ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable' + ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts' +} diff --git a/createdisk.sh b/createdisk.sh index 95601077..a54ae179 100755 --- a/createdisk.sh +++ b/createdisk.sh @@ -130,6 +130,8 @@ if [ "${ARCH}" == "aarch64" ] && [ ${BUNDLE_TYPE} != "okd" ]; then ${SSH} core@${VM_IP} -- "sudo rpm-ostree install https://kojipkgs.fedoraproject.org//packages/qemu/8.2.6/3.fc40/aarch64/qemu-user-static-x86-8.2.6-3.fc40.aarch64.rpm" fi +copy_systemd_units + cleanup_vm_image ${VM_NAME} ${VM_IP} # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state. diff --git a/systemd/crc-cluster-status.service b/systemd/crc-cluster-status.service new file mode 100644 index 00000000..789b7248 --- /dev/null +++ b/systemd/crc-cluster-status.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit checking if cluster is ready +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-cluster-status.sh +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-cluster-status.sh b/systemd/crc-cluster-status.sh new file mode 100644 index 00000000..e8afc8a5 --- /dev/null +++ b/systemd/crc-cluster-status.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -x + +export KUBECONFIG=/opt/kubeconfig + +function check_cluster_unhealthy() { + WAIT="authentication|console|etcd|ingress|openshift-apiserver" + + until `oc get co > /dev/null 2>&1` + do + sleep 2 + done + + for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }') + do + if [[ $i == "False" ]] + then + return 0 + fi + done + return 1 +} + +# rm -rf /tmp/.crc-cluster-ready + +COUNTER=0 +CLUSTER_HEALTH_SLEEP=8 +CLUSTER_HEALTH_RETRIES=500 + +while $(check_cluster_unhealthy) +do + sleep $CLUSTER_HEALTH_SLEEP + if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]] + then + return 1 + fi + ((COUNTER++)) +done + +# need to set a marker to let `crc` know the cluster is ready +# touch /tmp/.crc-cluster-ready + diff --git a/systemd/crc-dnsmasq.service b/systemd/crc-dnsmasq.service new file mode 100644 index 00000000..a8aa1480 --- /dev/null +++ b/systemd/crc-dnsmasq.service @@ -0,0 +1,13 @@ +[Unit] +Description=CRC Unit for configuring dnsmasq +Requires=ovs-configuration.service +After=ovs-configuration.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-dnsmasq.sh +ExecStartPost=/usr/bin/systemctl start dnsmasq.service + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-dnsmasq.sh b/systemd/crc-dnsmasq.sh new file mode 100644 index 00000000..908a10ce --- /dev/null +++ b/systemd/crc-dnsmasq.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -x + +hostName=$(hostname) +ip=$(ip -4 addr show br-ex | grep -oP '(?<=inet\s)192+(\.\d+){3}') +iip=$(hostname -i) + +cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf +listen-address=$ip +expand-hosts +log-queries +local=/crc.testing/ +domain=crc.testing +address=/apps-crc.testing/$ip +address=/api.crc.testing/$ip +address=/api-int.crc.testing/$ip +address=/$hostName.crc.testing/$iip +EOF + diff --git a/systemd/crc-pullsecret.path b/systemd/crc-pullsecret.path new file mode 100644 index 00000000..40cffec0 --- /dev/null +++ b/systemd/crc-pullsecret.path @@ -0,0 +1,9 @@ +[Unit] +Description=CRC Unit for monitoring the pull secret path +After=kubelet.service + +[Path] +PathExists=/opt/crc/pull-secret + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-pullsecret.service b/systemd/crc-pullsecret.service new file mode 100644 index 00000000..d1decf65 --- /dev/null +++ b/systemd/crc-pullsecret.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit for adding pull secret to cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-pullsecret.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-pullsecret.sh b/systemd/crc-pullsecret.sh new file mode 100644 index 00000000..16b2c66c --- /dev/null +++ b/systemd/crc-pullsecret.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -x + +export KUBECONFIG="/opt/kubeconfig" + +retry=0 +max_retry=20 +until `oc get secret > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret +existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}") +existingPs=$(echo "${existingPsB64}" | base64 -d) + +echo "${existingPs}" | jq -e '.auths' + +if [[ $? != 0 ]]; then + pullSecretB64=$(cat /opt/crc/pull-secret) + oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}" +fi + diff --git a/systemd/crc-routes-controller.service b/systemd/crc-routes-controller.service new file mode 100644 index 00000000..69748f58 --- /dev/null +++ b/systemd/crc-routes-controller.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit starting routes controller +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-routes-controller.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-routes-controller.sh b/systemd/crc-routes-controller.sh new file mode 100644 index 00000000..b6a3378e --- /dev/null +++ b/systemd/crc-routes-controller.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -x + +export KUBECONFIG=/opt/kubeconfig + +retry=0 +max_retry=20 +until `oc get pods > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +oc apply -f /opt/crc/routes-controller.yaml + diff --git a/systemd/ocp-cluster-ca.service b/systemd/ocp-cluster-ca.service new file mode 100644 index 00000000..06857d27 --- /dev/null +++ b/systemd/ocp-cluster-ca.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit setting custom cluster ca +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +EnvironmentFile=/opt/crc/crc-cloud +ExecStart=/usr/local/bin/ocp-cluster-ca.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-cluster-ca.sh b/systemd/ocp-cluster-ca.sh new file mode 100644 index 00000000..a1f8694a --- /dev/null +++ b/systemd/ocp-cluster-ca.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# To generate the custom-ca.crt +# USER="system:admin" +# GROUP="system:masters" +# USER_SUBJ="/O=${GROUP}/CN=${USER}" +# NAME="custom" +# CA_SUBJ="/OU=openshift/CN=admin-kubeconfig-signer-custom" +# VALIDITY=3650 + +# openssl genrsa -out $NAME-ca.key 4096 +# openssl req -x509 -new -nodes -key $NAME-ca.key -sha256 -days $VALIDITY -out $NAME-ca.crt -subj "$CA_SUBJ" +# openssl req -nodes -newkey rsa:2048 -keyout $USER.key -subj "$USER_SUBJ" -out $USER.csr +# openssl x509 -extfile <(printf "extendedKeyUsage = clientAuth") -req -in $USER.csr \ +# -CA $NAME-ca.crt -CAkey $NAME-ca.key -CAcreateserial -out $USER.crt -days $VALIDITY -sha256 + +set -x + +if [ -z $CRC_CLOUD ]; then + exit 1 +fi + +export KUBECONFIG="/opt/kubeconfig" + +retry=0 +max_retry=20 +until `oc get configmap > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +custom_ca_path=/opt/crc/custom-ca.crt + +retry=0 +max_retry=20 +until `ls ${custom_ca_path} > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +oc create configmap client-ca-custom -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} +oc patch apiserver cluster --type=merge -p '{"spec": {"clientCA": {"name": "client-ca-custom"}}}' +oc create configmap admin-kubeconfig-client-ca -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} \ +--dry-run -o yaml | oc replace -f - + diff --git a/systemd/ocp-clusterid.service b/systemd/ocp-clusterid.service new file mode 100644 index 00000000..22117073 --- /dev/null +++ b/systemd/ocp-clusterid.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit setting random cluster ID +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-clusterid.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-clusterid.sh b/systemd/ocp-clusterid.sh new file mode 100644 index 00000000..072312bc --- /dev/null +++ b/systemd/ocp-clusterid.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -x + +export KUBECONFIG="/opt/kubeconfig" +uuid=$(uuidgen) + +retry=0 +max_retry=20 +until `oc get clusterversion > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge diff --git a/systemd/ocp-custom-domain.service b/systemd/ocp-custom-domain.service new file mode 100644 index 00000000..e21e28d2 --- /dev/null +++ b/systemd/ocp-custom-domain.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit setting nip.io domain for cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +EnvironmentFile=/opt/crc/crc-cloud +ExecStart=/usr/local/bin/ocp-custom-domain.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-custom-domain.sh b/systemd/ocp-custom-domain.sh new file mode 100644 index 00000000..d9598598 --- /dev/null +++ b/systemd/ocp-custom-domain.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -x + +if [ -z $CRC_CLOUD ]; then + exit 1 +fi + +export KUBECONFIG="/opt/kubeconfig" +export EIP=$(hostname -i) + +STEPS_SLEEP_TIME=30 + +retry=0 +max_retry=20 +until `oc get secret > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +# create cert and add as secret +openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout nip.key -out nip.crt -subj "/CN=$EIP.nip.io" -addext "subjectAltName=DNS:apps.$EIP.nip.io,DNS:*.apps.$EIP.nip.io,DNS:api.$EIP.nip.io" +oc create secret tls nip-secret --cert=nip.crt --key=nip.key -n openshift-config +sleep $STEPS_SLEEP_TIME + +# patch ingress + cat < ingress-patch.yaml +spec: + appsDomain: apps.$EIP.nip.io + componentRoutes: + - hostname: console-openshift-console.apps.$EIP.nip.io + name: console + namespace: openshift-console + servingCertKeyPairSecret: + name: nip-secret + - hostname: oauth-openshift.apps.$EIP.nip.io + name: oauth-openshift + namespace: openshift-authentication + servingCertKeyPairSecret: + name: nip-secret +EOF +oc patch ingresses.config.openshift.io cluster --type=merge --patch-file=ingress-patch.yaml + +# patch API server to use new CA secret +oc patch apiserver cluster --type=merge -p '{"spec":{"servingCerts": {"namedCertificates":[{"names":["api.'$EIP'.nip.io"],"servingCertificate": {"name": "nip-secret"}}]}}}' + +# patch image registry route +oc patch -p '{"spec": {"host": "default-route-openshift-image-registry.'$EIP'.nip.io"}}' route default-route -n openshift-image-registry --type=merge + +#wait_cluster_become_healthy "authentication|console|etcd|ingress|openshift-apiserver" diff --git a/systemd/ocp-growfs.service b/systemd/ocp-growfs.service new file mode 100644 index 00000000..ce771ed6 --- /dev/null +++ b/systemd/ocp-growfs.service @@ -0,0 +1,9 @@ +[Unit] +Description=CRC Unit to grow the root filesystem + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-growfs.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-growfs.sh b/systemd/ocp-growfs.sh new file mode 100644 index 00000000..4c657bb2 --- /dev/null +++ b/systemd/ocp-growfs.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +root_partition=$(/usr/sbin/blkid -t TYPE=xfs -o device) +/usr/bin/growpart "${root_partition#?}" "${root_partition#/dev/???}" + +rootFS="/sysroot" +mount -o remount,rw "${rootFS}" +xfs_growfs "${rootFS}" +#mount -o remount,ro "${rootFS}" diff --git a/systemd/ocp-userpasswords.path b/systemd/ocp-userpasswords.path new file mode 100644 index 00000000..6080a717 --- /dev/null +++ b/systemd/ocp-userpasswords.path @@ -0,0 +1,10 @@ +[Unit] +Description=CRC Unit monitoring the developer and kubeadmin user password file paths +After=kubelet.service + +[Path] +PathExists=/opt/crc/pass_kubeadmin +PathExists=/opt/crc/pass_developer + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-userpasswords.service b/systemd/ocp-userpasswords.service new file mode 100644 index 00000000..a77521c8 --- /dev/null +++ b/systemd/ocp-userpasswords.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit setting the developer and kubeadmin user password +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-userpasswords.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-userpasswords.sh b/systemd/ocp-userpasswords.sh new file mode 100644 index 00000000..a35722b6 --- /dev/null +++ b/systemd/ocp-userpasswords.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -x + +export KUBECONFIG="/opt/kubeconfig" + +retry=0 +max_retry=20 +until `oc get secret > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +PASS_DEVELOPER=$(cat /opt/crc/pass_developer) +PASS_KUBEADMIN=$(cat /opt/crc/pass_kubeadmin) + +podman run --rm -ti xmartlabs/htpasswd developer $PASS_DEVELOPER > /tmp/htpasswd.developer +podman run --rm -ti xmartlabs/htpasswd kubeadmin $PASS_KUBEADMIN > /tmp/htpasswd.kubeadmin + +cat /tmp/htpasswd.developer > /tmp/htpasswd.txt +cat /tmp/htpasswd.kubeadmin >> /tmp/htpasswd.txt +sed -i '/^\s*$/d' /tmp/htpasswd.txt + +oc create secret generic htpass-secret --from-file=htpasswd=/tmp/htpasswd.txt -n openshift-config --dry-run=client -o yaml > /tmp/htpass-secret.yaml +oc replace -f /tmp/htpass-secret.yaml + +rm -rf /opt/crc/pass_developer /opt/crc/pass_kubeadmin