From 0ed44f421b888249d3cbaa2b4fff521e7f95fd7f Mon Sep 17 00:00:00 2001 From: Anjan Nath Date: Wed, 9 Oct 2024 18:41:51 +0530 Subject: [PATCH] add systemd services for configuration after start the services does the various needed tasks to setup the ocp or microshift cluster, these systemd units runs small shell scripts which are based on: https://github.com/crc-org/crc-cloud/blob/main/pkg/bundle/setup/clustersetup.sh and does the following tasks: - creates crc specific configurations for dnsmasq - sets a new uuid as cluster id - creates the pod for routes-controller - tries to grow the disk and filesystem - checks if the cluster operators are ready - adds the pull secret to the cluster - sets kubeadmin and developer user passwords - sets a custom ca for authentication - sets custom nip.io cluster domain --- createdisk-library.sh | 21 ++++++++++++ createdisk.sh | 2 ++ notes-self-sufficient.md | 24 ++++++++++++++ systemd/crc-cluster-status.service | 12 +++++++ systemd/crc-cluster-status.sh | 43 ++++++++++++++++++++++++ systemd/crc-dnsmasq.service | 13 ++++++++ systemd/crc-dnsmasq.sh | 19 +++++++++++ systemd/crc-pullsecret.path | 11 +++++++ systemd/crc-pullsecret.service | 11 +++++++ systemd/crc-pullsecret.sh | 21 ++++++++++++ systemd/crc-routes-controller.service | 12 +++++++ systemd/crc-routes-controller.sh | 11 +++++++ systemd/crc-systemd-common.sh | 12 +++++++ systemd/ocp-cluster-ca.path | 11 +++++++ systemd/ocp-cluster-ca.service | 11 +++++++ systemd/ocp-cluster-ca.sh | 26 +++++++++++++++ systemd/ocp-clusterid.service | 11 +++++++ systemd/ocp-clusterid.sh | 11 +++++++ systemd/ocp-custom-domain.service | 12 +++++++ systemd/ocp-custom-domain.sh | 47 +++++++++++++++++++++++++++ systemd/ocp-growfs.service | 9 +++++ systemd/ocp-growfs.sh | 11 +++++++ systemd/ocp-userpasswords.path | 12 +++++++ systemd/ocp-userpasswords.service | 11 +++++++ systemd/ocp-userpasswords.sh | 30 +++++++++++++++++ 25 files changed, 414 insertions(+) create mode 100644 notes-self-sufficient.md create mode 100644 systemd/crc-cluster-status.service create mode 100644 systemd/crc-cluster-status.sh create mode 100644 systemd/crc-dnsmasq.service create mode 100644 systemd/crc-dnsmasq.sh create mode 100644 systemd/crc-pullsecret.path create mode 100644 systemd/crc-pullsecret.service create mode 100644 systemd/crc-pullsecret.sh create mode 100644 systemd/crc-routes-controller.service create mode 100644 systemd/crc-routes-controller.sh create mode 100644 systemd/crc-systemd-common.sh create mode 100644 systemd/ocp-cluster-ca.path create mode 100644 systemd/ocp-cluster-ca.service create mode 100644 systemd/ocp-cluster-ca.sh create mode 100644 systemd/ocp-clusterid.service create mode 100644 systemd/ocp-clusterid.sh create mode 100644 systemd/ocp-custom-domain.service create mode 100644 systemd/ocp-custom-domain.sh create mode 100644 systemd/ocp-growfs.service create mode 100644 systemd/ocp-growfs.sh create mode 100644 systemd/ocp-userpasswords.path create mode 100644 systemd/ocp-userpasswords.service create mode 100644 systemd/ocp-userpasswords.sh diff --git a/createdisk-library.sh b/createdisk-library.sh index 282b6e1c..672088a2 100755 --- a/createdisk-library.sh +++ b/createdisk-library.sh @@ -216,6 +216,7 @@ function prepare_hyperV() { echo 'CONST{virt}=="microsoft", RUN{builtin}+="kmod load hv_sock"' > /etc/udev/rules.d/90-crc-vsock.rules EOF } + function prepare_qemu_guest_agent() { local vm_ip=$1 @@ -400,3 +401,23 @@ function remove_pull_secret_from_disk() { esac } +function copy_systemd_units() { + ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts' + ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/crc-*.path core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + + case "${BUNDLE_TYPE}" in + "snc"|"okd") + ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/ocp-*.path core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + ;; + esac + + ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%' + ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable' + ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts' +} diff --git a/createdisk.sh b/createdisk.sh index 95601077..a54ae179 100755 --- a/createdisk.sh +++ b/createdisk.sh @@ -130,6 +130,8 @@ if [ "${ARCH}" == "aarch64" ] && [ ${BUNDLE_TYPE} != "okd" ]; then ${SSH} core@${VM_IP} -- "sudo rpm-ostree install https://kojipkgs.fedoraproject.org//packages/qemu/8.2.6/3.fc40/aarch64/qemu-user-static-x86-8.2.6-3.fc40.aarch64.rpm" fi +copy_systemd_units + cleanup_vm_image ${VM_NAME} ${VM_IP} # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state. diff --git a/notes-self-sufficient.md b/notes-self-sufficient.md new file mode 100644 index 00000000..050da48c --- /dev/null +++ b/notes-self-sufficient.md @@ -0,0 +1,24 @@ +# Self sufficient bundles + +Since release 4.19.0 of OpenShift Local, the bundles generated by `snc` contains additional systemd services to provision the cluster and removes the need of +an outside entity for provision the cluster, although an outside process needs to create some files on pre-defined locations inside the VM for the systemd +services to do their work. + +## The following table lists the systemd services and the location of files they need to provision the cluster, users of SNC needs to create those files + +| Systemd unit | runs for (ocp, microshift, both) | Input files location | Marker env variables | +| :-------------------: | :------------------------------: | :----------------------------------: | :------------------: | +| crc-cluster-status | both | none | none | +| crc-pullsecret | both | /opt/crc/pull-secret | none | +| crc-dnsmasq | both | none | none | +| crc-routes-controller | both | none | none | +| ocp-cluster-ca | ocp | /opt/crc/custom-ca.crt | CRC_CLOUD=1 | +| ocp-clusterid | ocp | none | none | +| ocp-custom-domain | ocp | none | CRC_CLOUD=1 | +| ocp-growfs | ocp | none | none | +| ocp-userpasswords | ocp | /opt/crc/pass_{kubeadmin, developer} | none | + +> [!NOTE] +> "Marker env variable" is set using an env file, if the required env variable is not set then unit is skipped +> some units are ran only when CRC_CLOUD=1 is set, these are only needed when using the bundles with crc-cloud + diff --git a/systemd/crc-cluster-status.service b/systemd/crc-cluster-status.service new file mode 100644 index 00000000..789b7248 --- /dev/null +++ b/systemd/crc-cluster-status.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit checking if cluster is ready +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-cluster-status.sh +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-cluster-status.sh b/systemd/crc-cluster-status.sh new file mode 100644 index 00000000..8326701a --- /dev/null +++ b/systemd/crc-cluster-status.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -x + +export KUBECONFIG=/opt/kubeconfig + +function check_cluster_healthy() { + WAIT="authentication|console|etcd|ingress|openshift-apiserver" + + until `oc get co > /dev/null 2>&1` + do + sleep 2 + done + + for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }') + do + if [[ $i == "False" ]] + then + return 1 + fi + done + return 0 +} + +# rm -rf /tmp/.crc-cluster-ready + +COUNTER=0 +CLUSTER_HEALTH_SLEEP=8 +CLUSTER_HEALTH_RETRIES=500 + +while ! check_cluster_healthy +do + sleep $CLUSTER_HEALTH_SLEEP + if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]] + then + return 1 + fi + ((COUNTER++)) +done + +# need to set a marker to let `crc` know the cluster is ready +# touch /tmp/.crc-cluster-ready + diff --git a/systemd/crc-dnsmasq.service b/systemd/crc-dnsmasq.service new file mode 100644 index 00000000..0da3cc27 --- /dev/null +++ b/systemd/crc-dnsmasq.service @@ -0,0 +1,13 @@ +[Unit] +Description=CRC Unit for configuring dnsmasq +Requires=ovs-configuration.service +After=ovs-configuration.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/sbin/ip link show dev tap0 && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-dnsmasq.sh +ExecStartPost=/usr/bin/systemctl start dnsmasq.service + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-dnsmasq.sh b/systemd/crc-dnsmasq.sh new file mode 100644 index 00000000..3c959aaf --- /dev/null +++ b/systemd/crc-dnsmasq.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x + +hostName=$(hostname) +hostIp=$(hostname --all-ip-addresses | awk '{print $1}') + +cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf +interface=br-ex +expand-hosts +log-queries +local=/crc.testing/ +domain=crc.testing +address=/apps-crc.testing/192.168.130.11 +address=/api.crc.testing/192.168.130.11 +address=/api-int.crc.testing/192.168.130.11 +address=/$hostName.crc.testing/$hostIp +EOF + diff --git a/systemd/crc-pullsecret.path b/systemd/crc-pullsecret.path new file mode 100644 index 00000000..4b228d32 --- /dev/null +++ b/systemd/crc-pullsecret.path @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit for monitoring the pull secret path +After=kubelet.service + +[Path] +PathExists=/opt/crc/pull-secret +TriggerLimitIntervalSec=1min +TriggerLimitBurst=0 + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-pullsecret.service b/systemd/crc-pullsecret.service new file mode 100644 index 00000000..d1decf65 --- /dev/null +++ b/systemd/crc-pullsecret.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit for adding pull secret to cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-pullsecret.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-pullsecret.sh b/systemd/crc-pullsecret.sh new file mode 100644 index 00000000..7aebfae4 --- /dev/null +++ b/systemd/crc-pullsecret.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" + +wait_for_resource secret + +# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret +existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}") +existingPs=$(echo "${existingPsB64}" | base64 -d) + +echo "${existingPs}" | jq -e '.auths' + +if [[ $? != 0 ]]; then + pullSecretB64=$(cat /opt/crc/pull-secret | base64 -w0) + oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}" + rm -f /opt/crc/pull-secret +fi + diff --git a/systemd/crc-routes-controller.service b/systemd/crc-routes-controller.service new file mode 100644 index 00000000..69748f58 --- /dev/null +++ b/systemd/crc-routes-controller.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit starting routes controller +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-routes-controller.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-routes-controller.sh b/systemd/crc-routes-controller.sh new file mode 100644 index 00000000..a0af392f --- /dev/null +++ b/systemd/crc-routes-controller.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG=/opt/kubeconfig + +wait_for_resource pods + +oc apply -f /opt/crc/routes-controller.yaml + diff --git a/systemd/crc-systemd-common.sh b/systemd/crc-systemd-common.sh new file mode 100644 index 00000000..3ccec925 --- /dev/null +++ b/systemd/crc-systemd-common.sh @@ -0,0 +1,12 @@ +# $1 is the resource to check +# $2 is an optional maximum retry count; default 20 +function wait_for_resource() { + local retry=0 + local max_retry=${2:-20} + until `oc get "$1" > /dev/null 2>&1` + do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) + done +} diff --git a/systemd/ocp-cluster-ca.path b/systemd/ocp-cluster-ca.path new file mode 100644 index 00000000..b863cb74 --- /dev/null +++ b/systemd/ocp-cluster-ca.path @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit monitoring custom-ca.crt file path +After=kubelet.service + +[Path] +PathExists=/opt/crc/custom-ca.crt +TriggerLimitIntervalSec=1min +TriggerLimitBurst=0 + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-cluster-ca.service b/systemd/ocp-cluster-ca.service new file mode 100644 index 00000000..910cd2fe --- /dev/null +++ b/systemd/ocp-cluster-ca.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit setting custom cluster ca +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-cluster-ca.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-cluster-ca.sh b/systemd/ocp-cluster-ca.sh new file mode 100644 index 00000000..bf95de63 --- /dev/null +++ b/systemd/ocp-cluster-ca.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" + +wait_for_resource configmap + +custom_ca_path=/opt/crc/custom-ca.crt + +# retry=0 +# max_retry=20 +# until `ls ${custom_ca_path} > /dev/null 2>&1` +# do +# [ $retry == $max_retry ] && exit 1 +# sleep 5 +# ((retry++)) +# done + +oc create configmap client-ca-custom -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} +oc patch apiserver cluster --type=merge -p '{"spec": {"clientCA": {"name": "client-ca-custom"}}}' +oc create configmap admin-kubeconfig-client-ca -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} \ +--dry-run -o yaml | oc replace -f - + +rm -f /opt/crc/custom-ca.crt diff --git a/systemd/ocp-clusterid.service b/systemd/ocp-clusterid.service new file mode 100644 index 00000000..22117073 --- /dev/null +++ b/systemd/ocp-clusterid.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit setting random cluster ID +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-clusterid.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-clusterid.sh b/systemd/ocp-clusterid.sh new file mode 100644 index 00000000..686deaa5 --- /dev/null +++ b/systemd/ocp-clusterid.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" +uuid=$(uuidgen) + +wait_for_resource clusterversion + +oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge diff --git a/systemd/ocp-custom-domain.service b/systemd/ocp-custom-domain.service new file mode 100644 index 00000000..e21e28d2 --- /dev/null +++ b/systemd/ocp-custom-domain.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit setting nip.io domain for cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +EnvironmentFile=/opt/crc/crc-cloud +ExecStart=/usr/local/bin/ocp-custom-domain.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-custom-domain.sh b/systemd/ocp-custom-domain.sh new file mode 100644 index 00000000..bec018ec --- /dev/null +++ b/systemd/ocp-custom-domain.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -x + +if [ -z $CRC_CLOUD ]; then + echo "Not running in crc-cloud mode" + exit 0 +fi + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" +export EIP=$(hostname -i) + +STEPS_SLEEP_TIME=30 + +wait_for_resource secret + +# create cert and add as secret +openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout nip.key -out nip.crt -subj "/CN=$EIP.nip.io" -addext "subjectAltName=DNS:apps.$EIP.nip.io,DNS:*.apps.$EIP.nip.io,DNS:api.$EIP.nip.io" +oc create secret tls nip-secret --cert=nip.crt --key=nip.key -n openshift-config +sleep $STEPS_SLEEP_TIME + +# patch ingress + cat < ingress-patch.yaml +spec: + appsDomain: apps.$EIP.nip.io + componentRoutes: + - hostname: console-openshift-console.apps.$EIP.nip.io + name: console + namespace: openshift-console + servingCertKeyPairSecret: + name: nip-secret + - hostname: oauth-openshift.apps.$EIP.nip.io + name: oauth-openshift + namespace: openshift-authentication + servingCertKeyPairSecret: + name: nip-secret +EOF +oc patch ingresses.config.openshift.io cluster --type=merge --patch-file=ingress-patch.yaml + +# patch API server to use new CA secret +oc patch apiserver cluster --type=merge -p '{"spec":{"servingCerts": {"namedCertificates":[{"names":["api.'$EIP'.nip.io"],"servingCertificate": {"name": "nip-secret"}}]}}}' + +# patch image registry route +oc patch -p '{"spec": {"host": "default-route-openshift-image-registry.'$EIP'.nip.io"}}' route default-route -n openshift-image-registry --type=merge + +#wait_cluster_become_healthy "authentication|console|etcd|ingress|openshift-apiserver" diff --git a/systemd/ocp-growfs.service b/systemd/ocp-growfs.service new file mode 100644 index 00000000..ce771ed6 --- /dev/null +++ b/systemd/ocp-growfs.service @@ -0,0 +1,9 @@ +[Unit] +Description=CRC Unit to grow the root filesystem + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-growfs.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-growfs.sh b/systemd/ocp-growfs.sh new file mode 100644 index 00000000..4c657bb2 --- /dev/null +++ b/systemd/ocp-growfs.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +root_partition=$(/usr/sbin/blkid -t TYPE=xfs -o device) +/usr/bin/growpart "${root_partition#?}" "${root_partition#/dev/???}" + +rootFS="/sysroot" +mount -o remount,rw "${rootFS}" +xfs_growfs "${rootFS}" +#mount -o remount,ro "${rootFS}" diff --git a/systemd/ocp-userpasswords.path b/systemd/ocp-userpasswords.path new file mode 100644 index 00000000..f652e756 --- /dev/null +++ b/systemd/ocp-userpasswords.path @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit monitoring the developer and kubeadmin user password file paths +After=kubelet.service + +[Path] +PathExists=/opt/crc/pass_kubeadmin +PathExists=/opt/crc/pass_developer +TriggerLimitIntervalSec=1min +TriggerLimitBurst=0 + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-userpasswords.service b/systemd/ocp-userpasswords.service new file mode 100644 index 00000000..a77521c8 --- /dev/null +++ b/systemd/ocp-userpasswords.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit setting the developer and kubeadmin user password +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-userpasswords.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-userpasswords.sh b/systemd/ocp-userpasswords.sh new file mode 100644 index 00000000..9b3570bc --- /dev/null +++ b/systemd/ocp-userpasswords.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" + +function gen_htpasswd() { + if [ ! -z "${1}" ] && [ ! -z "${2}" ]; then + podman run --rm -ti xmartlabs/htpasswd $1 $2 >> /tmp/htpasswd.txt + fi +} + +wait_for_resource secret + +PASS_DEVELOPER=$(cat /opt/crc/pass_developer) +PASS_KUBEADMIN=$(cat /opt/crc/pass_kubeadmin) + +rm -f /tmp/htpasswd.txt +gen_htpasswd developer "${PASS_DEVELOPER}" +gen_htpasswd kubeadmin "${PASS_KUBEADMIN}" + +if [ -f /tmp/htpasswd.txt ]; then + sed -i '/^\s*$/d' /tmp/htpasswd.txt + + oc create secret generic htpass-secret --from-file=htpasswd=/tmp/htpasswd.txt -n openshift-config --dry-run=client -o yaml > /tmp/htpass-secret.yaml + oc replace -f /tmp/htpass-secret.yaml + + rm -f /opt/crc/pass_developer /opt/crc/pass_kubeadmin +fi