diff --git a/createdisk-library.sh b/createdisk-library.sh index 282b6e1c..672088a2 100755 --- a/createdisk-library.sh +++ b/createdisk-library.sh @@ -216,6 +216,7 @@ function prepare_hyperV() { echo 'CONST{virt}=="microsoft", RUN{builtin}+="kmod load hv_sock"' > /etc/udev/rules.d/90-crc-vsock.rules EOF } + function prepare_qemu_guest_agent() { local vm_ip=$1 @@ -400,3 +401,23 @@ function remove_pull_secret_from_disk() { esac } +function copy_systemd_units() { + ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts' + ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/crc-*.path core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + + case "${BUNDLE_TYPE}" in + "snc"|"okd") + ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/ocp-*.path core@${VM_IP}:/home/core/systemd-units/ + ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/ + ;; + esac + + ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%' + ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin' + ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable' + ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts' +} diff --git a/createdisk.sh b/createdisk.sh index 95601077..a54ae179 100755 --- a/createdisk.sh +++ b/createdisk.sh @@ -130,6 +130,8 @@ if [ "${ARCH}" == "aarch64" ] && [ ${BUNDLE_TYPE} != "okd" ]; then ${SSH} core@${VM_IP} -- "sudo rpm-ostree install https://kojipkgs.fedoraproject.org//packages/qemu/8.2.6/3.fc40/aarch64/qemu-user-static-x86-8.2.6-3.fc40.aarch64.rpm" fi +copy_systemd_units + cleanup_vm_image ${VM_NAME} ${VM_IP} # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state. diff --git a/notes-self-sufficient.md b/notes-self-sufficient.md new file mode 100644 index 00000000..050da48c --- /dev/null +++ b/notes-self-sufficient.md @@ -0,0 +1,24 @@ +# Self sufficient bundles + +Since release 4.19.0 of OpenShift Local, the bundles generated by `snc` contains additional systemd services to provision the cluster and removes the need of +an outside entity for provision the cluster, although an outside process needs to create some files on pre-defined locations inside the VM for the systemd +services to do their work. + +## The following table lists the systemd services and the location of files they need to provision the cluster, users of SNC needs to create those files + +| Systemd unit | runs for (ocp, microshift, both) | Input files location | Marker env variables | +| :-------------------: | :------------------------------: | :----------------------------------: | :------------------: | +| crc-cluster-status | both | none | none | +| crc-pullsecret | both | /opt/crc/pull-secret | none | +| crc-dnsmasq | both | none | none | +| crc-routes-controller | both | none | none | +| ocp-cluster-ca | ocp | /opt/crc/custom-ca.crt | CRC_CLOUD=1 | +| ocp-clusterid | ocp | none | none | +| ocp-custom-domain | ocp | none | CRC_CLOUD=1 | +| ocp-growfs | ocp | none | none | +| ocp-userpasswords | ocp | /opt/crc/pass_{kubeadmin, developer} | none | + +> [!NOTE] +> "Marker env variable" is set using an env file, if the required env variable is not set then unit is skipped +> some units are ran only when CRC_CLOUD=1 is set, these are only needed when using the bundles with crc-cloud + diff --git a/systemd/crc-cluster-status.service b/systemd/crc-cluster-status.service new file mode 100644 index 00000000..789b7248 --- /dev/null +++ b/systemd/crc-cluster-status.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit checking if cluster is ready +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-cluster-status.sh +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-cluster-status.sh b/systemd/crc-cluster-status.sh new file mode 100644 index 00000000..8326701a --- /dev/null +++ b/systemd/crc-cluster-status.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -x + +export KUBECONFIG=/opt/kubeconfig + +function check_cluster_healthy() { + WAIT="authentication|console|etcd|ingress|openshift-apiserver" + + until `oc get co > /dev/null 2>&1` + do + sleep 2 + done + + for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }') + do + if [[ $i == "False" ]] + then + return 1 + fi + done + return 0 +} + +# rm -rf /tmp/.crc-cluster-ready + +COUNTER=0 +CLUSTER_HEALTH_SLEEP=8 +CLUSTER_HEALTH_RETRIES=500 + +while ! check_cluster_healthy +do + sleep $CLUSTER_HEALTH_SLEEP + if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]] + then + return 1 + fi + ((COUNTER++)) +done + +# need to set a marker to let `crc` know the cluster is ready +# touch /tmp/.crc-cluster-ready + diff --git a/systemd/crc-dnsmasq.service b/systemd/crc-dnsmasq.service new file mode 100644 index 00000000..0da3cc27 --- /dev/null +++ b/systemd/crc-dnsmasq.service @@ -0,0 +1,13 @@ +[Unit] +Description=CRC Unit for configuring dnsmasq +Requires=ovs-configuration.service +After=ovs-configuration.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/sbin/ip link show dev tap0 && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-dnsmasq.sh +ExecStartPost=/usr/bin/systemctl start dnsmasq.service + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-dnsmasq.sh b/systemd/crc-dnsmasq.sh new file mode 100644 index 00000000..fcde7a8e --- /dev/null +++ b/systemd/crc-dnsmasq.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -x + +hostName=$(hostname) + +cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf +interface=br-ex +expand-hosts +log-queries +local=/crc.testing/ +domain=crc.testing +address=/apps-crc.testing/192.168.126.11 +address=/api.crc.testing/192.168.126.11 +address=/api-int.crc.testing/192.168.126.11 +address=/$hostName.crc.testing/192.168.126.11 +EOF + diff --git a/systemd/crc-pullsecret.path b/systemd/crc-pullsecret.path new file mode 100644 index 00000000..4b228d32 --- /dev/null +++ b/systemd/crc-pullsecret.path @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit for monitoring the pull secret path +After=kubelet.service + +[Path] +PathExists=/opt/crc/pull-secret +TriggerLimitIntervalSec=1min +TriggerLimitBurst=0 + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-pullsecret.service b/systemd/crc-pullsecret.service new file mode 100644 index 00000000..d1decf65 --- /dev/null +++ b/systemd/crc-pullsecret.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit for adding pull secret to cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crc-pullsecret.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-pullsecret.sh b/systemd/crc-pullsecret.sh new file mode 100644 index 00000000..0dddb2f4 --- /dev/null +++ b/systemd/crc-pullsecret.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" + +wait_for_resource secret + +# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret +existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}") +existingPs=$(echo "${existingPsB64}" | base64 -d) + +echo "${existingPs}" | jq -e '.auths' + +if [[ $? != 0 ]]; then + pullSecretB64=$(cat /opt/crc/pull-secret base64 -w0) + oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}" + rm -f /opt/crc/pull-secret +fi + diff --git a/systemd/crc-routes-controller.service b/systemd/crc-routes-controller.service new file mode 100644 index 00000000..69748f58 --- /dev/null +++ b/systemd/crc-routes-controller.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit starting routes controller +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0" +ExecStart=/usr/local/bin/crc-routes-controller.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/crc-routes-controller.sh b/systemd/crc-routes-controller.sh new file mode 100644 index 00000000..a0af392f --- /dev/null +++ b/systemd/crc-routes-controller.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG=/opt/kubeconfig + +wait_for_resource pods + +oc apply -f /opt/crc/routes-controller.yaml + diff --git a/systemd/crc-systemd-common.sh b/systemd/crc-systemd-common.sh new file mode 100644 index 00000000..3ccec925 --- /dev/null +++ b/systemd/crc-systemd-common.sh @@ -0,0 +1,12 @@ +# $1 is the resource to check +# $2 is an optional maximum retry count; default 20 +function wait_for_resource() { + local retry=0 + local max_retry=${2:-20} + until `oc get "$1" > /dev/null 2>&1` + do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) + done +} diff --git a/systemd/ocp-cluster-ca.service b/systemd/ocp-cluster-ca.service new file mode 100644 index 00000000..06857d27 --- /dev/null +++ b/systemd/ocp-cluster-ca.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit setting custom cluster ca +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +EnvironmentFile=/opt/crc/crc-cloud +ExecStart=/usr/local/bin/ocp-cluster-ca.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-cluster-ca.sh b/systemd/ocp-cluster-ca.sh new file mode 100644 index 00000000..903b0bd7 --- /dev/null +++ b/systemd/ocp-cluster-ca.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# To generate the custom-ca.crt +# USER="system:admin" +# GROUP="system:masters" +# USER_SUBJ="/O=${GROUP}/CN=${USER}" +# NAME="custom" +# CA_SUBJ="/OU=openshift/CN=admin-kubeconfig-signer-custom" +# VALIDITY=3650 + +# openssl genrsa -out $NAME-ca.key 4096 +# openssl req -x509 -new -nodes -key $NAME-ca.key -sha256 -days $VALIDITY -out $NAME-ca.crt -subj "$CA_SUBJ" +# openssl req -nodes -newkey rsa:2048 -keyout $USER.key -subj "$USER_SUBJ" -out $USER.csr +# openssl x509 -extfile <(printf "extendedKeyUsage = clientAuth") -req -in $USER.csr \ +# -CA $NAME-ca.crt -CAkey $NAME-ca.key -CAcreateserial -out $USER.crt -days $VALIDITY -sha256 + +set -x + +if [ -z $CRC_CLOUD ]; then + echo "Not running in crc-cloud mode" + exit 0 +fi + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" + +wait_for_resource configmap + +custom_ca_path=/opt/crc/custom-ca.crt + +retry=0 +max_retry=20 +until `ls ${custom_ca_path} > /dev/null 2>&1` +do + [ $retry == $max_retry ] && exit 1 + sleep 5 + ((retry++)) +done + +oc create configmap client-ca-custom -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} +oc patch apiserver cluster --type=merge -p '{"spec": {"clientCA": {"name": "client-ca-custom"}}}' +oc create configmap admin-kubeconfig-client-ca -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} \ +--dry-run -o yaml | oc replace -f - + diff --git a/systemd/ocp-clusterid.service b/systemd/ocp-clusterid.service new file mode 100644 index 00000000..22117073 --- /dev/null +++ b/systemd/ocp-clusterid.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit setting random cluster ID +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-clusterid.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-clusterid.sh b/systemd/ocp-clusterid.sh new file mode 100644 index 00000000..686deaa5 --- /dev/null +++ b/systemd/ocp-clusterid.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" +uuid=$(uuidgen) + +wait_for_resource clusterversion + +oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge diff --git a/systemd/ocp-custom-domain.service b/systemd/ocp-custom-domain.service new file mode 100644 index 00000000..e21e28d2 --- /dev/null +++ b/systemd/ocp-custom-domain.service @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit setting nip.io domain for cluster +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +EnvironmentFile=/opt/crc/crc-cloud +ExecStart=/usr/local/bin/ocp-custom-domain.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-custom-domain.sh b/systemd/ocp-custom-domain.sh new file mode 100644 index 00000000..bec018ec --- /dev/null +++ b/systemd/ocp-custom-domain.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -x + +if [ -z $CRC_CLOUD ]; then + echo "Not running in crc-cloud mode" + exit 0 +fi + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" +export EIP=$(hostname -i) + +STEPS_SLEEP_TIME=30 + +wait_for_resource secret + +# create cert and add as secret +openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout nip.key -out nip.crt -subj "/CN=$EIP.nip.io" -addext "subjectAltName=DNS:apps.$EIP.nip.io,DNS:*.apps.$EIP.nip.io,DNS:api.$EIP.nip.io" +oc create secret tls nip-secret --cert=nip.crt --key=nip.key -n openshift-config +sleep $STEPS_SLEEP_TIME + +# patch ingress + cat < ingress-patch.yaml +spec: + appsDomain: apps.$EIP.nip.io + componentRoutes: + - hostname: console-openshift-console.apps.$EIP.nip.io + name: console + namespace: openshift-console + servingCertKeyPairSecret: + name: nip-secret + - hostname: oauth-openshift.apps.$EIP.nip.io + name: oauth-openshift + namespace: openshift-authentication + servingCertKeyPairSecret: + name: nip-secret +EOF +oc patch ingresses.config.openshift.io cluster --type=merge --patch-file=ingress-patch.yaml + +# patch API server to use new CA secret +oc patch apiserver cluster --type=merge -p '{"spec":{"servingCerts": {"namedCertificates":[{"names":["api.'$EIP'.nip.io"],"servingCertificate": {"name": "nip-secret"}}]}}}' + +# patch image registry route +oc patch -p '{"spec": {"host": "default-route-openshift-image-registry.'$EIP'.nip.io"}}' route default-route -n openshift-image-registry --type=merge + +#wait_cluster_become_healthy "authentication|console|etcd|ingress|openshift-apiserver" diff --git a/systemd/ocp-growfs.service b/systemd/ocp-growfs.service new file mode 100644 index 00000000..ce771ed6 --- /dev/null +++ b/systemd/ocp-growfs.service @@ -0,0 +1,9 @@ +[Unit] +Description=CRC Unit to grow the root filesystem + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-growfs.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-growfs.sh b/systemd/ocp-growfs.sh new file mode 100644 index 00000000..4c657bb2 --- /dev/null +++ b/systemd/ocp-growfs.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +root_partition=$(/usr/sbin/blkid -t TYPE=xfs -o device) +/usr/bin/growpart "${root_partition#?}" "${root_partition#/dev/???}" + +rootFS="/sysroot" +mount -o remount,rw "${rootFS}" +xfs_growfs "${rootFS}" +#mount -o remount,ro "${rootFS}" diff --git a/systemd/ocp-userpasswords.path b/systemd/ocp-userpasswords.path new file mode 100644 index 00000000..f652e756 --- /dev/null +++ b/systemd/ocp-userpasswords.path @@ -0,0 +1,12 @@ +[Unit] +Description=CRC Unit monitoring the developer and kubeadmin user password file paths +After=kubelet.service + +[Path] +PathExists=/opt/crc/pass_kubeadmin +PathExists=/opt/crc/pass_developer +TriggerLimitIntervalSec=1min +TriggerLimitBurst=0 + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-userpasswords.service b/systemd/ocp-userpasswords.service new file mode 100644 index 00000000..a77521c8 --- /dev/null +++ b/systemd/ocp-userpasswords.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRC Unit setting the developer and kubeadmin user password +After=kubelet.service +Requires=kubelet.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ocp-userpasswords.sh + +[Install] +WantedBy=multi-user.target diff --git a/systemd/ocp-userpasswords.sh b/systemd/ocp-userpasswords.sh new file mode 100644 index 00000000..9b3570bc --- /dev/null +++ b/systemd/ocp-userpasswords.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -x + +source /usr/local/bin/crc-systemd-common.sh +export KUBECONFIG="/opt/kubeconfig" + +function gen_htpasswd() { + if [ ! -z "${1}" ] && [ ! -z "${2}" ]; then + podman run --rm -ti xmartlabs/htpasswd $1 $2 >> /tmp/htpasswd.txt + fi +} + +wait_for_resource secret + +PASS_DEVELOPER=$(cat /opt/crc/pass_developer) +PASS_KUBEADMIN=$(cat /opt/crc/pass_kubeadmin) + +rm -f /tmp/htpasswd.txt +gen_htpasswd developer "${PASS_DEVELOPER}" +gen_htpasswd kubeadmin "${PASS_KUBEADMIN}" + +if [ -f /tmp/htpasswd.txt ]; then + sed -i '/^\s*$/d' /tmp/htpasswd.txt + + oc create secret generic htpass-secret --from-file=htpasswd=/tmp/htpasswd.txt -n openshift-config --dry-run=client -o yaml > /tmp/htpass-secret.yaml + oc replace -f /tmp/htpass-secret.yaml + + rm -f /opt/crc/pass_developer /opt/crc/pass_kubeadmin +fi