images/krte: add Rootless Docker (and systemd as a dependency)

images/krte/Dockerfile

@@ -21,9 +21,16 @@ FROM debian:bookworm
 # arg that specifies the image name (for debugging)
 ARG IMAGE_ARG
 
-# arg that specifies the go version to install
+# arg that specifies the go version to install.
+# empty value specifies the latest version.
 ARG GO_VERSION
 
+# args for the user to run rootless docker (for KRTE_SYSTEMD_ROOTLESS mode)
+ARG ROOTLESS_USER=rootless
+ARG ROOTLESS_UID=5000
+ARG ROOTLESS_SUBUID_BEGIN=100000
+ARG ROOTLESS_SUBUID_COUNT=1048576
+
 # add envs:
 # - so we can debug with the image name:tag
 # - adding gsutil etc. to path (where we will install them)
@@ -36,9 +43,6 @@ ENV KRTE_IMAGE=${IMAGE_ARG} \
     CLOUDSDK_CORE_DISABLE_PROMPTS=1 \
     CONTAINER=docker
 
-# copy in image utility scripts
-COPY wrapper.sh /usr/local/bin/
-
 # Install tools needed to:
 # - install docker
 # - build kind
@@ -56,6 +60,8 @@ RUN echo "Installing Packages ..." \
             build-essential \
             ca-certificates \
             curl \
+            dbus \
+            dbus-user-session \
             file \
             git \
             gnupg2 \
@@ -67,10 +73,15 @@ RUN echo "Installing Packages ..." \
             procps \
             python3 \
             rsync \
+            slirp4netns \
             software-properties-common \
+            sudo \
+            systemd \
+            uidmap \
             unzip \
         && rm -rf /var/lib/apt/lists/* \
     && echo "Installing Go ..." \
+        && if [ -z "${GO_VERSION}" ]; then GO_VERSION=$(curl -fsSL https://go.dev/VERSION?m=text | grep -oP "go\K(.*)"); fi \
         && export GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz" \
         && curl -fsSL "https://go.dev/dl/${GO_TARBALL}" --output "${GO_TARBALL}" \
         && tar xzf "${GO_TARBALL}" -C /usr/local \
@@ -93,14 +104,30 @@ RUN echo "Installing Packages ..." \
             "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
             tee /etc/apt/sources.list.d/docker.list > /dev/null \
         && apt-get update \
-        && apt-get install -y --no-install-recommends docker-ce docker-buildx-plugin \
+        && apt-get install -y --no-install-recommends docker-ce docker-buildx-plugin docker-ce-rootless-extras \
+        && systemctl disable docker \
         && rm -rf /var/lib/apt/lists/* \
         && sed -i 's/cgroupfs_mount$/#cgroupfs_mount\n/' /etc/init.d/docker \
     && echo "Ensuring Legacy Iptables ..." \
         && update-alternatives --set iptables /usr/sbin/iptables-legacy \
-        && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+        && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy \
+    && echo "Setting up the systemd delegation for rootless" \
+        && mkdir -p /etc/systemd/system/[email protected] \
+        && echo "[Service]" > /etc/systemd/system/[email protected]/delegate.conf \
+        && echo "Delegate=yes" >> /etc/systemd/system/[email protected]/delegate.conf \
+    && echo "Setting up the rootless user" \
+        && useradd --create-home --home-dir /home/${ROOTLESS_USER} --uid ${ROOTLESS_UID} -G systemd-journal ${ROOTLESS_USER} \
+        && echo "${ROOTLESS_UID}:${ROOTLESS_SUBUID_BEGIN}:${ROOTLESS_SUBUID_COUNT}" >/etc/subuid \
+        && cp -f /etc/subuid /etc/subgid \
+        && mkdir -p /home/${ROOTLESS_USER}/.local/share/docker \
+        && chown -R ${ROOTLESS_USER} /home/${ROOTLESS_USER}
+
+# copy in image utility scripts
+COPY containerized-systemd.sh /usr/local/bin/
+COPY wrapper.sh /usr/local/bin/
 
 # entrypoint is our wrapper script, in Prow you will need to explicitly re-specify this
 ENTRYPOINT ["wrapper.sh"]
 # volume for docker in docker, use an emptyDir in Prow
 VOLUME ["/var/lib/docker"]
+VOLUME ["/home/${ROOTLESS_USER}/.local/share/docker"]

images/krte/README.md

@@ -8,3 +8,11 @@ is maintained for the sole purpose of testing Kubernetes with KIND.
 ## WARNING
 
 This image is _not_ supported for other use cases. Use at your own risk.
+
+## Build-time variables
+See the `ARG` instructions in [`Dockerfile`](./Dockerfile).
+
+## Run-time variables
+- `KRTE_SYSTEMD=true` (default: `false`): enable systemd
+- `KRTE_SYSTEMD_ROOTLESS=true` (default: `false`): switch to a non-root user via systemd.
+  The KRTE container itself still has to be run as the root, so DO NOT specify `securityContext.runAsUser`.

images/krte/containerized-systemd.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copied from https://github.com/AkihiroSuda/containerized-systemd/tree/v0.1.1 .
+# This script executes the args via systemd.
+# The stdio, workdir, and environment variables are preserved.
+
+set -e
+container=docker
+export container
+
+if [ $# -eq 0 ]; then
+	echo >&2 'ERROR: No command specified. You probably want to run `journalctl -f`, or maybe `bash`?'
+	exit 1
+fi
+
+if [ ! -t 0 ]; then
+	echo >&2 'ERROR: TTY needs to be enabled (`docker run -t ...`).'
+	exit 1
+fi
+
+env >/etc/docker-entrypoint-env
+
+cat >/etc/systemd/system/docker-entrypoint.target <<EOF
+[Unit]
+Description=the target for docker-entrypoint.service
+Requires=docker-entrypoint.service systemd-logind.service systemd-user-sessions.service
+EOF
+
+quoted_args="$(printf " %q" "${@}")"
+echo "${quoted_args}" >/etc/docker-entrypoint-cmd
+
+cat >/etc/systemd/system/docker-entrypoint.service <<EOF
+[Unit]
+Description=docker-entrypoint.service
+
+[Service]
+ExecStart=/bin/bash -exc "source /etc/docker-entrypoint-cmd"
+# EXIT_STATUS is either an exit code integer or a signal name string, see systemd.exec(5)
+ExecStopPost=/bin/bash -ec "if echo \${EXIT_STATUS} | grep [A-Z] > /dev/null; then echo >&2 \"got signal \${EXIT_STATUS}\"; systemctl exit \$(( 128 + \$( kill -l \${EXIT_STATUS} ) )); else systemctl exit \${EXIT_STATUS}; fi"
+StandardInput=tty-force
+StandardOutput=inherit
+StandardError=inherit
+WorkingDirectory=$(pwd)
+EnvironmentFile=/etc/docker-entrypoint-env
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl mask systemd-firstboot.service systemd-udevd.service systemd-modules-load.service
+systemctl unmask systemd-logind
+systemctl enable docker-entrypoint.service
+
+systemd=
+if [ -x /lib/systemd/systemd ]; then
+	systemd=/lib/systemd/systemd
+elif [ -x /usr/lib/systemd/systemd ]; then
+	systemd=/usr/lib/systemd/systemd
+elif [ -x /sbin/init ]; then
+	systemd=/sbin/init
+else
+	echo >&2 'ERROR: systemd is not installed'
+	exit 1
+fi
+systemd_args="--show-status=false --unit=docker-entrypoint.target"
+echo "$0: starting $systemd $systemd_args"
+exec $systemd $systemd_args

images/krte/wrapper.sh

@@ -30,6 +30,26 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+: "${KRTE_SYSTEMD_ROOTLESS:=false}"
+: "${KRTE_SYSTEMD_ROOTLESS_USER:=rootless}"
+: "${KRTE_SYSTEMD:=${KRTE_SYSTEMD_ROOTLESS}}"
+
+if [ "${KRTE_SYSTEMD}" = "true" ] && [ "$$" = "1" ]; then
+  >&2 echo "wrapper.sh] [INFO] Re-executing in systemd: \`$*\`"
+  exec /usr/local/bin/containerized-systemd.sh "$0" "$@"
+fi
+if [ "${KRTE_SYSTEMD_ROOTLESS}" = "true" ] && [ "$(id -u)" = "0" ]; then
+  >&2 echo "wrapper.sh] [INFO] Waiting for the systemd user session to start up"
+  loginctl enable-linger ${KRTE_SYSTEMD_ROOTLESS_USER}
+  sleep 3
+  >&2 echo "wrapper.sh] [INFO] Switching to rootless: \`$*\`"
+  rootless_uid=$(id -u ${KRTE_SYSTEMD_ROOTLESS_USER})
+  exec sudo -E -H -u ${KRTE_SYSTEMD_ROOTLESS_USER} \
+    XDG_RUNTIME_DIR=/run/user/${rootless_uid} \
+    DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/${rootless_uid}/bus \
+    "$0" "$@"
+fi
+
 >&2 echo "wrapper.sh] [INFO] Wrapping Test Command: \`$*\`"
 >&2 echo "wrapper.sh] [INFO] Running in: ${KRTE_IMAGE}"
 >&2 echo "wrapper.sh] [INFO] See: https://github.com/kubernetes/test-infra/blob/master/images/krte/wrapper.sh"
@@ -40,7 +60,11 @@ cleanup(){
   if [[ "${DOCKER_IN_DOCKER_ENABLED:-false}" == "true" ]]; then
     >&2 echo "wrapper.sh] [CLEANUP] Cleaning up after Docker in Docker ..."
     docker ps -aq | xargs -r docker rm -f || true
-    service docker stop || true
+    if [ "$(id -u)" = "0" ]; then
+      service docker stop || true
+    else
+      systemctl --user stop docker || true
+    fi
     >&2 echo "wrapper.sh] [CLEANUP] Done cleaning up after Docker in Docker."
   fi
 }
@@ -93,7 +117,11 @@ export DOCKER_IN_DOCKER_ENABLED=${DOCKER_IN_DOCKER_ENABLED:-false}
 if [[ "${DOCKER_IN_DOCKER_ENABLED}" == "true" ]]; then
   >&2 echo "wrapper.sh] [SETUP] Docker in Docker enabled, initializing ..."
   # If we have opted in to docker in docker, start the docker daemon,
-  service docker start
+  if [ "$(id -u)" = "0" ]; then
+    service docker start
+  else
+    dockerd-rootless-setuptool.sh install
+  fi
   # the service can be started but the docker socket not ready, wait for ready
   WAIT_N=0
   while true; do