ci(.github): enable self hosted runners for AMD64 E2E tasks (backport…

… of #10945) (#12276) Automatic cherry-pick of #10945 for branch release-2.6 Generated by [action](https://github.com/kumahq/kuma/actions/runs/12353055555) cherry-picked commit 126029d ⚠️ ⚠️ ⚠️ Conflicts happened when cherry-picking! ⚠️ ⚠️ ⚠️ ``` On branch release-2.6 Your branch is up to date with 'origin/release-2.6'. You are currently cherry-picking commit 126029d. (fix conflicts and run "git cherry-pick --continue") (use "git cherry-pick --skip" to skip this patch) (use "git cherry-pick --abort" to cancel the cherry-pick operation) Changes to be committed: modified: mk/kind.mk modified: test/kind/cluster-ipv6-kuma-1.yaml modified: test/kind/cluster-ipv6-kuma-2.yaml modified: test/kind/cluster-ipv6.yaml modified: test/kind/cluster-kuma-1.yaml modified: test/kind/cluster-kuma-2.yaml modified: test/kind/cluster.yaml Unmerged paths: (use "git add/rm <file>..." as appropriate to mark resolution) deleted by us: .github/workflows/_test.yaml both modified: .github/workflows/build-test-distribute.yaml both modified: .github/workflows/e2e.yaml both modified: mk/k3d.mk ``` --------- Signed-off-by: Jay Chen <[email protected]> Signed-off-by: Charly Molter <[email protected]> Signed-off-by: Lukasz Dziedziak <[email protected]> Co-authored-by: Jay Jijie Chen <[email protected]> Co-authored-by: Charly Molter <[email protected]> Co-authored-by: Lukasz Dziedziak <[email protected]>
kumahq · Dec 16, 2024 · 2a6e768 · 2a6e768
1 parent 186c053
commit 2a6e768
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 33 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -20,6 +20,8 @@ env:
   E2E_PARAM_PARALLELISM: ${{ fromJSON(inputs.matrix).parallelism }}
 jobs:
   e2e:
+    timeout-minutes: 60
+    # can't use env vars here
     runs-on: ${{ fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch] }}
     if: ${{ fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch] != '' }}
     strategy:
@@ -77,6 +79,8 @@ jobs:
           EOF
           sudo service docker restart
       - name: "Run E2E tests"
+        env:
+          DOCKERHUB_PULL_CREDENTIAL: ${{ secrets.DOCKERHUB_PULL_CREDENTIAL }}
         run: |
           if [[ "${{ env.E2E_PARAM_K8S_VERSION }}" == "kindIpv6" ]]; then
             export IPV6=true
@@ -102,6 +106,24 @@ jobs:
             export GINKGO_E2E_LABEL_FILTERS="job-${{ matrix.parallelRunnerId }}"
           fi
           env
+          if [[ "${{ env.E2E_PARAM_TARGET }}" == "multizone" ]]; then
+            export KUMA_DEFAULT_RETRIES=60
+          fi
+
+          function on_exit()
+          {
+            docker logout docker.io
+          }
+
+          # we pull a few images during the E2E run, sometimes we get rate-limited by docker hub
+          # to prevent this, we support specifying a pull credential here
+          if [[ "$DOCKERHUB_PULL_CREDENTIAL" != "" ]]; then
+            DOCKER_USER=$(echo "$DOCKERHUB_PULL_CREDENTIAL" | cut -d ':' -f 1)
+            DOCKER_PWD=$(echo "$DOCKERHUB_PULL_CREDENTIAL" | cut -d ':' -f 2)
+            echo -n "$DOCKER_PWD" | docker login -u "$DOCKER_USER" --password-stdin
+            trap "on_exit" EXIT
+          fi
+
           if [[ "${{ env.E2E_PARAM_TARGET }}" != "" ]]; then
             target="test/e2e-${{ env.E2E_PARAM_TARGET }}"
           else

diff --git a/mk/k3d.mk b/mk/k3d.mk
@@ -41,6 +41,7 @@ KUMA_NAMESPACE ?= kuma-system
 PORT_PREFIX := $$(($(patsubst 300-%,300+%-1,$(KIND_CLUSTER_NAME:kuma%=300%))))
 
 K3D_NETWORK_CNI ?= flannel
+K3D_REGISTRY_FILE ?=
 K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \
 	--k3s-arg '--disable=traefik@server:0' \
 	--k3s-arg '--disable=metrics-server@server:0' \
@@ -49,6 +50,7 @@ K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \
     --volume '$(subst @,\@,$(TOP)/$(KUMA_DIR))/test/framework/deployments:/tmp/deployments@server:0' \
 	--network kind \
 	--port "$(PORT_PREFIX)80-$(PORT_PREFIX)99:30080-30099@server:0" \
+	--registry-config "/tmp/.kuma-dev/k3d-registry.yaml" \
 	--timeout 120s
 
 ifeq ($(K3D_NETWORK_CNI),calico)
@@ -88,8 +90,23 @@ $(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml:
 		-o $(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml \
 		https://k3d.io/v$(K3D_VERSION)/usage/advanced/calico.yaml
 
+DOCKERHUB_PULL_CREDENTIAL ?=
+.PHONY: k3d/setup-docker-credentials
+k3d/setup-docker-credentials:
+	@mkdir -p /tmp/.kuma-dev ; \
+	echo '{"configs": {}}' > /tmp/.kuma-dev/k3d-registry.yaml ; \
+	if [[ "$(DOCKERHUB_PULL_CREDENTIAL)" != "" ]]; then \
+  		DOCKER_USER=$$(echo "$(DOCKERHUB_PULL_CREDENTIAL)" | cut -d ':' -f 1); \
+  		DOCKER_PWD=$$(echo "$(DOCKERHUB_PULL_CREDENTIAL)" | cut -d ':' -f 2); \
+  		echo "{\"configs\": {\"registry-1.docker.io\": {\"auth\": {\"username\": \"$${DOCKER_USER}\",\"password\":\"$${DOCKER_PWD}\"}}}}" > /tmp/.kuma-dev/k3d-registry.yaml ; \
+  	fi
+
+.PHONY: k3d/cleanup-docker-credentials
+k3d/cleanup-docker-credentials:
+	@rm -f /tmp/.kuma-dev/k3d-registry.yaml
+
 .PHONY: k3d/start
-k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create \
+k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create k3d/setup-docker-credentials
 	$(if $(findstring calico,$(K3D_NETWORK_CNI)),$(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml)
 	@echo "PORT_PREFIX=$(PORT_PREFIX)"
 	@KUBECONFIG=$(KIND_KUBECONFIG) \
@@ -136,7 +153,7 @@ k3d/wait:
     done
 
 .PHONY: k3d/stop
-k3d/stop:
+k3d/stop: k3d/cleanup-docker-credentials
 	@KUBECONFIG=$(KIND_KUBECONFIG) $(K3D_BIN) cluster delete "$(KIND_CLUSTER_NAME)"
 
 .PHONY: k3d/stop/all

diff --git a/mk/kind.mk b/mk/kind.mk
@@ -27,8 +27,21 @@ CI_KUBERNETES_VERSION ?= v1.23.17@sha256:59c989ff8a517a93127d4a536e7014d28e235fb
 KUMA_MODE ?= zone
 KUMA_NAMESPACE ?= kuma-system
 
+DOCKERHUB_PULL_CREDENTIAL ?=
+.PHONY: kind/setup-docker-credentials
+kind/setup-docker-credentials:
+	@mkdir -p /tmp/.kuma-dev ; \
+	echo '{"auths":{}}' > /tmp/.kuma-dev/kind-config.json ; \
+	if [[ "$(DOCKERHUB_PULL_CREDENTIAL)" != "" ]]; then \
+		echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$$(echo -n "$(DOCKERHUB_PULL_CREDENTIAL)" | base64)\"}}}" > /tmp/.kuma-dev/kind-config.json ; \
+	fi
+
+.PHONY: kind/cleanup-docker-credentials
+kind/cleanup-docker-credentials:
+	@rm -f /tmp/.kuma-dev/kind-config.json
+
 .PHONY: kind/start
-kind/start: ${KUBECONFIG_DIR}
+kind/start: ${KUBECONFIG_DIR} kind/setup-docker-credentials
 	$(KIND) get clusters | grep $(KIND_CLUSTER_NAME) >/dev/null 2>&1 && echo "Kind cluster already running." && exit 0 || \
 		($(KIND) create cluster \
 			--name "$(KIND_CLUSTER_NAME)" \
@@ -57,7 +70,7 @@ kind/wait:
     done
 
 .PHONY: kind/stop
-kind/stop:
+kind/stop: kind/cleanup-docker-credentials
 	@$(KIND) delete cluster --name $(KIND_CLUSTER_NAME)
 	@rm -f $(KUBECONFIG_DIR)/$(KIND_KUBECONFIG)
 

diff --git a/test/kind/cluster-ipv6-kuma-1.yaml b/test/kind/cluster-ipv6-kuma-1.yaml
@@ -5,17 +5,20 @@ networking:
   ipFamily: ipv6
   apiServerAddress: 127.0.0.1
 nodes:
-- role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: InitConfiguration
-    nodeRegistration:
-      kubeletExtraArgs:
-        node-labels: "ingress-ready=true"
-  extraPortMappings:
-  - containerPort: 30080
-    hostPort: 30080
-    protocol: TCP
-  - containerPort: 30443
-    hostPort: 30443
-    protocol: TCP
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 30080
+        hostPort: 30080
+        protocol: TCP
+      - containerPort: 30443
+        hostPort: 30443
+        protocol: TCP
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-ipv6-kuma-2.yaml b/test/kind/cluster-ipv6-kuma-2.yaml
@@ -4,3 +4,8 @@ apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   ipFamily: ipv6
   apiServerAddress: 127.0.0.1
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-ipv6.yaml b/test/kind/cluster-ipv6.yaml
@@ -4,3 +4,8 @@ apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   ipFamily: ipv6
   apiServerAddress: 127.0.0.1
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-kuma-1.yaml b/test/kind/cluster-kuma-1.yaml
@@ -1,18 +1,20 @@
-# an ipv6 cluster
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
-- role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: InitConfiguration
-    nodeRegistration:
-      kubeletExtraArgs:
-        node-labels: "ingress-ready=true"
-  extraPortMappings:
-  - containerPort: 30080
-    hostPort: 30080
-    protocol: TCP
-  - containerPort: 30443
-    hostPort: 30443
-    protocol: TCP
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 30080
+        hostPort: 30080
+        protocol: TCP
+      - containerPort: 30443
+        hostPort: 30443
+        protocol: TCP
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-kuma-2.yaml b/test/kind/cluster-kuma-2.yaml
@@ -1,3 +1,8 @@
 # an ipv6 cluster
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster.yaml b/test/kind/cluster.yaml
@@ -1,3 +1,8 @@
 # an ipv6 cluster
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json