diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 49b2d2e81264..741cc1539004 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -20,6 +20,8 @@ env: E2E_PARAM_PARALLELISM: ${{ fromJSON(inputs.matrix).parallelism }} jobs: e2e: + timeout-minutes: 60 + # can't use env vars here runs-on: ${{ fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch] }} if: ${{ fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch] != '' }} strategy: @@ -77,6 +79,8 @@ jobs: EOF sudo service docker restart - name: "Run E2E tests" + env: + DOCKERHUB_PULL_CREDENTIAL: ${{ secrets.DOCKERHUB_PULL_CREDENTIAL }} run: | if [[ "${{ env.E2E_PARAM_K8S_VERSION }}" == "kindIpv6" ]]; then export IPV6=true @@ -102,6 +106,24 @@ jobs: export GINKGO_E2E_LABEL_FILTERS="job-${{ matrix.parallelRunnerId }}" fi env + if [[ "${{ env.E2E_PARAM_TARGET }}" == "multizone" ]]; then + export KUMA_DEFAULT_RETRIES=60 + fi + + function on_exit() + { + docker logout docker.io + } + + # we pull a few images during the E2E run, sometimes we get rate-limited by docker hub + # to prevent this, we support specifying a pull credential here + if [[ "$DOCKERHUB_PULL_CREDENTIAL" != "" ]]; then + DOCKER_USER=$(echo "$DOCKERHUB_PULL_CREDENTIAL" | cut -d ':' -f 1) + DOCKER_PWD=$(echo "$DOCKERHUB_PULL_CREDENTIAL" | cut -d ':' -f 2) + echo -n "$DOCKER_PWD" | docker login -u "$DOCKER_USER" --password-stdin + trap "on_exit" EXIT + fi + if [[ "${{ env.E2E_PARAM_TARGET }}" != "" ]]; then target="test/e2e-${{ env.E2E_PARAM_TARGET }}" else diff --git a/mk/k3d.mk b/mk/k3d.mk index 88ab94cc8b8d..d5ce9b0ebbe6 100644 --- a/mk/k3d.mk +++ b/mk/k3d.mk @@ -41,6 +41,7 @@ KUMA_NAMESPACE ?= kuma-system PORT_PREFIX := $$(($(patsubst 300-%,300+%-1,$(KIND_CLUSTER_NAME:kuma%=300%)))) K3D_NETWORK_CNI ?= flannel +K3D_REGISTRY_FILE ?= K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \ --k3s-arg '--disable=traefik@server:0' \ --k3s-arg '--disable=metrics-server@server:0' \ @@ -49,6 +50,7 @@ K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \ --volume '$(subst @,\@,$(TOP)/$(KUMA_DIR))/test/framework/deployments:/tmp/deployments@server:0' \ --network kind \ --port "$(PORT_PREFIX)80-$(PORT_PREFIX)99:30080-30099@server:0" \ + --registry-config "/tmp/.kuma-dev/k3d-registry.yaml" \ --timeout 120s ifeq ($(K3D_NETWORK_CNI),calico) @@ -88,8 +90,23 @@ $(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml: -o $(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml \ https://k3d.io/v$(K3D_VERSION)/usage/advanced/calico.yaml +DOCKERHUB_PULL_CREDENTIAL ?= +.PHONY: k3d/setup-docker-credentials +k3d/setup-docker-credentials: + @mkdir -p /tmp/.kuma-dev ; \ + echo '{"configs": {}}' > /tmp/.kuma-dev/k3d-registry.yaml ; \ + if [[ "$(DOCKERHUB_PULL_CREDENTIAL)" != "" ]]; then \ + DOCKER_USER=$$(echo "$(DOCKERHUB_PULL_CREDENTIAL)" | cut -d ':' -f 1); \ + DOCKER_PWD=$$(echo "$(DOCKERHUB_PULL_CREDENTIAL)" | cut -d ':' -f 2); \ + echo "{\"configs\": {\"registry-1.docker.io\": {\"auth\": {\"username\": \"$${DOCKER_USER}\",\"password\":\"$${DOCKER_PWD}\"}}}}" > /tmp/.kuma-dev/k3d-registry.yaml ; \ + fi + +.PHONY: k3d/cleanup-docker-credentials +k3d/cleanup-docker-credentials: + @rm -f /tmp/.kuma-dev/k3d-registry.yaml + .PHONY: k3d/start -k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create \ +k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create k3d/setup-docker-credentials $(if $(findstring calico,$(K3D_NETWORK_CNI)),$(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml) @echo "PORT_PREFIX=$(PORT_PREFIX)" @KUBECONFIG=$(KIND_KUBECONFIG) \ @@ -136,7 +153,7 @@ k3d/wait: done .PHONY: k3d/stop -k3d/stop: +k3d/stop: k3d/cleanup-docker-credentials @KUBECONFIG=$(KIND_KUBECONFIG) $(K3D_BIN) cluster delete "$(KIND_CLUSTER_NAME)" .PHONY: k3d/stop/all diff --git a/mk/kind.mk b/mk/kind.mk index d8d1fd200121..d97cad7912e8 100644 --- a/mk/kind.mk +++ b/mk/kind.mk @@ -27,8 +27,21 @@ CI_KUBERNETES_VERSION ?= v1.23.17@sha256:59c989ff8a517a93127d4a536e7014d28e235fb KUMA_MODE ?= zone KUMA_NAMESPACE ?= kuma-system +DOCKERHUB_PULL_CREDENTIAL ?= +.PHONY: kind/setup-docker-credentials +kind/setup-docker-credentials: + @mkdir -p /tmp/.kuma-dev ; \ + echo '{"auths":{}}' > /tmp/.kuma-dev/kind-config.json ; \ + if [[ "$(DOCKERHUB_PULL_CREDENTIAL)" != "" ]]; then \ + echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$$(echo -n "$(DOCKERHUB_PULL_CREDENTIAL)" | base64)\"}}}" > /tmp/.kuma-dev/kind-config.json ; \ + fi + +.PHONY: kind/cleanup-docker-credentials +kind/cleanup-docker-credentials: + @rm -f /tmp/.kuma-dev/kind-config.json + .PHONY: kind/start -kind/start: ${KUBECONFIG_DIR} +kind/start: ${KUBECONFIG_DIR} kind/setup-docker-credentials $(KIND) get clusters | grep $(KIND_CLUSTER_NAME) >/dev/null 2>&1 && echo "Kind cluster already running." && exit 0 || \ ($(KIND) create cluster \ --name "$(KIND_CLUSTER_NAME)" \ @@ -57,7 +70,7 @@ kind/wait: done .PHONY: kind/stop -kind/stop: +kind/stop: kind/cleanup-docker-credentials @$(KIND) delete cluster --name $(KIND_CLUSTER_NAME) @rm -f $(KUBECONFIG_DIR)/$(KIND_KUBECONFIG) diff --git a/test/kind/cluster-ipv6-kuma-1.yaml b/test/kind/cluster-ipv6-kuma-1.yaml index fc471008139c..0cf149fde375 100644 --- a/test/kind/cluster-ipv6-kuma-1.yaml +++ b/test/kind/cluster-ipv6-kuma-1.yaml @@ -5,17 +5,20 @@ networking: ipFamily: ipv6 apiServerAddress: 127.0.0.1 nodes: -- role: control-plane - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" - extraPortMappings: - - containerPort: 30080 - hostPort: 30080 - protocol: TCP - - containerPort: 30443 - hostPort: 30443 - protocol: TCP + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 30080 + hostPort: 30080 + protocol: TCP + - containerPort: 30443 + hostPort: 30443 + protocol: TCP + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: /tmp/.kuma-dev/kind-config.json diff --git a/test/kind/cluster-ipv6-kuma-2.yaml b/test/kind/cluster-ipv6-kuma-2.yaml index 9586193f03a4..7f3a74ceaf84 100644 --- a/test/kind/cluster-ipv6-kuma-2.yaml +++ b/test/kind/cluster-ipv6-kuma-2.yaml @@ -4,3 +4,8 @@ apiVersion: kind.x-k8s.io/v1alpha4 networking: ipFamily: ipv6 apiServerAddress: 127.0.0.1 +nodes: + - role: control-plane + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: /tmp/.kuma-dev/kind-config.json diff --git a/test/kind/cluster-ipv6.yaml b/test/kind/cluster-ipv6.yaml index 9586193f03a4..7f3a74ceaf84 100644 --- a/test/kind/cluster-ipv6.yaml +++ b/test/kind/cluster-ipv6.yaml @@ -4,3 +4,8 @@ apiVersion: kind.x-k8s.io/v1alpha4 networking: ipFamily: ipv6 apiServerAddress: 127.0.0.1 +nodes: + - role: control-plane + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: /tmp/.kuma-dev/kind-config.json diff --git a/test/kind/cluster-kuma-1.yaml b/test/kind/cluster-kuma-1.yaml index 5d9d9bd481d6..edfb3f6d2f03 100644 --- a/test/kind/cluster-kuma-1.yaml +++ b/test/kind/cluster-kuma-1.yaml @@ -1,18 +1,20 @@ -# an ipv6 cluster kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: -- role: control-plane - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" - extraPortMappings: - - containerPort: 30080 - hostPort: 30080 - protocol: TCP - - containerPort: 30443 - hostPort: 30443 - protocol: TCP + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 30080 + hostPort: 30080 + protocol: TCP + - containerPort: 30443 + hostPort: 30443 + protocol: TCP + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: /tmp/.kuma-dev/kind-config.json diff --git a/test/kind/cluster-kuma-2.yaml b/test/kind/cluster-kuma-2.yaml index ed2a445a729d..a18ee416406e 100644 --- a/test/kind/cluster-kuma-2.yaml +++ b/test/kind/cluster-kuma-2.yaml @@ -1,3 +1,8 @@ # an ipv6 cluster kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: /tmp/.kuma-dev/kind-config.json diff --git a/test/kind/cluster.yaml b/test/kind/cluster.yaml index ed2a445a729d..a18ee416406e 100644 --- a/test/kind/cluster.yaml +++ b/test/kind/cluster.yaml @@ -1,3 +1,8 @@ # an ipv6 cluster kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: /tmp/.kuma-dev/kind-config.json