From f45667d10257b9aad48cf8c7b8b5780f9ad5289f Mon Sep 17 00:00:00 2001
From: Jay Jijie Chen <1180092+jijiechen@users.noreply.github.com>
Date: Wed, 29 Nov 2023 16:49:34 +0800
Subject: [PATCH] ci(actions): run amd64 E2E on GitHub actions and arm64 on
 CircleCI (#8476)

Signed-off-by: Jay Chen <1180092+jijiechen@users.noreply.github.com>
---
 .github/workflows/build-test-distribute.yaml |  21 ++--
 .github/workflows/e2e.yaml                   | 105 +++++++++++++++++--
 2 files changed, 109 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/build-test-distribute.yaml b/.github/workflows/build-test-distribute.yaml
index 13fbe7d2bc14..8b3b6c22fdbd 100644
--- a/.github/workflows/build-test-distribute.yaml
+++ b/.github/workflows/build-test-distribute.yaml
@@ -34,19 +34,14 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      # GitHub actions does not share cache across multiple jobs,
+      # so we have to operate cache in each job and action file
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: "Check if should run on all arch/os combinations"
-        if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'ci/run-full-matrix')
-        run: |
-          echo 'ENABLED_GOARCHES=arm64 amd64' >> $GITHUB_ENV
-          echo 'ENABLED_GOOSES=linux darwin' >> $GITHUB_ENV
       - uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
-      # GitHub actions does not share cache across multiple jobs,
-      # so we have to operate cache in each job and action file
       - uses: actions/cache@v3
         with:
           path: |
@@ -56,6 +51,11 @@ jobs:
             ${{ runner.os }}-devtools
       - run: |
           make dev/tools
+      - name: "Check if should run on all arch/os combinations"
+        if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'ci/run-full-matrix')
+        run: |
+          echo 'ENABLED_GOARCHES=arm64 amd64' >> $GITHUB_ENV
+          echo 'ENABLED_GOOSES=linux darwin' >> $GITHUB_ENV
       - run: |
           make build
       - run: |
@@ -120,9 +120,14 @@ jobs:
           retention-days: 7
   distributions:
     needs: ["check", "test", "test_e2e", "test_e2e_env"]
-    if: ${{ always() && !failure() && !cancelled() }}
+    if: ${{ always() }}
     runs-on: ubuntu-latest
     steps:
+      # see https://stackoverflow.com/a/67532120/4907315
+      - name: "Halt due to previous failures"
+        run: |
+          exit 1
+        if: ${{ contains(needs.*.result, 'failure')|| contains(needs.*.result, 'cancelled') }}
       - name: "Check if should run on all arch/os combinations"
         if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'ci/run-full-matrix')
         run: |
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 3c0c10b08b49..434973f3a9c4 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -42,16 +42,103 @@ jobs:
               arch: ${{ inputs.arch }} \
               cniNetworkPlugin: ${{ inputs.cniNetworkPlugin }} \
               "
-      - name: Expose github action artifact variables
+      - name: "GitHub Actions: check out code"
+        if: ${{ inputs.arch == 'amd64' }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: "GitHub Actions: setup go"
+        if: ${{ inputs.arch == 'amd64' }}
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+      - name: "GitHub Actions: set up cache"
+        if: ${{ inputs.arch == 'amd64' }}
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.kuma-dev
+          key: ${{ runner.os }}-devtools-${{ hashFiles('mk/dependencies/deps.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-devtools
+      - name: "GitHub Actions: download build artifacts"
+        if: ${{ inputs.arch == 'amd64' }}
+        uses: actions/download-artifact@v3
+        with:
+          name: build-output
+          path: build
+      - name: "GitHub Actions: extract artifacts"
+        if: ${{ inputs.arch == 'amd64' }}
+        run: |
+          echo "Extracting artifacts..."
+          ARCH='${{ inputs.arch }}'
+          ARTIFACT=$(find build/distributions/out/kuma-*-linux-$ARCH.tar.gz)
+          if [[ "$ARTIFACT" == "" ]]; then
+            echo "Could not find built artifact for linux($ARCH)."
+            exit 1
+          fi
+          cat $ARTIFACT.sha256 | sha256sum --check
+
+          mkdir build/distributions/artifacts
+          tar -xzf $ARTIFACT -C build/distributions/artifacts
+
+          SRC_DIR=$(find build/distributions/artifacts/*/bin -type d)
+          DEST_DIR=build/artifacts-linux-$ARCH
+          for BIN in $(ls $SRC_DIR); do
+            mkdir -p $DEST_DIR/$BIN
+            cp $SRC_DIR/$BIN $DEST_DIR/$BIN/
+          done
+      - name: "GitHub Actions: setup helm"
+        if: ${{ inputs.arch == 'amd64' }}
+        run: |
+          make dev/set-kuma-helm-repo
+      - name: "GitHub Actions: run E2E tests"
+        if: ${{ inputs.arch == 'amd64' }}
+        run: |
+          if [[ "${{ inputs.k8sVersion }}" == "kindIpv6" ]]; then
+            export IPV6=true
+            export K8S_CLUSTER_TOOL=kind
+            export KUMA_DEFAULT_RETRIES=60
+            export KUMA_DEFAULT_TIMEOUT="6s"
+          fi
+          if [[ "${{ inputs.k8sVersion }}" != "kind"* ]]; then
+            export CI_K3S_VERSION=${{ inputs.k8sVersion }}
+            export K3D_NETWORK_CNI=${{ inputs.cniNetworkPlugin }}
+          fi
+          if [[ "${{ inputs.arch }}" == "arm64" ]]; then
+            export MAKE_PARAMETERS="-j1"
+          else
+            export MAKE_PARAMETERS="-j2"
+          fi
+
+          if [[ "${{ inputs.legacyKDS }}" == true ]]; then
+            export KUMA_LEGACY_KDS=true
+          fi
+
+          if [[ "${{ inputs.target }}" == "" ]]; then
+            export GINKGO_E2E_LABEL_FILTERS="job-$CIRCLE_NODE_INDEX"
+          fi
+          env
+          if [[ "${{ inputs.target }}" != "" ]]; then
+            target="test/e2e-${{ inputs.target }}"
+          else
+            target="test/e2e"
+          fi
+          make ${MAKE_PARAMETERS} CI=true "${target}"
+
+      - name: "CircleCI: expose github action artifact variables"
+        if: ${{ inputs.arch != 'amd64' }}
         uses: actions/github-script@v6
         with:
           script: |
             core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
             core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN'])
-      - name: Install jq
+      - name: "CircleCI: install jq"
+        if: ${{ inputs.arch != 'amd64' }}
         run: |
           sudo apt-get install -y jq
-      - name: "Trigger a new pipeline workflow on CircleCI"
+      - name: "CircleCI: trigger a new pipeline workflow on CircleCI"
+        if: ${{ inputs.arch != 'amd64' }}
         id: circle-ci-trigger
         run: |
           # Trigger CircleCI manually, reference: https://github.com/CircleCI-Public/trigger-circleci-pipeline-action/blob/main/src/lib/CircleCIPipelineTrigger.ts#L82
@@ -129,8 +216,8 @@ jobs:
           echo ''
           echo "CircleCI pipeline triggered successfully, pipeline id: $PIPELINE_ID"
           echo "Check CircleCI workflow details at: https://app.circleci.com/pipelines/gh/${{ github.repository }}/$PIPELINE_NUMBER/workflows/$WORKFLOW_ID"
-      - name: "Check run status of pipeline workflow on CircleCI"
-        if: steps.circle-ci-trigger.outputs.workflow_id != ''
+      - name: "CircleCI: check run status of pipeline workflow on CircleCI"
+        if: ${{ inputs.arch != 'amd64' && steps.circle-ci-trigger.outputs.workflow_id != '' }}
         run: |
           set -e
           if [ "${{ runner.debug }}" == "1" ]; then
@@ -155,8 +242,8 @@ jobs:
             cat $OUTPUT_FILE
             rm $OUTPUT_FILE
             if [ "$STATUS_CODE" == "429" ]; then
-              # we are exceeding rate limit
-              echo "{}"
+              # we are exceeding rate limit, try again later
+              echo '{"status": ""}'
               return
             fi
             if [ $STATUS_CODE -lt 200 ] || [ $STATUS_CODE -gt 399 ] ; then
@@ -192,8 +279,8 @@ jobs:
           echo "Check CircleCI workflow details at: https://app.circleci.com/pipelines/gh/${{ github.repository }}/$PIPELINE_NUMBER/workflows/$WORKFLOW_ID"
           echo "Tracking workflow status:"
           check_workflow '${{ steps.circle-ci-trigger.outputs.workflow_id }}'
-      - name: Cancel CircleCI running if requested
-        if: cancelled() && steps.circle-ci-trigger.outputs.workflow_id != ''
+      - name: "CircleCI: cancel CircleCI running if requested"
+        if: ${{ inputs.arch != 'amd64' && cancelled() && steps.circle-ci-trigger.outputs.workflow_id != '' }}
         run: |
           set -e
           if [ "${{ runner.debug }}" == "1" ]; then