ci: refactor tests and benchmarks workflows (#548)

matter-labs · Mar 31, 2024 · 1351ac9 · 1351ac9
1 parent 8d81747
commit 1351ac9
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 153 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -2,45 +2,67 @@ name: Benchmarking
 
 on:
   workflow_dispatch:
+    inputs:
+      compiler_llvm_benchmark_mode:
+        description: "Mode filter for compiler-llvm benchmarks"
+        required: false
+        default: "+M^B3"
+      compiler_llvm_benchmark_path:
+        description: "Path filter for compiler-llvm benchmarks"
+        required: false
+        default: "tests/solidity/"
+      compiler_tester_reference_branch:
+        description: "compiler-tester branch to use as a benchmark reference"
+        required: false
+        default: "main"
+      compiler_tester_candidate_branch:
+        description: "compiler-tester branch to use as a benchmark candidate"
+        required: false
+        default: "main"
   pull_request:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  benchmarks_candidate:
+  benchmarks:
+    strategy:
+      matrix:
+        type: [ "reference", "candidate" ]
+    name: ${{ matrix.type }}
     runs-on: [self-hosted, ci-runner-compiler]
     container:
-      image: matterlabs/llvm_runner:latest
-      credentials:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+      image: matterlabs/llvm_runner:ubuntu22-llvm15-latest
+      options: -m 110g
+    env:
+      LLVM_BENCHMARK_MODE: ${{ github.event.inputs.compiler_llvm_benchmark_mode || '+M^B3' }}
+      LLVM_BENCHMARK_PATH: ${{ github.event.inputs.compiler_llvm_benchmark_path || 'tests/solidity/' }}
+      RUST_BACKTRACE: full
     steps:
-      - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1
-      - name: Preparing workspace. Setting environment.
-        run: |
-          echo "BENCHMARK_MODE=${{ github.event.inputs.compiler_llvm_benchmark_mode || '+M^B3' }}" >> $GITHUB_ENV
-          echo "BENCHMARK_PATH=${{ github.event.inputs.compiler_llvm_benchmark_path || 'tests/solidity/' }}" >> $GITHUB_ENV
-          echo "CANDIDATE_BRANCH_NAME=${{ github.event.inputs.compiler_tester_candidate_branch }}" >> $GITHUB_ENV
-
-      - name: Getting the branch name (pull request)
-        if: github.event_name == 'pull_request'
+      - name: Define branches
         shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV
-
-      - name: Checking out the Solidity repository
+        run: |
+          echo "TESTING_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_ENV
+          if [ "${{ matrix.type }}" = "candidate" ]; then
+            echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_candidate_branch || 'main' }}" >> $GITHUB_ENV
+          else
+            echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_reference_branch || 'main' }}" >> $GITHUB_ENV
+          fi
+
+      - name: Checking out the compiler-tester
         uses: actions/checkout@v4
         with:
+          repository: matter-labs/era-compiler-tester
           submodules: recursive
-          path: solidity
+          ref: ${{ env.COMPILER_TESTER_BRANCH_NAME }}
+          persist-credentials: false
 
-      - name: Checking out the compiler-tester candidate
+      - name: Checking out the Solidity repository
         uses: actions/checkout@v4
         with:
-          repository: matter-labs/era-compiler-tester
           submodules: recursive
-          path: compiler-tester
+          path: solidity
 
       - name: Building the Solidity compiler
         working-directory: solidity
@@ -50,133 +72,53 @@ jobs:
           cmake .. -DCMAKE_BUILD_TYPE='Release' -DSOLC_VERSION_ZKEVM='1.0.0-lts' -DPEDANTIC=OFF -DCMAKE_CXX_FLAGS='-w' -DUSE_Z3=OFF -DUSE_CVC4=OFF
           make -j$(nproc)
 
-      - name: Benchmarking the Solidity compiler candidate
+      - name: Build LLVM
+        uses: ./.github/actions/build-llvm
+
+      - name: Benchmarking ${{ matrix.type }}
         id: compiler_tester_run
-        working-directory: compiler-tester
         run: |
           mkdir -p './solc-bin/'
-          cp '../solidity/build/solc/solc' "./solc-bin/solc-${BRANCH_NAME}"
-          chmod +x "./solc-bin/solc-${BRANCH_NAME}"
-
-          cargo install compiler-llvm-builder
-          /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build
+          cp './solidity/build/solc/solc' "./solc-bin/solc-${{ env.TESTING_BRANCH }}"
+          chmod +x "./solc-bin/solc-${{ env.TESTING_BRANCH }}"
 
-          export RUST_BACKTRACE='full'
           export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/"
           cargo build --verbose --release --bin 'compiler-tester'
           cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/'
 
           ./target/release/compiler-tester \
             --zksolc './target-zksolc/release/zksolc' \
-            --zkvyper './target-zkvyper/release/zkvyper' \
             --path="${{ env.BENCHMARK_PATH || '' }}" \
             --mode="${{ env.BENCHMARK_MODE || '' }}" \
-            --benchmark='candidate.json' \
-            --solc-bin-config-path="configs/solc-bin-zkevm-candidate-${BRANCH_NAME}.json"
+            --benchmark='${{ matrix.type }}.json' \
+            --solc-bin-config-path="configs/solc-bin-zkevm-${{ matrix.type }}-${{ env.TESTING_BRANCH }}.json"
 
       - uses: actions/upload-artifact@v4
         with:
-          name: candidate-benchmark
-          path: compiler-tester/candidate.json
-
-  benchmarks_reference:
-    runs-on: [self-hosted, ci-runner-compiler]
-    container:
-      image: matterlabs/llvm_runner:latest
-      credentials:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    steps:
-      - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1
-      - name: Preparing workspace. Setting environment.
-        run: |
-          echo "BENCHMARK_MODE=${{ github.event.inputs.compiler_llvm_benchmark_mode || '+M^B3' }}" >> $GITHUB_ENV
-          echo "BENCHMARK_PATH=${{ github.event.inputs.compiler_llvm_benchmark_path || 'tests/solidity/' }}" >> $GITHUB_ENV
-          echo "REFERENCE_BRANCH_NAME=${{ github.event.inputs.compiler_tester_reference_branch }}" >> $GITHUB_ENV
-
-      - name: Getting the branch name (pull request)
-        if: github.event_name == 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV
+          name: solc-${{ matrix.type }}-benchmark
+          path: ${{ matrix.type }}.json
 
-      - name: Checking out the Solidity repository
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-          path: solidity
-
-      - name: Checking out the compiler-tester reference
-        uses: actions/checkout@v4
-        with:
-          repository: matter-labs/era-compiler-tester
-          ref: main
-          submodules: recursive
-          path: compiler-tester
-
-      - name: Benchmarking the Solidity compiler reference
-        working-directory: compiler-tester
-        id: compiler_tester_run
-        run: |
-          cargo install compiler-llvm-builder
-          /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build
-
-          export RUST_BACKTRACE='full'
-          export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/"
-          cargo build --verbose --release --bin 'compiler-tester'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/'
-
-          ./target/release/compiler-tester \
-            --zksolc './target-zksolc/release/zksolc' \
-            --zkvyper './target-zkvyper/release/zkvyper' \
-            --path="${{ env.BENCHMARK_PATH || '' }}" \
-            --mode="${{ env.BENCHMARK_MODE || '' }}" \
-            --benchmark='reference.json' \
-            --solc-bin-config-path="configs/solc-bin-zkevm-reference-${BRANCH_NAME}.json"
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: reference-benchmark
-          path: compiler-tester/reference.json
 
   benchmarks_analysis:
     runs-on: [matterlabs-ci-runner]
-    needs: [benchmarks_candidate, benchmarks_reference]
+    needs: benchmarks
     container:
-      image: matterlabs/llvm_runner:latest
-      credentials:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+      image: matterlabs/llvm_runner:ubuntu22-llvm15-latest
     steps:
-      - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1
-      - name: Getting the branch name (pull request)
-        if: github.event_name == 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV
-
       - name: Checking out the compiler-tester repository
         uses: actions/checkout@v4
         with:
           repository: matter-labs/era-compiler-tester
           submodules: recursive
-          path: compiler-tester
 
       - uses: actions/download-artifact@v4
         with:
-          name: candidate-benchmark
-          path: compiler-tester
-
-      - uses: actions/download-artifact@v4
-        with:
-          name: reference-benchmark
-          path: compiler-tester
+          pattern: solc-*
+          merge-multiple: true
 
       - name: Comparing the benchmark results
         id: compiler_tester_run
         run: |
-          cd compiler-tester && \
-          export RUST_BACKTRACE=full && \
           cargo run --release --bin benchmark-analyzer -- \
             --reference 'reference.json' \
             --candidate 'candidate.json' \
@@ -189,20 +131,12 @@ jobs:
           echo '```' >> $GITHUB_STEP_SUMMARY
           cat ./compiler-tester/result.txt >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
-          cat $GITHUB_STEP_SUMMARY > ./compiler-tester/result.txt
+          cat $GITHUB_STEP_SUMMARY > result.txt
 
       - name: Posting the benchmark results to a PR comment
         if: github.event_name == 'pull_request'
         uses: mshick/add-pr-comment@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          message-path: ./compiler-tester/result.txt
-
-      - uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message)
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.COMPILER_CI_MATTERMOST_WEBHOOK }} # required
-        if: ${{ failure() || success() }} # Skip canceled jobs
+          message-path: result.txt
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -1,8 +1,8 @@
 name: Integration testing
 
 on:
-  workflow_dispatch:
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -12,64 +12,59 @@ jobs:
   testing:
     runs-on: [self-hosted, ci-runner-compiler]
     container:
-      image: matterlabs/llvm_runner:latest
-      credentials:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+      image: matterlabs/llvm_runner:ubuntu22-llvm15-latest
     steps:
-      - name: Getting the branch name (pull request)
-        if: github.event_name == 'pull_request'
+      - name: Extract branch name
         shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV
+        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+        id: extract_branch
 
-      - name: Checking out the Solidity repository
+      - name: Checking out the compiler-tester repository
         uses: actions/checkout@v4
         with:
+          repository: matter-labs/era-compiler-tester
           submodules: recursive
-          path: solidity
+          persist-credentials: false
 
-      - name: Checking out the compiler-tester repository
+      - name: Checking out the Solidity repository
         uses: actions/checkout@v4
         with:
-          repository: matter-labs/era-compiler-tester
-          path: compiler-tester
           submodules: recursive
+          path: solidity
 
       - name: Building the Solidity compiler
         working-directory: solidity
         run: |
           mkdir -p ./build
           cd ./build
           cmake .. -DCMAKE_BUILD_TYPE='Release' -DSOLC_VERSION_ZKEVM='1.0.0-lts' -DPEDANTIC=OFF -DCMAKE_CXX_FLAGS='-w' -DUSE_Z3=OFF -DUSE_CVC4=OFF
-          make -j$(nproc)
+          make -j
+
+      - name: Build LLVM
+        uses: ./.github/actions/build-llvm
 
       - name: Building and running the compiler tester
-        id: compiler_tester_run
-        working-directory: compiler-tester
         run: |
           mkdir -p './solc-bin/'
-          cp '../solidity/build/solc/solc' "./solc-bin/solc-${BRANCH_NAME}"
-          chmod +x "./solc-bin/solc-${BRANCH_NAME}"
-
-          cargo install compiler-llvm-builder
-          /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build
+          cp './solidity/build/solc/solc' "./solc-bin/solc-${{ steps.extract_branch.outputs.branch }}"
+          chmod +x "./solc-bin/solc-${{ steps.extract_branch.outputs.branch }}"
 
           export RUST_BACKTRACE='full'
           export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/"
           cargo build --verbose --release --bin 'compiler-tester'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/'
+          cargo build --verbose --release \
+            --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml \
+            --target-dir './target-zksolc/'
 
           ./target/release/compiler-tester \
             --zksolc './target-zksolc/release/zksolc' \
-            --zkvyper './target-zkvyper/release/zkvyper' \
             --path='tests/solidity/' \
-            --solc-bin-config-path="configs/solc-bin-zkevm-candidate-${BRANCH_NAME}.json"
+            --solc-bin-config-path="configs/solc-bin-zkevm-candidate-${{ steps.extract_branch.outputs.branch }}.json"
 
       - uses: 8398a7/action-slack@v3
+        if: ${{ failure() || success() }} # Skip canceled jobs
         with:
           status: ${{ job.status }}
-          fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message)
+          fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest
         env:
-          SLACK_WEBHOOK_URL: ${{ secrets.COMPILER_CI_MATTERMOST_WEBHOOK }} # required
-        if: ${{ failure() || success() }} # Skip canceled jobs
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}