edr-benchmark.yml

name: EDR Benchmarks

on:
  push:
    branches:
      # TODO: change to `main` when we merging to main
      # https://github.com/NomicFoundation/edr/issues/685
      - "feat/solidity-tests"
  pull_request:
    branches:
      - "**"
  workflow_dispatch:

defaults:
  run:
    working-directory: crates/tools/js/benchmark

concurrency:
  group: ${{github.workflow}}-${{github.ref}}
  # Don't cancel in progress jobs in main
  # TODO: change to `main` when we merging to main
  # https://github.com/NomicFoundation/edr/issues/685
  cancel-in-progress: ${{ github.ref != 'refs/heads/feat/solidity-tests' }}

jobs:
  benchmarks-test:
    name: Benchmarks test
    environment: github-action-benchmark
    runs-on: self-hosted
    # Only run for trusted collaborators since third-parties could run malicious code on the self-hosted benchmark runner.
    # TODO: change ref comparison to `main` when we merging to main
    # https://github.com/NomicFoundation/edr/issues/685
    if: github.ref == 'refs/heads/feat/solidity-tests' || github.repository == github.event.pull_request.head.repo.full_name
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup-node
      - uses: ./.github/actions/setup-rust

      - name: Install packages
        run: pnpm install --frozen-lockfile --prefer-offline

      - name: Run benchmark tests
        run: pnpm test

  js-scenario-benchmark:
    name: Run JS scenario runner benchmark for Hardhat Node style workload
    environment: github-action-benchmark
    runs-on: self-hosted
    needs: benchmarks-test
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-node
      - uses: ./.github/actions/setup-rust

      - name: Rust cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            .cargo-cache
            target/
          save-always: true
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: ${{ runner.os }}-cargo-

      - name: Install packages
        run: pnpm install --frozen-lockfile --prefer-offline

      - name: Run benchmark
        run: pnpm run benchmark

      - name: Validate regressions
        run: pnpm run verify

      - name: Generate report for github-action-benchmark
        run: pnpm run --silent report | tee scenario-report.json

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: customSmallerIsBetter
          output-file-path: crates/tools/js/benchmark/scenario-report.json
          gh-repository: github.com/nomic-foundation-automation/edr-benchmark-results
          gh-pages-branch: main
          benchmark-data-dir-path: bench
          github-token: ${{ secrets.BENCHMARK_GITHUB_TOKEN }}
          # Only save the data for main branch pushes. For PRs we only compare
          auto-push: ${{ github.ref == 'refs/heads/main' && github.event_name != 'pull_request' }}
          alert-threshold: "110%"
          # Only fail on pull requests, don't break CI in main
          fail-on-alert: ${{ github.event_name == 'pull_request' }}
          # Enable Job Summary for PRs
          summary-always: true
          max-items-in-chart: 50

  js-soltests-benchmark:
    name: Run JS Solidity test runner benchmark
    environment: github-action-benchmark
    runs-on: self-hosted
    needs: benchmarks-test
    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-node
      - uses: ./.github/actions/setup-rust

      - name: Cache EDR RPC cache
        uses: actions/cache@v4
        with:
          path: |
            **/edr-cache
          key: edr-rs-rpc-cache-v1

      - name: Rust cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            .cargo-cache
            target/
          save-always: true
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: ${{ runner.os }}-cargo-

      - name: Install packages
        run: pnpm install --frozen-lockfile --prefer-offline

      - name: Run benchmark and generate report for github-action-benchmark
        run: pnpm run --silent soltests | tee soltest-report.json

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: customSmallerIsBetter
          output-file-path: crates/tools/js/benchmark/soltest-report.json
          gh-repository: github.com/nomic-foundation-automation/edr-benchmark-results
          gh-pages-branch: main
          benchmark-data-dir-path: soltests
          github-token: ${{ secrets.BENCHMARK_GITHUB_TOKEN }}
          # Only save the data for main branch pushes. For PRs we only compare
          # TODO: change to `main` when we merging to main
          # https://github.com/NomicFoundation/edr/issues/685
          auto-push: ${{ github.ref == 'refs/heads/feat/solidity-tests' && github.event_name != 'pull_request' }}
          alert-threshold: "110%"
          # Only fail on pull requests, don't break CI in main
          fail-on-alert: ${{ github.event_name == 'pull_request' }}
          # Enable Job Summary for PRs
          summary-always: true
          max-items-in-chart: 1000