Track benchmarks with Bencher

.github/workflows/benchmark.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  macro_benchmark:
+  macro_benchmarks:
     name: Macro Benchmarks
     if: "contains(github.event.pull_request.labels.*.name, 'ci: benchmark') || github.event_name == 'push'"
     runs-on: benchmarking-runner
@@ -68,68 +68,3 @@ jobs:
         with:
           name: body
           path: ci-benchmark/body.md
-
-  Cache_Benchmarks:
-    name: Cache Micro Benchmarks result
-    if: (github.event_name == 'push' && github.ref == 'refs/heads/main')
-    permissions:
-      pull-requests: write
-      contents: write
-    runs-on: benchmarking-runner
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-
-      - name: Install Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Run Benchmarks
-        run: |
-          cargo install cargo-criterion rust-script
-          cargo criterion --message-format=json > benches/main_benchmarks.json
-          ./scripts/json_to_md.rs benches/main_benchmarks.json > benches/main_benchmarks.md
-          cat benches/main_benchmarks.md
-
-      - name: Cache Criterion Benchmarks Json
-        uses: actions/cache@v4
-        with:
-          path: benches/main_benchmarks.json
-          key: criterion_benchmarks_${{ github.sha }}
-
-  Criterion_Compare:
-    name: Comparing Micro Benchmarks
-    if: "contains(github.event.pull_request.labels.*.name, 'ci: benchmark')"
-    runs-on: benchmarking-runner
-    permissions:
-      pull-requests: write
-      contents: write
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Install Rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Run Criterion Benchmarks
-        run: |
-          cargo install cargo-criterion rust-script
-          cargo criterion --message-format=json > benches/benchmarks.json
-          ./scripts/json_to_md.rs benches/benchmarks.json > benches/change_benchmarks.md
-
-      - name: Print Criterion Benchmarks
-        run: cat benches/change_benchmarks.md
-
-      - name: Restore file
-        uses: actions/cache@v4
-        with:
-          path: benches/main_benchmarks.json
-          key: criterion_benchmarks_${{ github.event.pull_request.base.sha }}
-          fail-on-cache-miss: true
-
-      - name: Print Benchmark Comparision
-        run: ./scripts/criterion_compare.rs benches/main_benchmarks.json benches/benchmarks.json table
-
-      - name: Check Degradation
-        run: ./scripts/criterion_compare.rs  benches/main_benchmarks.json benches/benchmarks.json check

.github/workflows/comment-commit.yml → .github/workflows/benchmark_comment.yml

@@ -1,4 +1,4 @@
-name: Comment on commit
+name: Benchmark comment on commit
 
 on:
   workflow_run:
@@ -7,8 +7,8 @@ on:
       - completed
 
 jobs:
-  comment:
-    name: Comment on commit
+  macro_benchmarks_comment:
+    name: Benchmark comment on commit
     runs-on: ubuntu-latest
     if: >
       github.event.workflow_run.conclusion == 'success'

.github/workflows/benchmark_main.yml

@@ -0,0 +1,35 @@
+name: Benchmark main
+
+on:
+  push:
+    paths-ignore: ["docs/**", "**.md"]
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  micro_benchmarks:
+    name: Micro Benchmarks
+    runs-on: benchmarking-runner
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Install Bencher CLI
+        uses: bencherdev/bencher@main
+
+      - name: Run Benchmarks
+        run: |
+          bencher run \
+          --project tailcall \
+          --branch main \
+          --testbed benchmarking-runner \
+          --token "${{ secrets.BENCHER_API_TOKEN }}" \
+          --adapter rust_criterion \
+          cargo bench

.github/workflows/benchmark_pr_run.yml

@@ -0,0 +1,30 @@
+name: Benchmark PR Run
+
+on:
+  pull_request:
+    paths-ignore: ["docs/**", "**.md"]
+    types: [opened, reopened, edited, synchronize, labeled]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  micro_benchmarks_pr_run:
+    name: Micro Benchmarks for PR
+    if: "contains(github.event.pull_request.labels.*.name, 'ci: benchmark')"
+    runs-on: benchmarking-runner
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Benchmarks
+        run: cargo bench > benchmark_results.txt
+      - name: Upload Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_results.txt
+          path: ./benchmark_results.txt
+      - name: Upload GitHub Pull Request Event
+        uses: actions/upload-artifact@v4
+        with:
+          name: event.json
+          path: ${{ github.event_path }}

.github/workflows/benchmark_pr_track.yml

@@ -0,0 +1,75 @@
+name: Benchmark PR Track
+
+on:
+  workflow_run:
+    workflows: [Benchmark PR Run]
+    types: [completed]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  micro_benchmarks_pr_track:
+    if: github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    env:
+      BENCHMARK_RESULTS: benchmark_results.txt
+      PR_EVENT: event.json
+    steps:
+      - name: Download Benchmark Results
+        uses: actions/github-script@v6
+        with:
+          script: |
+            async function downloadArtifact(artifactName) {
+              let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.payload.workflow_run.id,
+              });
+              let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                return artifact.name == artifactName
+              })[0];
+              if (!matchArtifact) {
+                core.setFailed(`Failed to find artifact: ${artifactName}`);
+              }
+              let download = await github.rest.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+              });
+              let fs = require('fs');
+              fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data));
+            }
+            await downloadArtifact(process.env.BENCHMARK_RESULTS);
+            await downloadArtifact(process.env.PR_EVENT);
+      - name: Unzip Benchmark Results
+        run: |
+          unzip $BENCHMARK_RESULTS.zip
+          unzip $PR_EVENT.zip
+      - name: Export PR Event Data
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let fs = require('fs');
+            let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'}));
+            core.exportVariable("PR_HEAD", `${prEvent.number}/merge`);
+            core.exportVariable("PR_BASE", prEvent.pull_request.base.ref);
+            core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha);
+            core.exportVariable("PR_NUMBER", prEvent.number);
+      - uses: bencherdev/bencher@main
+      - name: Track Benchmarks with Bencher
+        run: |
+          bencher run \
+          --project tailcall \
+          --token '${{ secrets.BENCHER_API_TOKEN }}' \
+          --branch '${{ env.PR_HEAD }}' \
+          --branch-start-point '${{ env.PR_BASE }}' \
+          --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \
+          --testbed benchmarking-runner \
+          --adapter rust_criterion \
+          --err \
+          --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+          --ci-number '${{ env.PR_NUMBER }}' \
+          --file "$BENCHMARK_RESULTS"