Relative Benchmarking

diesel-rs · Dec 17, 2023 · a33fc07 · a33fc07
1 parent 7989b07
commit a33fc07
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 32 deletions.
diff --git a/.github/workflows/run_benches.yml b/.github/workflows/run_benches.yml
@@ -1,14 +1,12 @@
 on:
-  push:
-    branches: master
   pull_request:
     types: [labeled, opened, reopened, synchronize]
 
 name: Run and Cache Benchmarks
 
 jobs:
   run_benchmarks:
-    if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
+    if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -48,14 +46,29 @@ jobs:
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@stable
 
-      - name: Benchmark ${{ matrix.backend }}
-        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > ${{ matrix.backend }}.txt
+      - name: Benchmark PR ${{ matrix.backend }}
+        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > pr_${{ matrix.backend }}.txt
 
-      - name: Upload ${{ matrix.backend }} Benchmark Results
+      - name: Upload PR ${{ matrix.backend }} Benchmark Results
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.backend }}.txt
-          path: ./${{ matrix.backend }}.txt
+          name: pr_${{ matrix.backend }}.txt
+          path: ./pr_${{ matrix.backend }}.txt
+
+      - name: Checkout base branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          repository: ${{ github.event.pull_request.base.repo.full_name }}
+
+      - name: Benchmark base ${{ matrix.backend }}
+        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > base_${{ matrix.backend }}.txt
+
+      - name: Upload base ${{ matrix.backend }} Benchmark Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: base_${{ matrix.backend }}.txt
+          path: ./base_${{ matrix.backend }}.txt
 
       - name: Upload GitHub Event
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/track_benches.yml b/.github/workflows/track_benches.yml
@@ -18,7 +18,8 @@ jobs:
       BENCHER_PROJECT: diesel
       BENCHER_ADAPTER: rust_criterion
       BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
-      BENCHMARK_RESULTS: ${{ matrix.backend }}.txt
+      PR_BENCHMARK_RESULTS: pr_${{ matrix.backend }}.txt
+      BASE_BENCHMARK_RESULTS: base_${{ matrix.backend }}.txt
       GITHUB_EVENT: event.json
     steps:
       - name: Download Benchmark Results
@@ -46,11 +47,13 @@ jobs:
               let fs = require('fs');
               fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data));
             }
-            await downloadArtifact(process.env.BENCHMARK_RESULTS);
+            await downloadArtifact(process.env.PR_BENCHMARK_RESULTS);
+            await downloadArtifact(process.env.BASE_BENCHMARK_RESULTS);
             await downloadArtifact(process.env.GITHUB_EVENT);
       - name: Unzip Benchmark Results
         run: |
-          unzip $BENCHMARK_RESULTS.zip
+          unzip $PR_BENCHMARK_RESULTS.zip
+          unzip $BASE_BENCHMARK_RESULTS.zip
           unzip $GITHUB_EVENT.zip
       - name: Export GitHub Event Data
         uses: actions/github-script@v6
@@ -59,32 +62,34 @@ jobs:
             let fs = require('fs');
             let githubEvent = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT, {encoding: 'utf8'}));
             console.log(githubEvent);
-            if (githubEvent.ref) {
-              core.exportVariable("EVENT_NAME", "push");
-              core.exportVariable("PUSH_REF", githubEvent.ref);
-            } else {
-              core.exportVariable("EVENT_NAME", "pull_request");
-              core.exportVariable("PR_HEAD", githubEvent.pull_request.head.ref);
-              core.exportVariable("PR_BASE", githubEvent.pull_request.base.ref);
-              core.exportVariable("PR_DEFAULT", githubEvent.pull_request.base.repo.default_branch);
-              core.exportVariable("PR_NUMBER", githubEvent.number);
-            }
+            core.exportVariable("PR_HEAD", `${githubEvent.pull_request.head.ref}-${githubEvent.pull_request.head.sha.slice(0, 8)}`);
+            core.exportVariable("PR_ID", `${githubEvent.pull_request.head.ref}/${process.env.BENCHER_TESTBED}/${process.env.BENCHER_ADAPTER}`);
+            core.exportVariable("PR_NUMBER", githubEvent.number);
       - uses: bencherdev/bencher@main
-      - name: Track Benchmarks
+      - name: Track base Benchmarks
         run: |
-          (${{ env.EVENT_NAME == 'push' && env.PUSH_REF == 'refs/heads/master' }} && \
           bencher run \
-          --branch "master" \
+          --if-branch '${{ env.PR_HEAD }}' \
+          --else-branch \
           --token "${{ secrets.BENCHER_API_TOKEN }}" \
-          --err \
-          --file "$BENCHMARK_RESULTS") || \
-          (${{ env.EVENT_NAME == 'pull_request'}} && \
+          --file "$BASE_BENCHMARK_RESULTS"
+      - name: Create PR threshold
+        run: |
+          bencher threshold create \
+          --project "$BENCHER_PROJECT" \
+          --branch '${{ env.PR_HEAD }}' \
+          --testbed "$BENCHER_TESTBED" \
+          --measure latency \
+          --test t \
+          --upper-boundary 0.98 \
+          --token "${{ secrets.BENCHER_API_TOKEN }}"
+      - name: Track PR Benchmarks
+        run: |
           bencher run \
-          --if-branch '${{ env.PR_HEAD }}' \
-          --else-if-branch '${{ env.PR_BASE }}' \
-          --else-if-branch '${{ env.PR_DEFAULT }}' \
+          --branch '${{ env.PR_HEAD }}' \
+          --token "${{ secrets.BENCHER_API_TOKEN }}" \
+          --ci-id '${{ env.PR_ID }}' \
           --ci-number '${{ env.PR_NUMBER }}' \
           --github-actions "${{ secrets.GITHUB_TOKEN }}" \
-          --token "${{ secrets.BENCHER_API_TOKEN }}" \
           --err \
-          --file "$BENCHMARK_RESULTS")
+          --file "$PR_BENCHMARK_RESULTS"