benchmark.yml

name: Nightly Benchmark

on:
  push:
    branches: [ main ]
  workflow_dispatch:
    inputs:
      ec2_instance_type:
        description: 'Select EC2 instance type'
        required: true
        default: 'c7a-4xlarge'
        type: choice
        options:
          - c7a-2xlarge
          - c7a-4xlarge
          - c8g-2xlarge

permissions:
  contents: write
  checks: write
  pull-requests: write

jobs:
    benchmark:
      name: Continuous Benchmarking with Bencher
      container: rustlang/rust:nightly
      permissions:
        checks: write
        actions: write
      runs-on: ${{ github.event_name == 'push' && github.ref_name == 'main' && 'c7a-4xlarge' || github.event.inputs.ec2_instance_type }}
      steps:
        - name: Checkout Repository
          uses: actions/checkout@v4
        - name: Setup Bencher
          uses: bencherdev/bencher@main
        - name: Create Output Directory
          run: mkdir output
        - name: Execute Benchmark Tests
          run: ./scripts/nightly_benchmarks.py --export-file output/result.json
        - name: Track base branch benchmarks with Bencher
          run: |
            bencher run \
            --project ben \
            --token '${{ secrets.BENCHER_API_TOKEN }}' \
            --branch main \
            --testbed c7a-4xlarge \
            --threshold-measure latency \
            --threshold-test t_test \
            --threshold-max-sample-size 64 \
            --threshold-upper-boundary 0.99 \
            --thresholds-reset \
            --err \
            --adapter json \
            --github-actions '${{ secrets.GITHUB_TOKEN }}' \
            --file output/result.json
        - name: Upload artifact
          uses: actions/upload-artifact@v4
          with:
            name: gh-pages
            path: output/
    publish_results:
      name: Publish Results to Github Page
      needs: [benchmark]
      runs-on: ubuntu-latest
      steps:
        - name: Download artifact
          uses: actions/download-artifact@v4
          with:
            name: gh-pages
        - name: Deploy to GitHub Pages
          uses: crazy-max/ghaction-github-pages@v4
          with:
            repo: irreducibleoss/binius-benchmark
            fqdn: benchmark.binius.xyz
            target_branch: main
            build_dir: ./
          env:
            GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}