e2e_runtime-reproducibility.yml

name: e2e test runtime reproducibility

on:
  workflow_dispatch:
  schedule:
    - cron: "0 19 * * 0" # 7pm UTC on Sundays

jobs:
  os-matrix:
    strategy:
      matrix:
        os: [ubuntu-22.04, ubuntu-20.04]
        # As we do not use the Cachix cache containing the artifacts built by developers in this workflow,
        # building the node-installer-image ensures that the whole transitive closure of the packages we build (i.e. everything
        # except what's in cache.nixos.org). Therefore, building this ensures that the kernel, image, IGVM, runtime and the image itself
        # is reproducible across individual builds (as the --rebuild flag is used, causing Nix to rebuild the node-installer-image derivation)
        # and across independent builds on Ubuntu 20.04 and 22.04 (which also test the reproducibility of the transitive closure of our packages, as no shared
        # cache is present between the two machines)
        #
        # If adjusting the build-target, remember to also adjust the matrix for the collect-checksums job
        build-target:
          [
            "microsoft.contrast-node-installer-image",
            "kata.contrast-node-installer-image",
          ]
      fail-fast: false
    # Usually we would define the matrix outputs here, but as GitHub Actions don't seem to allow per-combination outputs,
    # we'll write the outputs without defining them here. See https://github.com/orgs/community/discussions/17245#discussioncomment-3814009.
    runs-on: ${{ matrix.os }}
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: ./.github/actions/setup_nix
        with:
          githubToken: ${{ secrets.GITHUB_TOKEN }}
          cachixToken: "" # Don't use the cachix cache
      - uses: nicknovitski/nix-develop@8b263330c8f0999d6ce564372b099f00af6f8e8a # v1.1.1
      - name: Build
        id: build
        run: |
          nix build .#${{ matrix.build-target }} --option substituters https://cache.nixos.org --builders ""
          reference_checksum="$(jq  -r '.manifests[0].digest' result/index.json)"
          echo "$reference_checksum" > ${{ matrix.build-target }}-${{ matrix.os }}-reference_checksum.txt

          nix build .#${{ matrix.build-target }} --rebuild --option substituters https://cache.nixos.org --builders "" -o rebuild
          rebuild_checksum="$(jq  -r '.manifests[0].digest' rebuild/index.json)"
          echo "$rebuild_checksum" > ${{ matrix.build-target }}-${{ matrix.os }}-rebuild_checksum.txt
      - name: Upload Build Artifacts
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: ${{ matrix.build-target }}-${{ matrix.os }}
          path: result
      - name: Upload Build Artifacts (Rebuild)
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: ${{ matrix.build-target }}-${{ matrix.os }}-rebuild
          path: rebuild
      - name: Upload checksums
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: ${{ matrix.build-target }}-${{ matrix.os }}-checksums
          path: ${{ matrix.build-target }}-${{ matrix.os }}-*_checksum.txt
      - name: Notify teams channel of failure
        if: ${{ failure() && github.ref == 'main' && github.run_attempt == 1 }}
        uses: ./.github/actions/post_to_teams
        with:
          webhook: ${{ secrets.TEAMS_CI_WEBHOOK }}
          title: "Runtime reproducibility test failed"
          message: "Runtime reproducibility test failed on ${{ matrix.build-target}}-${{matrix.os }}"
          additionalFields: '[{"title": "Build target", "value": "${{ matrix.build-target }}"}, {"title": "OS", "value": "${{ matrix.os }}"}]'

  collect-checksums:
    runs-on: ubuntu-22.04
    permissions:
      contents: read
    needs: os-matrix
    strategy:
      matrix:
        build-target:
          [
            "microsoft.contrast-node-installer-image",
            "kata.contrast-node-installer-image",
          ]
    steps:
      - name: Download all checksum artifacts
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: "${{matrix.build-target}}-*-checksums"
          path: "./checksums"
          merge-multiple: true
      - name: Collect checksums
        id: collect
        shell: python
        run: |
          import json, os, pathlib
          checksum_mismatch = False
          seen = {}
          for file in os.listdir("./checksums"):
              checksum = pathlib.Path("./checksums", file).read_text()
              if not checksum in seen:
                  seen[checksum] = []
          assert len(seen) > 0
          if len(seen) > 1:
              print(f"At least one checksum mismatched for {target}:")
              print(json.dumps(seen, indent=2))
              checksum_mismatch = True
          if checksum_mismatch:
              exit(1)
          print("All checksums were equal")
      - name: Notify teams channel of failure
        if: ${{ failure() && github.ref == 'main' && github.run_attempt == 1 }}
        uses: ./.github/actions/post_to_teams
        with:
          webhook: ${{ secrets.TEAMS_CI_WEBHOOK }}
          title: "Runtime reproducibility test failed"
          message: "failed to collect checksums"
          additionalFields: '[{"title": "Build target", "value": "${{matrix.build-target}}"}]'