ci: update cargo-dist 0.5.0 -> 0.13.3

Bumps the cargo-dist dependency, and regenerates the relevant CI files. Tested these changes in a private fork, so we can be confident they won't break builds. Motivated by deprecation warnings in cargo-dist declaring that pinning rust versions should use `rust-toolchain.toml`, updated ours to pin 1.75. Standardizing on the toolchain file lets us remove repeated versions from a few different places, such as the containerbuild. Removed custom action helpers from Rust CI workflows, because GHA runners already come with rustup, and will honor the toolchain file. This change ensures that our dev, build, and test environments all use the same version of rust, which should minimize surprises like we saw in #4231. Included in the release artifacts now is a complete source tarball with associated checksum, as requested in #4195. We're also distributing binaries compressed as .tar.gz, rather than .tar.xz, to satisfy #4256, and opting into a custom install script to satisfy #4255. Therefore: * closes #4195 * closes #4255 * closes #4256 (cherry picked from commit 36280c4) Involved some manual conflict-resolution, particularly around RUSTFLAGS, preserving the overrides since we haven't dropped the requirement for the tokio unstable flag in the release branches yet. ci: fix rust-toolchain.toml in container Follow-up to #4282, ensuring that the rust-toolchain.toml makes it into the container image, otherwise it'll throw an error on image build. Observed this on the post-merge preview deploy. (cherry picked from commit 8d3f4f9)
penumbra-zone · Apr 29, 2024 · 1fa479e · 1fa479e
1 parent aba5601
commit 1fa479e
Show file tree

Hide file tree

Showing 12 changed files with 202 additions and 112 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -6,6 +6,7 @@
 !Cargo.toml
 !Cargo.lock
 !.cargo/
+!rust-toolchain.toml
 
 # testnets for 'pd testnet generate' defaults
 !testnets/

diff --git a/.github/workflows/buf-pull-request.yml b/.github/workflows/buf-pull-request.yml
@@ -54,13 +54,6 @@ jobs:
         with:
           lfs: true
 
-      - name: Install Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: false
-
       - uses: bufbuild/buf-setup-action@v1
         with:
           buf_api_token: ${{ secrets.BUF_TOKEN }}

diff --git a/.github/workflows/docs-lint.yml b/.github/workflows/docs-lint.yml
@@ -13,9 +13,9 @@ jobs:
         with:
           lfs: false
 
-      - name: Install rust toolchain
+      - name: Install nightly rust toolchain
         # The script for rustdoc build requires nightly toolchain.
-        uses: dtolnay/rust-toolchain@nightly
+        run: rustup toolchain install nightly
 
       # Loading cache takes ~15s, but saves us minutes of build.
       - name: Load rust cache
@@ -36,9 +36,6 @@ jobs:
         with:
           lfs: false
 
-      - name: Install rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
       - name: Load rust cache
         uses: astriaorg/[email protected]
 

diff --git a/.github/workflows/notes.yml b/.github/workflows/notes.yml
@@ -17,8 +17,10 @@ jobs:
         uses: actions/checkout@v4
         with:
           lfs: true
+
       - name: Install rust toolchain
-        uses: dtolnay/rust-toolchain@nightly
+        run: rustup toolchain install nightly
+
       - name: Load Rust caching
         uses: astriaorg/[email protected]
       - name: Load get-version action to grab version component of deployment path

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,15 +1,16 @@
-# Copyright 2022-2023, axodotdev
+# Copyright 2022-2024, axodotdev
 # SPDX-License-Identifier: MIT or Apache-2.0
 #
 # CI that:
 #
 # * checks for a Git Tag that looks like a release
 # * builds artifacts with cargo-dist (archives, installers, hashes)
 # * uploads those artifacts to temporary workflow zip
-# * on success, uploads the artifacts to a Github Release™
+# * on success, uploads the artifacts to a GitHub Release
 #
-# Note that the Github Release™ will be created with a generated
+# Note that the GitHub Release will be created with a generated
 # title/body based on your changelogs.
+
 name: Release
 
 permissions:
@@ -21,28 +22,29 @@ permissions:
 # PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
 # must be a Cargo-style SemVer Version (must have at least major.minor.patch).
 #
-# If PACKAGE_NAME is specified, then the release will be for that
+# If PACKAGE_NAME is specified, then the announcement will be for that
 # package (erroring out if it doesn't have the given version or isn't cargo-dist-able).
 #
-# If PACKAGE_NAME isn't specified, then the release will be for all
+# If PACKAGE_NAME isn't specified, then the announcement will be for all
 # (cargo-dist-able) packages in the workspace with that version (this mode is
 # intended for workspaces with only one dist-able package, or with all dist-able
 # packages versioned/released in lockstep).
 #
 # If you push multiple tags at once, separate instances of this workflow will
-# spin up, creating an independent Github Release™ for each one. However Github
+# spin up, creating an independent announcement for each one. However, GitHub
 # will hard limit this to 3 tags per commit, as it will assume more tags is a
 # mistake.
 #
-# If there's a prerelease-style suffix to the version, then the Github Release™
+# If there's a prerelease-style suffix to the version, then the release(s)
 # will be marked as a prerelease.
 on:
   push:
     tags:
       - '**[0-9]+.[0-9]+.[0-9]+*'
+  pull_request:
 
 jobs:
-  # Run 'cargo dist plan' to determine what tasks we need to do
+  # Run 'cargo dist plan' (or host) to determine what tasks we need to do
   plan:
     runs-on: ubuntu-latest
     outputs:
@@ -56,65 +58,73 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Install Rust
-        run: rustup update "1.73" --no-self-update && rustup default "1.73"
       - name: Install cargo-dist
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.5.0/cargo-dist-installer.sh | sh"
+        # we specify bash to get pipefail; it guards against the `curl` command
+        # failing. otherwise `sh` won't catch that `curl` returned non-0
+        shell: bash
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.3/cargo-dist-installer.sh | sh"
+      # sure would be cool if github gave us proper conditionals...
+      # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
+      # functionality based on whether this is a pull_request, and whether it's from a fork.
+      # (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
+      # but also really annoying to build CI around when it needs secrets to work right.)
       - id: plan
         run: |
-          cargo dist plan ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} --output-format=json > dist-manifest.json
-          echo "cargo dist plan ran successfully"
-          cat dist-manifest.json
-          echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
+          cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
+          echo "cargo dist ran successfully"
+          cat plan-dist-manifest.json
+          echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
       - name: "Upload dist-manifest.json"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: artifacts
-          path: dist-manifest.json
+          name: artifacts-plan-dist-manifest
+          path: plan-dist-manifest.json
 
   # Build and packages all the platform-specific things
-  upload-local-artifacts:
+  build-local-artifacts:
+    name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
     # Let the initial task tell us to not run (currently very blunt)
-    needs: plan
-    if: ${{ fromJson(needs.plan.outputs.val).releases != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
+    needs:
+      - plan
+    if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
     strategy:
       fail-fast: false
-      # We override the generated `matrix` so we can specify custom runners,
-      # for faster build times. This works for Linux & macOS. To generate the base template, run:
-      # `cargo dist plan --output-format json`. That JSON content has been adapted to YAML below.
-      # matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
-      matrix:
-        include:
-        - runner: buildjet-16vcpu-ubuntu-2204
-          dist_args: --artifacts=local --target=x86_64-unknown-linux-gnu
-          install_dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.5.0/cargo-dist-installer.sh | sh
-          targets:
-            - x86_64-unknown-linux-gnu
-        - runner: macos-12-xl
-          dist_args: --artifacts=local --target=aarch64-apple-darwin
-          install_dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.5.0/cargo-dist-installer.sh | sh
-          targets:
-            - aarch64-apple-darwin
-        - runner: macos-12-xl
-          dist_args: --artifacts=local --target=x86_64-apple-darwin
-          install_dist: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.5.0/cargo-dist-installer.sh | sh
-          targets:
-            - x86_64-apple-darwin
-
+      # Target platforms/runners are computed by cargo-dist in create-release.
+      # Each member of the matrix has the following arguments:
+      #
+      # - runner: the github runner
+      # - dist-args: cli flags to pass to cargo dist
+      # - install-dist: expression to run to install cargo-dist on the runner
+      #
+      # Typically there will be:
+      # - 1 "global" task that builds universal installers
+      # - N "local" tasks that build each platform's binaries and platform-specific installers
+      matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
     runs-on: ${{ matrix.runner }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
       RUSTFLAGS: "--cfg tokio_unstable"
     steps:
+      - name: enable windows longpaths
+        run: |
+          git config --global core.longpaths true
       - uses: actions/checkout@v4
         with:
+          submodules: recursive
           lfs: true
-      - name: Install Rust
-        run: rustup update "1.73" --no-self-update && rustup default "1.73"
       - uses: swatinem/rust-cache@v2
+        with:
+          key: ${{ join(matrix.targets, '-') }}
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
+      # Get the dist-manifest
+      - name: Fetch local artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: target/distrib/
+          merge-multiple: true
       - name: Install dependencies
         run: |
           ${{ matrix.packages_install }}
@@ -130,54 +140,135 @@ jobs:
         # inconsistent syntax between shell and powershell.
         shell: bash
         run: |
-          # Parse out what we just built and upload it to the Github Release™
+          # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT"
+          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
       - name: "Upload artifacts"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: artifacts
+          name: artifacts-build-local-${{ join(matrix.targets, '_') }}
           path: |
             ${{ steps.cargo-dist.outputs.paths }}
             ${{ env.BUILD_MANIFEST_NAME }}
 
-  should-publish:
+  # Build and package all the platform-agnostic(ish) things
+  build-global-artifacts:
     needs:
       - plan
-      - upload-local-artifacts
-    if: ${{ needs.plan.outputs.publishing == 'true' }}
-    runs-on: ubuntu-latest
+      - build-local-artifacts
+    runs-on: "ubuntu-20.04"
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
     steps:
-      - name: print tag
-        run: echo "ok we're publishing!"
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          lfs: true
+      - name: Install cargo-dist
+        shell: bash
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.3/cargo-dist-installer.sh | sh"
+      # Get all the local artifacts for the global tasks to use (for e.g. checksums)
+      - name: Fetch local artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: target/distrib/
+          merge-multiple: true
+      - id: cargo-dist
+        shell: bash
+        run: |
+          cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
+          echo "cargo dist ran successfully"
 
-  # Create a Github Release with all the results once everything is done
-  publish-release:
-    needs: [plan, should-publish]
-    runs-on: ubuntu-latest
+          # Parse out what we just built and upload it to scratch storage
+          echo "paths<<EOF" >> "$GITHUB_OUTPUT"
+          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+
+          cp dist-manifest.json "$BUILD_MANIFEST_NAME"
+      - name: "Upload artifacts"
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts-build-global
+          path: |
+            ${{ steps.cargo-dist.outputs.paths }}
+            ${{ env.BUILD_MANIFEST_NAME }}
+  # Determines if we should publish/announce
+  host:
+    needs:
+      - plan
+      - build-local-artifacts
+      - build-global-artifacts
+    # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
+    if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    runs-on: "ubuntu-20.04"
+    outputs:
+      val: ${{ steps.host.outputs.manifest }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install cargo-dist
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.13.3/cargo-dist-installer.sh | sh"
+      # Fetch artifacts from scratch-storage
+      - name: Fetch artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: target/distrib/
+          merge-multiple: true
+      # This is a harmless no-op for GitHub Releases, hosting for that happens in "announce"
+      - id: host
+        shell: bash
+        run: |
+          cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
+          echo "artifacts uploaded and released successfully"
+          cat dist-manifest.json
+          echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
+      - name: "Upload dist-manifest.json"
+        uses: actions/upload-artifact@v4
+        with:
+          # Overwrite the previous copy
+          name: artifacts-dist-manifest
+          path: dist-manifest.json
+
+  # Create a GitHub Release while uploading all files to it
+  announce:
+    needs:
+      - plan
+      - host
+    # use "always() && ..." to allow us to wait for all publish jobs while
+    # still allowing individual publish jobs to skip themselves (for prereleases).
+    # "host" however must run to completion, no skipping allowed!
+    if: ${{ always() && needs.host.result == 'success' }}
+    runs-on: "ubuntu-20.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: "Download artifacts"
-        uses: actions/download-artifact@v3
+      - name: "Download GitHub Artifacts"
+        uses: actions/download-artifact@v4
         with:
-          name: artifacts
+          pattern: artifacts-*
           path: artifacts
+          merge-multiple: true
       - name: Cleanup
         run: |
           # Remove the granular manifests
-          rm artifacts/*-dist-manifest.json
-      - name: Create Release
+          rm -f artifacts/*-dist-manifest.json
+      - name: Create GitHub Release
         uses: ncipollo/release-action@v1
         with:
           tag: ${{ needs.plan.outputs.tag }}
-          name: ${{ fromJson(needs.plan.outputs.val).announcement_title }}
-          body: ${{ fromJson(needs.plan.outputs.val).announcement_github_body }}
-          prerelease: ${{ fromJson(needs.plan.outputs.val).announcement_is_prerelease }}
+          name: ${{ fromJson(needs.host.outputs.val).announcement_title }}
+          body: ${{ fromJson(needs.host.outputs.val).announcement_github_body }}
+          prerelease: ${{ fromJson(needs.host.outputs.val).announcement_is_prerelease }}
           artifacts: "artifacts/*"