Skip to content

Commit

Permalink
chore(IDX): simplify build-determinism.sh
Browse files Browse the repository at this point in the history
This tries to simplify the `build-determinism.sh` to make it more robust
and more readable. In particular:

* Avoid returning successfully (without checking checksums) if
  `bazel-targets` does not exist
* Remove outdated test ignores
* Unpack `build-ic.tar` in the GHA
* Simplify the GHA step
* Homogenize bash opts and avoid `set -x` for better readability
  • Loading branch information
nmattia committed Feb 6, 2025
1 parent 38236fc commit b10b234
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 58 deletions.
4 changes: 1 addition & 3 deletions .github/workflows-source/ci-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -343,10 +343,8 @@ jobs:
name: build-ic
- name: Build Determinism Test
id: build-determinism
shell: bash
run: |
set -eExuo pipefail
sudo apt update && sudo apt install -y curl
tar -xf build-ic.tar # build-determinism.sh expects ./build-ic/
"$CI_PROJECT_DIR"/ci/scripts/build-determinism.sh
env:
TARGET: ${{ matrix.TARGET }}
Expand Down
4 changes: 1 addition & 3 deletions .github/workflows/ci-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -408,10 +408,8 @@ jobs:
name: build-ic
- name: Build Determinism Test
id: build-determinism
shell: bash
run: |
set -eExuo pipefail
sudo apt update && sudo apt install -y curl
tar -xf build-ic.tar # build-determinism.sh expects ./build-ic/
"$CI_PROJECT_DIR"/ci/scripts/build-determinism.sh
env:
TARGET: ${{ matrix.TARGET }}
Expand Down
78 changes: 26 additions & 52 deletions ci/scripts/build-determinism.sh
Original file line number Diff line number Diff line change
@@ -1,57 +1,31 @@
#!/usr/bin/env bash

set -eExuo pipefail

# bazel-targets file is expected from bazel-test-all CI job
if [ ! -e bazel-targets ]; then
echo "Missing 'bazel-targets' file!"
set -euo pipefail

VERSION="$(git rev-parse HEAD)"

# PATH0
mkdir -p "$PATH0"
curl -sfS --retry 5 --retry-delay 10 \
"https://download.dfinity.systems/ic/$VERSION/$PATH0/SHA256SUMS" \
-o "$PATH0/SHA256SUMS"

# ignore *.wasm.gz.did files (these are checksummed by upload_artifacts but
# not by build-ic.sh)
sed -i -e '/.wasm.gz.did/d' "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"

# for hostos / guestos we only care about update-img
sed -i -e '/disk-img/d' "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"

if ! diff -u "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"; then
cat build-ic/info
echo "Build Determinism Check Failed!"
echo "Contact IDX or investigate by yourself using diffoscope:"
echo " * [bazel-test-all]: curl -sfS https://download.dfinity.systems/ic/$VERSION/$PATH0/<artifact> -O"
echo " * [build-ic]: curl $(cat build-ic/url) -O"
echo "See info for pull the artifacts from both CI jobs above. Specify <artifact> based on logs (e.g. 'ic-admin.gz', 'disk-img.tar.zst')."
echo "Note that [build-ic] artifacts.tar contains all the build artifacts (binaries, canisters and IC images)."
exit 1
fi

if grep -q "$TARGET" bazel-targets || grep -qF "//..." bazel-targets; then
VERSION="$(git rev-parse HEAD)"

# build-ic.tar with SHA256SUMS files is expected from build-ic CI job
if [ ! -e build-ic.tar ]; then
echo "Missing 'build-ic.tar' file!"
exit 1
fi

# PATH0
mkdir -p "$PATH0"
curl -sfS --retry 5 --retry-delay 10 \
"https://download.dfinity.systems/ic/$VERSION/$PATH0/SHA256SUMS" \
-o "$PATH0/SHA256SUMS"

# PATH1
tar -xf build-ic.tar

# ignore *.wasm.gz.did files
sed -i -e '/.wasm.gz.did/d' "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"
# TODO(IDX): remove when issue is identified and addressed
# https://gitlab.com/dfinity-lab/public/ic/-/snippets/3697069
sed -i -e '/wasm.wasm.gz/d' "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"
# TODO(IDX): remove when issue is identified and addressed
# https://gitlab.com/dfinity-lab/public/ic/-/snippets/3704313
sed -i -e '/genesis-token-canister.wasm.gz/d' "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"

# for hostos / guestos we only care about update-img
if [ "${DISKIMG:-}" != "true" ]; then
sed -i -e '/disk-img/d' "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"
fi

if ! diff -u "$PATH0/SHA256SUMS" "$PATH1/SHA256SUMS"; then
set -x
cat build-ic/info
echo "Build Determinism Check Failed!"
echo "Contact IDX or investigate by yourself using diffoscope:"
echo " * [bazel-test-all]: curl -sfS https://download.dfinity.systems/ic/$VERSION/$PATH0/<artifact> -O"
echo " * [build-ic]: curl $(cat build-ic/url) -O"
echo "See info for pull the artifacts from both CI jobs above. Specify <artifact> based on logs (e.g. 'ic-admin.gz', 'disk-img.tar.zst')."
echo "Note that [build-ic] artifacts.tar contains all the build artifacts (binaries, canisters and IC images)."
set +x
exit 1
else
echo "Build Determinism Check Successful"
fi
fi
echo "Build Determinism Check Successful"

0 comments on commit b10b234

Please sign in to comment.