diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8628c222dd7..a35f3460cba 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d", + "image": "ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c", "remoteUser": "ubuntu", "privileged": true, "runArgs": [ diff --git a/.github/workflows-source/ci-main.yml b/.github/workflows-source/ci-main.yml index 3d43f17dbc3..44395ef5e98 100644 --- a/.github/workflows-source/ci-main.yml +++ b/.github/workflows-source/ci-main.yml @@ -32,7 +32,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c dind-large-setup: &dind-large-setup runs-on: labels: dind-large diff --git a/.github/workflows-source/ci-pr-only.yml b/.github/workflows-source/ci-pr-only.yml index 2072e4571ea..e80c41f3add 100644 --- a/.github/workflows-source/ci-pr-only.yml +++ b/.github/workflows-source/ci-pr-only.yml @@ -17,7 +17,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c dind-small-setup: &dind-small-setup timeout-minutes: 30 runs-on: diff --git a/.github/workflows-source/release-testing.yml b/.github/workflows-source/release-testing.yml index 4af1efc2eb1..61fe219cb2c 100644 --- a/.github/workflows-source/release-testing.yml +++ b/.github/workflows-source/release-testing.yml @@ -23,7 +23,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c dind-large-setup: &dind-large-setup runs-on: group: zh1 diff --git a/.github/workflows-source/schedule-daily.yml b/.github/workflows-source/schedule-daily.yml index 4cd59af5f67..eab90bc2e2e 100644 --- a/.github/workflows-source/schedule-daily.yml +++ b/.github/workflows-source/schedule-daily.yml @@ -16,7 +16,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c dind-large-setup: &dind-large-setup runs-on: group: zh1 diff --git a/.github/workflows-source/schedule-hourly.yml b/.github/workflows-source/schedule-hourly.yml index 23a2a1a24a3..6bb9fd54a5e 100644 --- a/.github/workflows-source/schedule-hourly.yml +++ b/.github/workflows-source/schedule-hourly.yml @@ -16,7 +16,7 @@ env: anchors: image: &image - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c dind-large-setup: &dind-large-setup runs-on: labels: dind-large diff --git a/.github/workflows/anonymization-backend-release.yml b/.github/workflows/anonymization-backend-release.yml index 3f44f4cd37f..d759f66921d 100644 --- a/.github/workflows/anonymization-backend-release.yml +++ b/.github/workflows/anonymization-backend-release.yml @@ -33,7 +33,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 09640664450..2aac71a38e5 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -30,7 +30,7 @@ jobs: bazel-test-all: name: Bazel Test All container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -113,7 +113,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -179,7 +179,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -201,7 +201,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -223,7 +223,7 @@ jobs: runs-on: labels: dind-small container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c timeout-minutes: 30 steps: - name: Checkout @@ -253,7 +253,7 @@ jobs: build-ic: name: Build IC container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info timeout-minutes: 90 @@ -355,7 +355,7 @@ jobs: cargo-clippy-linux: name: Cargo Clippy Linux container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c timeout-minutes: 30 runs-on: group: ch1 @@ -392,7 +392,7 @@ jobs: cargo-build-release-linux: name: Cargo Build Release Linux container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c timeout-minutes: 30 runs-on: group: ch1 diff --git a/.github/workflows/ci-pr-only.yml b/.github/workflows/ci-pr-only.yml index 91a8301c95b..191febcfeb8 100644 --- a/.github/workflows/ci-pr-only.yml +++ b/.github/workflows/ci-pr-only.yml @@ -20,7 +20,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME if: ${{ github.event_name != 'merge_group' }} @@ -50,7 +50,7 @@ jobs: runs-on: labels: dind-small container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME if: ${{ github.event_name != 'merge_group' }} @@ -91,7 +91,7 @@ jobs: runs-on: labels: dind-small container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME if: ${{ github.event_name != 'merge_group' }} @@ -139,7 +139,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME if: contains(github.event.pull_request.labels.*.name, 'CI_COVERAGE') diff --git a/.github/workflows/rate-limits-backend-release.yml b/.github/workflows/rate-limits-backend-release.yml index 02948d0370f..63f9cd3ea38 100644 --- a/.github/workflows/rate-limits-backend-release.yml +++ b/.github/workflows/rate-limits-backend-release.yml @@ -33,7 +33,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info diff --git a/.github/workflows/release-testing.yml b/.github/workflows/release-testing.yml index 34b13a0cd10..f2087b3b9ff 100644 --- a/.github/workflows/release-testing.yml +++ b/.github/workflows/release-testing.yml @@ -28,7 +28,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours @@ -57,7 +57,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours @@ -86,7 +86,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 90 @@ -115,7 +115,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 60 @@ -162,7 +162,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours @@ -190,7 +190,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 180 # 3 hours diff --git a/.github/workflows/rosetta-release.yml b/.github/workflows/rosetta-release.yml index cd99e734580..e9b20f237d1 100644 --- a/.github/workflows/rosetta-release.yml +++ b/.github/workflows/rosetta-release.yml @@ -22,7 +22,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/schedule-daily.yml b/.github/workflows/schedule-daily.yml index 1ea0f4b894b..53475c570ce 100644 --- a/.github/workflows/schedule-daily.yml +++ b/.github/workflows/schedule-daily.yml @@ -15,7 +15,7 @@ jobs: bazel-test-bare-metal: name: Bazel Test Bare Metal container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 @@ -72,7 +72,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 720 # 12 hours @@ -100,7 +100,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 20 @@ -128,7 +128,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 480 @@ -169,7 +169,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 60 @@ -221,7 +221,7 @@ jobs: group: zh1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 diff --git a/.github/workflows/schedule-hourly.yml b/.github/workflows/schedule-hourly.yml index 30425c6be15..35e4ee8f705 100644 --- a/.github/workflows/schedule-hourly.yml +++ b/.github/workflows/schedule-hourly.yml @@ -17,7 +17,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 @@ -40,7 +40,7 @@ jobs: bazel-system-test-hourly: name: Bazel System Tests Hourly container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 @@ -69,7 +69,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp timeout-minutes: 120 diff --git a/.github/workflows/schedule-rust-bench.yml b/.github/workflows/schedule-rust-bench.yml index eb8e515e004..3cbe1b28d33 100644 --- a/.github/workflows/schedule-rust-bench.yml +++ b/.github/workflows/schedule-rust-bench.yml @@ -20,7 +20,7 @@ jobs: # see linux-x86-64 runner group labels: rust-benchmarks container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c # running on bare metal machine using ubuntu user options: --user ubuntu -v /cache:/cache timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/schedule-weekly.yml b/.github/workflows/schedule-weekly.yml index e0ac84c0026..04393d2545c 100644 --- a/.github/workflows/schedule-weekly.yml +++ b/.github/workflows/schedule-weekly.yml @@ -14,7 +14,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME -v /cache:/cache diff --git a/.github/workflows/system-tests-k8s.yml b/.github/workflows/system-tests-k8s.yml index c9135ff2ea3..9155025c5f6 100644 --- a/.github/workflows/system-tests-k8s.yml +++ b/.github/workflows/system-tests-k8s.yml @@ -55,7 +55,7 @@ jobs: group: ln1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME -e KUBECONFIG --privileged --cgroupns host @@ -97,7 +97,7 @@ jobs: group: ln1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME -e KUBECONFIG --privileged --cgroupns host diff --git a/.github/workflows/update-mainnet-revisions.yaml b/.github/workflows/update-mainnet-revisions.yaml index 605c82259e1..62eb26f19b2 100644 --- a/.github/workflows/update-mainnet-revisions.yaml +++ b/.github/workflows/update-mainnet-revisions.yaml @@ -38,7 +38,7 @@ jobs: runs-on: labels: dind-small container: - image: ghcr.io/dfinity/ic-build@sha256:2e8185171700872d48fdfb4b08e175fca5be27b3fbbc4d7bed681ec8486f8b1d + image: ghcr.io/dfinity/ic-build@sha256:5926f073e5d8c1b37d5905697107f13dd3876a44863225910e146f5e2d26de2c options: >- -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info steps: diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 69b1933e477..6102ee4100c 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "12261e4e43785de1ae9b245dee18085d41c7b97556016ba5e1b71191acea6117", + "checksum": "34229526fddbaaf52bbba5ee88323761d029952999db026978bc53729c6ea2b2", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -2468,6 +2468,44 @@ ], "license_file": "LICENSE-APACHE" }, + "arrayref 0.3.9": { + "name": "arrayref", + "version": "0.3.9", + "package_url": "https://github.com/droundy/arrayref", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/arrayref/0.3.9/download", + "sha256": "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + } + }, + "targets": [ + { + "Library": { + "crate_name": "arrayref", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "arrayref", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.3.9" + }, + "license": "BSD-2-Clause", + "license_ids": [ + "BSD-2-Clause" + ], + "license_file": "LICENSE" + }, "arrayvec 0.5.2": { "name": "arrayvec", "version": "0.5.2", @@ -7957,6 +7995,106 @@ ], "license_file": "LICENSE.txt" }, + "blake3 1.5.5": { + "name": "blake3", + "version": "1.5.5", + "package_url": "https://github.com/BLAKE3-team/BLAKE3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/blake3/1.5.5/download", + "sha256": "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "blake3", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "blake3", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "arrayref 0.3.9", + "target": "arrayref" + }, + { + "id": "arrayvec 0.7.4", + "target": "arrayvec" + }, + { + "id": "blake3 1.5.5", + "target": "build_script_build" + }, + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "constant_time_eq 0.3.1", + "target": "constant_time_eq" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.5.5" + }, + "build_script_attrs": { + "compile_data_glob": [ + "**" + ], + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cc 1.1.37", + "target": "cc" + } + ], + "selects": {} + } + }, + "license": "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception", + "license_ids": [ + "Apache-2.0", + "CC0-1.0" + ], + "license_file": "LICENSE_A2" + }, "block-buffer 0.9.0": { "name": "block-buffer", "version": "0.9.0", @@ -13897,6 +14035,46 @@ ], "license_file": "LICENSE-APACHE" }, + "constant_time_eq 0.3.1": { + "name": "constant_time_eq", + "version": "0.3.1", + "package_url": "https://github.com/cesarb/constant_time_eq", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/constant_time_eq/0.3.1/download", + "sha256": "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + } + }, + "targets": [ + { + "Library": { + "crate_name": "constant_time_eq", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "constant_time_eq", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.3.1" + }, + "license": "CC0-1.0 OR MIT-0 OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "CC0-1.0", + "MIT-0" + ], + "license_file": "LICENSE-APACHE" + }, "convert_case 0.4.0": { "name": "convert_case", "version": "0.4.0", @@ -18534,6 +18712,10 @@ "id": "bitflags 1.3.2", "target": "bitflags" }, + { + "id": "blake3 1.5.5", + "target": "blake3" + }, { "id": "bs58 0.5.0", "target": "bs58" @@ -87767,6 +87949,7 @@ "bitcoincore-rpc 0.19.0", "bitcoind 0.32.0", "bitflags 1.3.2", + "blake3 1.5.5", "bs58 0.5.0", "build-info 0.0.27", "build-info-build 0.0.27", diff --git a/Cargo.Bazel.Fuzzing.toml.lock b/Cargo.Bazel.Fuzzing.toml.lock index 2eb3bdb7886..e437164565b 100644 --- a/Cargo.Bazel.Fuzzing.toml.lock +++ b/Cargo.Bazel.Fuzzing.toml.lock @@ -432,6 +432,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.5.2" @@ -1344,6 +1350,19 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec 0.7.4", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -2283,6 +2302,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -3009,6 +3034,7 @@ dependencies = [ "bitcoincore-rpc", "bitcoind", "bitflags 1.3.2", + "blake3", "bs58", "build-info", "build-info-build", diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index f8413db2d1b..3c2101b570f 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "536b2baa9e3ea1956ceedabcbea2b38d092ad5b579a5e7e61df6c84cae348752", + "checksum": "b6a812f36ad8264c09beaa2d4739ad501453c502bed65913d798247ecac4a54f", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -2472,6 +2472,44 @@ ], "license_file": "LICENSE-APACHE" }, + "arrayref 0.3.9": { + "name": "arrayref", + "version": "0.3.9", + "package_url": "https://github.com/droundy/arrayref", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/arrayref/0.3.9/download", + "sha256": "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + } + }, + "targets": [ + { + "Library": { + "crate_name": "arrayref", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "arrayref", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "0.3.9" + }, + "license": "BSD-2-Clause", + "license_ids": [ + "BSD-2-Clause" + ], + "license_file": "LICENSE" + }, "arrayvec 0.5.2": { "name": "arrayvec", "version": "0.5.2", @@ -7874,6 +7912,106 @@ ], "license_file": "LICENSE.txt" }, + "blake3 1.5.5": { + "name": "blake3", + "version": "1.5.5", + "package_url": "https://github.com/BLAKE3-team/BLAKE3", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/blake3/1.5.5/download", + "sha256": "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" + } + }, + "targets": [ + { + "Library": { + "crate_name": "blake3", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "blake3", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "arrayref 0.3.9", + "target": "arrayref" + }, + { + "id": "arrayvec 0.7.4", + "target": "arrayvec" + }, + { + "id": "blake3 1.5.5", + "target": "build_script_build" + }, + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "constant_time_eq 0.3.1", + "target": "constant_time_eq" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.5.5" + }, + "build_script_attrs": { + "compile_data_glob": [ + "**" + ], + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cc 1.1.37", + "target": "cc" + } + ], + "selects": {} + } + }, + "license": "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception", + "license_ids": [ + "Apache-2.0", + "CC0-1.0" + ], + "license_file": "LICENSE_A2" + }, "block-buffer 0.9.0": { "name": "block-buffer", "version": "0.9.0", @@ -13725,6 +13863,46 @@ ], "license_file": "LICENSE-APACHE" }, + "constant_time_eq 0.3.1": { + "name": "constant_time_eq", + "version": "0.3.1", + "package_url": "https://github.com/cesarb/constant_time_eq", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/constant_time_eq/0.3.1/download", + "sha256": "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + } + }, + "targets": [ + { + "Library": { + "crate_name": "constant_time_eq", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "constant_time_eq", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.3.1" + }, + "license": "CC0-1.0 OR MIT-0 OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "CC0-1.0", + "MIT-0" + ], + "license_file": "LICENSE-APACHE" + }, "convert_case 0.4.0": { "name": "convert_case", "version": "0.4.0", @@ -18362,6 +18540,10 @@ "id": "bitflags 1.3.2", "target": "bitflags" }, + { + "id": "blake3 1.5.5", + "target": "blake3" + }, { "id": "bs58 0.5.0", "target": "bs58" @@ -87647,6 +87829,7 @@ "bitcoincore-rpc 0.19.0", "bitcoind 0.32.0", "bitflags 1.3.2", + "blake3 1.5.5", "bs58 0.5.0", "build-info 0.0.27", "build-info-build 0.0.27", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index ccf252a795a..9a398fdb614 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -433,6 +433,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.5.2" @@ -1345,6 +1351,19 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec 0.7.4", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -2272,6 +2291,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "795bc6e66a8e340f075fcf6227e417a2dc976b92b91f3cdc778bb858778b6747" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -2998,6 +3023,7 @@ dependencies = [ "bitcoincore-rpc", "bitcoind", "bitflags 1.3.2", + "blake3", "bs58", "build-info", "build-info-build", diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index 998576ff820..5ddef1421d0 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -1,6 +1,8 @@ # A .bazelrc needing build configuration. Without this configuration, the # build will most likely fail. +startup --unix_digest_hash_attribute_name=user.icsum + # To require no rustfmt issues, pass --config=fmt. # To require no clippy issues, pass --config=clippy. Without this, warnings will still be generated. # To enable both of the above, pass --config=lint. diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index faee354defd..2c9798c9b6a 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -292,6 +292,9 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable "bitflags": crate.spec( version = "^1.2.1", ), + "blake3": crate.spec( + version = "^1.5", + ), "bs58": crate.spec( version = "^0.5.0", ), diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile index 04c9e77b6c4..d33c7f63f41 100644 --- a/ci/container/Dockerfile +++ b/ci/container/Dockerfile @@ -180,6 +180,14 @@ RUN curl --fail https://sh.rustup.rs -sSf \ rustup component add clippy RUN rm -rf /tmp/rust-version +# Compile and add icsum +COPY --chown=ubuntu:ubuntu ./rs/ic_os/build_tools/icsum /tmp/icsum +RUN cd /tmp/icsum && \ + cargo build --profile release && \ + mv target/release/icsum /usr/local/bin && \ + chmod +x /usr/local/bin/icsum && \ + rm -r /tmp/icsum + # Add cargo-audit ARG CARGO_AUDIT_VERSION=0.21.0 RUN cargo install cargo-audit --version ${CARGO_AUDIT_VERSION} diff --git a/ci/container/TAG b/ci/container/TAG index 510327d7002..a385d39119b 100644 --- a/ci/container/TAG +++ b/ci/container/TAG @@ -1 +1 @@ -b3869ba20c4e9ba2915be302f5b5a4b23da0545af2f1cb896441055372950389 +9afdf4979b00656c1ad15198488721a1055170c1b8b45323c60a434574327670 diff --git a/ci/container/files/packages.common b/ci/container/files/packages.common index 68954d42409..9f39fe1d9c0 100644 --- a/ci/container/files/packages.common +++ b/ci/container/files/packages.common @@ -1,4 +1,5 @@ # Essentials +attr curl build-essential git diff --git a/rs/ic_os/build_tools/icsum/BUILD.bazel b/rs/ic_os/build_tools/icsum/BUILD.bazel new file mode 100644 index 00000000000..b971e6256d8 --- /dev/null +++ b/rs/ic_os/build_tools/icsum/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test") + +package(default_visibility = ["//rs:ic-os-pkg"]) + +DEPENDENCIES = [ + # Keep sorted. + "@crate_index//:anyhow", + "@crate_index//:blake3", + "@crate_index//:nix", + "@crate_index//:rand", + "@crate_index//:tempfile", +] + +MACRO_DEPENDENCIES = [ + # Keep sorted. +] + +rust_binary( + name = "icsum", + srcs = glob(["src/**/*.rs"]), + proc_macro_deps = MACRO_DEPENDENCIES, + target_compatible_with = [ + "@platforms//os:linux", + ], + version = "0.1.0", + deps = DEPENDENCIES, +) + +rust_test( + name = "icsum_test", + crate = ":icsum", + proc_macro_deps = MACRO_DEPENDENCIES, + deps = DEPENDENCIES, +) diff --git a/rs/ic_os/build_tools/icsum/Cargo.toml b/rs/ic_os/build_tools/icsum/Cargo.toml new file mode 100644 index 00000000000..f31539801db --- /dev/null +++ b/rs/ic_os/build_tools/icsum/Cargo.toml @@ -0,0 +1,17 @@ +[workspace] + +[package] +name = "icsum" +version = "0.1.0" +edition = "2021" + +# We are not referencing deps from workspace using workspace = true because we want to be +# able to compile icsum in separation. +[dependencies] +anyhow = "1.0" +nix = { version = "0.29", features = ["fs"] } +blake3 = "1.5" + +[dev-dependencies] +rand = "0.9" +tempfile = "3.15" diff --git a/rs/ic_os/build_tools/icsum/src/main.rs b/rs/ic_os/build_tools/icsum/src/main.rs new file mode 100644 index 00000000000..594a6fccfc5 --- /dev/null +++ b/rs/ic_os/build_tools/icsum/src/main.rs @@ -0,0 +1,342 @@ +use anyhow::{bail, Context}; +use blake3::{Hash, Hasher}; +use nix::errno::Errno; +use nix::unistd::Whence; +use std::env; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::FileExt; +use std::path::Path; + +const BLOCK_LEN: usize = 1024 * 1024; +const BLOCK_LEN_U64: u64 = BLOCK_LEN as u64; + +/// `icsum [filename]` +/// A tool for quickly calculating file checksums. This tool is much +/// faster than common checksum tools when run on sparse files. +fn main() -> anyhow::Result<()> { + let args: Vec<_> = env::args().collect(); + // The first arg is the own executable path. + if args.len() != 2 { + bail!( + "The program takes 1 argument, the file name, received {}", + args.len() - 1 + ); + } + let path = &args[1]; + + let hash = calculate_digest(&mut File::open(Path::new(path)).context("Could not open file")?) + .context("Hash calculation failed")?; + + println!("{}", hash.to_hex()); + + Ok(()) +} + +fn calculate_digest(file: &mut File) -> std::io::Result { + const HASH_TAG: &[u8] = b"icsum-hash"; + // The algorithm works by chunking the input into blocks of [BLOCK_LEN]. + // Each such block's hash is calculated separately and combined by outer_hasher. + // Since the hash of two equal blocks is the same, we can precalculate the hash of blocks of + // holes (a block with zeros only). We then use the OS's file seek API to avoid actually having + // to read the zeros from holes. + // + // Outer hash: + // hash = blake3(len(HASH_TAG) || HASH_TAG || len(file) || h_block_0 || .. || h_block_n) + // Block hash: + // h_block_n = blake3(len(BLOCK_TAG) || BLOCK_TAG || block_n) + // + // Research paper: https://www.scitepress.org/Papers/2024/127645/127645.pdf + let mut outer_hasher = Hasher::new(); + outer_hasher.update(&HASH_TAG.len().to_be_bytes()); + outer_hasher.update(HASH_TAG); + outer_hasher.update(&file.metadata()?.len().to_be_bytes()); + let hole_hash = block_hash(vec![0; BLOCK_LEN].as_slice()); + iterate_blocks(file, |block| match block { + Block::Hole => { + outer_hasher.update(hole_hash.as_bytes()); + } + Block::Data(bytes) => { + outer_hasher.update(block_hash(bytes).as_bytes()); + } + })?; + + Ok(outer_hasher.finalize()) +} + +fn block_hash(block: &[u8]) -> Hash { + const BLOCK_TAG: &[u8] = b"icsum-block"; + let mut hasher = Hasher::new(); + hasher.update(&BLOCK_TAG.len().to_be_bytes()); + hasher.update(BLOCK_TAG); + hasher.update(block); + hasher.finalize() +} + +#[derive(Eq, PartialEq)] +enum Block<'a> { + /// A hole of [BLOCK_LEN] bytes. If the last block is a hole of less than [BLOCK_LEN], it will + /// be represented by a [Block::Data] block containing zeros instead. + Hole, + /// Data block. This is always [BLOCK_LEN] long, except in the last block where it may be + /// shorter. + Data(&'a [u8]), +} + +fn iterate_blocks(file: &mut File, mut callback: impl FnMut(Block)) -> std::io::Result<()> { + let file_len = file.metadata()?.len(); + let mut buf = vec![0; BLOCK_LEN]; + let mut state = state_at(file, 0, file_len)?; + + for block in 0..file_len / BLOCK_LEN_U64 { + let block_start_offset = block * BLOCK_LEN_U64; + + match state { + // If we are in a hole but would pass `next_data` in this block, switch to data state. + State::InHole { next_data } if block_start_offset + BLOCK_LEN_U64 > next_data => { + state = State::InData { + next_hole: seek_hole(file, block_start_offset + BLOCK_LEN_U64)? + .unwrap_or(file_len), + }; + } + // If we are in data state but already passed `next_hole`, switch to hole state. + State::InData { next_hole } if block_start_offset >= next_hole => { + state = State::InHole { + next_data: seek_data(file, block_start_offset)?.unwrap_or(file_len), + }; + } + _ => {} + } + + match state { + State::InHole { .. } => callback(Block::Hole), + State::InData { .. } => { + file.read_exact_at(&mut buf, block_start_offset)?; + callback(Block::Data(&buf)); + } + } + } + + // Handle remaining data at the end of file that may be shorter than a full block. + buf.clear(); + file.seek(SeekFrom::Start(file_len / BLOCK_LEN_U64 * BLOCK_LEN_U64))?; + file.read_to_end(&mut buf)?; + callback(Block::Data(&buf)); + + Ok(()) +} + +enum State { + InHole { next_data: u64 }, + InData { next_hole: u64 }, +} + +fn state_at(file: &mut File, offset: u64, file_len: u64) -> std::io::Result { + let next_data = seek_data(file, offset)?.unwrap_or(file_len); + let next_hole = seek_hole(file, offset)?.unwrap_or(file_len); + file.seek(SeekFrom::Start(offset))?; + let result = if next_data == offset { + State::InData { next_hole } + } else { + State::InHole { next_data } + }; + + Ok(result) +} + +/// Seek hole in `file` starting from offset `from`. If `from` is outside the file or if there +/// are no more holes in the file, the return value Ok(None) otherwise the return value is +/// Ok(start_of_hole) and the file is seeked to that position. +fn seek_hole(file: &mut File, from: u64) -> std::io::Result> { + // NOTE: u64 does not fully fit in i64, but i64::MAX is 9_223_372_036_854_775_807 + // (8 exbibytes) and we won't be seeing files this large any time soon. + let unsized_from = from.try_into().unwrap(); + + match nix::unistd::lseek(file.as_raw_fd(), unsized_from, Whence::SeekHole) { + Ok(result) => Ok(Some(result as u64)), + Err(Errno::ENXIO) => Ok(None), + Err(err) => Err(err.into()), + } +} + +/// See [seek_hole] for documentation. +fn seek_data(file: &mut File, from: u64) -> std::io::Result> { + // NOTE: u64 does not fully fit in i64, but i64::MAX is 9_223_372_036_854_775_807 + // (8 exbibytes) and we won't be seeing files this large any time soon. + let unsized_from = from.try_into().unwrap(); + + match nix::unistd::lseek(file.as_raw_fd(), unsized_from, Whence::SeekData) { + Ok(result) => Ok(Some(result as u64)), + Err(Errno::ENXIO) => Ok(None), + Err(err) => Err(err.into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::StdRng; + use rand::{RngCore, SeedableRng}; + use std::io::Write; + use std::path::PathBuf; + + fn create_tempfile() -> File { + let tempdir = env::var_os("TEST_TMPDIR").map_or(env::temp_dir(), PathBuf::from); + tempfile::tempfile_in(tempdir).expect("Could not create tempfile") + } + + mod iterate_blocks { + use super::*; + + #[test] + /// This test ensures that the test environment supports sparse files. + /// Otherwise, the rest of the test cases would be useless. + fn file_system_supports_sparse_files() { + let mut file = create_tempfile(); + file.seek(SeekFrom::Start(BLOCK_LEN_U64)).unwrap(); + writeln!(&mut file, "hello").unwrap(); + + file.rewind().unwrap(); + + let mut holes = 0; + iterate_blocks(&mut file, |block| { + if block == Block::Hole { + holes += 1; + } + }) + .unwrap(); + + assert!( + holes >= 1, + "Expected at least 1 hole, filesystem likely doesn't support sparse files." + ); + } + + #[test] + fn no_holes_prime_bytes() { + let mut file = create_tempfile(); + let mut rng = StdRng::seed_from_u64(0); + let mut data = vec![0u8; 26205239]; + rng.fill_bytes(&mut data); + + file.write_all(&data).unwrap(); + + verify_file(file); + } + + #[test] + fn no_holes_32mb() { + let mut file = create_tempfile(); + let mut rng = StdRng::seed_from_u64(0); + let mut data = vec![0u8; 32 * 1024 * 1024]; + rng.fill_bytes(&mut data); + + file.write_all(&data).unwrap(); + + verify_file(file); + } + + #[test] + fn prime_holes_and_sizes() { + let mut file = create_tempfile(); + let mut rng = StdRng::seed_from_u64(0); + let mut data = vec![0u8; 26205239]; + rng.fill_bytes(&mut data); + + file.seek(SeekFrom::Current(9590653)).unwrap(); + file.write_all(&data).unwrap(); + + file.seek(SeekFrom::Current(26205239)).unwrap(); + file.write_all(&data).unwrap(); + + verify_file(file); + } + + #[test] + fn power_of_2_holes_and_sizes() { + let eight_mb = (8 * 1024 * 1024) as i64; + let mut file = create_tempfile(); + let mut rng = StdRng::seed_from_u64(0); + let mut data = vec![0u8; eight_mb as usize]; + rng.fill_bytes(&mut data); + + file.seek(SeekFrom::Current(eight_mb)).unwrap(); + file.write_all(&data).unwrap(); + + file.seek(SeekFrom::Current(eight_mb)).unwrap(); + file.write_all(&data).unwrap(); + + verify_file(file); + } + + #[test] + fn power_of_2_mb_hole_at_end() { + let eight_mb = (8 * 1024 * 1024) as i64; + let mut file = create_tempfile(); + let mut rng = StdRng::seed_from_u64(0); + let mut data = vec![0u8; eight_mb as usize]; + rng.fill_bytes(&mut data); + + file.write_all(&data).unwrap(); + + file.seek(SeekFrom::Current(eight_mb)).unwrap(); + file.write_all(&data).unwrap(); + + file.set_len(eight_mb as u64 * 8).unwrap(); + + verify_file(file); + } + + fn verify_file(mut file: File) { + let mut contents_actual = vec![]; + file.rewind().unwrap(); + iterate_blocks(&mut file, |block| match block { + Block::Hole => contents_actual.extend_from_slice(&[0; BLOCK_LEN]), + Block::Data(data) => contents_actual.extend_from_slice(data), + }) + .unwrap(); + + file.rewind().unwrap(); + let mut contents_expected = vec![]; + file.read_to_end(&mut contents_expected).unwrap(); + + // We don't use assert_eq because it would print the contents which is too long. + assert!(contents_actual == contents_expected); + } + } + + #[test] + fn zeros_have_same_digest_as_hole() { + let mut file1 = create_tempfile(); + let mut file2 = create_tempfile(); + + write!(&mut file1, "hello").unwrap(); + write!(&mut file2, "hello").unwrap(); + + file1.write_all(&vec![0; 9590653]).unwrap(); + file2.seek(SeekFrom::Current(9590653)).unwrap(); + + write!(&mut file1, "foo").unwrap(); + write!(&mut file2, "foo").unwrap(); + + assert_eq!( + calculate_digest(&mut file1).unwrap(), + calculate_digest(&mut file2).unwrap() + ); + } + + #[test] + fn only_zeros_have_same_digest_as_hole() { + let mut file1 = create_tempfile(); + let mut file2 = create_tempfile(); + + file1.write_all(&vec![0; 9590653]).unwrap(); + file2.set_len(9590653).unwrap(); + + assert_eq!( + calculate_digest(&mut file1).unwrap(), + calculate_digest(&mut file2).unwrap() + ); + } +} diff --git a/toolchains/sysimage/proc_wrapper.sh b/toolchains/sysimage/proc_wrapper.sh index 4bf11957840..6233dbfd655 100755 --- a/toolchains/sysimage/proc_wrapper.sh +++ b/toolchains/sysimage/proc_wrapper.sh @@ -9,3 +9,11 @@ set -euo pipefail tmpdir=$(mktemp -d --tmpdir "icosbuildXXXX") trap 'sudo rm -rf "$tmpdir"' INT TERM EXIT ICOS_TMPDIR="$tmpdir" "$@" + +# Calculate the checksum for every output created by Bazel. For the calculation, we use icsum which is much +# faster than Bazel's built-in checksum for sparse files (e.g. disk images). +for arg in ${@:2}; do + if [[ -f "$arg" ]] && [[ -w "$arg" ]] && ! getfattr -n user.icsum "$arg" >/dev/null 2>&1; then + setfattr -n user.icsum -v $(icsum "$arg") "$arg" + fi +done