feat(IDX): enable bzlmod (#1959)

This turns on [Bazel modules](https://bazel.build/external/module) and
moves some of our (Bazel) dependency declarations to `MODULE.bazel`.
This means those dependencies are specified declaratively and not
imperatively as was the case in the `WORKSPACE`. This also allows us to
update some dependencies, like `rules_python`, which previously created
conflicts in transitive dependencies.

Some dependencies were also replaced (`rules_gazelle` -> `gazelle`).

This also adds a new workflow that ensures the Bazel lockfile,
`MODULE.bazel.lock`, is up to date on all the platforms we support
(Apple Intel, Apple Silicon, Linux x86). See workflow file for logic and
details.

---------

Co-authored-by: Marko Kosmerl <[email protected]>

.github/workflows/bazel-lockfile.yml

@@ -0,0 +1,221 @@
+# Workflow to automatically update the bazel lockfile on pull requests and dev
+# branches. In general, fails if lockfile is out of date.
+# For each platform we support, we run a dry (--nobuild) Bazel build, and then
+# upload the resulting lockfile as an artifact. The next platform downloads the
+# lockfile artifact, and then also runs a dry build, rinse, repeat.
+name: Update Bazel Lockfile
+on:
+  # Merge groups should be checked for up-to-date lockfile
+  merge_group:
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      # master is checked for up-to-date lockfile
+      - master
+      # dev branches get a new commit with updated lockfile (when necessary)
+      - 'dev-gh-*'
+
+# runs for the same workflow are cancelled on PRs but not on master
+# (logic copied from main workflow)
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  bazel-bzlmod-lockfile-apple-silicon:
+    name: Apple Silicon
+    runs-on: namespace-profile-darwin # profile created in namespace console
+    steps:
+      - uses: actions/checkout@v4
+      - name: Bazel dry run
+        run: |
+          bazel \
+            --noworkspace_rc \
+            --bazelrc=./bazel/conf/.bazelrc.build \
+            build \
+            --config=ci --config=macos_ci \
+            --test_tag_filters="test_macos,test_macos_slow,-upload" \
+            //rs/... //publish/binaries/... \
+            --lockfile_mode=update \
+            --nobuild
+      - uses: actions/upload-artifact@v4
+        with:
+          name: bazel-module-lock-apple-silicon
+          path: ./MODULE.bazel.lock
+
+  bazel-bzlmod-lockfile-linux:
+    name: Linux
+    container:
+      image: ghcr.io/dfinity/ic-build@sha256:115daa5ad5149182bb0416cbe5730f305be3bb2f48df576bc2c23067eefce84b
+      options: >-
+        -e NODE_NAME -v /cache:/cache
+    runs-on:
+      labels: dind-small
+    needs: bazel-bzlmod-lockfile-apple-silicon
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Before script
+        id: before-script
+        shell: bash
+        run: |
+          [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
+      - name: Login to Dockerhub
+        shell: bash
+        run: ./ci/scripts/docker-login.sh
+        env:
+          DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
+          DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: bazel-module-lock-apple-silicon
+
+      # Run a "build" with --nobuild so that Bazel updates the lockfile (if
+      # necessary). No targets will be built.
+      # The rest is copied from the main workflow.
+      - name: Bazel dry run
+        id: bazel-test-all
+        uses: ./.github/actions/bazel-test-all/
+        env:
+          CI_COMMIT_SHA: ${{ github.sha }}
+          CI_JOB_NAME: ${{ github.job }}
+          CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CI_PIPELINE_SOURCE: ${{ github.event_name }}
+          CI_PROJECT_DIR: ${{ github.workspace }}
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+          CI_RUN_ID: ${{ github.run_id }}
+          RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
+          BUILDEVENT_DATASET: "github-ci-dfinity"
+        with:
+          BAZEL_COMMAND: "build"
+          BAZEL_TARGETS: "//..."
+          BAZEL_EXTRA_ARGS: '--lockfile_mode=update --nobuild'
+          BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
+          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: bazel-module-lock-linux
+          path: ./MODULE.bazel.lock
+
+  # Same a Linux above but with our macOS self-hosted runners.
+  bazel-bzlmod-lockfile-macos-intel:
+    name: Apple Intel
+    needs: bazel-bzlmod-lockfile-linux
+    runs-on:
+      labels: macOS
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set PATH
+        run: |
+          echo "/usr/local/bin" >> $GITHUB_PATH
+          echo "$HOME/.cargo/bin:" >> $GITHUB_PATH
+      - name: Login to Dockerhub
+        shell: bash
+        run: ./ci/scripts/docker-login.sh
+        env:
+          DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
+          DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: bazel-module-lock-linux
+      - name: Run Bazel Test Darwin x86-64
+        id: bazel-test-darwin-x86-64
+        uses: ./.github/actions/bazel-test-all/
+        env:
+          CI_COMMIT_SHA: ${{ github.sha }}
+          CI_JOB_NAME: ${{ github.job }}
+          CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CI_PIPELINE_SOURCE: ${{ github.event_name }}
+          CI_PROJECT_DIR: ${{ github.workspace }}
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+          CI_RUN_ID: ${{ github.run_id }}
+          RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
+          BUILDEVENT_DATASET: "github-ci-dfinity"
+        with:
+          BAZEL_CI_CONFIG: "--config=ci --config macos_ci"
+          BAZEL_COMMAND: build
+          BAZEL_EXTRA_ARGS: '--test_tag_filters=test_macos --lockfile_mode=update --nobuild'
+          BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output/${CI_RUN_ID}"
+          BAZEL_TARGETS: "//rs/... //publish/binaries/..."
+          BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: bazel-module-lock-apple-intel
+          path: ./MODULE.bazel.lock
+
+  # Finally download the lockfile that went through every platform and check it
+  # for changes. See steps for actual logic depending on GHA event.
+  bazel-bzlmod-lockfile-update:
+    name: Check lockfile changes
+    needs: bazel-bzlmod-lockfile-macos-intel # the last platform
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref }}
+          # set the token in case we need to push the updated lockfile
+          token: ${{ secrets.IDX_PUSH_TO_PR }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: bazel-module-lock-apple-intel
+      - name: check and push
+        run: |
+          set -e
+          echo git status
+          git status
+          echo checking state
+
+          if git diff --quiet; then
+            # if diff is clean, then the lockfile did not need updating
+            echo "MODULE.bazel.lock is up to date"
+            exit 0
+          fi
+
+          # on merge groups & on master, fail the job
+
+          if [ "${{ github.event_name }}" == "merge_group" ]; then
+            echo "denying merge because of out-of-date MODULE.bazel.lock"
+            exit 1
+          fi
+
+          if [ "${{ github.event_name }}" == "push" ] && \
+              [ "${{ github.ref_name }}" == "master" ]; then
+            echo "MODULE.bazel.lock is out of date on master branch"
+            exit 1
+          fi
+
+          # on dev branches and PRs, update the lockfile
+
+          if [ "${{ github.event_name }}" == "push" ] && \
+              [[ "${{ github.ref_name }}" =~ ^dev-gh-* ]]; then
+
+            echo "updating MODULE.bazel.lock"
+            git config user.email "[email protected]"
+            git config user.name "IDX GitHub Automation"
+            git add MODULE.bazel.lock
+            git commit -m "Update MODULE.bazel.lock"
+            git push
+
+            exit 1
+          fi
+
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+
+            echo "updating MODULE.bazel.lock"
+            git config user.email "[email protected]"
+            git config user.name "IDX GitHub Automation"
+            git add MODULE.bazel.lock
+            git commit -m "Update MODULE.bazel.lock"
+            git push
+
+            exit 1
+          fi
+
+          # Unknown case; just fail
+          echo "MODULE.bazel.lock is out of date"
+          exit 1

BUILD.bazel

@@ -1,5 +1,5 @@
-load("@bazel_gazelle//:def.bzl", "gazelle")
 load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
+load("@gazelle//:def.bzl", "gazelle")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 
 package(default_visibility = ["//visibility:public"])
@@ -79,19 +79,19 @@ gazelle(
 
 alias(
     name = "gobin",
-    actual = "@go_sdk//:bin/go",
+    actual = "@rules_go//go",
     visibility = ["//visibility:public"],
 )
 
-# Builds python dependencies
+# Builds python dependencies. To update the lockfile:
+# $ bazel run //:python-requirements.update
 compile_pip_requirements(
     name = "python-requirements",
-    timeout = "moderate",
-    requirements_in = "requirements.in",
+    src = "requirements.in",
+    # NOTE: the requirements.txt reports the system python's version, not
+    # the one specified in MODULE.bazel.
+    # https://rules-python.readthedocs.io/en/latest/pypi-dependencies.html#pip-rules
     requirements_txt = "requirements.txt",
-    tags = [
-        "requires-network",
-    ],
 )
 
 test_suite(