Merge branch 'main' into reapply-latest-cuda-12.2

README.md

@@ -9,3 +9,17 @@ This repository includes the following CI images for RAPIDS:
 ## `latest` tag
 
 The `latest` image tags are controlled by the values in `latest.yaml`.
+
+## Building the dockerfiles locally
+
+To easily build the dockerfiles locally, you may use the following snippets:
+
+```sh
+export LINUX_VER=ubuntu22.04
+export CUDA_VER=12.2.2
+export PYTHON_VER=3.11
+export ARCH=amd64
+docker build $(ci/compute-build-args.sh) -f ci-conda.Dockerfile context/
+docker build $(ci/compute-build-args.sh) -f ci-wheel.Dockerfile context/
+docker build $(ci/compute-build-args.sh) -f citestwheel.Dockerfile context/
+```

ci-conda.Dockerfile

@@ -1,6 +1,13 @@
 ARG CUDA_VER=notset
 ARG LINUX_VER=notset
 ARG PYTHON_VER=notset
+ARG YQ_VER
+ARG AWS_CLI_VER
+
+FROM mikefarah/yq:${YQ_VER} as yq
+
+FROM amazon/aws-cli:${AWS_CLI_VER} as aws-cli
+
 FROM rapidsai/miniforge-cuda:cuda${CUDA_VER}-base-${LINUX_VER}-py${PYTHON_VER}
 
 ARG TARGETPLATFORM
@@ -93,6 +100,7 @@ esac
 EOF
 
 # Install gha-tools
+ARG SCCACHE_VER
 RUN wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - \
   | tar -xz -C /usr/local/bin
 
@@ -105,21 +113,21 @@ rapids-mamba-retry install -y \
   gh \
   git \
   jq \
-  "sccache==0.7.6" \
+  "sccache==${SCCACHE_VER}" \
   "python=${PYTHON_VERSION}.*=*_cpython"
 conda clean -aipty
 EOF
 
 # Install codecov binary
+ARG CODECOV_VER
 RUN <<EOF
 case "${TARGETPLATFORM}" in
   "linux/amd64")
-    CODECOV_VERSION=v0.3.2
     curl https://uploader.codecov.io/verification.gpg --max-time 10 --retry 5 \
       | gpg --no-default-keyring --keyring trustedkeys.gpg --import
-    curl -Os --max-time 10 --retry 5 https://uploader.codecov.io/${CODECOV_VERSION}/linux/codecov
-    curl -Os --max-time 10 --retry 5 https://uploader.codecov.io/${CODECOV_VERSION}/linux/codecov.SHA256SUM
-    curl -Os --max-time 10 --retry 5 https://uploader.codecov.io/${CODECOV_VERSION}/linux/codecov.SHA256SUM.sig
+    curl -Os --max-time 10 --retry 5 https://uploader.codecov.io/v${CODECOV_VER}/linux/codecov
+    curl -Os --max-time 10 --retry 5 https://uploader.codecov.io/v${CODECOV_VER}/linux/codecov.SHA256SUM
+    curl -Os --max-time 10 --retry 5 https://uploader.codecov.io/v${CODECOV_VER}/linux/codecov.SHA256SUM.sig
     gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
     shasum -a 256 -c codecov.SHA256SUM
     chmod +x codecov
@@ -145,8 +153,8 @@ RUN /opt/conda/bin/git config --system --add safe.directory '*'
 RUN pip install dunamai "rapids-dependency-file-generator==1.*" \
     && pip cache purge
 
-COPY --from=mikefarah/yq:4.41.1 /usr/bin/yq /usr/local/bin/yq
-COPY --from=amazon/aws-cli /usr/local/aws-cli/ /usr/local/aws-cli/
-COPY --from=amazon/aws-cli /usr/local/bin/ /usr/local/bin/
+COPY --from=yq /usr/bin/yq /usr/local/bin/yq
+COPY --from=aws-cli /usr/local/aws-cli/ /usr/local/aws-cli/
+COPY --from=aws-cli /usr/local/bin/ /usr/local/bin/
 
 CMD ["/bin/bash"]

ci-wheel.Dockerfile

@@ -2,6 +2,10 @@ ARG CUDA_VER=notset
 ARG LINUX_VER=notset
 
 ARG BASE_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VER}-devel-${LINUX_VER}
+ARG AWS_CLI_VER
+
+FROM amazon/aws-cli:${AWS_CLI_VER} as aws-cli
+
 FROM ${BASE_IMAGE}
 
 ARG CUDA_VER
@@ -108,24 +112,24 @@ case "${LINUX_VER}" in
 esac
 EOF
 
-# Download and install GH CLI tool v2.32.0
-ARG GH_VERSION=2.32.0
+# Download and install GH CLI tool
+ARG GH_CLI_VER=notset
 RUN <<EOF
 set -e
-wget https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${CPU_ARCH}.tar.gz
+wget https://github.com/cli/cli/releases/download/v${GH_CLI_VER}/gh_${GH_CLI_VER}_linux_${CPU_ARCH}.tar.gz
 tar -xf gh_*.tar.gz
 mv gh_*/bin/gh /usr/local/bin
 rm -rf gh_*
 EOF
 
 # Install sccache
-ARG SCCACHE_VERSION=0.7.6
+ARG SCCACHE_VER=notset
 
 RUN <<EOF
 curl -o /tmp/sccache.tar.gz \
-  -L "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-"${REAL_ARCH}"-unknown-linux-musl.tar.gz"
+  -L "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VER}/sccache-v${SCCACHE_VER}-"${REAL_ARCH}"-unknown-linux-musl.tar.gz"
 tar -C /tmp -xvf /tmp/sccache.tar.gz
-mv "/tmp/sccache-v${SCCACHE_VERSION}-"${REAL_ARCH}"-unknown-linux-musl/sccache" /usr/bin/sccache
+mv "/tmp/sccache-v${SCCACHE_VER}-"${REAL_ARCH}"-unknown-linux-musl/sccache" /usr/bin/sccache
 chmod +x /usr/bin/sccache
 EOF
 
@@ -134,11 +138,11 @@ ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${REAL_ARCH} AUDITWHEEL_PLAT=${P
 
 
 # Install ucx
-ARG UCX_VERSION=1.14.1
+ARG UCX_VER=notset
 RUN <<EOF
 mkdir -p /ucx-src
 cd /ucx-src
-git clone https://github.com/openucx/ucx -b v${UCX_VERSION} ucx-git-repo
+git clone https://github.com/openucx/ucx -b v${UCX_VER} ucx-git-repo
 cd ucx-git-repo
 ./autogen.sh
 ./contrib/configure-release \
@@ -199,8 +203,8 @@ pip cache purge
 EOF
 
 # Install the AWS CLI
-COPY --from=amazon/aws-cli /usr/local/aws-cli/ /usr/local/aws-cli/
-COPY --from=amazon/aws-cli /usr/local/bin/ /usr/local/bin/
+COPY --from=aws-cli /usr/local/aws-cli/ /usr/local/aws-cli/
+COPY --from=aws-cli /usr/local/bin/ /usr/local/bin/
 
 # Mark all directories as safe for git so that GHA clones into the root don't
 # run into issues

ci/compute-build-args.sh

@@ -13,19 +13,22 @@ elif [[
   MANYLINUX_VER="manylinux_2_28"
 fi
 
-ARGS=(
-  # common args
-  "CUDA_VER=${CUDA_VER}"
-  "LINUX_VER=${LINUX_VER}"
-  "PYTHON_VER=${PYTHON_VER}"
-  # wheel args
-  "CPU_ARCH=${ARCH}"
-  "REAL_ARCH=$(arch)"
-  "MANYLINUX_VER=${MANYLINUX_VER}"
-)
+ARGS="
+CUDA_VER: ${CUDA_VER}
+LINUX_VER: ${LINUX_VER}
+PYTHON_VER: ${PYTHON_VER}
+CPU_ARCH: ${ARCH}
+REAL_ARCH: $(arch)
+MANYLINUX_VER: ${MANYLINUX_VER}
+"
+export ARGS
 
+if [ -n "${GITHUB_ACTIONS:-}" ]; then
 cat <<EOF > "${GITHUB_OUTPUT:-/dev/stdout}"
 ARGS<<EOT
-$(printf "%s\n" "${ARGS[@]}")
+$(yq -r '. + env(ARGS) | to_entries | map(.key + "=" + .value) | join(" \n")' versions.yaml)
 EOT
 EOF
+else
+  yq -r '. + env(ARGS) | to_entries | map("--build-arg " + .key + "=" + .value) | join(" ")' versions.yaml
+fi

citestwheel.Dockerfile

@@ -2,6 +2,10 @@ ARG CUDA_VER=notset
 ARG LINUX_VER=notset
 
 ARG BASE_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VER}-devel-${LINUX_VER}
+ARG AWS_CLI_VER
+
+FROM amazon/aws-cli:${AWS_CLI_VER} as aws-cli
+
 FROM ${BASE_IMAGE}
 
 ARG CUDA_VER
@@ -52,8 +56,8 @@ ENV PATH="/pyenv/versions/${PYTHON_VER}/bin/:$PATH"
 
 # Install the AWS CLI
 # Needed to download wheels for running tests
-COPY --from=amazon/aws-cli /usr/local/aws-cli/ /usr/local/aws-cli/
-COPY --from=amazon/aws-cli /usr/local/bin/ /usr/local/bin/
+COPY --from=aws-cli /usr/local/aws-cli/ /usr/local/aws-cli/
+COPY --from=aws-cli /usr/local/bin/ /usr/local/bin/
 
 # Install latest gha-tools
 RUN wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - \

latest.yaml

@@ -1,9 +1,9 @@
 # Define the values used for the "latest" tag
 conda:
   CUDA_VER: "12.2.2"
-  PYTHON_VER: "3.10"
+  PYTHON_VER: "3.11"
   LINUX_VER: "ubuntu22.04"
 wheels:
   CUDA_VER: "12.2.2"
-  PYTHON_VER: "3.10"
+  PYTHON_VER: "3.11"
   LINUX_VER: "ubuntu20.04"

renovate.json

@@ -2,5 +2,16 @@
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
     "config:base"
+  ],
+  "regexManagers": [
+    {
+      "fileMatch": "versions.yaml",
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>[^ ]+) depName=(?<depName>[^\\s]+)(?:\\s+versioning=(?<versioning>[^\\n]+))?\\n(?<depNameCapture>[^:]+): (?<currentValue>[^\\n]+)"
+      ],
+      "depNameTemplate": "{{depName}}",
+      "datasourceTemplate": "{{datasource}}",
+      "versioningTemplate": "{{#if versioning}}{{versioning}}{{else}}semver{{/if}}"
+    }
   ]
 }

versions.yaml

@@ -0,0 +1,12 @@
+# renovate: datasource=github-releases depName=mozilla/sccache
+SCCACHE_VER: 0.7.6
+# renovate: datasource=github-releases depName=cli/cli
+GH_CLI_VER: 2.45.0
+# renovate: datasource=github-releases depName=codecov/uploader
+CODECOV_VER: 0.3.2
+# renovate: datasource=docker depName=mikefarah/yq versioning=docker
+YQ_VER: 4.42.1
+# renovate: datasource=github-releases depName=openucx/ucx
+UCX_VER: 1.14.1
+# renovate: datasource=docker depName=amazon/aws-cli versioning=docker
+AWS_CLI_VER: 2.15.25