From a652c011eeac4fcdf3d2a50d287cb1877eddc84e Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 5 Oct 2023 15:18:10 -0400 Subject: [PATCH 01/14] Refactor `git` upgrade for `citestwheel` (#84) Co-authored-by: Ray Douglass Co-authored-by: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Co-authored-by: Bradley Dice --- ci-conda.Dockerfile | 3 ++- ci-wheel.Dockerfile | 29 +++++++++++++++++++++++++---- citestwheel.Dockerfile | 28 +++++++++++++++++----------- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/ci-conda.Dockerfile b/ci-conda.Dockerfile index ff9d9e3..a8d861b 100644 --- a/ci-conda.Dockerfile +++ b/ci-conda.Dockerfile @@ -29,7 +29,8 @@ RUN \ PKG_CUDA_VER="$(echo ${CUDA_VER} | cut -d '.' -f1,2 | tr '.' '-')"; \ case "${LINUX_VER}" in \ "ubuntu"*) \ - apt-get update \ + echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors \ + && apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ cuda-gdb-${PKG_CUDA_VER} \ diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index b276e76..848bae5 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -27,13 +27,34 @@ ENV PATH="/pyenv/bin:/pyenv/shims:$PATH" RUN case "${LINUX_VER}" in \ "ubuntu"*) \ - apt update -y && apt install -y jq build-essential software-properties-common wget gcc zlib1g-dev libbz2-dev libssl-dev libreadline-dev libsqlite3-dev libffi-dev curl git libncurses5-dev libnuma-dev openssh-client libcudnn8-dev zip libopenblas-dev liblapack-dev protobuf-compiler autoconf automake libtool cmake && rm -rf /var/lib/apt/lists/* \ - && add-apt-repository ppa:git-core/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test && apt update -y && apt install -y git gcc-9 g++-9 && add-apt-repository -r ppa:git-core/ppa && add-apt-repository -r ppa:ubuntu-toolchain-r/test \ + echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors \ + && apt update -y \ + && apt install -y \ + jq build-essential software-properties-common wget gcc zlib1g-dev libbz2-dev \ + libssl-dev libreadline-dev libsqlite3-dev libffi-dev curl git libncurses5-dev \ + libnuma-dev openssh-client libcudnn8-dev zip libopenblas-dev liblapack-dev \ + protobuf-compiler autoconf automake libtool cmake \ + && add-apt-repository ppa:git-core/ppa \ + && add-apt-repository ppa:ubuntu-toolchain-r/test \ + && apt update -y \ + && apt install -y git gcc-9 g++-9 \ + && add-apt-repository -r ppa:git-core/ppa \ + && add-apt-repository -r ppa:ubuntu-toolchain-r/test \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 --slave /usr/bin/g++ g++ /usr/bin/g++-9 --slave /usr/bin/gcov gcov /usr/bin/gcov-9 \ + && rm -rf /var/lib/apt/lists/* \ ;; \ "centos"*) \ - yum update --exclude=libnccl* -y && yum install -y epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl numactl-devel openssh-clients libcudnn8-devel zip blas-devel lapack-devel protobuf-compiler autoconf automake libtool centos-release-scl scl-utils cmake && yum clean all \ - && yum remove -y git && yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm && yum install -y git jq devtoolset-11 && yum remove -y endpoint-repo \ + yum update --exclude=libnccl* -y \ + && yum install -y \ + epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ + sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \ + numactl-devel openssh-clients libcudnn8-devel zip blas-devel lapack-devel \ + protobuf-compiler autoconf automake libtool centos-release-scl scl-utils cmake \ + && yum remove -y git \ + && yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm \ + && yum install -y git jq devtoolset-11 \ + && yum remove -y endpoint-repo \ + && yum clean all \ && echo -e ' \ #!/bin/bash\n \ source scl_source enable devtoolset-11\n \ diff --git a/citestwheel.Dockerfile b/citestwheel.Dockerfile index b14a703..061b5a4 100644 --- a/citestwheel.Dockerfile +++ b/citestwheel.Dockerfile @@ -19,14 +19,23 @@ ARG DEBIAN_FRONTEND=noninteractive ENV PYENV_ROOT="/pyenv" ENV PATH="/pyenv/bin:/pyenv/shims:$PATH" -RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends \ - wget curl git jq ssh \ - make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget \ - curl llvm libncursesw5-dev xz-utils tk-dev unzip \ - libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev +RUN < /etc/apt/apt.conf.d/warnings-as-errors +apt-get update +apt-get install -y software-properties-common +# update git > 2.17 +add-apt-repository ppa:git-core/ppa -y +apt-get update +apt-get upgrade -y +apt-get install -y --no-install-recommends \ + wget curl git jq ssh \ + make build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev wget \ + curl llvm libncursesw5-dev xz-utils tk-dev unzip \ + libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev +rm -rf /var/cache/apt/archives /var/lib/apt/lists/* +EOF # Install pyenv RUN curl https://pyenv.run | bash @@ -49,9 +58,6 @@ RUN mkdir -p /aws_install && cd /aws_install && \ cd / && \ rm -rf /aws_install -# update git > 2.17 -RUN grep '18.04' /etc/issue && bash -c "apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa -y && apt-get update && apt-get install --upgrade -y git" || true; - # Install latest gha-tools RUN wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - \ | tar -xz -C /usr/local/bin From ca7457b543b7921c912913a6469f3b2c65029eab Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 9 Oct 2023 13:33:54 -0500 Subject: [PATCH 02/14] Use yaml instead of yml. (#79) --- ...ild-and-publish-images.yml => build-and-publish-images.yaml} | 2 +- .github/workflows/{build-image.yml => build-image.yaml} | 0 .github/workflows/{pr.yml => pr.yaml} | 2 +- .github/workflows/{push.yml => push.yaml} | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{build-and-publish-images.yml => build-and-publish-images.yaml} (98%) rename .github/workflows/{build-image.yml => build-image.yaml} (100%) rename .github/workflows/{pr.yml => pr.yaml} (80%) rename .github/workflows/{push.yml => push.yaml} (89%) diff --git a/.github/workflows/build-and-publish-images.yml b/.github/workflows/build-and-publish-images.yaml similarity index 98% rename from .github/workflows/build-and-publish-images.yml rename to .github/workflows/build-and-publish-images.yaml index 854a9b9..3632abd 100644 --- a/.github/workflows/build-and-publish-images.yml +++ b/.github/workflows/build-and-publish-images.yaml @@ -32,7 +32,7 @@ jobs: matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }} fail-fast: false secrets: inherit - uses: ./.github/workflows/build-image.yml + uses: ./.github/workflows/build-image.yaml with: ARCHES: ${{ toJSON(matrix.ARCHES) }} CUDA_VER: ${{ matrix.CUDA_VER }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yaml similarity index 100% rename from .github/workflows/build-image.yml rename to .github/workflows/build-image.yaml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yaml similarity index 80% rename from .github/workflows/pr.yml rename to .github/workflows/pr.yaml index 82ffc77..d7a0afe 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yaml @@ -11,7 +11,7 @@ concurrency: jobs: build-images: - uses: ./.github/workflows/build-and-publish-images.yml + uses: ./.github/workflows/build-and-publish-images.yaml with: build_type: pull-request secrets: inherit diff --git a/.github/workflows/push.yml b/.github/workflows/push.yaml similarity index 89% rename from .github/workflows/push.yml rename to .github/workflows/push.yaml index ffb858b..10f2ff8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yaml @@ -20,7 +20,7 @@ concurrency: jobs: build-images: - uses: ./.github/workflows/build-and-publish-images.yml + uses: ./.github/workflows/build-and-publish-images.yaml with: build_type: branch secrets: inherit From b4dcc6813e3280a45566e596f3cb7225effb128d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 16 Oct 2023 06:24:23 -0700 Subject: [PATCH 03/14] Only install CUDA packages on CUDA 11. (#77) Co-authored-by: jakirkham --- ci-conda.Dockerfile | 61 ++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/ci-conda.Dockerfile b/ci-conda.Dockerfile index a8d861b..bf7217b 100644 --- a/ci-conda.Dockerfile +++ b/ci-conda.Dockerfile @@ -26,42 +26,25 @@ ENV SCCACHE_S3_NO_CREDENTIALS=false # Install system packages depending on the LINUX_VER RUN \ - PKG_CUDA_VER="$(echo ${CUDA_VER} | cut -d '.' -f1,2 | tr '.' '-')"; \ case "${LINUX_VER}" in \ "ubuntu"*) \ echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors \ && apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ - cuda-gdb-${PKG_CUDA_VER} \ - cuda-cudart-dev-${PKG_CUDA_VER} \ - cuda-cupti-dev-${PKG_CUDA_VER} \ file \ unzip \ wget \ - # ignore the build-essential package since it installs dependencies like gcc/g++ - # we don't need them since we use conda compilers, so this keeps our images smaller - && apt-get download cuda-nvcc-${PKG_CUDA_VER} \ - && dpkg -i --ignore-depends="build-essential" ./cuda-nvcc-*.deb \ - && rm ./cuda-nvcc-*.deb \ - # apt will not work correctly if it thinks it needs the build-essential dependency - # so we patch it out with a sed command - && sed -i 's/, build-essential//g' /var/lib/dpkg/status \ && rm -rf "/var/lib/apt/lists/*"; \ ;; \ "centos"* | "rockylinux"*) \ yum -y update \ && yum -y install --setopt=install_weak_deps=False \ - cuda-cudart-devel-${PKG_CUDA_VER} \ - cuda-driver-devel-${PKG_CUDA_VER} \ - cuda-gdb-${PKG_CUDA_VER} \ - cuda-cupti-${PKG_CUDA_VER} \ file \ unzip \ wget \ which \ yum-utils \ - && rpm -Uvh --nodeps $(repoquery --location cuda-nvcc-${PKG_CUDA_VER}) \ && yum clean all; \ ;; \ *) \ @@ -69,6 +52,50 @@ RUN \ ;; \ esac +# Install CUDA packages, only for CUDA 11 (CUDA 12+ should fetch from conda) +RUN \ + case "${CUDA_VER}" in \ + "11"*) \ + PKG_CUDA_VER="$(echo ${CUDA_VER} | cut -d '.' -f1,2 | tr '.' '-')"; \ + echo "Attempting to install CUDA Toolkit ${PKG_CUDA_VER}"; \ + case "${LINUX_VER}" in \ + "ubuntu"*) \ + apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + cuda-gdb-${PKG_CUDA_VER} \ + cuda-cudart-dev-${PKG_CUDA_VER} \ + cuda-cupti-dev-${PKG_CUDA_VER} \ + # ignore the build-essential package since it installs dependencies like gcc/g++ + # we don't need them since we use conda compilers, so this keeps our images smaller + && apt-get download cuda-nvcc-${PKG_CUDA_VER} \ + && dpkg -i --ignore-depends="build-essential" ./cuda-nvcc-*.deb \ + && rm ./cuda-nvcc-*.deb \ + # apt will not work correctly if it thinks it needs the build-essential dependency + # so we patch it out with a sed command + && sed -i 's/, build-essential//g' /var/lib/dpkg/status \ + && rm -rf "/var/lib/apt/lists/*"; \ + ;; \ + "centos"* | "rockylinux"*) \ + yum -y update \ + && yum -y install --setopt=install_weak_deps=False \ + cuda-cudart-devel-${PKG_CUDA_VER} \ + cuda-driver-devel-${PKG_CUDA_VER} \ + cuda-gdb-${PKG_CUDA_VER} \ + cuda-cupti-${PKG_CUDA_VER} \ + && rpm -Uvh --nodeps $(repoquery --location cuda-nvcc-${PKG_CUDA_VER}) \ + && yum clean all; \ + ;; \ + *) \ + echo "Unsupported LINUX_VER: ${LINUX_VER}" && exit 1; \ + ;; \ + esac \ + ;; \ + *) \ + echo "Skipping CUDA Toolkit installation for CUDA ${CUDA_VER}"; \ + ;; \ + esac + # Install gha-tools RUN wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - \ | tar -xz -C /usr/local/bin From d28a59a46c17d50993e70b758aae34f0dd1f37bb Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:26:09 -0400 Subject: [PATCH 04/14] Use nvcr.io for parent images (#85) --- ci-wheel.Dockerfile | 2 +- citestwheel.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index 848bae5..d21e448 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -2,7 +2,7 @@ ARG CUDA_VER=11.8.0 ARG LINUX_VER=ubuntu20.04 ARG REAL_ARCH=x86_64 -ARG BASE_IMAGE=nvidia/cuda:${CUDA_VER}-devel-${LINUX_VER} +ARG BASE_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VER}-devel-${LINUX_VER} FROM ${BASE_IMAGE} ARG CUDA_VER diff --git a/citestwheel.Dockerfile b/citestwheel.Dockerfile index 061b5a4..cb0e586 100644 --- a/citestwheel.Dockerfile +++ b/citestwheel.Dockerfile @@ -1,7 +1,7 @@ ARG CUDA_VER=11.8.0 ARG LINUX_VER=ubuntu18.04 -ARG BASE_IMAGE=nvidia/cuda:${CUDA_VER}-devel-${LINUX_VER} +ARG BASE_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VER}-devel-${LINUX_VER} FROM ${BASE_IMAGE} ARG CUDA_VER From 3da9927b25be2968830817a8c1df5fd01ebc00ff Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:03:29 -0500 Subject: [PATCH 05/14] Update mambaforge to miniforge (#83) --- ci-conda.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-conda.Dockerfile b/ci-conda.Dockerfile index bf7217b..ef5dfb8 100644 --- a/ci-conda.Dockerfile +++ b/ci-conda.Dockerfile @@ -1,7 +1,7 @@ ARG CUDA_VER=11.8.0 ARG LINUX_VER=ubuntu22.04 ARG PYTHON_VER=3.10 -FROM rapidsai/mambaforge-cuda:cuda${CUDA_VER}-base-${LINUX_VER}-py${PYTHON_VER} +FROM rapidsai/miniforge-cuda:cuda${CUDA_VER}-base-${LINUX_VER}-py${PYTHON_VER} ARG TARGETPLATFORM ARG CUDA_VER From b55a7497a0a3596c9815341a8a2e183314d10b71 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:40:49 -0400 Subject: [PATCH 06/14] Update mikefarah/yq Docker tag to v4.35.2 (#75) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- ci-conda.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-conda.Dockerfile b/ci-conda.Dockerfile index ef5dfb8..4a969e3 100644 --- a/ci-conda.Dockerfile +++ b/ci-conda.Dockerfile @@ -147,6 +147,6 @@ RUN /opt/conda/bin/git config --system --add safe.directory '*' RUN pip install dunamai "rapids-dependency-file-generator==1.*" \ && pip cache purge -COPY --from=mikefarah/yq:4.35.1 /usr/bin/yq /usr/local/bin/yq +COPY --from=mikefarah/yq:4.35.2 /usr/bin/yq /usr/local/bin/yq CMD ["/bin/bash"] From b058f3bfa449662fdda8237e273085346b5af9da Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:41:11 -0400 Subject: [PATCH 07/14] Update docker/login-action action to v3 (#70) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build-and-publish-images.yaml | 2 +- .github/workflows/build-image.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-publish-images.yaml b/.github/workflows/build-and-publish-images.yaml index 3632abd..b68216a 100644 --- a/.github/workflows/build-and-publish-images.yaml +++ b/.github/workflows/build-and-publish-images.yaml @@ -50,7 +50,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 9825626..eab20bc 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -37,7 +37,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} From de4800a495394d9bceb470df62aec6b547141d36 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 27 Oct 2023 17:03:40 -0400 Subject: [PATCH 08/14] Update docker/setup-buildx-action action to v3 (#71) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index eab20bc..59cc7f3 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -45,7 +45,7 @@ jobs: run: | docker context create builders - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver: docker endpoint: builders From b0053687f13701d5e9a73283bb6bd5ac7d906a38 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 27 Oct 2023 17:18:26 -0400 Subject: [PATCH 09/14] Update docker/build-push-action action to v5 (#69) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index 59cc7f3..dc7e632 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -58,7 +58,7 @@ jobs: PYTHON_VER: ${{ inputs.PYTHON_VER }} ARCH: ${{ matrix.ARCH }} - name: Build image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: context file: ${{ inputs.IMAGE_REPO }}.Dockerfile From 0b7347917614d7c058fafd5069e3ceb520f409aa Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Fri, 27 Oct 2023 16:46:36 -0500 Subject: [PATCH 10/14] [DO NOT MERGE] Build ciwheel on rockylinux8 (#87) --- ci-wheel.Dockerfile | 27 +++++++++++++++++++++++++++ matrix.yaml | 4 +--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index d21e448..057f535 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -68,6 +68,30 @@ RUN case "${LINUX_VER}" in \ && make install \ && popd \ ;; \ + "rockylinux"*) \ + dnf update -y \ + && dnf install -y \ + epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ + sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \ + numactl-devel openssh-clients libcudnn8-devel zip \ + protobuf-compiler autoconf automake libtool dnf-plugins-core cmake \ + && dnf config-manager --set-enabled powertools \ + && dnf install -y blas-devel lapack-devel \ + && dnf -y install gcc-toolset-11-gcc gcc-toolset-11-gcc-c++ \ + && dnf clean all \ + && echo -e ' \ + #!/bin/bash\n \ + source /opt/rh/gcc-toolset-11/enable \ + ' > /etc/profile.d/enable_devtools.sh \ + && pushd tmp \ + && wget https://ftp.openssl.org/source/openssl-1.1.1k.tar.gz \ + && tar -xzvf openssl-1.1.1k.tar.gz \ + && cd openssl-1.1.1k \ + && ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic \ + && make \ + && make install \ + && popd \ + ;; \ *) \ echo "Unsupported LINUX_VER: ${LINUX_VER}" && exit 1; \ ;; \ @@ -141,6 +165,9 @@ RUN case "${LINUX_VER}" in \ # Need to specify the openssl location because of the install from source CPPFLAGS="-I/usr/include/openssl" LDFLAGS="-L/usr/lib" pyenv install --verbose "${RAPIDS_PY_VERSION}" \ ;; \ + "rockylinux"*) \ + CPPFLAGS="-I/usr/include/openssl" LDFLAGS="-L/usr/lib" pyenv install --verbose "${RAPIDS_PY_VERSION}" \ + ;; \ *) \ echo "Unsupported LINUX_VER: ${LINUX_VER}" && exit 1; \ ;; \ diff --git a/matrix.yaml b/matrix.yaml index a43e67c..5774b83 100644 --- a/matrix.yaml +++ b/matrix.yaml @@ -51,11 +51,9 @@ exclude: - CUDA_VER: "12.1.1" IMAGE_REPO: "ci-wheel" - # exclude citestwheel and ci-wheel for rockylinux8 + # exclude citestwheel for rockylinux8 - LINUX_VER: "rockylinux8" IMAGE_REPO: "citestwheel" - - LINUX_VER: "rockylinux8" - IMAGE_REPO: "ci-wheel" # exclude citestwheel and ci-wheel for ubuntu22.04 - LINUX_VER: "ubuntu22.04" From bf3858fceb896d29e33906a405916e18eae0678d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sun, 29 Oct 2023 16:27:03 -0700 Subject: [PATCH 11/14] Edits to Rockylinux ci-wheel images (#88) --- ci-wheel.Dockerfile | 2 +- ci/compute-build-args.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index 057f535..5c5af5e 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -73,7 +73,7 @@ RUN case "${LINUX_VER}" in \ && dnf install -y \ epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \ - numactl-devel openssh-clients libcudnn8-devel zip \ + numactl-devel openssh-clients libcudnn8-devel zip jq \ protobuf-compiler autoconf automake libtool dnf-plugins-core cmake \ && dnf config-manager --set-enabled powertools \ && dnf install -y blas-devel lapack-devel \ diff --git a/ci/compute-build-args.sh b/ci/compute-build-args.sh index 49905a4..75ca6d6 100755 --- a/ci/compute-build-args.sh +++ b/ci/compute-build-args.sh @@ -7,6 +7,10 @@ if [[ "${LINUX_VER}" == "ubuntu20.04" ]]; then MANYLINUX_VER="manylinux_2_31" +elif [[ + "${LINUX_VER}" == "rockylinux8" +]]; then + MANYLINUX_VER="manylinux_2_28" fi ARGS=( From 4be8b788efba965820392187483c1736a4168c3d Mon Sep 17 00:00:00 2001 From: jakirkham Date: Mon, 30 Oct 2023 13:01:18 -0700 Subject: [PATCH 12/14] Add `which` to `ci-wheel` (#89) --- ci-wheel.Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index 5c5af5e..0b900e1 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -30,7 +30,8 @@ RUN case "${LINUX_VER}" in \ echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors \ && apt update -y \ && apt install -y \ - jq build-essential software-properties-common wget gcc zlib1g-dev libbz2-dev \ + debianutils build-essential software-properties-common \ + jq wget gcc zlib1g-dev libbz2-dev \ libssl-dev libreadline-dev libsqlite3-dev libffi-dev curl git libncurses5-dev \ libnuma-dev openssh-client libcudnn8-dev zip libopenblas-dev liblapack-dev \ protobuf-compiler autoconf automake libtool cmake \ @@ -46,7 +47,7 @@ RUN case "${LINUX_VER}" in \ "centos"*) \ yum update --exclude=libnccl* -y \ && yum install -y \ - epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ + epel-release which wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \ numactl-devel openssh-clients libcudnn8-devel zip blas-devel lapack-devel \ protobuf-compiler autoconf automake libtool centos-release-scl scl-utils cmake \ @@ -71,7 +72,7 @@ RUN case "${LINUX_VER}" in \ "rockylinux"*) \ dnf update -y \ && dnf install -y \ - epel-release wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ + epel-release which wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \ numactl-devel openssh-clients libcudnn8-devel zip jq \ protobuf-compiler autoconf automake libtool dnf-plugins-core cmake \ From bf7d540855e7e94695a54e40080c3acb8681764a Mon Sep 17 00:00:00 2001 From: jakirkham Date: Fri, 3 Nov 2023 15:37:57 -0700 Subject: [PATCH 13/14] Add `yasm` & `openslide` for cuCIM wheel builds (#91) Co-authored-by: Gregory Lee --- ci-wheel.Dockerfile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index 0b900e1..4f16891 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -34,7 +34,7 @@ RUN case "${LINUX_VER}" in \ jq wget gcc zlib1g-dev libbz2-dev \ libssl-dev libreadline-dev libsqlite3-dev libffi-dev curl git libncurses5-dev \ libnuma-dev openssh-client libcudnn8-dev zip libopenblas-dev liblapack-dev \ - protobuf-compiler autoconf automake libtool cmake \ + protobuf-compiler autoconf automake libtool cmake yasm libopenslide-dev \ && add-apt-repository ppa:git-core/ppa \ && add-apt-repository ppa:ubuntu-toolchain-r/test \ && apt update -y \ @@ -46,11 +46,14 @@ RUN case "${LINUX_VER}" in \ ;; \ "centos"*) \ yum update --exclude=libnccl* -y \ + && yum install -y epel-release\ + && yum update --exclude=libnccl* -y \ && yum install -y \ - epel-release which wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ + which wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \ numactl-devel openssh-clients libcudnn8-devel zip blas-devel lapack-devel \ protobuf-compiler autoconf automake libtool centos-release-scl scl-utils cmake \ + yasm openslide-devel \ && yum remove -y git \ && yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm \ && yum install -y git jq devtoolset-11 \ @@ -71,14 +74,17 @@ RUN case "${LINUX_VER}" in \ ;; \ "rockylinux"*) \ dnf update -y \ + && dnf install -y epel-release \ + && dnf update -y \ && dnf install -y \ - epel-release which wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ + which wget gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite \ sqlite-devel xz xz-devel libffi-devel curl git ncurses-devel numactl \ - numactl-devel openssh-clients libcudnn8-devel zip jq \ + numactl-devel openssh-clients libcudnn8-devel zip jq openslide-devel \ protobuf-compiler autoconf automake libtool dnf-plugins-core cmake \ && dnf config-manager --set-enabled powertools \ && dnf install -y blas-devel lapack-devel \ && dnf -y install gcc-toolset-11-gcc gcc-toolset-11-gcc-c++ \ + && dnf -y install yasm \ && dnf clean all \ && echo -e ' \ #!/bin/bash\n \ From 3761deca99c7f78928ac4cef675c23a3b1675cc7 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:30:16 -0600 Subject: [PATCH 14/14] Switch to anaconda index (#92) --- ci-wheel.Dockerfile | 6 +++++- citestwheel.Dockerfile | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci-wheel.Dockerfile b/ci-wheel.Dockerfile index 4f16891..8c6b6b0 100644 --- a/ci-wheel.Dockerfile +++ b/ci-wheel.Dockerfile @@ -20,7 +20,7 @@ ENV RAPIDS_CUDA_VERSION="${CUDA_VER}" ENV RAPIDS_PY_VERSION="${PYTHON_VER}" # RAPIDS pip index -ENV PIP_EXTRA_INDEX_URL="https://pypi.k8s.rapids.ai/simple" +ENV PIP_EXTRA_INDEX_URL="https://pypi.anaconda.org/rapidsai-wheels-nightly/simple" ENV PYENV_ROOT="/pyenv" ENV PATH="/pyenv/bin:/pyenv/shims:$PATH" @@ -185,6 +185,10 @@ RUN pyenv global ${PYTHON_VER} && python -m pip install auditwheel patchelf twin # Install latest gha-tools RUN wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin +# Install anaconda-client +RUN pip install git+https://github.com/Anaconda-Platform/anaconda-client && \ + pip cache purge + # Install the AWS CLI RUN mkdir -p /aws_install && cd /aws_install && \ curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" && \ diff --git a/citestwheel.Dockerfile b/citestwheel.Dockerfile index cb0e586..beee144 100644 --- a/citestwheel.Dockerfile +++ b/citestwheel.Dockerfile @@ -12,7 +12,7 @@ ENV RAPIDS_CUDA_VERSION="${CUDA_VER}" ENV RAPIDS_PY_VERSION="${PYTHON_VER}" # RAPIDS pip index -ENV PIP_EXTRA_INDEX_URL="https://pypi.k8s.rapids.ai/simple" +ENV PIP_EXTRA_INDEX_URL="https://pypi.anaconda.org/rapidsai-wheels-nightly/simple" ARG DEBIAN_FRONTEND=noninteractive