diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.10.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.10.____cpython.yaml similarity index 87% rename from .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.10.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.10.____cpython.yaml index e2f780e7..bdae2c04 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.10.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.10.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -15,7 +15,7 @@ channel_targets: cuda_compiler: - cuda-nvcc cuda_compiler_version: -- '12.0' +- '12.6' cudnn: - '9' cxx_compiler: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '12' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.22' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.11.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.11.____cpython.yaml similarity index 87% rename from .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.11.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.11.____cpython.yaml index a5c49903..f2ddac7b 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.11.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.11.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -15,7 +15,7 @@ channel_targets: cuda_compiler: - cuda-nvcc cuda_compiler_version: -- '12.0' +- '12.6' cudnn: - '9' cxx_compiler: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '12' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.23' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.12.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.12.____cpython.yaml similarity index 87% rename from .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.12.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.12.____cpython.yaml index 687f4339..ffa9f2e8 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.12.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.12.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -15,7 +15,7 @@ channel_targets: cuda_compiler: - cuda-nvcc cuda_compiler_version: -- '12.0' +- '12.6' cudnn: - '9' cxx_compiler: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '12' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.26' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.9.____cpython.yaml similarity index 87% rename from .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.9.____cpython.yaml index 519ac3da..12bbb5ac 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.9.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -15,7 +15,7 @@ channel_targets: cuda_compiler: - cuda-nvcc cuda_compiler_version: -- '12.0' +- '12.6' cudnn: - '9' cxx_compiler: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '12' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.22' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.10.____cpython.yaml b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.10.____cpython.yaml index ece2ee59..81a48556 100644 --- a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.10.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.10.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '13' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.22' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.11.____cpython.yaml b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.11.____cpython.yaml index 53d5ba0e..e2c5fcad 100644 --- a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.11.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.11.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '13' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.23' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.12.____cpython.yaml b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.12.____cpython.yaml index 2ea107be..e7cb9613 100644 --- a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.12.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.12.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '13' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.26' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.9.____cpython.yaml index 00d5b9a3..d7716436 100644 --- a/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.9.____cpython.yaml @@ -7,7 +7,7 @@ c_stdlib: c_stdlib_version: - '2.17' cdt_name: -- cos7 +- conda channel_sources: - conda-forge channel_targets: @@ -23,13 +23,13 @@ cxx_compiler: cxx_compiler_version: - '13' docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 +- quay.io/condaforge/linux-anvil-x86_64:alma9 flatbuffers: - 24.3.25 giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -47,7 +47,7 @@ libprotobuf: nccl: - '2' numpy: -- '1.22' +- '2.0' openssl: - '3' perl: @@ -71,8 +71,6 @@ tensorflow: zip_keys: - - c_compiler_version - cxx_compiler_version - - c_stdlib_version - - cdt_name - cuda_compiler - cuda_compiler_version - docker_image diff --git a/.ci_support/migrations/cuda120.yaml b/.ci_support/migrations/cuda120.yaml deleted file mode 100644 index abfb9cb8..00000000 --- a/.ci_support/migrations/cuda120.yaml +++ /dev/null @@ -1,90 +0,0 @@ -migrator_ts: 1682985063 -__migrator: - kind: - version - migration_number: - 3 - build_number: - 1 - paused: false - override_cbc_keys: - - cuda_compiler_stub - operation: key_add - check_solvable: false - primary_key: cuda_compiler_version - ordering: - cxx_compiler_version: - - 9 - - 8 - - 7 - c_compiler_version: - - 9 - - 8 - - 7 - fortran_compiler_version: - - 9 - - 8 - - 7 - docker_image: - - quay.io/condaforge/linux-anvil-comp7 # [os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-aarch64 # [os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] - - quay.io/condaforge/linux-anvil-ppc64le # [os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] - - quay.io/condaforge/linux-anvil-armv7l # [os.environ.get("BUILD_PLATFORM") == "linux-armv7l"] - - quay.io/condaforge/linux-anvil-cuda:9.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cuda:10.0 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cuda:10.1 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cuda:10.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cuda:11.0 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cuda:11.1 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cuda:11.2 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - # case: native compilation (build == target) - - quay.io/condaforge/linux-anvil-ppc64le-cuda:11.2 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] - - quay.io/condaforge/linux-anvil-aarch64-cuda:11.2 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] - # case: cross-compilation (build != target) - - quay.io/condaforge/linux-anvil-cuda:11.2 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cuda:11.2 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - # case: non-CUDA builds - - quay.io/condaforge/linux-anvil-cos7-x86_64 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] - cuda_compiler_version: - - None - - 10.2 # [(linux64 or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 11.0 # [(linux64 or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 11.1 # [(linux64 or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 11.2 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 12.0 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - commit_message: | - Rebuild for CUDA 12 w/arch + Windows support - - The transition to CUDA 12 SDK includes new packages for all CUDA libraries and - build tools. Notably, the cudatoolkit package no longer exists, and packages - should depend directly on the specific CUDA libraries (libcublas, libcusolver, - etc) as needed. For an in-depth overview of the changes and to report problems - [see this issue]( https://github.com/conda-forge/conda-forge.github.io/issues/1963 ). - Please feel free to raise any issues encountered there. Thank you! :pray: - -cuda_compiler: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - cuda-nvcc # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - -cuda_compiler_version: # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 12.0 # [(linux or win64) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - -c_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 12 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - -cxx_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 12 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - -fortran_compiler_version: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - 12 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - -cdt_name: # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - cos7 # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - -docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux-") and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - - quay.io/condaforge/linux-anvil-cos7-x86_64 # [linux64 and os.environ.get("BUILD_PLATFORM") == "linux-64" and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] - # case: native compilation (build == target) - - quay.io/condaforge/linux-anvil-ppc64le # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] - - quay.io/condaforge/linux-anvil-aarch64 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] - # case: cross-compilation (build != target) - - quay.io/condaforge/linux-anvil-cos7-x86_64 # [ppc64le and os.environ.get("BUILD_PLATFORM") == "linux-64"] - - quay.io/condaforge/linux-anvil-cos7-x86_64 # [aarch64 and os.environ.get("BUILD_PLATFORM") == "linux-64"] diff --git a/.ci_support/migrations/cudnn9.yaml b/.ci_support/migrations/cudnn9.yaml deleted file mode 100644 index 30ba547e..00000000 --- a/.ci_support/migrations/cudnn9.yaml +++ /dev/null @@ -1,8 +0,0 @@ -__migrator: - build_number: 1 - commit_message: Rebuild for cudnn 9 - kind: version - migration_number: 1 -cudnn: -- '9' -migrator_ts: 1724175586.3831317 diff --git a/.ci_support/migrations/libabseil20240722_grpc165_libprotobuf5275.yaml b/.ci_support/migrations/libabseil20240722_grpc165_libprotobuf5275.yaml deleted file mode 100644 index 85540a1f..00000000 --- a/.ci_support/migrations/libabseil20240722_grpc165_libprotobuf5275.yaml +++ /dev/null @@ -1,18 +0,0 @@ -__migrator: - build_number: 1 - commit_message: Rebuild for libabseil 20240722, libgrp 1.65 & libprotobuf 5.27.5 - kind: version - migration_number: 1 - exclude: - - abseil-cpp - - grpc-cpp - - libprotobuf - - protobuf - - re2 -libabseil: -- 20240722 -libgrpc: -- "1.65" -libprotobuf: -- 5.27.5 -migrator_ts: 1727040240.8650293 diff --git a/.ci_support/migrations/numpy2.yaml b/.ci_support/migrations/numpy2.yaml new file mode 100644 index 00000000..fd981b85 --- /dev/null +++ b/.ci_support/migrations/numpy2.yaml @@ -0,0 +1,52 @@ +__migrator: + build_number: 1 + kind: version + commit_message: | + Rebuild for numpy 2.0 + + TL;DR: The way we build against numpy has changed as of numpy 2.0. This bot + PR has updated the recipe to account for the changes (see below for details). + + The biggest change is that we no longer need to use the oldest available numpy + version at build time in order to support old numpy version at runtime - numpy + will by default use a compatible ABI for the oldest still-supported numpy versions. + + Additionally, we no longer need to use `{{ pin_compatible("numpy") }}` as a + run requirement - this has been handled for more than two years now by a + run-export on the numpy package itself. The migrator will therefore remove + any occurrences of this. + + However, you will still need to add the lower bound for the numpy version, + in line with what the upstream package requires. The default lower bound from + the run-export is `>=1.19`; if your package needs a newer version than that, + please add `numpy >=x.y` under `run:`. + + Finally, by default, building against numpy 2.0 will assume that the package + is compatible with numpy 2.0, which is not necessarily the case. You should + check that the upstream package explicitly supports numpy 2.0, otherwise you + need to add a `- numpy <2.0dev0` run requirement until that happens (check numpy + issue 26191 for an overview of the most important packages). + + ### To-Dos: + * [ ] Match run-requirements for numpy (i.e. check upstream `pyproject.toml` or however the project specifies numpy compatibility) + * If upstream is not yet compatible with numpy 2.0, add `numpy <2.0dev0` upper bound under `run:`. + * If upstream is already compatible with numpy 2.0, double-check their supported numpy versions. + * If upstream requires a minimum numpy version newer than 1.19, you need to add `numpy >=x.y` under `run:`. + * [ ] Remove any remaining occurrences of `{{ pin_compatible("numpy") }}` that the bot may have missed. + + PS. If the build does not compile anymore, this is almost certainly a sign that + the upstream project is not yet ready for numpy 2.0; do not close this PR until + a version compatible with numpy 2.0 has been released upstream and on this + feedstock (in the meantime, you can keep the bot from reopening this PR in + case of git conflicts by marking it as a draft). + + migration_number: 1 + +# needs to match length of zip {python, python_impl, numpy} +# as it is in global CBC in order to override it +numpy: + - 2.0 + - 2.0 + - 2.0 + - 2.0 +migrator_ts: 1713572489.295986 diff --git a/.ci_support/migrations/python312.yaml b/.ci_support/migrations/python312.yaml deleted file mode 100644 index 784a0a25..00000000 --- a/.ci_support/migrations/python312.yaml +++ /dev/null @@ -1,38 +0,0 @@ -migrator_ts: 1695046563 -__migrator: - migration_number: 1 - operation: key_add - primary_key: python - ordering: - python: - - 3.6.* *_cpython - - 3.7.* *_cpython - - 3.8.* *_cpython - - 3.9.* *_cpython - - 3.10.* *_cpython - - 3.11.* *_cpython - - 3.12.* *_cpython # new entry - - 3.6.* *_73_pypy - - 3.7.* *_73_pypy - - 3.8.* *_73_pypy - - 3.9.* *_73_pypy - paused: false - longterm: True - pr_limit: 30 - max_solver_attempts: 6 # this will make the bot retry "not solvable" stuff 6 times - exclude: - # this shouldn't attempt to modify the python feedstocks - - python - - pypy3.6 - - pypy-meta - - cross-python - - python_abi - exclude_pinned_pkgs: false - -python: - - 3.12.* *_cpython -# additional entries to add for zip_keys -numpy: - - 1.26 -python_impl: - - cpython diff --git a/.ci_support/osx_64_python3.10.____cpython.yaml b/.ci_support/osx_64_python3.10.____cpython.yaml index 80799435..87cf7fea 100644 --- a/.ci_support/osx_64_python3.10.____cpython.yaml +++ b/.ci_support/osx_64_python3.10.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.22' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.ci_support/osx_64_python3.11.____cpython.yaml b/.ci_support/osx_64_python3.11.____cpython.yaml index 8881b7b2..ebcd1df4 100644 --- a/.ci_support/osx_64_python3.11.____cpython.yaml +++ b/.ci_support/osx_64_python3.11.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.23' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.ci_support/osx_64_python3.12.____cpython.yaml b/.ci_support/osx_64_python3.12.____cpython.yaml index 9d0f31c8..b7205850 100644 --- a/.ci_support/osx_64_python3.12.____cpython.yaml +++ b/.ci_support/osx_64_python3.12.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.26' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.ci_support/osx_64_python3.9.____cpython.yaml b/.ci_support/osx_64_python3.9.____cpython.yaml index 97dd727c..7d6272f6 100644 --- a/.ci_support/osx_64_python3.9.____cpython.yaml +++ b/.ci_support/osx_64_python3.9.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.22' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.ci_support/osx_arm64_python3.10.____cpython.yaml b/.ci_support/osx_arm64_python3.10.____cpython.yaml index c6fa878a..63730328 100644 --- a/.ci_support/osx_arm64_python3.10.____cpython.yaml +++ b/.ci_support/osx_arm64_python3.10.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - arm64-apple-darwin20.0.0 numpy: -- '1.22' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.ci_support/osx_arm64_python3.11.____cpython.yaml b/.ci_support/osx_arm64_python3.11.____cpython.yaml index 33dde1c7..5c4a691d 100644 --- a/.ci_support/osx_arm64_python3.11.____cpython.yaml +++ b/.ci_support/osx_arm64_python3.11.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - arm64-apple-darwin20.0.0 numpy: -- '1.23' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.ci_support/osx_arm64_python3.12.____cpython.yaml b/.ci_support/osx_arm64_python3.12.____cpython.yaml index 93dfb0b7..7ba51274 100644 --- a/.ci_support/osx_arm64_python3.12.____cpython.yaml +++ b/.ci_support/osx_arm64_python3.12.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - arm64-apple-darwin20.0.0 numpy: -- '1.26' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.ci_support/osx_arm64_python3.9.____cpython.yaml b/.ci_support/osx_arm64_python3.9.____cpython.yaml index 900e33f6..355e58a2 100644 --- a/.ci_support/osx_arm64_python3.9.____cpython.yaml +++ b/.ci_support/osx_arm64_python3.9.____cpython.yaml @@ -27,7 +27,7 @@ flatbuffers: giflib: - '5.2' github_actions_labels: -- cirun-openstack-cpu-xlarge +- cirun-openstack-gpu-xlarge icu: - '75' libabseil: @@ -45,7 +45,7 @@ libprotobuf: macos_machine: - arm64-apple-darwin20.0.0 numpy: -- '1.22' +- '2.0' openssl: - '3' pin_run_as_build: diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index 0535f6aa..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,17 +0,0 @@ -on: - status: {} - check_suite: - types: - - completed - -jobs: - automerge-action: - runs-on: ubuntu-latest - name: automerge - steps: - - name: automerge-action - id: automerge-action - uses: conda-forge/automerge-action@main - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - rerendering_github_token: ${{ secrets.RERENDERING_GITHUB_TOKEN }} diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 22aa3549..54d33423 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -21,46 +21,54 @@ jobs: fail-fast: false matrix: include: - - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.10.____cpython + - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.10.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h91fe0f008d', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.11.____cpython + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h9bec57d5dc', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.11.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_hed7027e077', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.12.____cpython + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h7e1b4662c4', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.12.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h11ed5249c8', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpython + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_hdf3f9bfbcc', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + - CONFIG: linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h625cfcfe2e', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version12cuda_c_h6bc6036033', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.10.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_h1a635e5433', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_h1a635e5433', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.11.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_h70857f1454', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_h70857f1454', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.12.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_h2dff17f9f4', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_h2dff17f9f4', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - CONFIG: linux_64_c_compiler_version13cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version13python3.9.____cpython UPLOAD_PACKAGES: True os: ubuntu - runs_on: ['cirun-openstack-cpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_hbe4b9baad6', 'linux', 'x64', 'self-hosted'] - DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 + runs_on: ['cirun-openstack-gpu-xlarge--${{ github.run_id }}-linux_64_c_compiler_version13cuda_c_hbe4b9baad6', 'linux', 'x64', 'self-hosted'] + DOCKER_IMAGE: quay.io/condaforge/linux-anvil-x86_64:alma9 + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" steps: - name: Checkout code @@ -117,12 +125,6 @@ jobs: fi ./.scripts/run_osx_build.sh - - name: Install Miniconda for windows - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 - with: - miniforge-version: latest - if: matrix.os == 'windows' - - name: Build on windows shell: cmd run: | @@ -131,6 +133,7 @@ jobs: set "sha=%GITHUB_SHA%" call ".scripts\run_win_build.bat" env: + MINIFORGE_HOME: D:\Miniforge PYTHONUNBUFFERED: 1 CONFIG: ${{ matrix.CONFIG }} CI: github_actions diff --git a/.github/workflows/webservices.yml b/.github/workflows/webservices.yml deleted file mode 100644 index d6f06b5c..00000000 --- a/.github/workflows/webservices.yml +++ /dev/null @@ -1,13 +0,0 @@ -on: repository_dispatch - -jobs: - webservices: - runs-on: ubuntu-latest - name: webservices - steps: - - name: webservices - id: webservices - uses: conda-forge/webservices-dispatch-action@main - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - rerendering_github_token: ${{ secrets.RERENDERING_GITHUB_TOKEN }} diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 8601d531..6970424c 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -31,13 +31,13 @@ pkgs_dirs: solver: libmamba CONDARC +mv /opt/conda/conda-meta/history /opt/conda/conda-meta/history.$(date +%Y-%m-%d-%H-%M-%S) +echo > /opt/conda/conda-meta/history +micromamba install --root-prefix ~/.conda --prefix /opt/conda \ + --yes --override-channels --channel conda-forge --strict-channel-priority \ + pip python=3.12 conda-build conda-libmamba-solver conda-forge-ci-setup=4 "conda-build>=24.1" export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1 -mamba install --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ - pip mamba conda-build conda-libmamba-solver conda-forge-ci-setup=4 "conda-build>=24.1" -mamba update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ - pip mamba conda-build conda-libmamba-solver conda-forge-ci-setup=4 "conda-build>=24.1" - # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" @@ -76,8 +76,8 @@ else --extra-meta flow_run_id="${flow_run_id:-}" remote_url="${remote_url:-}" sha="${sha:-}" ( startgroup "Inspecting artifacts" ) 2> /dev/null - # inspect_artifacts was only added in conda-forge-ci-setup 4.6.0 - command -v inspect_artifacts >/dev/null 2>&1 && inspect_artifacts || echo "inspect_artifacts needs conda-forge-ci-setup >=4.6.0" + # inspect_artifacts was only added in conda-forge-ci-setup 4.9.4 + command -v inspect_artifacts >/dev/null 2>&1 && inspect_artifacts --recipe-dir "${RECIPE_ROOT}" -m "${CONFIG_FILE}" || echo "inspect_artifacts needs conda-forge-ci-setup >=4.9.4" ( endgroup "Inspecting artifacts" ) 2> /dev/null ( startgroup "Validating outputs" ) 2> /dev/null diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 353866d9..14bc5e47 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -6,29 +6,41 @@ source .scripts/logging_utils.sh set -xe -MINIFORGE_HOME=${MINIFORGE_HOME:-${HOME}/miniforge3} - -( startgroup "Installing a fresh version of Miniforge" ) 2> /dev/null - -MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" -MINIFORGE_FILE="Miniforge3-MacOSX-$(uname -m).sh" -curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" -rm -rf ${MINIFORGE_HOME} -bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME} - -( endgroup "Installing a fresh version of Miniforge" ) 2> /dev/null +MINIFORGE_HOME="${MINIFORGE_HOME:-${HOME}/miniforge3}" +MINIFORGE_HOME="${MINIFORGE_HOME%/}" # remove trailing slash +export CONDA_BLD_PATH="${CONDA_BLD_PATH:-${MINIFORGE_HOME}/conda-bld}" + +( startgroup "Provisioning base env with micromamba" ) 2> /dev/null +MICROMAMBA_VERSION="1.5.10-0" +if [[ "$(uname -m)" == "arm64" ]]; then + osx_arch="osx-arm64" +else + osx_arch="osx-64" +fi +MICROMAMBA_URL="https://github.com/mamba-org/micromamba-releases/releases/download/${MICROMAMBA_VERSION}/micromamba-${osx_arch}" +MAMBA_ROOT_PREFIX="${MINIFORGE_HOME}-micromamba-$(date +%s)" +echo "Downloading micromamba ${MICROMAMBA_VERSION}" +micromamba_exe="$(mktemp -d)/micromamba" +curl -L -o "${micromamba_exe}" "${MICROMAMBA_URL}" +chmod +x "${micromamba_exe}" +echo "Creating environment" +"${micromamba_exe}" create --yes --root-prefix "${MAMBA_ROOT_PREFIX}" --prefix "${MINIFORGE_HOME}" \ + --channel conda-forge \ + pip python=3.12 conda-build conda-libmamba-solver conda-forge-ci-setup=4 "conda-build>=24.1" +echo "Moving pkgs cache from ${MAMBA_ROOT_PREFIX} to ${MINIFORGE_HOME}" +mv "${MAMBA_ROOT_PREFIX}/pkgs" "${MINIFORGE_HOME}" +echo "Cleaning up micromamba" +rm -rf "${MAMBA_ROOT_PREFIX}" "${micromamba_exe}" || true +( endgroup "Provisioning base env with micromamba" ) 2> /dev/null ( startgroup "Configuring conda" ) 2> /dev/null - -source ${MINIFORGE_HOME}/etc/profile.d/conda.sh +echo "Activating environment" +source "${MINIFORGE_HOME}/etc/profile.d/conda.sh" conda activate base export CONDA_SOLVER="libmamba" export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1 -mamba install --update-specs --quiet --yes --channel conda-forge --strict-channel-priority \ - pip mamba conda-build conda-libmamba-solver conda-forge-ci-setup=4 "conda-build>=24.1" -mamba update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \ - pip mamba conda-build conda-libmamba-solver conda-forge-ci-setup=4 "conda-build>=24.1" + @@ -88,8 +100,8 @@ else ( startgroup "Inspecting artifacts" ) 2> /dev/null - # inspect_artifacts was only added in conda-forge-ci-setup 4.6.0 - command -v inspect_artifacts >/dev/null 2>&1 && inspect_artifacts || echo "inspect_artifacts needs conda-forge-ci-setup >=4.6.0" + # inspect_artifacts was only added in conda-forge-ci-setup 4.9.4 + command -v inspect_artifacts >/dev/null 2>&1 && inspect_artifacts --recipe-dir ./recipe -m ./.ci_support/${CONFIG}.yaml || echo "inspect_artifacts needs conda-forge-ci-setup >=4.9.4" ( endgroup "Inspecting artifacts" ) 2> /dev/null ( startgroup "Validating outputs" ) 2> /dev/null diff --git a/README.md b/README.md index 1ea77e28..31e94534 100644 --- a/README.md +++ b/README.md @@ -37,31 +37,31 @@ Current build status - + - + - + - + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e21431e5..a13f0ae2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,14 +13,13 @@ stages: RET: 'true' steps: - checkout: self - fetchDepth: '2' - bash: | git_log=`git log --max-count=1 --skip=1 --pretty=format:"%B" | tr "\n" " "` echo "##vso[task.setvariable variable=log]$git_log" displayName: Obtain commit message - bash: echo "##vso[task.setvariable variable=RET]false" - condition: or(contains(variables.log, '[skip azp]'), contains(variables.log, '[azp skip]'), contains(variables.log, '[skip ci]'), contains(variables.log, '[ci skip]')) + condition: and(eq(variables['Build.Reason'], 'PullRequest'), or(contains(variables.log, '[skip azp]'), contains(variables.log, '[azp skip]'), contains(variables.log, '[skip ci]'), contains(variables.log, '[ci skip]'))) displayName: Skip build? - bash: echo "##vso[task.setvariable variable=start_main;isOutput=true]$RET" name: result diff --git a/build-locally.py b/build-locally.py index 6788aea6..c4a56c66 100755 --- a/build-locally.py +++ b/build-locally.py @@ -26,6 +26,13 @@ def setup_environment(ns): os.path.dirname(__file__), "miniforge3" ) + # The default cache location might not be writable using docker on macOS. + if ns.config.startswith("linux") and platform.system() == "Darwin": + os.environ["CONDA_FORGE_DOCKER_RUN_ARGS"] = ( + os.environ.get("CONDA_FORGE_DOCKER_RUN_ARGS", "") + + " -e RATTLER_CACHE_DIR=/tmp/rattler_cache" + ) + def run_docker_build(ns): script = ".scripts/run_docker_build.sh" diff --git a/recipe/build.sh b/recipe/build.sh index 409215a2..a51c9435 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -96,16 +96,18 @@ if [[ ${cuda_compiler_version} != "None" ]]; then export TF_NEED_CUDA=1 export TF_CUDA_VERSION="${cuda_compiler_version}" export TF_CUDNN_VERSION="${cudnn}" + export HERMETIC_CUDA_VERSION="${cuda_compiler_version}" + export HERMETIC_CUDNN_VERSION="${cudnn}" export TF_NCCL_VERSION=$(pkg-config nccl --modversion | grep -Po '\d+\.\d+') export LDFLAGS="${LDFLAGS//-Wl,-z,now/-Wl,-z,lazy}" export CC_OPT_FLAGS="-march=nocona -mtune=haswell" if [[ ${cuda_compiler_version} == 11.8 ]]; then - export TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_50,sm_60,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,sm_87,sm_89,sm_90,compute_90 + export HERMETIC_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_50,sm_60,sm_62,sm_70,sm_72,sm_75,sm_80,sm_86,sm_87,sm_89,sm_90,compute_90 export TF_CUDA_PATHS="${PREFIX},${CUDA_HOME}" elif [[ "${cuda_compiler_version}" == 12* ]]; then - export TF_CUDA_COMPUTE_CAPABILITIES=sm_60,sm_70,sm_75,sm_80,sm_86,sm_89,sm_90,compute_90 + export HERMETIC_CUDA_COMPUTE_CAPABILITIES=sm_60,sm_70,sm_75,sm_80,sm_86,sm_89,sm_90,compute_90 export CUDNN_INSTALL_PATH=$PREFIX export NCCL_INSTALL_PATH=$PREFIX export CUDA_HOME="${BUILD_PREFIX}/targets/x86_64-linux" @@ -113,12 +115,38 @@ if [[ ${cuda_compiler_version} != "None" ]]; then # XLA can only cope with a single cuda header include directory, merge both rsync -a ${PREFIX}/targets/x86_64-linux/include/ ${BUILD_PREFIX}/targets/x86_64-linux/include/ + # Although XLA supports a non-hermetic build, it still tries to find headers in the hermetic locations. + # We do this in the BUILD_PREFIX to not have any impact on the resulting jaxlib package. + # Otherwise, these copied files would be included in the package. + rm -rf ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party + mkdir -p ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party/gpus/cuda/extras/CUPTI + cp -r ${PREFIX}/targets/x86_64-linux/include ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party/gpus/cuda/ + cp -r ${PREFIX}/targets/x86_64-linux/include ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party/gpus/cuda/extras/CUPTI/ + mkdir -p ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party/gpus/cudnn + cp ${PREFIX}/include/cudnn*.h ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party/gpus/cudnn/ + mkdir -p ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party/nccl + cp ${PREFIX}/include/nccl.h ${BUILD_PREFIX}/targets/x86_64-linux/include/third_party/nccl/ + rsync -a ${PREFIX}/targets/x86_64-linux/lib/ ${BUILD_PREFIX}/targets/x86_64-linux/lib/ + ln -s ${BUILD_PREFIX}/bin/fatbinary ${BUILD_PREFIX}/targets/x86_64-linux/bin/fatbinary + ln -s ${BUILD_PREFIX}/bin/nvlink ${BUILD_PREFIX}/targets/x86_64-linux/bin/nvlink + ln -s ${BUILD_PREFIX}/bin/ptxas ${BUILD_PREFIX}/targets/x86_64-linux/bin/ptxas + + export LOCAL_CUDA_PATH="${BUILD_PREFIX}/targets/x86_64-linux" + export LOCAL_CUDNN_PATH="${PREFIX}" + export LOCAL_NCCL_PATH="${PREFIX}" + # hmaarrfk -- 2023/12/30 # This logic should be safe to keep in even when the underlying issue is resolved # xref: https://github.com/conda-forge/cuda-nvcc-impl-feedstock/issues/9 if [[ -x ${BUILD_PREFIX}/nvvm/bin/cicc ]]; then cp ${BUILD_PREFIX}/nvvm/bin/cicc ${BUILD_PREFIX}/bin/cicc fi + + # Needs GCC 13+ + echo "build --define=xnn_enable_avxvnniint8=false" >> .bazelrc + + # cuda-compat is used for providing libcuda.so.1 temporarily + cp $PREFIX/cuda-compat/libcuda.so.1 $PREFIX/lib/libcuda.so.1 else echo "unsupported cuda version." exit 1 @@ -187,11 +215,11 @@ fi cat >> .bazelrc <={{ ml_dtypes_version }},<0.5 outputs: # 2021/12/29: hmaarrfk @@ -191,6 +196,7 @@ outputs: - libcurand-dev # [(cuda_compiler_version or "").startswith("12")] - libcufft-dev # [(cuda_compiler_version or "").startswith("12")] - libcusparse-dev # [(cuda_compiler_version or "").startswith("12")] + - libnvjitlink-dev # [(cuda_compiler_version or "").startswith("12")] # conda build requirements - python - pip @@ -222,7 +228,7 @@ outputs: - astunparse >=1.6.0 - gast >=0.2.1,!=0.5.0,!=0.5.1,!=0.5.2 - google-pasta >=0.1.1 - - h5py >=3.10 + - h5py >=3.11 - ml_dtypes >={{ ml_dtypes_version }},<0.5 - numpy - opt_einsum >=2.3.2 @@ -235,7 +241,7 @@ outputs: - typing_extensions >=3.6.6 - wrapt >=1.11.0 # TF-API needs to move in sync - - tensorboard >=2.17,<2.18 + - tensorboard >=2.18,<2.19 - keras >=3.0 run: - python @@ -245,7 +251,7 @@ outputs: - gast >=0.2.1,!=0.5.0,!=0.5.1,!=0.5.2 - google-pasta >=0.1.1 - grpcio {{ libgrpc }}.* - - h5py >=3.10 + - h5py >=3.11 - ml_dtypes >={{ ml_dtypes_version }},<0.5 - opt_einsum >=2.3.2 - protobuf >=5.26,<6 @@ -256,7 +262,7 @@ outputs: - typing_extensions >=3.6.6 - wrapt >=1.11.0 # TF-API needs to move in sync - - tensorboard >=2.17,<2.18 + - tensorboard >=2.18,<2.19 - keras >=3.0 # avoid that people without GPUs needlessly download ~0.5-1GB - __cuda # [cuda_compiler_version != "None"] @@ -441,7 +447,7 @@ outputs: - sqlite - zlib run: - - libml_dtypes-headers >={{ ml_dtypes_version }},<0.5 + - libml_dtypes-headers >={{ libml_dtypes_version }},<0.6 # avoid that people without GPUs needlessly download ~200-300MB - __cuda # [cuda_compiler_version != "None"] test: diff --git a/recipe/patches/0001-loosen-requirements.patch b/recipe/patches/0001-loosen-requirements.patch index c6df4eb8..2d84ef88 100644 --- a/recipe/patches/0001-loosen-requirements.patch +++ b/recipe/patches/0001-loosen-requirements.patch @@ -1,4 +1,4 @@ -From e6a16ebf1cb912b32f8fa6f9b26dbe8add0d930b Mon Sep 17 00:00:00 2001 +From e38b64198fe3f683895817c3685f9da939aadfd3 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 30 Dec 2021 06:57:09 +0200 Subject: [PATCH 01/27] loosen requirements @@ -11,32 +11,22 @@ Co-authored-by: Mark Harfouche Co-authored-by: ngam Co-authored-by: Uwe L. Korn --- - tensorflow/tools/pip_package/setup.py | 8 +------- - 1 file changed, 1 insertion(+), 7 deletions(-) + tensorflow/tools/pip_package/setup.py | 6 ------ + 1 file changed, 6 deletions(-) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py -index 88da8b54..ac24634f 100644 +index fb37ef9b..61f6ce1f 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py -@@ -82,16 +82,12 @@ REQUIRED_PACKAGES = [ +@@ -81,7 +81,6 @@ REQUIRED_PACKAGES = [ + 'flatbuffers >= 24.3.25', 'gast >=0.2.1,!=0.5.0,!=0.5.1,!=0.5.2', 'google_pasta >= 0.1.1', - 'h5py >= 3.10.0', - 'libclang >= 13.0.0', - 'ml_dtypes >= 0.3.1, < 0.5.0', -- # TODO(b/304751256): Adjust the numpy pin to a single version, when ready -- 'numpy >= 1.23.5, < 2.0.0 ; python_version <= "3.11"', -- 'numpy >= 1.26.0, < 2.0.0 ; python_version >= "3.12"', 'opt_einsum >= 2.3.2', 'packaging', # pylint:disable=line-too-long - ( -- 'protobuf>=3.20.3,<5.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5' -+ 'protobuf>=3.20.3,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5' - ), - 'requests >= 2.21.0, < 3', - 'setuptools', -@@ -99,8 +95,6 @@ REQUIRED_PACKAGES = [ +@@ -94,8 +93,6 @@ REQUIRED_PACKAGES = [ 'termcolor >= 1.1.0', 'typing_extensions >= 3.6.6', 'wrapt >= 1.11.0', @@ -45,3 +35,13 @@ index 88da8b54..ac24634f 100644 # grpcio does not build correctly on big-endian machines due to lack of # BoringSSL support. # See https://github.com/tensorflow/tensorflow/issues/17882. +@@ -111,9 +108,6 @@ REQUIRED_PACKAGES = [ + # 'keras >= 2.14.0rc0, < 2.15' on the release branch after the branch cut. + 'tensorboard >= 2.18, < 2.19', + 'keras >= 3.5.0', +- # TODO(b/367877753): Update the upper bound to <2.2.0 once the compatibility +- # issues with numpy 2.1.0 is fixed. +- 'numpy >= 1.26.0, < 2.1.0', + 'h5py >= 3.11.0', + 'ml_dtypes >= 0.4.0, < 0.5.0', + ] diff --git a/recipe/patches/0002-Add-additional-absl_synchronization-linkage-to-gRPC.patch b/recipe/patches/0002-Add-additional-absl_synchronization-linkage-to-gRPC.patch index db5cd2ba..16408324 100644 --- a/recipe/patches/0002-Add-additional-absl_synchronization-linkage-to-gRPC.patch +++ b/recipe/patches/0002-Add-additional-absl_synchronization-linkage-to-gRPC.patch @@ -1,4 +1,4 @@ -From 14a1b6162855bfe8d8d16cbd054f98afba4f16e5 Mon Sep 17 00:00:00 2001 +From f80db539ec99bf062c5e7b935d0e2391225627fc Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 30 Jun 2023 08:39:32 +0000 Subject: [PATCH 02/27] Add additional absl_synchronization linkage to gRPC diff --git a/recipe/patches/0003-Fix-missing-abseil-linkages.patch b/recipe/patches/0003-Fix-missing-abseil-linkages.patch index 047d0fb0..fd4a2e76 100644 --- a/recipe/patches/0003-Fix-missing-abseil-linkages.patch +++ b/recipe/patches/0003-Fix-missing-abseil-linkages.patch @@ -1,4 +1,4 @@ -From 7dc829cca3ae349278ca7f9186c5c5db5a81a390 Mon Sep 17 00:00:00 2001 +From 9452e8ed5d2f79b06a205807687c492e3524ade5 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 3 Jul 2023 04:45:46 +0000 Subject: [PATCH 03/27] Fix missing abseil linkages diff --git a/recipe/patches/0004-Fix-protobuf_python-for-systemlibs.patch b/recipe/patches/0004-Fix-protobuf_python-for-systemlibs.patch index 925704b2..de5739c7 100644 --- a/recipe/patches/0004-Fix-protobuf_python-for-systemlibs.patch +++ b/recipe/patches/0004-Fix-protobuf_python-for-systemlibs.patch @@ -1,4 +1,4 @@ -From d70aa0acf092f069a154502100abd8c6f230129d Mon Sep 17 00:00:00 2001 +From 53fce719a6f0c64954b48d4b59b3be046b709493 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 30 Aug 2023 16:58:41 +0200 Subject: [PATCH 04/27] Fix protobuf_python for systemlibs @@ -12,10 +12,10 @@ Subject: [PATCH 04/27] Fix protobuf_python for systemlibs create mode 100644 third_party/systemlibs/protobuf_python.bzl diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl -index fd29dff0..dc78011d 100644 +index be83c971..5320aa18 100644 --- a/tensorflow/workspace2.bzl +++ b/tensorflow/workspace2.bzl -@@ -382,6 +382,7 @@ def _tf_repositories(): +@@ -390,6 +390,7 @@ def _tf_repositories(): system_link_files = { "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", @@ -23,7 +23,7 @@ index fd29dff0..dc78011d 100644 }, urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), ) -@@ -786,10 +787,10 @@ def _tf_repositories(): +@@ -794,10 +795,10 @@ def _tf_repositories(): tf_http_archive( name = "pybind11_protobuf", diff --git a/recipe/patches/0005-Add-absl_log-systemlib.patch b/recipe/patches/0005-Add-absl_log-systemlib.patch index fffca4eb..2a9da913 100644 --- a/recipe/patches/0005-Add-absl_log-systemlib.patch +++ b/recipe/patches/0005-Add-absl_log-systemlib.patch @@ -1,4 +1,4 @@ -From 6d4c056204f6417cc7b01706d0a6b063a50e40e3 Mon Sep 17 00:00:00 2001 +From bb623beeb54b94c22a2b019a935e684cc1dff2c8 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 30 Aug 2023 16:59:28 +0200 Subject: [PATCH 05/27] Add absl_log systemlib @@ -38,7 +38,7 @@ index 00000000..0248bda0 + ], +) diff --git a/third_party/absl/workspace.bzl b/third_party/absl/workspace.bzl -index 06f75166..c54df3cf 100644 +index 9565a82c..addb2410 100644 --- a/third_party/absl/workspace.bzl +++ b/third_party/absl/workspace.bzl @@ -20,6 +20,7 @@ def repo(): diff --git a/recipe/patches/0006-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch b/recipe/patches/0006-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch index 77c293cb..3db49c7b 100644 --- a/recipe/patches/0006-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch +++ b/recipe/patches/0006-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch @@ -1,4 +1,4 @@ -From 5c31fdbd5aec3c392ae578bcac615c4d2c77e564 Mon Sep 17 00:00:00 2001 +From a33625860c147ebd81b99a06830dfc3f70a27e98 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 15 Sep 2023 11:06:27 +0200 Subject: [PATCH 06/27] Omit linking to layout_proto_cc if protobuf linkage is @@ -9,10 +9,10 @@ Subject: [PATCH 06/27] Omit linking to layout_proto_cc if protobuf linkage is 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD -index c3f9a377..b0432b14 100644 +index 0ad8e84b..43843f32 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD -@@ -1366,7 +1366,6 @@ tf_python_pybind_extension( +@@ -1363,7 +1363,6 @@ tf_python_pybind_extension( features = ["-layering_check"], deps = [ ":pywrap_densor_device_headers", diff --git a/recipe/patches/0007-Fix-further-abseil-linkage.patch b/recipe/patches/0007-Fix-further-abseil-linkage.patch index 000b2239..2bc6fb46 100644 --- a/recipe/patches/0007-Fix-further-abseil-linkage.patch +++ b/recipe/patches/0007-Fix-further-abseil-linkage.patch @@ -1,4 +1,4 @@ -From 46645306963d53e96f4bde85a7a7fd4960a5a447 Mon Sep 17 00:00:00 2001 +From 206c175e081b4cedf15d2aa07c8fd2b21a5272ee Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 18 Sep 2023 14:47:08 +0000 Subject: [PATCH 07/27] Fix further abseil linkage diff --git a/recipe/patches/0008-Add-constraint-to-pybind11-systemlib.patch b/recipe/patches/0008-Add-constraint-to-pybind11-systemlib.patch index 64e6905e..9e4bc7c0 100644 --- a/recipe/patches/0008-Add-constraint-to-pybind11-systemlib.patch +++ b/recipe/patches/0008-Add-constraint-to-pybind11-systemlib.patch @@ -1,4 +1,4 @@ -From 6d8aae7e5e303b9b0bb6731e13032388d4e38d50 Mon Sep 17 00:00:00 2001 +From 6cc878513d44e08cd39b8f55f39f52b3c13893c8 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 27 Oct 2023 11:20:12 +0200 Subject: [PATCH 08/27] Add constraint to pybind11 systemlib diff --git a/recipe/patches/0009-Different-file-ending-for-flatbuffers-LICENSE.patch b/recipe/patches/0009-Different-file-ending-for-flatbuffers-LICENSE.patch index 430ed382..784ae1c5 100644 --- a/recipe/patches/0009-Different-file-ending-for-flatbuffers-LICENSE.patch +++ b/recipe/patches/0009-Different-file-ending-for-flatbuffers-LICENSE.patch @@ -1,4 +1,4 @@ -From a2a3cc6043d732aa17498ef46735405498c59bf7 Mon Sep 17 00:00:00 2001 +From f7c3d89c2339cdf2517621e42666553d5ffd36da Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 27 Oct 2023 11:21:11 +0200 Subject: [PATCH 09/27] Different file ending for flatbuffers LICENSE diff --git a/recipe/patches/0010-Use-correct-hermetic-python.patch b/recipe/patches/0010-Use-correct-hermetic-python.patch index 757f74b9..5fc07b60 100644 --- a/recipe/patches/0010-Use-correct-hermetic-python.patch +++ b/recipe/patches/0010-Use-correct-hermetic-python.patch @@ -1,4 +1,4 @@ -From 95485b5d00566df1982a410672821a951697d05f Mon Sep 17 00:00:00 2001 +From 7b162495d8efdc0fd6e0aad8f598cdeed3774af4 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 27 Oct 2023 11:21:38 +0200 Subject: [PATCH 10/27] Use correct, hermetic python diff --git a/recipe/patches/0011-Add-well_known_types_py_pb2-to-protobuf-systemlib.patch b/recipe/patches/0011-Add-well_known_types_py_pb2-to-protobuf-systemlib.patch index ee28b292..7cf96f88 100644 --- a/recipe/patches/0011-Add-well_known_types_py_pb2-to-protobuf-systemlib.patch +++ b/recipe/patches/0011-Add-well_known_types_py_pb2-to-protobuf-systemlib.patch @@ -1,4 +1,4 @@ -From a878cc2a275f4887f0309ace423a049a30b1a39e Mon Sep 17 00:00:00 2001 +From 1e33a03dfdaac70dbee540b901a1fee950b814a6 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 15 Nov 2023 09:18:28 +0100 Subject: [PATCH 11/27] Add well_known_types_py_pb2 to protobuf systemlib diff --git a/recipe/patches/0012-Add-protobuf-toolchain.patch b/recipe/patches/0012-Add-protobuf-toolchain.patch index 4cc9dea8..58fbd916 100644 --- a/recipe/patches/0012-Add-protobuf-toolchain.patch +++ b/recipe/patches/0012-Add-protobuf-toolchain.patch @@ -1,4 +1,4 @@ -From de24030836aeecfa290e5f5ba11e61f9c6dbcbc5 Mon Sep 17 00:00:00 2001 +From 70bda39b9de02070156308d9e4070c9d0e651c38 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 15 Nov 2023 10:32:41 +0000 Subject: [PATCH 12/27] Add protobuf toolchain diff --git a/recipe/patches/0013-fix-genproto.patch b/recipe/patches/0013-fix-genproto.patch index e6da4e23..d4966c43 100644 --- a/recipe/patches/0013-fix-genproto.patch +++ b/recipe/patches/0013-fix-genproto.patch @@ -1,4 +1,4 @@ -From f9bf16ddf1cb2d288694871fa3383d90aa874a1a Mon Sep 17 00:00:00 2001 +From a6a1ee1af7e7829adc68a8f05b2ad8a9273982fb Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 16 Nov 2023 06:38:27 +0000 Subject: [PATCH 13/27] fix genproto @@ -8,10 +8,10 @@ Subject: [PATCH 13/27] fix genproto 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl b/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl -index 35cdcdc5..ca4bdd0e 100644 +index 726f5463..62837c84 100644 --- a/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl +++ b/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl -@@ -401,10 +401,7 @@ def py_proto_library( +@@ -400,10 +400,7 @@ def py_proto_library( genproto_deps = [] for dep in deps: diff --git a/recipe/patches/0014-Remove-some-usage-of-absl-str_format-in-CUDA.patch b/recipe/patches/0014-Remove-some-usage-of-absl-str_format-in-CUDA.patch index ad416592..de9a6a8d 100644 --- a/recipe/patches/0014-Remove-some-usage-of-absl-str_format-in-CUDA.patch +++ b/recipe/patches/0014-Remove-some-usage-of-absl-str_format-in-CUDA.patch @@ -1,4 +1,4 @@ -From ae92d44a00de6f162bdc045052be0d6140fc081f Mon Sep 17 00:00:00 2001 +From 0472aaf3932e1ea815cfd45582c001e159fc1052 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 23 Nov 2023 09:05:37 +0000 Subject: [PATCH 14/27] Remove some usage of absl::str_format in CUDA diff --git a/recipe/patches/0015-Adjust-relative-path-for-libdevice.patch b/recipe/patches/0015-Adjust-relative-path-for-libdevice.patch index 3cc7bdd0..f465fab5 100644 --- a/recipe/patches/0015-Adjust-relative-path-for-libdevice.patch +++ b/recipe/patches/0015-Adjust-relative-path-for-libdevice.patch @@ -1,4 +1,4 @@ -From 38ca4efe105b78c1622f2246cc5b66c5c6aa5e7a Mon Sep 17 00:00:00 2001 +From f44de9a6a74b949da5681b31eb51b7bc601110fb Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Sat, 27 Jan 2024 22:27:56 -0500 Subject: [PATCH 15/27] Adjust relative path for libdevice @@ -8,15 +8,15 @@ Subject: [PATCH 15/27] Adjust relative path for libdevice 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/xla/third_party/tsl/tsl/platform/default/cuda_libdevice_path.cc b/third_party/xla/third_party/tsl/tsl/platform/default/cuda_libdevice_path.cc -index 46321e74..fd0cab0d 100644 +index ac0a804b..fe8305fd 100644 --- a/third_party/xla/third_party/tsl/tsl/platform/default/cuda_libdevice_path.cc +++ b/third_party/xla/third_party/tsl/tsl/platform/default/cuda_libdevice_path.cc -@@ -51,7 +51,7 @@ std::vector CandidateCudaRoots() { +@@ -69,7 +69,7 @@ std::vector CandidateCudaRoots() { // TF lib binaries are located in both the package's root dir and within a // 'python' subdirectory (for pywrap libs). So we check two possible paths // relative to the current binary for the wheel-based nvcc package. - for (auto path : {"../nvidia/cuda_nvcc", "../../nvidia/cuda_nvcc"}) + for (auto path : {"../nvidia/cuda_nvcc", "../../nvidia/cuda_nvcc", "../../../.."}) roots.emplace_back(io::JoinPath(dir, path)); - } - #endif // defined(PLATFORM_POSIX) && !defined(__APPLE__) + + // Also add the path to the copy of libdevice.10.bc that we include within diff --git a/recipe/patches/0016-Link-to-absl_log_flags-instead-of-absl_flags.patch b/recipe/patches/0016-Link-to-absl_log_flags-instead-of-absl_flags.patch index a60d2f5a..76429170 100644 --- a/recipe/patches/0016-Link-to-absl_log_flags-instead-of-absl_flags.patch +++ b/recipe/patches/0016-Link-to-absl_log_flags-instead-of-absl_flags.patch @@ -1,4 +1,4 @@ -From 346c37a3c4273ee4d65a9275c26617b391876c33 Mon Sep 17 00:00:00 2001 +From 786e194933fc0aedf991ed6159868e4a06b72ea3 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Mon, 26 Feb 2024 22:30:08 -0500 Subject: [PATCH 16/27] Link to absl_log_flags instead of absl_flags diff --git a/recipe/patches/0017-Update-ABSL-Log-Definition-for-libabsl_vlog_config_i.patch b/recipe/patches/0017-Update-ABSL-Log-Definition-for-libabsl_vlog_config_i.patch index a69d1c97..4b71c8ba 100644 --- a/recipe/patches/0017-Update-ABSL-Log-Definition-for-libabsl_vlog_config_i.patch +++ b/recipe/patches/0017-Update-ABSL-Log-Definition-for-libabsl_vlog_config_i.patch @@ -1,4 +1,4 @@ -From 81ff1b5ec7052ab02abc008d031b9a85a82069e0 Mon Sep 17 00:00:00 2001 +From b911e2b40323f89503fa8475abf017d3a42516f6 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Tue, 27 Feb 2024 18:01:19 -0500 Subject: [PATCH 17/27] Update ABSL Log Definition for diff --git a/recipe/patches/0018-add-absl_string_view-target.patch b/recipe/patches/0018-add-absl_string_view-target.patch index c0b4ad49..3f0b175d 100644 --- a/recipe/patches/0018-add-absl_string_view-target.patch +++ b/recipe/patches/0018-add-absl_string_view-target.patch @@ -1,4 +1,4 @@ -From 3d79a13204bb6fa626dc1cb82e03506280a53590 Mon Sep 17 00:00:00 2001 +From 918451e9f02087c2891d58d4969216a3a702c2ac Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 12 May 2024 18:45:03 +1100 Subject: [PATCH 18/27] add absl_string_view target diff --git a/recipe/patches/0019-add-absl_nullability-target.patch b/recipe/patches/0019-add-absl_nullability-target.patch index f094ab3c..dd4899ec 100644 --- a/recipe/patches/0019-add-absl_nullability-target.patch +++ b/recipe/patches/0019-add-absl_nullability-target.patch @@ -1,4 +1,4 @@ -From c1ab28a6ca9909f0310cac9008569a5df03ef4a1 Mon Sep 17 00:00:00 2001 +From 938c10d189c2bc47d891ce46ca9ff92d197aa226 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 12 May 2024 19:30:44 +1100 Subject: [PATCH 19/27] add absl_nullability target diff --git a/recipe/patches/0020-add-absl_prefetch-target.patch b/recipe/patches/0020-add-absl_prefetch-target.patch index 32b94416..0740450d 100644 --- a/recipe/patches/0020-add-absl_prefetch-target.patch +++ b/recipe/patches/0020-add-absl_prefetch-target.patch @@ -1,4 +1,4 @@ -From 16b223f630d00a46d1c90ad129924afb6f7143e4 Mon Sep 17 00:00:00 2001 +From ec3fcb3a2bb1139ed3a7dee824edf0c7170e349d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 12 May 2024 22:29:25 +1100 Subject: [PATCH 20/27] add absl_prefetch target diff --git a/recipe/patches/0021-add-absl_die_if_null-target.patch b/recipe/patches/0021-add-absl_die_if_null-target.patch index 43e54b7d..f231c2d8 100644 --- a/recipe/patches/0021-add-absl_die_if_null-target.patch +++ b/recipe/patches/0021-add-absl_die_if_null-target.patch @@ -1,17 +1,17 @@ -From cf4e36f47e874278b60c2fdab575bfb33fe803ac Mon Sep 17 00:00:00 2001 +From 027858614ecf06428e284eb7f32699eda84ae07f Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 12 May 2024 22:46:04 +1100 Subject: [PATCH 21/27] add absl_die_if_null target --- - third_party/absl/system.absl.log.BUILD | 11 +++++++++++ - 1 file changed, 11 insertions(+) + third_party/absl/system.absl.log.BUILD | 16 ++++++++++++++++ + 1 file changed, 16 insertions(+) diff --git a/third_party/absl/system.absl.log.BUILD b/third_party/absl/system.absl.log.BUILD -index 1fe1e342..6606aaa5 100644 +index 1fe1e342..141bb319 100644 --- a/third_party/absl/system.absl.log.BUILD +++ b/third_party/absl/system.absl.log.BUILD -@@ -2,6 +2,17 @@ load("@rules_cc//cc:defs.bzl", "cc_library") +@@ -2,6 +2,22 @@ load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//visibility:public"]) @@ -25,6 +25,11 @@ index 1fe1e342..6606aaa5 100644 + "//absl/strings", + ], +) ++ ++alias( ++ name = "absl_log", ++ actual = ":log", ++) + cc_library( name = "log", diff --git a/recipe/patches/0022-add-absl_crc32c-targets.patch b/recipe/patches/0022-add-absl_crc32c-targets.patch index 5cfc1229..8a421c04 100644 --- a/recipe/patches/0022-add-absl_crc32c-targets.patch +++ b/recipe/patches/0022-add-absl_crc32c-targets.patch @@ -1,4 +1,4 @@ -From 3d268a099e1a3f62922c913f049f503fb77ad613 Mon Sep 17 00:00:00 2001 +From 4d7b4e4c13759923bbf033c4b7c5e71fa33e22dd Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 13 May 2024 09:41:09 +1100 Subject: [PATCH 22/27] add absl_crc32c targets @@ -86,7 +86,7 @@ index 00000000..487c36ff + ], +) diff --git a/third_party/absl/workspace.bzl b/third_party/absl/workspace.bzl -index c54df3cf..94ee77bb 100644 +index addb2410..e71aa167 100644 --- a/third_party/absl/workspace.bzl +++ b/third_party/absl/workspace.bzl @@ -14,6 +14,7 @@ def repo(): diff --git a/recipe/patches/0023-add-kernel_timeout_internal-target.patch b/recipe/patches/0023-add-kernel_timeout_internal-target.patch index e83dca61..f5a15726 100644 --- a/recipe/patches/0023-add-kernel_timeout_internal-target.patch +++ b/recipe/patches/0023-add-kernel_timeout_internal-target.patch @@ -1,4 +1,4 @@ -From 63e72374d50094425d0675059c2b11e3c95d3135 Mon Sep 17 00:00:00 2001 +From f98ad31ffbb83ba849a8a97592c8c6f78e6f7fd0 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 13 May 2024 23:36:48 +1100 Subject: [PATCH 23/27] add kernel_timeout_internal target diff --git a/recipe/patches/0024-work-around-for-warning-that-clang-falsely-treats-as.patch b/recipe/patches/0024-work-around-for-warning-that-clang-falsely-treats-as.patch index 55b7c7ed..188230f3 100644 --- a/recipe/patches/0024-work-around-for-warning-that-clang-falsely-treats-as.patch +++ b/recipe/patches/0024-work-around-for-warning-that-clang-falsely-treats-as.patch @@ -1,4 +1,4 @@ -From a89bade09f737a1cd5f7e2ee7ec26b38dfb771dd Mon Sep 17 00:00:00 2001 +From 24eeae23f9140fddfbfe2568e49f9d3359f2a71d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 19 May 2024 20:52:35 +1100 Subject: [PATCH 24/27] work around for warning that clang falsely treats as @@ -10,10 +10,10 @@ see https://github.com/llvm/llvm-project/issues/92630 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc -index fb80a081..01668b2c 100644 +index f48811d1..0d1c0305 100644 --- a/.bazelrc +++ b/.bazelrc -@@ -313,7 +313,7 @@ build:ios --noenable_platform_specific_config +@@ -310,7 +310,7 @@ build:ios --noenable_platform_specific_config build:android --copt=-w build:ios --copt=-w build:linux --host_copt=-w diff --git a/recipe/patches/0025-Hardcode-BUILD_PREFIX-in-build_pip_package.patch b/recipe/patches/0025-Hardcode-BUILD_PREFIX-in-build_pip_package.patch index c4796ac8..86a3ad5e 100644 --- a/recipe/patches/0025-Hardcode-BUILD_PREFIX-in-build_pip_package.patch +++ b/recipe/patches/0025-Hardcode-BUILD_PREFIX-in-build_pip_package.patch @@ -1,4 +1,4 @@ -From ed916a2d3c5399dbf6c71b1f6f7f07b8b1ddb8fc Mon Sep 17 00:00:00 2001 +From 7985aee1f8f9efc8832c7ec8b4e13485928a669c Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 12 Jul 2024 12:52:38 +0000 Subject: [PATCH 25/27] Hardcode BUILD_PREFIX in build_pip_package @@ -8,10 +8,10 @@ Subject: [PATCH 25/27] Hardcode BUILD_PREFIX in build_pip_package 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/tools/pip_package/build_pip_package.py b/tensorflow/tools/pip_package/build_pip_package.py -index 9588fc19..2d0718af 100644 +index d882db09..091db3f4 100644 --- a/tensorflow/tools/pip_package/build_pip_package.py +++ b/tensorflow/tools/pip_package/build_pip_package.py -@@ -240,12 +240,12 @@ def patch_so(srcs_dir: str) -> None: +@@ -285,12 +285,12 @@ def patch_so(srcs_dir: str) -> None: } for file, path in to_patch.items(): rpath = subprocess.check_output( diff --git a/recipe/patches/0026-Only-link-to-sparse_core_layout_proto_cc-headers.patch b/recipe/patches/0026-Only-link-to-sparse_core_layout_proto_cc-headers.patch index 82feb5d3..ccca83cd 100644 --- a/recipe/patches/0026-Only-link-to-sparse_core_layout_proto_cc-headers.patch +++ b/recipe/patches/0026-Only-link-to-sparse_core_layout_proto_cc-headers.patch @@ -1,4 +1,4 @@ -From c3205a7e020e1e9ee30581b2b869ed83746136b7 Mon Sep 17 00:00:00 2001 +From 4b271291f3811779838efcbb90448b3b06c45922 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 16 Jul 2024 17:18:23 +0200 Subject: [PATCH 26/27] Only link to sparse_core_layout_proto_cc headers @@ -8,10 +8,10 @@ Subject: [PATCH 26/27] Only link to sparse_core_layout_proto_cc headers 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD -index 7d55de7b..c409c1b0 100644 +index 5c1dc5da..4d467227 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD -@@ -1487,7 +1487,7 @@ cc_library( +@@ -1483,7 +1483,7 @@ cc_library( hdrs = ["sparse_core_layout.h"], visibility = ["//tensorflow/python/tpu:__pkg__"], # ONLY for `_pywrap_sparse_core_layout`. deps = [ diff --git a/recipe/patches/0027-Protobuf-5-compatability.patch b/recipe/patches/0027-Protobuf-5-compatability.patch index 9783d137..78b14d04 100644 --- a/recipe/patches/0027-Protobuf-5-compatability.patch +++ b/recipe/patches/0027-Protobuf-5-compatability.patch @@ -1,4 +1,4 @@ -From d98ec2c4eb0a0b58f0003d3b9d7ba7aeb413fe78 Mon Sep 17 00:00:00 2001 +From 61c8a1306bb7bbc8919a237602e6dd158c80ff23 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 10 Oct 2024 19:59:31 +0200 Subject: [PATCH 27/27] Protobuf 5 compatability @@ -23,12 +23,12 @@ Subject: [PATCH 27/27] Protobuf 5 compatability create mode 100644 third_party/0002-Fix-compatibility-with-protobuf-v26-migrate-from-dep.patch diff --git a/tensorflow/compiler/mlir/lite/python/jax_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/python/jax_to_tfl_flatbuffer.cc -index ed968b54..71e16db6 100644 +index eba1ebc4..dc9f7b81 100644 --- a/tensorflow/compiler/mlir/lite/python/jax_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/python/jax_to_tfl_flatbuffer.cc @@ -57,7 +57,7 @@ namespace { // Error collector that simply ignores errors reported. - class NoOpErrorCollector : public protobuf::io::ErrorCollector { + class NoOpErrorCollector : public tsl::protobuf::io::ErrorCollector { public: - void AddError(int line, int column, const std::string& message) override {} + void RecordError(int line, int column, const absl::string_view message) override {} @@ -110,7 +110,7 @@ index b289f54b..38e89170 100644 json_options); if (!status.ok()) { diff --git a/tensorflow/core/profiler/convert/xplane_to_tools_data.cc b/tensorflow/core/profiler/convert/xplane_to_tools_data.cc -index b623c6a9..624f2c04 100644 +index dcc7cea4..a995db64 100644 --- a/tensorflow/core/profiler/convert/xplane_to_tools_data.cc +++ b/tensorflow/core/profiler/convert/xplane_to_tools_data.cc @@ -218,7 +218,11 @@ absl::StatusOr ConvertMultiXSpacesToPodViewer( @@ -197,10 +197,10 @@ index 406e6641..d97eae73 100644 << column << " - " << message; } diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl -index dc78011d..a4faad8f 100644 +index 5320aa18..32f0a39c 100644 --- a/tensorflow/workspace2.bzl +++ b/tensorflow/workspace2.bzl -@@ -870,6 +870,10 @@ def _tf_repositories(): +@@ -878,6 +878,10 @@ def _tf_repositories(): name = "riegeli", sha256 = "1d216d5c97fa60632143d209a1bb48c2a83788efdb876902e7bbc06396d5ee1f", strip_prefix = "riegeli-5d75119232cd4f6db8dfa69a1503289f050e9643", @@ -309,7 +309,7 @@ index 00000000..b5af8d2d +2.34.1 + diff --git a/third_party/xla/third_party/tsl/tsl/platform/default/human_readable_json.cc b/third_party/xla/third_party/tsl/tsl/platform/default/human_readable_json.cc -index 4f70c155..ae746f33 100644 +index 167cdd2b..dc1ebdd1 100644 --- a/third_party/xla/third_party/tsl/tsl/platform/default/human_readable_json.cc +++ b/third_party/xla/third_party/tsl/tsl/platform/default/human_readable_json.cc @@ -33,7 +33,11 @@ absl::StatusOr ProtoToHumanReadableJson( diff --git a/recipe/patches/0028-Fix-ResolvePadding-to-be-consistent-in-the-size-of-t.patch b/recipe/patches/0028-Fix-ResolvePadding-to-be-consistent-in-the-size-of-t.patch new file mode 100644 index 00000000..002610a8 --- /dev/null +++ b/recipe/patches/0028-Fix-ResolvePadding-to-be-consistent-in-the-size-of-t.patch @@ -0,0 +1,38 @@ +From b3804de29e8b40b1cb18b940656307e24cd7a457 Mon Sep 17 00:00:00 2001 +From: "A. Unique TensorFlower" +Date: Thu, 14 Nov 2024 15:25:44 -0800 +Subject: [PATCH 28/28] Fix ResolvePadding to be consistent in the size of the + small vector + +PiperOrigin-RevId: 696670235 +--- + .../transforms/legalize_hlo_conversions/op_util_common.cc | 2 +- + .../transforms/legalize_hlo_conversions/op_util_common.h | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.cc b/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.cc +index 3d67bbfd..df67b39d 100644 +--- a/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.cc ++++ b/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.cc +@@ -68,7 +68,7 @@ llvm::SmallVector ResolveStridesOrDilations( + return llvm::SmallVector(attr.getValues()); + } + +-llvm::SmallVector ResolvePadding( ++llvm::SmallVector ResolvePadding( + int64_t rank, std::optional opt_padding) { + llvm::SmallVector res; + if (!opt_padding.has_value()) { +diff --git a/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.h b/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.h +index 3c2c8ae5..b522b41c 100644 +--- a/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.h ++++ b/tensorflow/compiler/mlir/lite/stablehlo/transforms/legalize_hlo_conversions/op_util_common.h +@@ -127,7 +127,7 @@ llvm::SmallVector ResolveStridesOrDilations( + + // Resolves optional paddings attributes. If not present, will return + // trivial [0, 0] paddings on each dim. +-llvm::SmallVector ResolvePadding( ++llvm::SmallVector ResolvePadding( + int64_t rank, std::optional opt_padding); + + // Does the padding correspond to "SAME" on given dimension configuration.
VariantStatus
linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.10.____cpythonlinux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.10.____cpython - variant + variant
linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.11.____cpythonlinux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.11.____cpython - variant + variant
linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.12.____cpythonlinux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.12.____cpython - variant + variant
linux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.0cxx_compiler_version12python3.9.____cpythonlinux_64_c_compiler_version12cuda_compilercuda-nvcccuda_compiler_version12.6cxx_compiler_version12python3.9.____cpython - variant + variant