From 8115f88682dcf94ae420ef902e8968ed14d7325c Mon Sep 17 00:00:00 2001 From: Mike Bland Date: Thu, 28 Nov 2024 13:54:25 -0500 Subject: [PATCH 1/2] Extract dev_deps_repositories() from WORKSPACE `dev_deps_repositories()` encapsulates the instantiation of repositories used only for `rules_scala` development. Also removes the unused `//private` package and its `WORKSPACE` statements. Part of #1482. Both `WORKSPACE` and Bzlmod builds can use this macro, though how Bzlmod will use it will depend on whether we continue building with Bazel 6. `@bazel_tools//tools/build_defs/repo:local.bzl` isn't available under Bazel 6. To continue building with Bazel 6 under Bzlmod, we will need to call `dev_deps_repositories()` from `WORKSPACE.bzlmod` to continue using `native.{,new_}local_repository()`. If we switch to Bazel 7, we can load `local.bzl` and strip the `native.` prefix from the `{,new_}local_repository()` calls. Then we can call `dev_deps_repositories()` from a module extension instead of from `WORKSPACE.bzlmod`. Another alternative would be updating the local repositories to become proper nested repositories. Then we can call `local_repository()` from `WORKSPACE` and call `bazel_dep()` and `local_path_override()` from `MODULE.bazel`. In that case, we'd remove the `{,new_}local_repository` calls from `dev_deps_dependencies()`, and remove `proto_cross_repo_boundary_repository()` entirely. --- WORKSPACE | 106 +++------------------ private/BUILD | 7 -- private/format.bzl | 71 -------------- scala/private/extensions/dev_deps.bzl | 117 ++++++++++++++++++++++++ test/proto_cross_repo_boundary/repo.bzl | 12 ++- 5 files changed, 141 insertions(+), 172 deletions(-) delete mode 100644 private/BUILD delete mode 100644 private/format.bzl create mode 100644 scala/private/extensions/dev_deps.bzl diff --git a/WORKSPACE b/WORKSPACE index 96628c248..94b447f05 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -26,15 +26,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() -_build_tools_release = "5.1.0" - -http_archive( - name = "com_github_bazelbuild_buildtools", - sha256 = "e3bb0dc8b0274ea1aca75f1f8c0c835adbe589708ea89bf698069d0790701ea3", - strip_prefix = "buildtools-%s" % _build_tools_release, - url = "https://github.com/bazelbuild/buildtools/archive/%s.tar.gz" % _build_tools_release, -) - load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config") scala_config(enable_compiler_dependency_tracking = True) @@ -66,7 +57,6 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") protobuf_deps() -load("//scala:scala_cross_version.bzl", "default_maven_server_urls") load("//twitter_scrooge:twitter_scrooge.bzl", "twitter_scrooge") twitter_scrooge() @@ -93,53 +83,6 @@ scalafmt_default_config() scalafmt_repositories() -MAVEN_SERVER_URLS = default_maven_server_urls() - -# needed for the cross repo proto test -load("//test/proto_cross_repo_boundary:repo.bzl", "proto_cross_repo_boundary_repository") - -proto_cross_repo_boundary_repository() - -new_local_repository( - name = "test_new_local_repo", - build_file_content = - """ -filegroup( - name = "data", - srcs = glob(["**/*.txt"]), - visibility = ["//visibility:public"], -) -""", - path = "third_party/test/new_local_repo", -) - -local_repository( - name = "example_external_workspace", - path = "third_party/test/example_external_workspace", -) - -load("//scala:scala_maven_import_external.bzl", "java_import_external") - -# bazel's java_import_external has been altered in rules_scala to be a macro based on jvm_import_external -# in order to allow for other jvm-language imports (e.g. scala_import) -# the 3rd-party dependency below is using the java_import_external macro -# in order to make sure no regression with the original java_import_external -java_import_external( - name = "org_apache_commons_commons_lang_3_5_without_file", - generated_linkable_rule_name = "linkable_org_apache_commons_commons_lang_3_5_without_file", - jar_sha256 = "8ac96fc686512d777fca85e144f196cd7cfe0c0aec23127229497d1a38ff651c", - jar_urls = ["https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.jar"], - licenses = ["notice"], # Apache 2.0 - neverlink = True, - testonly_ = True, -) - -## Linting - -load("//private:format.bzl", "format_repositories") - -format_repositories() - http_archive( name = "io_bazel_rules_go", sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996", @@ -159,12 +102,6 @@ go_rules_dependencies() go_register_toolchains(version = "1.19.5") -load("@rules_java//java:repositories.bzl", "remote_jdk8_repos") - -# We need to select based on platform when we use these -# https://github.com/bazelbuild/bazel/issues/11655 -remote_jdk8_repos() - http_archive( name = "bazelci_rules", sha256 = "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e", @@ -179,35 +116,20 @@ rbe_preconfig( toolchain = "ubuntu2004-bazel-java11", ) -load("//third_party/repositories:repositories.bzl", "repositories") - -repositories( - fetch_sources = False, - for_artifact_ids = [ - # test adding a scala jar: - "com_twitter__scalding_date", - # test of strict deps (scalac plugin UT + E2E) - "com_google_guava_guava_21_0_with_file", - "com_github_jnr_jffi_native", - "org_apache_commons_commons_lang_3_5", - "com_google_guava_guava_21_0", - # test of import external - # scala maven import external decodes maven artifacts to its parts - # (group id, artifact id, packaging, version and classifier). To make sure - # the decoding and then the download url composition are working the artifact example - # must contain all the different parts and sha256s so the downloaded content will be - # validated against it - "org_springframework_spring_core", - "org_springframework_spring_tx", - "org_typelevel_kind_projector", - # For testing that we don't include sources jars to the classpath - "org_typelevel__cats_core", - ], - maven_servers = MAVEN_SERVER_URLS, -) +load("//scala/private/extensions:dev_deps.bzl", "dev_deps_repositories") -load("//test/toolchains:jdk.bzl", "remote_jdk21_repositories", "remote_jdk21_toolchains") +dev_deps_repositories() -remote_jdk21_repositories() +# Copied from bazel_tools/tools/jdk/remote_java_repository.bzl. +[ + register_toolchains( + "@remotejdk21_" + platform + "_toolchain_config_repo//:all", + ) + for platform in [ + "linux", + "macos", + "win", + ] +] -remote_jdk21_toolchains() +register_toolchains("//test/toolchains:java21_toolchain_definition") diff --git a/private/BUILD b/private/BUILD deleted file mode 100644 index 469006a8d..000000000 --- a/private/BUILD +++ /dev/null @@ -1,7 +0,0 @@ -load("@rules_java//java:defs.bzl", "java_binary") - -java_binary( - name = "java_format", - main_class = "com.google.googlejavaformat.java.Main", - runtime_deps = ["@google_java_format"], -) diff --git a/private/format.bzl b/private/format.bzl deleted file mode 100644 index 35b7ba3c0..000000000 --- a/private/format.bzl +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2018 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Dependencies for linting/formatting. -""" - -load( - "@bazel_tools//tools/build_defs/repo:http.bzl", - "http_archive", -) -load( - "@bazel_tools//tools/build_defs/repo:java.bzl", - "java_import_external", -) - -def _com_github_google_yapf_repository_impl(rctx): - rctx.download_and_extract( - url = "https://github.com/google/yapf/archive/v0.21.0.tar.gz", - stripPrefix = "yapf-0.21.0", - ) - rctx.file("BUILD", """ -alias( - name="yapf", - actual="//yapf:yapf", - visibility = ["//visibility:public"], -) -""") - rctx.file("yapf/BUILD", """ -py_binary( - name="yapf", - srcs=glob(["**/*.py"]), - main="__main__.py", - visibility = ["//visibility:public"], -)""") - -_com_github_google_yapf_repository = repository_rule( - attrs = {}, - implementation = _com_github_google_yapf_repository_impl, -) - -def format_repositories(): - _com_github_google_yapf_repository(name = "com_github_google_yapf") - - http_archive( - name = "io_bazel", - urls = [ - "https://github.com/bazelbuild/bazel/releases/download/0.11.1/bazel-0.11.1-dist.zip", - ], - sha256 = ( - "e8d762bcc01566fa50952c8028e95cfbe7545a39b8ceb3a0d0d6df33b25b333f" - ), - ) - - java_import_external( - name = "google_java_format", - licenses = ["notice"], # Apache 2.0 - jar_urls = [ - "https://github.com/google/google-java-format/releases/download/google-java-format-1.5/google-java-format-1.5-all-deps.jar", - ], - jar_sha256 = ("7b839bb7534a173f0ed0cd0e9a583181d20850fcec8cf6e3800e4420a1fad184"), - ) diff --git a/scala/private/extensions/dev_deps.bzl b/scala/private/extensions/dev_deps.bzl new file mode 100644 index 000000000..e01d45a9d --- /dev/null +++ b/scala/private/extensions/dev_deps.bzl @@ -0,0 +1,117 @@ +"""Repositories for testing rules_scala itself""" + +load("//scala:scala_cross_version.bzl", "default_maven_server_urls") +load("//scala:scala_maven_import_external.bzl", "java_import_external") +load( + "//test/proto_cross_repo_boundary:repo.bzl", + "proto_cross_repo_boundary_repository", +) +load("//test/toolchains:jdk.bzl", "remote_jdk21_repositories") +load("//third_party/repositories:repositories.bzl", "repositories") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# Once we switch to Bazel 7, uncomment this `load` statement remove the +# `native.` prefix from `local_repository` and `new_local_repository`. +# Update //test/proto_cross_repo_boundary:repo.bzl in the same way. +#load( +# "@bazel_tools//tools/build_defs/repo:local.bzl", +# "local_repository", +# "new_local_repository", +#) +load("@rules_java//java:repositories.bzl", "remote_jdk8_repos") + +_BUILD_TOOLS_RELEASE = "5.1.0" + +def dev_deps_repositories( + name = "unused_dev_deps_name", + maven_servers = default_maven_server_urls(), + fetch_sources = False): + """Instantiates internal only repos for development and testing + + Once we're using Bazel 7, uncomment the `load()` statement for `local.bzl` + in this file and remove `native.` from the `native.{,new_}local_repository` + calls. + + Until then, `dev_deps_repositories()` must be called from `WORKSPACE` or + `WORKSPACE.bzlmod`, as module extensions won't be able to use it. + + Args: + name: unused macro parameter to satisfy Buildifier lint rules + maven_servers: servers to use when resolving Maven artifacts + fetch_sources: retrieve Maven artifact sources when True + """ + http_archive( + name = "com_github_bazelbuild_buildtools", + sha256 = "e3bb0dc8b0274ea1aca75f1f8c0c835adbe589708ea89bf698069d0790701ea3", + strip_prefix = "buildtools-%s" % _BUILD_TOOLS_RELEASE, + url = ( + "https://github.com/bazelbuild/buildtools/archive/%s.tar.gz" % + _BUILD_TOOLS_RELEASE + ), + ) + + # needed for the cross repo proto test + proto_cross_repo_boundary_repository() + + native.local_repository( + name = "example_external_workspace", + path = "third_party/test/example_external_workspace", + ) + + native.new_local_repository( + name = "test_new_local_repo", + build_file_content = """ +filegroup( + name = "data", + srcs = glob(["**/*.txt"]), + visibility = ["//visibility:public"], +) +""", + path = "third_party/test/new_local_repo", + ) + + # bazel's java_import_external has been altered in rules_scala to be a macro + # based on jvm_import_external in order to allow for other jvm-language + # imports (e.g. scala_import) the 3rd-party dependency below is using the + # java_import_external macro in order to make sure no regression with the + # original java_import_external + java_import_external( + name = "org_apache_commons_commons_lang_3_5_without_file", + generated_linkable_rule_name = "linkable_org_apache_commons_commons_lang_3_5_without_file", + jar_sha256 = "8ac96fc686512d777fca85e144f196cd7cfe0c0aec23127229497d1a38ff651c", + jar_urls = ["https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.jar"], + licenses = ["notice"], # Apache 2.0 + neverlink = True, + testonly_ = True, + ) + + # We need to select based on platform when we use these + # https://github.com/bazelbuild/bazel/issues/11655 + remote_jdk8_repos() + + repositories( + fetch_sources = fetch_sources, + for_artifact_ids = [ + # test adding a scala jar: + "com_twitter__scalding_date", + # test of strict deps (scalac plugin UT + E2E) + "com_google_guava_guava_21_0_with_file", + "com_github_jnr_jffi_native", + "org_apache_commons_commons_lang_3_5", + "com_google_guava_guava_21_0", + # test of import external + # scala maven import external decodes maven artifacts to its parts + # (group id, artifact id, packaging, version and classifier). To make sure + # the decoding and then the download url composition are working the artifact example + # must contain all the different parts and sha256s so the downloaded content will be + # validated against it + "org_springframework_spring_core", + "org_springframework_spring_tx", + "org_typelevel_kind_projector", + # For testing that we don't include sources jars to the classpath + "org_typelevel__cats_core", + ], + maven_servers = maven_servers, + ) + + remote_jdk21_repositories() diff --git a/test/proto_cross_repo_boundary/repo.bzl b/test/proto_cross_repo_boundary/repo.bzl index fe3fecf04..f780f18e9 100644 --- a/test/proto_cross_repo_boundary/repo.bzl +++ b/test/proto_cross_repo_boundary/repo.bzl @@ -1,6 +1,14 @@ -def proto_cross_repo_boundary_repository(): +"""Macro for creating the @proto_cross_repo_boundary repo for testing""" + +# Once we switch to Bazel 7, uncomment this `load` statement remove the +# `native.` prefix from `new_local_repository`. +#load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +def proto_cross_repo_boundary_repository( + name = "proto_cross_repo_boundary"): + """Creates the @proto_cross_repo_boundary repo for rules_scala testing.""" native.new_local_repository( - name = "proto_cross_repo_boundary", + name = name, path = "test/proto_cross_repo_boundary/repo", build_file = "//test/proto_cross_repo_boundary:repo/BUILD.repo", ) From 0bbf8c6bcdb91a6913e1108a9e879b9aceef9b2f Mon Sep 17 00:00:00 2001 From: Mike Bland Date: Thu, 28 Nov 2024 15:06:34 -0500 Subject: [PATCH 2/2] Add test local_repository calls to WORKSPACE Removes `native.{,new_}local_repository()` calls in macros in favor of `local_repository` calls from `WORKSPACE`. Part of #1482. `native.{,new_}local_repository()` isn't available under Bzlmod, `@bazel_tools//tools/build_defs/repo:local.bzl` with the Starlarkified definitions isn't available under Bazel 6, and Bazel 8 compatibility work is imminent. Redefining the repositories in this way will be compatible with Bazel 6, 7, and 8, both under `WORKSPACE` and Bzlmod. (`MODULE.bazel` will use a combination of `bazel_dep()` and `local_path_override()`.) --- WORKSPACE | 16 ++++++++ scala/private/extensions/dev_deps.bzl | 40 ------------------- test/proto_cross_repo_boundary/repo.bzl | 14 ------- .../repo/.bazelversion | 1 + .../repo/{BUILD.repo => BUILD} | 0 test/proto_cross_repo_boundary/repo/WORKSPACE | 0 third_party/test/new_local_repo/.bazelversion | 1 + third_party/test/new_local_repo/BUILD | 5 +++ third_party/test/new_local_repo/WORKSPACE | 0 9 files changed, 23 insertions(+), 54 deletions(-) delete mode 100644 test/proto_cross_repo_boundary/repo.bzl create mode 100644 test/proto_cross_repo_boundary/repo/.bazelversion rename test/proto_cross_repo_boundary/repo/{BUILD.repo => BUILD} (100%) create mode 100644 test/proto_cross_repo_boundary/repo/WORKSPACE create mode 100644 third_party/test/new_local_repo/.bazelversion create mode 100644 third_party/test/new_local_repo/BUILD create mode 100644 third_party/test/new_local_repo/WORKSPACE diff --git a/WORKSPACE b/WORKSPACE index 94b447f05..b111cf67e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -83,6 +83,22 @@ scalafmt_default_config() scalafmt_repositories() +# needed for the cross repo proto test +local_repository( + name = "proto_cross_repo_boundary", + path = "test/proto_cross_repo_boundary/repo", +) + +local_repository( + name = "test_new_local_repo", + path = "third_party/test/new_local_repo", +) + +local_repository( + name = "example_external_workspace", + path = "third_party/test/example_external_workspace", +) + http_archive( name = "io_bazel_rules_go", sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996", diff --git a/scala/private/extensions/dev_deps.bzl b/scala/private/extensions/dev_deps.bzl index e01d45a9d..798d4afdc 100644 --- a/scala/private/extensions/dev_deps.bzl +++ b/scala/private/extensions/dev_deps.bzl @@ -2,22 +2,9 @@ load("//scala:scala_cross_version.bzl", "default_maven_server_urls") load("//scala:scala_maven_import_external.bzl", "java_import_external") -load( - "//test/proto_cross_repo_boundary:repo.bzl", - "proto_cross_repo_boundary_repository", -) load("//test/toolchains:jdk.bzl", "remote_jdk21_repositories") load("//third_party/repositories:repositories.bzl", "repositories") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -# Once we switch to Bazel 7, uncomment this `load` statement remove the -# `native.` prefix from `local_repository` and `new_local_repository`. -# Update //test/proto_cross_repo_boundary:repo.bzl in the same way. -#load( -# "@bazel_tools//tools/build_defs/repo:local.bzl", -# "local_repository", -# "new_local_repository", -#) load("@rules_java//java:repositories.bzl", "remote_jdk8_repos") _BUILD_TOOLS_RELEASE = "5.1.0" @@ -28,13 +15,6 @@ def dev_deps_repositories( fetch_sources = False): """Instantiates internal only repos for development and testing - Once we're using Bazel 7, uncomment the `load()` statement for `local.bzl` - in this file and remove `native.` from the `native.{,new_}local_repository` - calls. - - Until then, `dev_deps_repositories()` must be called from `WORKSPACE` or - `WORKSPACE.bzlmod`, as module extensions won't be able to use it. - Args: name: unused macro parameter to satisfy Buildifier lint rules maven_servers: servers to use when resolving Maven artifacts @@ -50,26 +30,6 @@ def dev_deps_repositories( ), ) - # needed for the cross repo proto test - proto_cross_repo_boundary_repository() - - native.local_repository( - name = "example_external_workspace", - path = "third_party/test/example_external_workspace", - ) - - native.new_local_repository( - name = "test_new_local_repo", - build_file_content = """ -filegroup( - name = "data", - srcs = glob(["**/*.txt"]), - visibility = ["//visibility:public"], -) -""", - path = "third_party/test/new_local_repo", - ) - # bazel's java_import_external has been altered in rules_scala to be a macro # based on jvm_import_external in order to allow for other jvm-language # imports (e.g. scala_import) the 3rd-party dependency below is using the diff --git a/test/proto_cross_repo_boundary/repo.bzl b/test/proto_cross_repo_boundary/repo.bzl deleted file mode 100644 index f780f18e9..000000000 --- a/test/proto_cross_repo_boundary/repo.bzl +++ /dev/null @@ -1,14 +0,0 @@ -"""Macro for creating the @proto_cross_repo_boundary repo for testing""" - -# Once we switch to Bazel 7, uncomment this `load` statement remove the -# `native.` prefix from `new_local_repository`. -#load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") - -def proto_cross_repo_boundary_repository( - name = "proto_cross_repo_boundary"): - """Creates the @proto_cross_repo_boundary repo for rules_scala testing.""" - native.new_local_repository( - name = name, - path = "test/proto_cross_repo_boundary/repo", - build_file = "//test/proto_cross_repo_boundary:repo/BUILD.repo", - ) diff --git a/test/proto_cross_repo_boundary/repo/.bazelversion b/test/proto_cross_repo_boundary/repo/.bazelversion new file mode 100644 index 000000000..f22d756da --- /dev/null +++ b/test/proto_cross_repo_boundary/repo/.bazelversion @@ -0,0 +1 @@ +6.5.0 diff --git a/test/proto_cross_repo_boundary/repo/BUILD.repo b/test/proto_cross_repo_boundary/repo/BUILD similarity index 100% rename from test/proto_cross_repo_boundary/repo/BUILD.repo rename to test/proto_cross_repo_boundary/repo/BUILD diff --git a/test/proto_cross_repo_boundary/repo/WORKSPACE b/test/proto_cross_repo_boundary/repo/WORKSPACE new file mode 100644 index 000000000..e69de29bb diff --git a/third_party/test/new_local_repo/.bazelversion b/third_party/test/new_local_repo/.bazelversion new file mode 100644 index 000000000..f22d756da --- /dev/null +++ b/third_party/test/new_local_repo/.bazelversion @@ -0,0 +1 @@ +6.5.0 diff --git a/third_party/test/new_local_repo/BUILD b/third_party/test/new_local_repo/BUILD new file mode 100644 index 000000000..07b439abc --- /dev/null +++ b/third_party/test/new_local_repo/BUILD @@ -0,0 +1,5 @@ +filegroup( + name = "data", + srcs = glob(["**/*.txt"]), + visibility = ["//visibility:public"], +) diff --git a/third_party/test/new_local_repo/WORKSPACE b/third_party/test/new_local_repo/WORKSPACE new file mode 100644 index 000000000..e69de29bb