Port DependencyAnalyzer plugin to Scala 3 (#1640)

* Adapt DependencyAnalyzer/OptionsParsers.tokenize tailrec to a bug when compiling under Scala 3.1.3 (fixed in 3.2) * Setup dt_patches for Scala 3 * Port dependency-analyzer compiler plugin, setup and unify tests. * Add .bazelbsp and .bsp to .gitignore * Add test_thridparty_version.sh as (actually working) subset of it's dangerous counterpart * Show errors of failing tests * Use Java17 for test_thirdparty_version - latest supported JDK version by Scala 3.1.3 is JDK 18 * Remove debug printlns * Simplify resolving version of Scala 3 compiler
bazelbuild · Nov 11, 2024 · 9fccc25 · 9fccc25
1 parent 716210f
commit 9fccc25
Show file tree

Hide file tree

Showing 34 changed files with 1,137 additions and 496 deletions.
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
@@ -80,6 +80,11 @@ tasks:
     platform: macos
     shell_commands:
     - "./test_version.sh"
+  thirdparty_version_linux:
+    name: "./test_thirdparty_version.sh"      
+    platform: ubuntu2204_java17
+    shell_commands:
+    - "./test_thirdparty_version.sh"
   examples_linux:
     name: "./test_examples"
     platform: ubuntu2004

diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,8 @@ hash2
 .DS_store
 .bazel_cache
 .ijwb
+.bazelbsp
+.bsp
 .metals
 .vscode
 unformatted-*.backup.scala

diff --git a/dt_patches/dt_compiler_3.patch b/dt_patches/dt_compiler_3.patch
@@ -0,0 +1,19 @@
+
+diff --git a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala
+--- dotty/tools/dotc/core/SymbolLoaders.scala	(revision 91ef92159c628eaeab8311dc82bed7ed4fe03c63)
++++ dotty/tools/dotc/core/SymbolLoaders.scala	(date 1730331517451)
+@@ -353,6 +353,14 @@
+     try {
+       val start = System.currentTimeMillis
+       trace.onDebug("loading") {
++        root.symbol.associatedFile match {
++          case assocFile: AbstractFile => assocFile.underlyingSource.foreach {
++            case jar: dotty.tools.io.ZipArchive =>
++              report.echo(s"DT:${jar.path}")
++            case _ => ()
++          }
++          case null => ()
++        }
+         doComplete(root)
+       }
+       report.informTime("loaded " + description, start)
diff --git a/dt_patches/dt_patch_test.sh b/dt_patches/dt_patch_test.sh
@@ -116,6 +116,13 @@ run_test_local test_compiler_patch 2.13.12
 run_test_local test_compiler_patch 2.13.14
 run_test_local test_compiler_patch 2.13.15
 
+run_test_local test_compiler_patch 3.1.0 # Minimal supported version
+run_test_local test_compiler_patch 3.1.3
+run_test_local test_compiler_patch 3.2.2
+run_test_local test_compiler_patch 3.3.4 
+run_test_local test_compiler_patch 3.4.3
+run_test_local test_compiler_patch 3.5.2
+
 run_test_local test_compiler_srcjar_error 2.12.11
 run_test_local test_compiler_srcjar_error 2.12.12
 run_test_local test_compiler_srcjar_error 2.12.13
@@ -139,3 +146,9 @@ run_test_local test_compiler_srcjar_nonhermetic 2.13.11
 run_test_local test_compiler_srcjar_nonhermetic 2.13.12
 run_test_local test_compiler_srcjar_nonhermetic 2.13.14
 run_test_local test_compiler_srcjar_nonhermetic 2.13.15
+
+run_test_local test_compiler_srcjar 3.1.3
+run_test_local test_compiler_srcjar 3.2.2
+run_test_local test_compiler_srcjar_nonhermetic 3.3.4
+run_test_local test_compiler_srcjar 3.4.3
+run_test_local test_compiler_srcjar_nonhermetic 3.5.2
diff --git a/dt_patches/test_dt_patches/BUILD b/dt_patches/test_dt_patches/BUILD
@@ -2,11 +2,18 @@ load(
     "@io_bazel_rules_scala//scala:scala.bzl",
     "setup_scala_toolchain",
 )
+load("@io_bazel_rules_scala//scala:scala_cross_version_select.bzl", "select_for_scala_version")
 
-SCALA_LIBS = [
-    "@scala_library",
-    "@scala_reflect",
-]
+SCALA_LIBS = ["@scala_library"] + select_for_scala_version(
+    any_2 = ["@scala_reflect"],
+    any_3 = [
+        "@scala3_interfaces",
+        "@scala2_library",
+        "@tasty_core",
+        "@scala_asm",
+    ],
+    since_3_4 = ["@sbt_compiler_interface"],
+)
 
 setup_scala_toolchain(
     name = "dt_scala_toolchain",

diff --git a/dt_patches/test_dt_patches/WORKSPACE b/dt_patches/test_dt_patches/WORKSPACE
@@ -48,25 +48,62 @@ load("@io_bazel_rules_scala_config//:config.bzl", "SCALA_VERSION")
 
 scala_maven_import_external(
     name = "scala_library",
-    artifact = "org.scala-lang:scala-library:%s" % SCALA_VERSION,
+    artifact = "org.scala-lang:scala-library:%s" % SCALA_VERSION if SCALA_VERSION.startswith("2.") else "org.scala-lang:scala3-library_3:%s" % SCALA_VERSION,
     licenses = ["notice"],
     server_urls = default_maven_server_urls(),
 )
 
 scala_maven_import_external(
     name = "scala_compiler",
-    artifact = "org.scala-lang:scala-compiler:%s" % SCALA_VERSION,
+    artifact = "org.scala-lang:scala-compiler:%s" % SCALA_VERSION if SCALA_VERSION.startswith("2.") else "org.scala-lang:scala3-compiler_3:%s" % SCALA_VERSION,
     licenses = ["notice"],
     server_urls = default_maven_server_urls(),
 )
 
+# Scala 2 only
 scala_maven_import_external(
     name = "scala_reflect",
     artifact = "org.scala-lang:scala-reflect:%s" % SCALA_VERSION,
     licenses = ["notice"],
     server_urls = default_maven_server_urls(),
 )
 
+# Scala 3 only
+scala_maven_import_external(
+    name = "scala3_interfaces",
+    artifact = "org.scala-lang:scala3-interfaces:%s" % SCALA_VERSION,
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "scala2_library",
+    artifact = "org.scala-lang:scala-library:2.13.15",
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "tasty_core",
+    artifact = "org.scala-lang:tasty-core_3:%s" % SCALA_VERSION,
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "scala_asm",
+    artifact = "org.scala-lang.modules:scala-asm:9.7.0-scala-2",
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "sbt_compiler_interface",
+    artifact = "org.scala-sbt:compiler-interface:1.9.6",
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
 rules_scala_setup()
 
 rules_scala_toolchain_deps_repositories(

diff --git a/dt_patches/test_dt_patches_user_srcjar/BUILD b/dt_patches/test_dt_patches_user_srcjar/BUILD
@@ -2,11 +2,18 @@ load(
     "@io_bazel_rules_scala//scala:scala.bzl",
     "setup_scala_toolchain",
 )
+load("@io_bazel_rules_scala//scala:scala_cross_version_select.bzl", "select_for_scala_version")
 
-SCALA_LIBS = [
-    "@scala_library",
-    "@scala_reflect",
-]
+SCALA_LIBS = ["@scala_library"] + select_for_scala_version(
+    any_2 = ["@scala_reflect"],
+    any_3 = [
+        "@scala3_interfaces",
+        "@scala2_library",
+        "@tasty_core",
+        "@scala_asm",
+    ],
+    since_3_4 = ["@sbt_compiler_interface"],
+)
 
 setup_scala_toolchain(
     name = "dt_scala_toolchain",

diff --git a/dt_patches/test_dt_patches_user_srcjar/WORKSPACE b/dt_patches/test_dt_patches_user_srcjar/WORKSPACE
@@ -52,27 +52,70 @@ http_jar(
     url = "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.16/scala-compiler-2.12.16-sources.jar",
 )
 
+http_jar(
+    name = "scala3_compiler_srcjar",
+    sha256 = "3c413efa9a2921ef59da7f065c445ae1b6b97057cbbc6b16957ad052a575a3ce",
+    url = "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.4.3/scala3-compiler_3-3.4.3-sources.jar",
+)
+
 scala_maven_import_external(
     name = "scala_library",
-    artifact = "org.scala-lang:scala-library:%s" % SCALA_VERSION,
+    artifact = "org.scala-lang:scala-library:%s" % SCALA_VERSION if SCALA_VERSION.startswith("2.") else "org.scala-lang:scala3-library_3:%s" % SCALA_VERSION,
     licenses = ["notice"],
     server_urls = default_maven_server_urls(),
 )
 
 scala_maven_import_external(
     name = "scala_compiler",
-    artifact = "org.scala-lang:scala-compiler:%s" % SCALA_VERSION,
+    artifact = "org.scala-lang:scala-compiler:%s" % SCALA_VERSION if SCALA_VERSION.startswith("2.") else "org.scala-lang:scala3-compiler_3:%s" % SCALA_VERSION,
     licenses = ["notice"],
     server_urls = default_maven_server_urls(),
 )
 
+# Scala 2 only
 scala_maven_import_external(
     name = "scala_reflect",
     artifact = "org.scala-lang:scala-reflect:%s" % SCALA_VERSION,
     licenses = ["notice"],
     server_urls = default_maven_server_urls(),
 )
 
+# Scala 3 only
+scala_maven_import_external(
+    name = "scala3_interfaces",
+    artifact = "org.scala-lang:scala3-interfaces:%s" % SCALA_VERSION,
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "scala2_library",
+    artifact = "org.scala-lang:scala-library:2.13.15",
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "tasty_core",
+    artifact = "org.scala-lang:tasty-core_3:%s" % SCALA_VERSION,
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "scala_asm",
+    artifact = "org.scala-lang.modules:scala-asm:9.7.0-scala-2",
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
+scala_maven_import_external(
+    name = "sbt_compiler_interface",
+    artifact = "org.scala-sbt:compiler-interface:1.9.6",
+    licenses = ["notice"],
+    server_urls = default_maven_server_urls(),
+)
+
 srcjars_by_version = {
     # Invalid
     "2.12.11": [],
@@ -120,6 +163,23 @@ srcjars_by_version = {
     "2.13.15": {
         "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.15/scala-compiler-2.13.15-sources.jar?foo",
     },
+    "3.1.3": {
+        "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.1.3/scala3-compiler_3-3.1.3-sources.jar",
+        "integrity": "sha384-4J2ihR1QSdP5cvL3y2OUfw4uUX/hsQqcPlJV+IrQdsM/soiIAYfoEeIEt6vl3xBk",
+    },
+    "3.2.2": {
+        "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.2/scala3-compiler_3-3.2.2-sources.jar",
+        "sha256": "669d580fc4a8d3c2e2d13d5735ae9be05d567613fe44482de5bcc5e2e2ee89ea",
+    },
+    "3.3.4": {
+        "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.4/scala3-compiler_3-3.3.4-sources.jar",
+    },
+    "3.4.3": {
+        "label": "@scala3_compiler_srcjar//jar:downloaded.jar",
+    },
+    "3.5.2": {
+        "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.5.2/scala3-compiler_3-3.5.2-sources.jar",
+    },
 }
 
 rules_scala_setup(scala_compiler_srcjar = srcjars_by_version[SCALA_VERSION])

diff --git a/scala/private/macros/scala_repositories.bzl b/scala/private/macros/scala_repositories.bzl
@@ -75,23 +75,21 @@ def dt_patched_compiler_setup(scala_version, scala_compiler_srcjar = None):
 
     if scala_major_version == "2.12":
         if minor_version >= 1 and minor_version <= 7:
-            patch = Label(
-                "//dt_patches:dt_compiler_%s.1.patch" % scala_major_version,
-            )
+            patch = Label("//dt_patches:dt_compiler_%s.1.patch" % scala_major_version)
         elif minor_version <= 11:
-            patch = Label(
-                "//dt_patches:dt_compiler_%s.8.patch" % scala_major_version,
-            )
+            patch = Label("//dt_patches:dt_compiler_%s.8.patch" % scala_major_version)
+    elif scala_major_version.startswith("3."):
+        patch = Label("//dt_patches:dt_compiler_3.patch")
 
     build_file_content = "\n".join([
         "package(default_visibility = [\"//visibility:public\"])",
         "filegroup(",
         "    name = \"src\",",
-        "    srcs=[\"scala/tools/nsc/symtab/SymbolLoaders.scala\"],",
+        "    srcs=[\"scala/tools/nsc/symtab/SymbolLoaders.scala\"]," if scala_major_version.startswith("2.") else "    srcs=[\"dotty/tools/dotc/core/SymbolLoaders.scala\"],",
         ")",
     ])
     default_scalac_srcjar = {
-        "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/%s/scala-compiler-%s-sources.jar" % (scala_version, scala_version),
+        "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/%s/scala-compiler-%s-sources.jar" % (scala_version, scala_version) if scala_major_version.startswith("2.") else "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/%s/scala3-compiler_3-%s-sources.jar" % (scala_version, scala_version),
     }
     srcjar = scala_compiler_srcjar if scala_compiler_srcjar != None else default_scalac_srcjar
     _validate_scalac_srcjar(srcjar) or fail(

diff --git a/src/java/io/bazel/rulesscala/scalac/BUILD b/src/java/io/bazel/rulesscala/scalac/BUILD
@@ -24,11 +24,9 @@ java_binary(
     ],
     main_class = "io.bazel.rulesscala.scalac.ScalacWorker",
     visibility = ["//visibility:public"],
-    deps = (select_for_scala_version(
-        any_2 = [
-            "//third_party/dependency_analyzer/src/main/io/bazel/rulesscala/dependencyanalyzer/compiler:dep_reporting_compiler",
-        ],
-    ) if ENABLE_COMPILER_DEPENDENCY_TRACKING else []) + SCALAC_DEPS,
+    deps = ([
+        "//third_party/dependency_analyzer/src/main/io/bazel/rulesscala/dependencyanalyzer/compiler:dep_reporting_compiler",
+    ] if ENABLE_COMPILER_DEPENDENCY_TRACKING else []) + SCALAC_DEPS,
 )
 
 java_binary(

diff --git a/test_thirdparty_version.sh b/test_thirdparty_version.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+test_scala_version() {
+  SCALA_VERSION=$1
+  bazel test --test_output=errors //third_party/... --repo_env=SCALA_VERSION=${SCALA_VERSION}
+}
+
+dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+test_dir=$dir/test/shell
+# shellcheck source=./test_runner.sh
+. "${test_dir}"/test_runner.sh
+runner=$(get_test_runner "${1:-local}")
+
+
+# Latest version of each major version
+$runner test_scala_version "3.5.2" # Latest Next version
+$runner test_scala_version "3.3.4" # Latest LTS version
+$runner test_scala_version "3.1.3" # First supported major for Scala 3, max supported JDK=18
+$runner test_scala_version "2.13.15"
+$runner test_scala_version "2.12.20"
+$runner test_scala_version "2.11.12"
+
+# Tests for other versions should be placed in dangerous_test_thirdparty_version.sh 
+# However that script is outdated and uses only default Scala version for each minor