diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52f4804c5..dcb10b404 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -109,10 +109,12 @@ jobs: with: ref: main - - name: Setup Scala - uses: olafurpg/setup-scala@v14 + - name: Setup JDK and sbt + uses: actions/setup-java@v4.2.1 with: - java-version: "adopt@1.8" + distribution: temurin + java-version: 8 + cache: sbt - name: Import GPG Key run: | diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index f3e832af3..bb64b1134 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -18,26 +18,30 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.11.12, 2.12.18, 2.13.12] - spark: [2.4.8, 3.2.4, 3.3.3, 3.4.1] + scala: [2.11.12, 2.12.19, 2.13.13] + spark: [2.4.8, 3.3.4, 3.4.2, 3.5.1] exclude: - scala: 2.11.12 - spark: 3.2.4 + spark: 3.3.4 - scala: 2.11.12 - spark: 3.3.3 + spark: 3.4.2 - scala: 2.11.12 - spark: 3.4.1 - - scala: 2.13.12 + spark: 3.5.1 + - scala: 2.12.19 + spark: 2.4.8 + - scala: 2.13.13 spark: 2.4.8 name: Test Spark ${{matrix.spark}} on Scala ${{matrix.scala}} steps: - name: Checkout code uses: actions/checkout@v2 - uses: coursier/cache-action@v5 - - name: Setup Scala - uses: olafurpg/setup-scala@v10 + - name: Setup JDK and sbt + uses: actions/setup-java@v4.2.1 with: - java-version: "adopt@1.8" + distribution: temurin + java-version: 8 + cache: sbt - name: Build and run unit tests working-directory: ./pramen run: sbt ++${{matrix.scala}} test -DSPARK_VERSION=${{matrix.spark}} diff --git a/pramen/build.sbt b/pramen/build.sbt index cae16fa63..7c8b11892 100644 --- a/pramen/build.sbt +++ b/pramen/build.sbt @@ -20,8 +20,8 @@ import BuildInfoTemplateSettings._ import com.github.sbt.jacoco.report.JacocoReportSettings val scala211 = "2.11.12" -val scala212 = "2.12.18" -val scala213 = "2.13.12" +val scala212 = "2.12.19" +val scala213 = "2.13.13" ThisBuild / organization := "za.co.absa.pramen" diff --git a/pramen/examples/combined_example.sh b/pramen/examples/combined_example.sh index 43ccb55c1..1aeb2aa48 100755 --- a/pramen/examples/combined_example.sh +++ b/pramen/examples/combined_example.sh @@ -3,7 +3,7 @@ # Prerequisites: # 1. Download Spark 3.4.1 (Scala 2.12) and install it in /opt/spark/spark-3.4.1 or some other directory # 2. At repo_root/pramen, run -# sbt -DSPARK_VERSION="3.4.1" ++2.12.18 assembly +# sbt -DSPARK_VERSION="3.4.1" ++2.12.19 assembly # 3. Run # ./examples/combined_example.sh diff --git a/pramen/project/Dependencies.scala b/pramen/project/Dependencies.scala index 7a203c702..26b129560 100644 --- a/pramen/project/Dependencies.scala +++ b/pramen/project/Dependencies.scala @@ -50,7 +50,7 @@ object Dependencies { "org.apache.httpcomponents" % "httpclient" % httpClientVersion, "org.scalatest" %% "scalatest" % scalatestVersion % Test ) ++ Seq( - getAbrisDependency(scalaVersion), + getAbrisDependency(sparkVersion(scalaVersion)), getDeltaDependency(sparkVersion(scalaVersion), isCompile = false, isTest = true) ) diff --git a/pramen/project/Versions.scala b/pramen/project/Versions.scala index 16bcad7a7..40ea40fbc 100644 --- a/pramen/project/Versions.scala +++ b/pramen/project/Versions.scala @@ -18,8 +18,8 @@ import sbt.* object Versions { val defaultSparkVersionForScala211 = "2.4.8" - val defaultSparkVersionForScala212 = "3.3.3" - val defaultSparkVersionForScala213 = "3.4.1" + val defaultSparkVersionForScala212 = "3.3.4" + val defaultSparkVersionForScala213 = "3.4.2" val typesafeConfigVersion = "1.4.0" val postgreSqlDriverVersion = "42.3.8" @@ -93,13 +93,15 @@ object Versions { } } - def getAbrisDependency(scalaVersion: String): ModuleID = { - // According to this: https://docs.delta.io/latest/releases.html - val abrisVersion = scalaVersion match { - case version if version.startsWith("2.11.") => "5.1.1" - case version if version.startsWith("2.12.") => "5.1.1" - case version if version.startsWith("2.13.") => "6.0.0" - case _ => throw new IllegalArgumentException(s"Scala $scalaVersion not supported for Abris dependency.") + def getAbrisDependency(sparkVersion: String): ModuleID = { + // According to this: https://github.com/AbsaOSS/ABRiS?tab=readme-ov-file#supported-versions + val abrisVersion = sparkVersion match { + case version if version.startsWith("2.4.") => "5.1.1" + case version if version.startsWith("3.0.") => "5.1.1" + case version if version.startsWith("3.1.") => "5.1.1" + case version if version == "3.2.0" => "6.1.1" + case version if version.startsWith("3.") => "6.4.0" + case _ => throw new IllegalArgumentException(s"Spark $sparkVersion not supported for Abris dependency.") } println(s"Using Abris version $abrisVersion") diff --git a/pramen/project/build.properties b/pramen/project/build.properties index dd156299b..b0d0dd4d0 100644 --- a/pramen/project/build.properties +++ b/pramen/project/build.properties @@ -13,4 +13,4 @@ # limitations under the License. # -sbt.version=1.9.7 +sbt.version=1.9.9