diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df933e5..1c0bbbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,11 +18,11 @@ on: required: true jobs: - spark_versions_validation: + spark-versions-validation: if: ${{ github.event_name != 'workflow_dispatch' }} strategy: matrix: - spark_version: ['3.2.4', '3.3.2', '3.4.1'] + spark_version: ['3.4.1', '3.5.0'] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -32,17 +32,14 @@ jobs: distribution: "temurin" cache: "sbt" - env: - SPARK_VERSION_OVERRIDE: ${{ matrix.spark_version }} - run: | - echo "$SPARK_VERSION_OVERRIDE" > project/spark-version.conf - - run: cat project/spark-version.conf + SPARK_VERSION: ${{ matrix.spark-version }} + run: echo "SPARK_VERSION_OVERRIDE=${SPARK_VERSION}" >> $GITHUB_ENV + - run: sbt "show sparkVersion" - run: sbt compile - run: sbt test build: runs-on: ubuntu-latest - env: - SPARK_VERSION_OVERRIDE: ${{ inputs.spark-version }} steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v3 @@ -50,15 +47,18 @@ jobs: java-version: 8 distribution: "temurin" cache: "sbt" - - if: ${{ env.SPARK_VERSION_OVERRIDE != '' }} - run: echo "$SPARK_VERSION_OVERRIDE" > project/spark-version.conf + - if: ${{ github.event.inputs.spark-version != '' }} + env: + SPARK_VERSION: ${{ inputs.spark-version }} + run: echo "SPARK_VERSION_OVERRIDE=${SPARK_VERSION}" >> $GITHUB_ENV + - run: sbt "show sparkVersion" - run: sbt compile - run: sbt test - run: sbt package - - run: tar cf artefacts.tar target/scala-*/*.jar */target/scala-*/*.jar + - run: tar cf artefacts.tar target/ */target/ - uses: actions/upload-artifact@v3.1.2 with: name: Artefacts path: artefacts.tar -run-name: ${{ github.event_name == 'workflow_dispatch' && format('Building for Spark {0}', inputs.spark-version) || ' '}} \ No newline at end of file +run-name: ${{ github.event_name == 'workflow_dispatch' && format('Building for Spark {0}', inputs.spark-version) || ' '}} diff --git a/build.sbt b/build.sbt index 0bbaa30..b50432b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ ThisBuild / versionScheme := Some("strict") ThisBuild / scalaVersion := "2.12.12" ThisBuild / organization := "com.amadeus.dataio" val sparkVersion = settingKey[String]("The version of Spark used for building.") -ThisBuild / sparkVersion := "3.4.1" +ThisBuild / sparkVersion := sys.env.getOrElse("SPARK_VERSION_OVERRIDE", "3.4.1") // Common dependencies ThisBuild / libraryDependencies ++= Seq(