From f493a509cc96dc597358260eb3bcd3440d67f336 Mon Sep 17 00:00:00 2001 From: Michael Johns Date: Fri, 5 Jan 2024 11:47:45 -0500 Subject: [PATCH] numpy to 1.22.4 (avoid scipy conflict). R off. --- .github/actions/python_build/action.yml | 2 +- .github/actions/scala_build/action.yml | 2 +- .github/workflows/build_main.yml | 6 +++--- .github/workflows/build_python.yml | 2 +- .github/workflows/build_r.yml | 2 +- .github/workflows/build_scala.yml | 2 +- .github/workflows/pypi-release.yml | 2 +- scripts/mosaic-gdal-init.sh | 7 ++----- .../mosaic/expressions/raster/RST_MergeAggBehaviors.scala | 1 + .../mosaic/expressions/raster/RST_MergeBehaviors.scala | 1 + 10 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/actions/python_build/action.yml b/.github/actions/python_build/action.yml index 72306a29b..3b7f84880 100644 --- a/.github/actions/python_build/action.yml +++ b/.github/actions/python_build/action.yml @@ -12,7 +12,7 @@ runs: run: | cd python pip install build wheel pyspark==${{ matrix.spark }} - pip install numpy~=${{ matrix.numpy }} + pip install numpy==${{ matrix.numpy }} pip install --no-cache-dir --force-reinstall GDAL[numpy]==${{ matrix.gdal }} pip install . - name: Test and build python package diff --git a/.github/actions/scala_build/action.yml b/.github/actions/scala_build/action.yml index e94c5c7db..30936ba7e 100644 --- a/.github/actions/scala_build/action.yml +++ b/.github/actions/scala_build/action.yml @@ -27,7 +27,7 @@ runs: sudo apt-get update -y # - install numpy first pip install --upgrade pip - pip install numpy~=${{ matrix.numpy }} + pip install numpy==${{ matrix.numpy }} # - install natives sudo apt-get install -y unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7 sudo apt-get install -y gdal-bin libgdal-dev diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml index aa8da5158..4f0c676cd 100644 --- a/.github/workflows/build_main.yml +++ b/.github/workflows/build_main.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: python: [ 3.10.12 ] - numpy: [ 1.26.3 ] + numpy: [ 1.22.4 ] gdal: [ 3.4.1 ] spark: [ 3.4.0 ] R: [ 4.2.2 ] @@ -28,7 +28,7 @@ jobs: uses: ./.github/actions/scala_build - name: build python uses: ./.github/actions/python_build - - name: build R - uses: ./.github/actions/r_build + # - name: build R + # uses: ./.github/actions/r_build - name: upload artefacts uses: ./.github/actions/upload_artefacts diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 9da376e0b..30d62cb3c 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: python: [ 3.10.12 ] - numpy: [ 1.26.3 ] + numpy: [ 1.22.4 ] gdal: [ 3.4.1 ] spark: [ 3.4.0 ] R: [ 4.2.2 ] diff --git a/.github/workflows/build_r.yml b/.github/workflows/build_r.yml index ee25dacf5..986ca744d 100644 --- a/.github/workflows/build_r.yml +++ b/.github/workflows/build_r.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: python: [ 3.10.12 ] - numpy: [ 1.26.3 ] + numpy: [ 1.22.4 ] gdal: [ 3.4.1 ] spark: [ 3.4.0 ] R: [ 4.2.2 ] diff --git a/.github/workflows/build_scala.yml b/.github/workflows/build_scala.yml index e8da8373e..0269130d8 100644 --- a/.github/workflows/build_scala.yml +++ b/.github/workflows/build_scala.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: python: [ 3.10.12 ] - numpy: [ 1.26.3 ] + numpy: [ 1.22.4 ] gdal: [ 3.4.1 ] spark: [ 3.4.0 ] R: [ 4.2.2 ] diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index c5a11c4f0..251ead879 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python: [ 3.10.12 ] - numpy: [ 1.26.2 ] + numpy: [ 1.22.4 ] gdal: [ 3.4.1 ] spark: [ 3.4.0 ] R: [ 4.2.2 ] diff --git a/scripts/mosaic-gdal-init.sh b/scripts/mosaic-gdal-init.sh index b97524545..190faa412 100644 --- a/scripts/mosaic-gdal-init.sh +++ b/scripts/mosaic-gdal-init.sh @@ -11,7 +11,7 @@ # - setup_gdal(...) # [4] this script has conditional logic based on variables # Author: Michael Johns | mjohns@databricks.com -# Last Modified: 04 JAN, 2024 +# Last Modified: 05 JAN, 2024 # TEMPLATE-BASED REPLACEMENT # - can also be manually specified @@ -28,8 +28,7 @@ WITH_FUSE_SO=0 # <- use fuse dir shared objects (vs wget) # SPECIFIED VERSIONS # - may be changed by conditional logic # - https://docs.scipy.org/doc/scipy/dev/toolchain.html#toolchain-roadmap -NUMPY_VERSION='<1.27,>=1.26' # <- for GDAL -SCIPY_VERSION='<1.12,>=1.11' # <- adjusted for numpy +NUMPY_VERSION='<1.23,>=1.22' # <- for GDAL + Mosaic GDAL_VERSION=3.4.1 # <- ubuntugis is 3.4.3 # - optional: install Mosaic @@ -54,10 +53,8 @@ then sudo apt-get update -y # - install specific numpy version - # - install scipy version (dep conflict) pip install --upgrade pip pip install --no-cache-dir --force-reinstall "numpy$NUMPY_VERSION" - pip install --no-cache-dir --force-reinstall "scipy$SCIPY_VERSION" # - install natives sudo apt-get install -y gdal-bin libgdal-dev diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala index 38d5d3ed9..8698b46af 100644 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala +++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala @@ -20,6 +20,7 @@ trait RST_MergeAggBehaviors extends QueryTest { val rastersInMemory = spark.read .format("gdal") .option("raster_storage", "in-memory") + .option("pathGlobFilter", "*_B01.TIF") .load("src/test/resources/modis") val gridTiles = rastersInMemory diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala index fae8e5913..893d6bdf4 100644 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala +++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala @@ -21,6 +21,7 @@ trait RST_MergeBehaviors extends QueryTest { val rastersInMemory = spark.read .format("gdal") .option("raster_storage", "in-memory") + .option("pathGlobFilter", "*_B01.TIF") .load("src/test/resources/modis") val gridTiles = rastersInMemory