-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add tests for RST_stats expressions.
- Loading branch information
milos.colic
committed
Jan 17, 2024
1 parent
9c3c7cb
commit fa2fc5c
Showing
6 changed files
with
161 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
48 changes: 48 additions & 0 deletions
48
src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_AvgBehaviors.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
package com.databricks.labs.mosaic.expressions.raster | ||
|
||
import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI | ||
import com.databricks.labs.mosaic.core.index.IndexSystem | ||
import com.databricks.labs.mosaic.functions.MosaicContext | ||
import org.apache.spark.sql.QueryTest | ||
import org.apache.spark.sql.functions._ | ||
import org.scalatest.matchers.should.Matchers._ | ||
|
||
trait RST_AvgBehaviors extends QueryTest { | ||
|
||
def behavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = { | ||
val mc = MosaicContext.build(indexSystem, geometryAPI) | ||
mc.register() | ||
val sc = spark | ||
import mc.functions._ | ||
import sc.implicits._ | ||
|
||
val rastersInMemory = spark.read | ||
.format("gdal") | ||
.option("raster_storage", "in-memory") | ||
.load("src/test/resources/modis") | ||
|
||
val df = rastersInMemory | ||
.withColumn("tile", rst_tessellate($"tile", lit(3))) | ||
.withColumn("result", rst_avg($"tile")) | ||
.select("result") | ||
.select(explode($"result").as("result")) | ||
|
||
rastersInMemory | ||
.withColumn("tile", rst_tessellate($"tile", lit(3))) | ||
.createOrReplaceTempView("source") | ||
|
||
noException should be thrownBy spark.sql(""" | ||
|select rst_avg(tile) from source | ||
|""".stripMargin) | ||
|
||
val result = df.as[Double].collect().max | ||
|
||
result > 0 shouldBe true | ||
|
||
an[Exception] should be thrownBy spark.sql(""" | ||
|select rst_avg() from source | ||
|""".stripMargin) | ||
|
||
} | ||
|
||
} |
32 changes: 32 additions & 0 deletions
32
src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_AvgTest.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
package com.databricks.labs.mosaic.expressions.raster | ||
|
||
import com.databricks.labs.mosaic.core.geometry.api.JTS | ||
import com.databricks.labs.mosaic.core.index.H3IndexSystem | ||
import org.apache.spark.sql.QueryTest | ||
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode | ||
import org.apache.spark.sql.internal.SQLConf | ||
import org.apache.spark.sql.test.SharedSparkSessionGDAL | ||
|
||
import scala.util.Try | ||
|
||
class RST_AvgTest extends QueryTest with SharedSparkSessionGDAL with RST_AvgBehaviors { | ||
|
||
private val noCodegen = | ||
withSQLConf( | ||
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", | ||
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString | ||
) _ | ||
|
||
// Hotfix for SharedSparkSession afterAll cleanup. | ||
override def afterAll(): Unit = Try(super.afterAll()) | ||
|
||
// These tests are not index system nor geometry API specific. | ||
// Only testing one pairing is sufficient. | ||
test("Testing rst_avg behavior with H3IndexSystem and JTS") { | ||
noCodegen { | ||
assume(System.getProperty("os.name") == "Linux") | ||
behavior(H3IndexSystem, JTS) | ||
} | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
48 changes: 48 additions & 0 deletions
48
src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MinBehaviors.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
package com.databricks.labs.mosaic.expressions.raster | ||
|
||
import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI | ||
import com.databricks.labs.mosaic.core.index.IndexSystem | ||
import com.databricks.labs.mosaic.functions.MosaicContext | ||
import org.apache.spark.sql.QueryTest | ||
import org.apache.spark.sql.functions._ | ||
import org.scalatest.matchers.should.Matchers._ | ||
|
||
trait RST_MinBehaviors extends QueryTest { | ||
|
||
def behavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = { | ||
val mc = MosaicContext.build(indexSystem, geometryAPI) | ||
mc.register() | ||
val sc = spark | ||
import mc.functions._ | ||
import sc.implicits._ | ||
|
||
val rastersInMemory = spark.read | ||
.format("gdal") | ||
.option("raster_storage", "in-memory") | ||
.load("src/test/resources/modis") | ||
|
||
val df = rastersInMemory | ||
.withColumn("tile", rst_tessellate($"tile", lit(3))) | ||
.withColumn("result", rst_min($"tile")) | ||
.select("result") | ||
.select(explode($"result").as("result")) | ||
|
||
rastersInMemory | ||
.withColumn("tile", rst_tessellate($"tile", lit(3))) | ||
.createOrReplaceTempView("source") | ||
|
||
noException should be thrownBy spark.sql(""" | ||
|select rst_min(tile) from source | ||
|""".stripMargin) | ||
|
||
val result = df.as[Double].collect().min | ||
|
||
result < 0 shouldBe true | ||
|
||
an[Exception] should be thrownBy spark.sql(""" | ||
|select rst_min() from source | ||
|""".stripMargin) | ||
|
||
} | ||
|
||
} |
32 changes: 32 additions & 0 deletions
32
src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MinTest.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
package com.databricks.labs.mosaic.expressions.raster | ||
|
||
import com.databricks.labs.mosaic.core.geometry.api.JTS | ||
import com.databricks.labs.mosaic.core.index.H3IndexSystem | ||
import org.apache.spark.sql.QueryTest | ||
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode | ||
import org.apache.spark.sql.internal.SQLConf | ||
import org.apache.spark.sql.test.SharedSparkSessionGDAL | ||
|
||
import scala.util.Try | ||
|
||
class RST_MinTest extends QueryTest with SharedSparkSessionGDAL with RST_MinBehaviors { | ||
|
||
private val noCodegen = | ||
withSQLConf( | ||
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", | ||
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString | ||
) _ | ||
|
||
// Hotfix for SharedSparkSession afterAll cleanup. | ||
override def afterAll(): Unit = Try(super.afterAll()) | ||
|
||
// These tests are not index system nor geometry API specific. | ||
// Only testing one pairing is sufficient. | ||
test("Testing rst_min behavior with H3IndexSystem and JTS") { | ||
noCodegen { | ||
assume(System.getProperty("os.name") == "Linux") | ||
behavior(H3IndexSystem, JTS) | ||
} | ||
} | ||
|
||
} |