Skip to content

Commit

Permalink
Add tests for RST_stats expressions.
Browse files Browse the repository at this point in the history
  • Loading branch information
milos.colic committed Jan 17, 2024
1 parent 9c3c7cb commit fa2fc5c
Show file tree
Hide file tree
Showing 6 changed files with 161 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ case class RST_Avg(raster: Expression, expressionConfig: MosaicExpressionConfig)
/** Expression info required for the expression registration for spark SQL. */
object RST_Avg extends WithExpressionInfo {

override def name: String = "rst_mean"
override def name: String = "rst_avg"

override def usage: String = "_FUNC_(expr1) - Returns an array containing mean values for each band."

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package com.databricks.labs.mosaic.expressions.raster

import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI
import com.databricks.labs.mosaic.core.index.IndexSystem
import com.databricks.labs.mosaic.functions.MosaicContext
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.functions._
import org.scalatest.matchers.should.Matchers._

trait RST_AvgBehaviors extends QueryTest {

def behavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
import mc.functions._
import sc.implicits._

val rastersInMemory = spark.read
.format("gdal")
.option("raster_storage", "in-memory")
.load("src/test/resources/modis")

val df = rastersInMemory
.withColumn("tile", rst_tessellate($"tile", lit(3)))
.withColumn("result", rst_avg($"tile"))
.select("result")
.select(explode($"result").as("result"))

rastersInMemory
.withColumn("tile", rst_tessellate($"tile", lit(3)))
.createOrReplaceTempView("source")

noException should be thrownBy spark.sql("""
|select rst_avg(tile) from source
|""".stripMargin)

val result = df.as[Double].collect().max

result > 0 shouldBe true

an[Exception] should be thrownBy spark.sql("""
|select rst_avg() from source
|""".stripMargin)

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package com.databricks.labs.mosaic.expressions.raster

import com.databricks.labs.mosaic.core.geometry.api.JTS
import com.databricks.labs.mosaic.core.index.H3IndexSystem
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSessionGDAL

import scala.util.Try

class RST_AvgTest extends QueryTest with SharedSparkSessionGDAL with RST_AvgBehaviors {

private val noCodegen =
withSQLConf(
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString
) _

// Hotfix for SharedSparkSession afterAll cleanup.
override def afterAll(): Unit = Try(super.afterAll())

// These tests are not index system nor geometry API specific.
// Only testing one pairing is sufficient.
test("Testing rst_avg behavior with H3IndexSystem and JTS") {
noCodegen {
assume(System.getProperty("os.name") == "Linux")
behavior(H3IndexSystem, JTS)
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,6 @@ trait RST_MaxBehaviors extends QueryTest {
|select rst_max(tile) from source
|""".stripMargin)

noException should be thrownBy rastersInMemory
.withColumn("result", rst_rastertogridmax($"tile", lit(3)))
.select("result")

val result = df.as[Double].collect().max

result > 0 shouldBe true
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package com.databricks.labs.mosaic.expressions.raster

import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI
import com.databricks.labs.mosaic.core.index.IndexSystem
import com.databricks.labs.mosaic.functions.MosaicContext
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.functions._
import org.scalatest.matchers.should.Matchers._

trait RST_MinBehaviors extends QueryTest {

def behavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
import mc.functions._
import sc.implicits._

val rastersInMemory = spark.read
.format("gdal")
.option("raster_storage", "in-memory")
.load("src/test/resources/modis")

val df = rastersInMemory
.withColumn("tile", rst_tessellate($"tile", lit(3)))
.withColumn("result", rst_min($"tile"))
.select("result")
.select(explode($"result").as("result"))

rastersInMemory
.withColumn("tile", rst_tessellate($"tile", lit(3)))
.createOrReplaceTempView("source")

noException should be thrownBy spark.sql("""
|select rst_min(tile) from source
|""".stripMargin)

val result = df.as[Double].collect().min

result < 0 shouldBe true

an[Exception] should be thrownBy spark.sql("""
|select rst_min() from source
|""".stripMargin)

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package com.databricks.labs.mosaic.expressions.raster

import com.databricks.labs.mosaic.core.geometry.api.JTS
import com.databricks.labs.mosaic.core.index.H3IndexSystem
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSessionGDAL

import scala.util.Try

class RST_MinTest extends QueryTest with SharedSparkSessionGDAL with RST_MinBehaviors {

private val noCodegen =
withSQLConf(
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString
) _

// Hotfix for SharedSparkSession afterAll cleanup.
override def afterAll(): Unit = Try(super.afterAll())

// These tests are not index system nor geometry API specific.
// Only testing one pairing is sufficient.
test("Testing rst_min behavior with H3IndexSystem and JTS") {
noCodegen {
assume(System.getProperty("os.name") == "Linux")
behavior(H3IndexSystem, JTS)
}
}

}

0 comments on commit fa2fc5c

Please sign in to comment.