-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor RasterToGrid readers.
- Loading branch information
milos.colic
committed
Jan 15, 2024
1 parent
76bc69f
commit 9c3c7cb
Showing
21 changed files
with
837 additions
and
134 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
38 changes: 38 additions & 0 deletions
38
src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALInfo.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package com.databricks.labs.mosaic.core.raster.operator.gdal | ||
|
|
||
| import com.databricks.labs.mosaic.core.raster.gdal.MosaicRasterGDAL | ||
| import org.gdal.gdal.{InfoOptions, gdal} | ||
|
|
||
| /** GDALBuildVRT is a wrapper for the GDAL BuildVRT command. */ | ||
| object GDALInfo { | ||
|
|
||
| /** | ||
| * Executes the GDAL BuildVRT command. For flags check the way gdalinfo.py | ||
| * script is called, InfoOptions expects a collection of same flags. | ||
| * | ||
| * @param raster | ||
| * The raster to get info from. | ||
| * @param command | ||
| * The GDAL Info command. | ||
| * @return | ||
| * A result json string. | ||
| */ | ||
| def executeInfo(raster: MosaicRasterGDAL, command: String): String = { | ||
| require(command.startsWith("gdalinfo"), "Not a valid GDAL Info command.") | ||
|
|
||
| val infoOptionsVec = OperatorOptions.parseOptions(command) | ||
| val infoOptions = new InfoOptions(infoOptionsVec) | ||
| val gdalInfo = gdal.GDALInfo(raster.getRaster, infoOptions) | ||
|
|
||
| if (gdalInfo == null) { | ||
| throw new Exception(s""" | ||
| |GDAL Info failed. | ||
| |Command: $command | ||
| |Error: ${gdal.GetLastErrorMsg} | ||
| |""".stripMargin) | ||
| } | ||
|
|
||
| gdalInfo | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
57 changes: 57 additions & 0 deletions
57
src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Avg.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package com.databricks.labs.mosaic.expressions.raster | ||
|
|
||
| import com.databricks.labs.mosaic.core.raster.operator.gdal.GDALInfo | ||
| import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile | ||
| import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} | ||
| import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression | ||
| import com.databricks.labs.mosaic.functions.MosaicExpressionConfig | ||
| import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder | ||
| import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback | ||
| import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} | ||
| import org.apache.spark.sql.catalyst.util.ArrayData | ||
| import org.apache.spark.sql.types._ | ||
|
|
||
|
|
||
| /** Returns the upper left x of the raster. */ | ||
| case class RST_Avg(raster: Expression, expressionConfig: MosaicExpressionConfig) | ||
| extends RasterExpression[RST_Avg](raster, ArrayType(DoubleType), returnsRaster = false, expressionConfig) | ||
| with NullIntolerant | ||
| with CodegenFallback { | ||
|
|
||
| /** Returns the upper left x of the raster. */ | ||
| override def rasterTransform(tile: MosaicRasterTile): Any = { | ||
| import org.json4s._ | ||
| import org.json4s.jackson.JsonMethods._ | ||
| implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats | ||
|
|
||
| val command = s"gdalinfo -stats -json -mm -nogcp -nomd -norat -noct" | ||
| val gdalInfo = GDALInfo.executeInfo(tile.raster, command) | ||
| // parse json from gdalinfo | ||
| val json = parse(gdalInfo).extract[Map[String, Any]] | ||
| val maxValues = json("bands").asInstanceOf[List[Map[String, Any]]].map { band => | ||
| band("mean").asInstanceOf[Double] | ||
| } | ||
| ArrayData.toArrayData(maxValues.toArray) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| /** Expression info required for the expression registration for spark SQL. */ | ||
| object RST_Avg extends WithExpressionInfo { | ||
|
|
||
| override def name: String = "rst_mean" | ||
|
|
||
| override def usage: String = "_FUNC_(expr1) - Returns an array containing mean values for each band." | ||
|
|
||
| override def example: String = | ||
| """ | ||
| | Examples: | ||
| | > SELECT _FUNC_(raster_tile); | ||
| | [1.123, 2.123, 3.123] | ||
| | """.stripMargin | ||
|
|
||
| override def builder(expressionConfig: MosaicExpressionConfig): FunctionBuilder = { | ||
| GenericExpressionFactory.getBaseBuilder[RST_Avg](1, expressionConfig) | ||
| } | ||
|
|
||
| } |
48 changes: 48 additions & 0 deletions
48
src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Max.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package com.databricks.labs.mosaic.expressions.raster | ||
|
|
||
| import com.databricks.labs.mosaic.core.raster.operator.gdal.GDALInfo | ||
| import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile | ||
| import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} | ||
| import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression | ||
| import com.databricks.labs.mosaic.functions.MosaicExpressionConfig | ||
| import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder | ||
| import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback | ||
| import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} | ||
| import org.apache.spark.sql.catalyst.util.ArrayData | ||
| import org.apache.spark.sql.types._ | ||
|
|
||
|
|
||
| /** Returns the upper left x of the raster. */ | ||
| case class RST_Max(raster: Expression, expressionConfig: MosaicExpressionConfig) | ||
| extends RasterExpression[RST_Max](raster, ArrayType(DoubleType), returnsRaster = false, expressionConfig) | ||
| with NullIntolerant | ||
| with CodegenFallback { | ||
|
|
||
| /** Returns the upper left x of the raster. */ | ||
| override def rasterTransform(tile: MosaicRasterTile): Any = { | ||
| val nBands = tile.raster.raster.GetRasterCount() | ||
| val maxValues = (1 to nBands).map(tile.raster.getBand(_).maxPixelValue) | ||
| ArrayData.toArrayData(maxValues.toArray) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| /** Expression info required for the expression registration for spark SQL. */ | ||
| object RST_Max extends WithExpressionInfo { | ||
|
|
||
| override def name: String = "rst_max" | ||
|
|
||
| override def usage: String = "_FUNC_(expr1) - Returns an array containing max values for each band." | ||
|
|
||
| override def example: String = | ||
| """ | ||
| | Examples: | ||
| | > SELECT _FUNC_(raster_tile); | ||
| | [1.123, 2.123, 3.123] | ||
| | """.stripMargin | ||
|
|
||
| override def builder(expressionConfig: MosaicExpressionConfig): FunctionBuilder = { | ||
| GenericExpressionFactory.getBaseBuilder[RST_Max](1, expressionConfig) | ||
| } | ||
|
|
||
| } |
60 changes: 60 additions & 0 deletions
60
src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| package com.databricks.labs.mosaic.expressions.raster | ||
|
|
||
| import com.databricks.labs.mosaic.core.raster.api.GDAL | ||
| import com.databricks.labs.mosaic.core.raster.gdal.MosaicRasterGDAL | ||
| import com.databricks.labs.mosaic.core.raster.operator.gdal.{GDALCalc, GDALInfo, GDALWarp} | ||
| import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile | ||
| import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} | ||
| import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression | ||
| import com.databricks.labs.mosaic.functions.MosaicExpressionConfig | ||
| import com.databricks.labs.mosaic.utils.PathUtils | ||
| import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder | ||
| import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback | ||
| import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} | ||
| import org.apache.spark.sql.catalyst.util.ArrayData | ||
| import org.apache.spark.sql.types._ | ||
|
|
||
| /** Returns the upper left x of the raster. */ | ||
| case class RST_Median(rasterExpr: Expression, expressionConfig: MosaicExpressionConfig) | ||
| extends RasterExpression[RST_Median](rasterExpr, ArrayType(DoubleType), returnsRaster = false, expressionConfig) | ||
| with NullIntolerant | ||
| with CodegenFallback { | ||
|
|
||
| /** Returns the upper left x of the raster. */ | ||
| override def rasterTransform(tile: MosaicRasterTile): Any = { | ||
| val raster = tile.raster | ||
| val width = raster.xSize * raster.pixelXSize | ||
| val height = raster.ySize * raster.pixelYSize | ||
| val outShortName = raster.getDriversShortName | ||
| val resultFileName = PathUtils.createTmpFilePath(GDAL.getExtension(outShortName)) | ||
| val medRaster = GDALWarp.executeWarp( | ||
| resultFileName, | ||
| Seq(raster), | ||
| command = s"gdalwarp -r med -tr $width $height -of $outShortName" | ||
| ) | ||
| // Max pixel is a hack since we get a 1x1 raster back | ||
| val maxValues = (1 to medRaster.raster.GetRasterCount()).map(medRaster.getBand(_).maxPixelValue) | ||
| ArrayData.toArrayData(maxValues.toArray) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| /** Expression info required for the expression registration for spark SQL. */ | ||
| object RST_Median extends WithExpressionInfo { | ||
|
|
||
| override def name: String = "rst_median" | ||
|
|
||
| override def usage: String = "_FUNC_(expr1) - Returns an array containing mean values for each band." | ||
|
|
||
| override def example: String = | ||
| """ | ||
| | Examples: | ||
| | > SELECT _FUNC_(raster_tile); | ||
| | [1.123, 2.123, 3.123] | ||
| | """.stripMargin | ||
|
|
||
| override def builder(expressionConfig: MosaicExpressionConfig): FunctionBuilder = { | ||
| GenericExpressionFactory.getBaseBuilder[RST_Median](1, expressionConfig) | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.