Skip to content

Commit

Permalink
PathUtils handle dbfs:/Volumes paths.
Browse files Browse the repository at this point in the history
  • Loading branch information
mjohns-databricks committed Jan 3, 2024
1 parent 7705812 commit b05f342
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 3 deletions.
1 change: 0 additions & 1 deletion python/mosaic/config/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,3 @@
display_handler: DisplayHandler
ipython_hook: InteractiveShell
notebook_utils = None
default_gdal_init_script_path: str = "/dbfs/FileStore/geospatial/mosaic/gdal/"
26 changes: 24 additions & 2 deletions src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,18 @@ import java.nio.file.{Files, Paths}
object PathUtils {

def getCleanPath(path: String): String = {
val cleanPath = path.replace("file:/", "/").replace("dbfs:/", "/dbfs/")
//val cleanPath = path.replace("file:/", "/").replace("dbfs:/", "/dbfs/")
val cleanPath = {
if (path.startsWith("file:/")) {
path.replace("file:/", "/")
} else if (path.startsWith("dbfs:/Volumes")) {
path.replace("dbfs:/Volumes", "/Volumes")

Check warning on line 17 in src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala#L17

Added line #L17 was not covered by tests
} else if (path.startsWith("dbfs:/")) {
path.replace("dbfs:/", "/dbfs/")

Check warning on line 19 in src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala#L19

Added line #L19 was not covered by tests
} else {
path
}
}
if (cleanPath.endsWith(".zip") || cleanPath.contains(".zip:")) {
getZipPath(cleanPath)
} else {
Expand Down Expand Up @@ -59,7 +70,18 @@ object PathUtils {

def copyToTmp(inPath: String): String = {
val cleanPath = getCleanPath(inPath)
val copyFromPath = inPath.replace("file:/", "/").replace("dbfs:/", "/dbfs/")
//val copyFromPath = inPath.replace("file:/", "/").replace("dbfs:/", "/dbfs/")
val copyFromPath = {
if (inPath.startsWith("file:/")) {
inPath.replace("file:/", "/")
} else if (inPath.startsWith("dbfs:/Volumes")) {
inPath.replace("dbfs:/Volumes", "/Volumes")
} else if (inPath.startsWith("dbfs:/")) {
inPath.replace("dbfs:/", "/dbfs/")

Check warning on line 80 in src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala#L77-L80

Added lines #L77 - L80 were not covered by tests
} else {
inPath

Check warning on line 82 in src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala#L82

Added line #L82 was not covered by tests
}
}
val driver = MosaicRasterGDAL.identifyDriver(cleanPath)
val extension = if (inPath.endsWith(".zip")) "zip" else GDAL.getExtension(driver)
val tmpPath = createTmpFilePath(extension)
Expand Down

0 comments on commit b05f342

Please sign in to comment.