Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport release-3_38] Allow postgresraster layers as gdal processing tools input #57973

Merged
merged 6 commits into from
Jul 18, 2024
Merged

[Backport release-3_38] Allow postgresraster layers as gdal processing tools input #57973

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4b8d036
fix postgresraster provider reporting wrong block size when tiled
uclaros Jul 2, 2024
fde6317
Test postgresraster blocksize
uclaros Jul 3, 2024
2de0bd8
backport most of [processing] Fix running commands on layers with ope…
uclaros Jul 11, 2024
277a2f0
Allow postgresraster layers as gdal processing tools input
uclaros Jul 2, 2024
0edb270
GdalUtils test added
uclaros Jul 3, 2024
7147a23
use temporary auth db for test
uclaros Jul 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions python/plugins/processing/algs/gdal/AssignProjection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,20 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
if inLayer is None:
raise QgsProcessingException(self.invalidRasterError(parameters, self.INPUT))

input_details = GdalUtils.gdal_connection_details_from_layer(inLayer)
fileName = inLayer.source()

crs = self.parameterAsCrs(parameters, self.CRS, context)

arguments = [
'-a_srs',
GdalUtils.gdal_crs_string(crs),

fileName
input_details.connection_string
]

if input_details.open_options:
arguments.extend(input_details.open_options_as_arguments())

self.setOutputValue(self.OUTPUT, fileName)

return [self.commandName() + ('.bat' if isWindows() else '.py'), GdalUtils.escapeAndJoin(arguments)]
Expand Down
20 changes: 12 additions & 8 deletions python/plugins/processing/algs/gdal/Buffer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
if source is None:
raise QgsProcessingException(self.invalidSourceError(parameters, self.INPUT))
fields = source.fields()
ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
source_details = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)

geometry = self.parameterAsString(parameters, self.GEOMETRY, context)
distance = self.parameterAsDouble(parameters, self.DISTANCE, context)
fieldName = self.parameterAsString(parameters, self.FIELD, context)
Expand All @@ -109,7 +110,7 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
outFile = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
self.setOutputValue(self.OUTPUT, outFile)

output, outputFormat = GdalUtils.ogrConnectionStringAndFormat(outFile, context)
output_details = GdalUtils.gdal_connection_details_from_uri(outFile, context)

other_fields_exist = any(
True for f in fields
Expand All @@ -119,17 +120,17 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
other_fields = ',*' if other_fields_exist else ''

arguments = [
output,
ogrLayer,
output_details.connection_string,
source_details.connection_string,
'-dialect',
'sqlite',
'-sql'
]

if dissolve or fieldName:
sql = f'SELECT ST_Union(ST_Buffer({geometry}, {distance})) AS {geometry}{other_fields} FROM "{layerName}"'
sql = f'SELECT ST_Union(ST_Buffer({geometry}, {distance})) AS {geometry}{other_fields} FROM "{source_details.layer_name}"'
else:
sql = f'SELECT ST_Buffer({geometry}, {distance}) AS {geometry}{other_fields} FROM "{layerName}"'
sql = f'SELECT ST_Buffer({geometry}, {distance}) AS {geometry}{other_fields} FROM "{source_details.layer_name}"'

if fieldName:
sql = f'{sql} GROUP BY "{fieldName}"'
Expand All @@ -139,10 +140,13 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
if self.parameterAsBoolean(parameters, self.EXPLODE_COLLECTIONS, context):
arguments.append('-explodecollections')

if source_details.open_options:
arguments.extend(source_details.open_options_as_arguments())

if options:
arguments.append(options)

if outputFormat:
arguments.append(f'-f {outputFormat}')
if output_details.format:
arguments.append(f'-f {output_details.format}')

return [self.commandName(), GdalUtils.escapeAndJoin(arguments)]
6 changes: 5 additions & 1 deletion python/plugins/processing/algs/gdal/ClipRasterByExtent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
if inLayer is None:
raise QgsProcessingException('Invalid input layer {}'.format(parameters[self.INPUT] if self.INPUT in parameters else 'INPUT'))
input_details = GdalUtils.gdal_connection_details_from_layer(inLayer)

bbox = self.parameterAsExtent(parameters, self.EXTENT, context, inLayer.crs())
override_crs = self.parameterAsBoolean(parameters, self.OVERCRS, context)
Expand Down Expand Up @@ -169,7 +170,10 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
extra = self.parameterAsString(parameters, self.EXTRA, context)
arguments.append(extra)

arguments.append(inLayer.source())
arguments.append(input_details.connection_string)
arguments.append(out)

if input_details.open_options:
arguments.extend(input_details.open_options_as_arguments())

return [self.commandName(), GdalUtils.escapeAndJoin(arguments)]
12 changes: 8 additions & 4 deletions python/plugins/processing/algs/gdal/ClipRasterByMask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,9 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
if inLayer is None:
raise QgsProcessingException(self.invalidRasterError(parameters, self.INPUT))
input_details = GdalUtils.gdal_connection_details_from_layer(inLayer)

maskLayer, maskLayerName = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback, executing)
mask_details = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback, executing)

sourceCrs = self.parameterAsCrs(parameters, self.SOURCE_CRS, context)
targetCrs = self.parameterAsCrs(parameters, self.TARGET_CRS, context)
Expand Down Expand Up @@ -237,9 +238,9 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
arguments.append('-tap')

arguments.append('-cutline')
arguments.append(maskLayer)
arguments.append(mask_details.connection_string)
arguments.append('-cl')
arguments.append(maskLayerName)
arguments.append(mask_details.layer_name)

if self.parameterAsBoolean(parameters, self.CROP_TO_CUTLINE, context):
arguments.append('-crop_to_cutline')
Expand All @@ -260,7 +261,10 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
extra = self.parameterAsString(parameters, self.EXTRA, context)
arguments.append(extra)

arguments.append(inLayer.source())
arguments.append(input_details.connection_string)
arguments.append(out)

if input_details.open_options:
arguments.extend(input_details.open_options_as_arguments())

return [self.commandName(), GdalUtils.escapeAndJoin(arguments)]
19 changes: 12 additions & 7 deletions python/plugins/processing/algs/gdal/ClipVectorByExtent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def commandName(self):
return 'ogr2ogr'

def getConsoleCommands(self, parameters, context, feedback, executing=True):
ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
input_details = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
source = self.parameterAsSource(parameters, self.INPUT, context)
if source is None:
raise QgsProcessingException(self.invalidSourceError(parameters, self.INPUT))
Expand All @@ -82,7 +82,9 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
outFile = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
self.setOutputValue(self.OUTPUT, outFile)

output, outputFormat = GdalUtils.ogrConnectionStringAndFormat(outFile, context)
output_details = GdalUtils.gdal_connection_details_from_uri(
outFile,
context)

arguments = [
'-spat',
Expand All @@ -92,15 +94,18 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
str(extent.yMinimum()),
'-clipsrc spat_extent',

output,
ogrLayer,
layerName
output_details.connection_string,
input_details.connection_string,
input_details.layer_name
]

if input_details.open_options:
arguments.extend(input_details.open_options_as_arguments())

if options:
arguments.append(options)

if outputFormat:
arguments.append(f'-f {outputFormat}')
if output_details.format:
arguments.append(f'-f {output_details.format}')

return [self.commandName(), GdalUtils.escapeAndJoin(arguments)]
23 changes: 13 additions & 10 deletions python/plugins/processing/algs/gdal/ClipVectorByMask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,29 +74,32 @@ def commandName(self):
return 'ogr2ogr'

def getConsoleCommands(self, parameters, context, feedback, executing=True):
inLayer, inLayerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
maskLayer, maskLayerName = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback, executing)
input_details = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
mask_details = self.getOgrCompatibleSource(self.MASK, parameters, context, feedback, executing)
options = self.parameterAsString(parameters, self.OPTIONS, context)
outFile = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
self.setOutputValue(self.OUTPUT, outFile)

output, outputFormat = GdalUtils.ogrConnectionStringAndFormat(outFile, context)
output_details = GdalUtils.gdal_connection_details_from_uri(outFile, context)

arguments = [
'-clipsrc',
maskLayer,
mask_details.connection_string,
'-clipsrclayer',
maskLayerName,
mask_details.layer_name,

output,
inLayer,
inLayerName,
output_details.connection_string,
input_details.connection_string,
input_details.layer_name,
]

if input_details.open_options:
arguments.extend(input_details.open_options_as_arguments())

if options:
arguments.append(options)

if outputFormat:
arguments.append(f'-f {outputFormat}')
if output_details.format:
arguments.append(f'-f {output_details.format}')

return [self.commandName(), GdalUtils.escapeAndJoin(arguments)]
3 changes: 2 additions & 1 deletion python/plugins/processing/algs/gdal/ColorRelief.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,9 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
inLayer = self.parameterAsRasterLayer(parameters, self.INPUT, context)
if inLayer is None:
raise QgsProcessingException(self.invalidRasterError(parameters, self.INPUT))
input_details = GdalUtils.gdal_connection_details_from_layer(inLayer)

arguments.append(inLayer.source())
arguments.append(input_details.connection_string)
arguments.append(self.parameterAsFile(parameters, self.COLOR_TABLE, context))

out = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
Expand Down
3 changes: 2 additions & 1 deletion python/plugins/processing/algs/gdal/Datasources2Vrt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,8 @@ def processAlgorithm(self, parameters, context, feedback):
if feedback.isCanceled():
break

basePath = GdalUtils.ogrConnectionStringFromLayer(layer)
connection_details = GdalUtils.gdal_connection_details_from_layer(layer)
basePath = connection_details.connection_string
layerName = GdalUtils.ogrLayerName(layer.source())

vrt += f'<OGRVRTLayer name="{html.escape(layerName, True)}">'
Expand Down
19 changes: 11 additions & 8 deletions python/plugins/processing/algs/gdal/Dissolve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,15 +112,15 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
raise QgsProcessingException(self.invalidSourceError(parameters, self.INPUT))

fields = source.fields()
ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
input_details = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
geometry = self.parameterAsString(parameters, self.GEOMETRY, context)
fieldName = self.parameterAsString(parameters, self.FIELD, context)

options = self.parameterAsString(parameters, self.OPTIONS, context)
outFile = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
self.setOutputValue(self.OUTPUT, outFile)

output, outputFormat = GdalUtils.ogrConnectionStringAndFormat(outFile, context)
output_details = GdalUtils.gdal_connection_details_from_uri(outFile, context)

other_fields_exist = any(
True for f in fields
Expand All @@ -130,8 +130,8 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
other_fields = ',*' if other_fields_exist else ''

arguments = [
output,
ogrLayer,
output_details.connection_string,
input_details.connection_string,
'-nlt PROMOTE_TO_MULTI',
'-dialect',
'sqlite',
Expand All @@ -158,20 +158,23 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
group_by = f' GROUP BY "{fieldName}"'

if self.parameterAsBoolean(parameters, self.KEEP_ATTRIBUTES, context):
sql = f'SELECT ST_Union({geometry}) AS {geometry}{other_fields}{params} FROM "{layerName}"{group_by}'
sql = f'SELECT ST_Union({geometry}) AS {geometry}{other_fields}{params} FROM "{input_details.layer_name}"{group_by}'
else:
sql = 'SELECT ST_Union({}) AS {}{}{} FROM "{}"{}'.format(geometry, geometry, f', "{fieldName}"' if fieldName else '',
params, layerName, group_by)
params, input_details.layer_name, group_by)

arguments.append(sql)

if self.parameterAsBoolean(parameters, self.EXPLODE_COLLECTIONS, context):
arguments.append('-explodecollections')

if input_details.open_options:
arguments.extend(input_details.open_options_as_arguments())

if options:
arguments.append(options)

if outputFormat:
arguments.append(f'-f {outputFormat}')
if output_details.format:
arguments.append(f'-f {output_details.format}')

return [self.commandName(), GdalUtils.escapeAndJoin(arguments)]
17 changes: 11 additions & 6 deletions python/plugins/processing/algs/gdal/ExecuteSql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,23 @@ def commandName(self):
return "ogr2ogr"

def getConsoleCommands(self, parameters, context, feedback, executing=True):
ogrLayer, layerName = self.getOgrCompatibleSource(self.INPUT, parameters, context, feedback, executing)
input_details = self.getOgrCompatibleSource(self.INPUT,
parameters, context,
feedback, executing)
sql = self.parameterAsString(parameters, self.SQL, context)
options = self.parameterAsString(parameters, self.OPTIONS, context)
outFile = self.parameterAsOutputLayer(parameters, self.OUTPUT, context)
self.setOutputValue(self.OUTPUT, outFile)

output, outputFormat = GdalUtils.ogrConnectionStringAndFormat(outFile, context)
output_details = GdalUtils.gdal_connection_details_from_uri(outFile, context)

if not sql:
raise QgsProcessingException(
self.tr('Empty SQL. Please enter valid SQL expression and try again.'))

arguments = [
output,
ogrLayer,
output_details.connection_string,
input_details.connection_string,
'-sql',
sql
]
Expand All @@ -104,10 +106,13 @@ def getConsoleCommands(self, parameters, context, feedback, executing=True):
arguments.append('-dialect')
arguments.append(dialect)

if input_details.open_options:
arguments.extend(input_details.open_options_as_arguments())

if options:
arguments.append(options)

if outputFormat:
arguments.append(f'-f {outputFormat}')
if output_details.format:
arguments.append(f'-f {output_details.format}')

return [self.commandName(), GdalUtils.escapeAndJoin(arguments)]
Loading
Loading