diff --git a/.github/workflows/asan/build.sh b/.github/workflows/asan/build.sh index 20385b49bc34..340e22e6b704 100755 --- a/.github/workflows/asan/build.sh +++ b/.github/workflows/asan/build.sh @@ -23,6 +23,7 @@ cmake ${GDAL_SOURCE_DIR:=..} \ -DUSE_CCACHE=ON \ -DGDAL_USE_GEOTIFF_INTERNAL=ON \ -DGDAL_USE_TIFF_INTERNAL=ON \ + -DGDAL_ENABLE_DRIVER_PDF_PLUGIN=ON \ -DGDAL_USE_LIBKML=OFF -DOGR_ENABLE_DRIVER_LIBKML=OFF \ -DFileGDB_ROOT=/usr/local/FileGDB_API make -j$NPROC diff --git a/.github/workflows/asan/test.sh b/.github/workflows/asan/test.sh index ffbfbb0878d0..b51a96ef8fcc 100755 --- a/.github/workflows/asan/test.sh +++ b/.github/workflows/asan/test.sh @@ -16,6 +16,14 @@ export PYTHONMALLOC=malloc gdalinfo autotest/gcore/data/byte.tif python3 -c "from osgeo import gdal; print('yes')" +# Check fix for https://github.com/rasterio/rasterio/issues/3250 +mv ${GDAL_DRIVER_PATH}/gdal_PDF.so ${GDAL_DRIVER_PATH}/gdal_PDF.so.disabled +echo "from osgeo import gdal" > register_many_times.py +echo "for i in range(1000):" >> register_many_times.py +echo " gdal.AllRegister()" >> register_many_times.py +python3 register_many_times.py +mv ${GDAL_DRIVER_PATH}/gdal_PDF.so.disabled ${GDAL_DRIVER_PATH}/gdal_PDF.so + cd autotest # Run each module in its own pytest process. diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index 56c4324caa31..c05a672b7f54 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -319,7 +319,7 @@ jobs: - name: Checkout GDAL uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Install development packages - uses: msys2/setup-msys2@ddf331adaebd714795f1042345e6ca57bd66cea8 # v2.24.1 + uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 with: msystem: MINGW64 update: true @@ -418,7 +418,7 @@ jobs: shell: pwsh run: | echo "JAVA_HOME=$env:JAVA_HOME_11_X64" >> %GITHUB_ENV% - - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 + - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: activate-environment: gdalenv miniforge-version: latest @@ -525,7 +525,7 @@ jobs: git config --global core.autocrlf false - name: Checkout GDAL uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 + - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: activate-environment: gdalenv miniforge-version: latest @@ -696,7 +696,7 @@ jobs: git config --global core.autocrlf false - name: Checkout GDAL uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 + - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: activate-environment: gdalenv python-version: 3.9 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f26c7e499044..4c3712e9dba6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -109,7 +109,7 @@ jobs: # We do that after running CMake to avoid CodeQL to trigger during CMake time, # in particular during HDF5 detection which is terribly slow (https://github.com/OSGeo/gdal/issues/9549) - name: Initialize CodeQL - uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -129,6 +129,6 @@ jobs: cmake --build build -j$(nproc) - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 4ef81cea9244..73d59f93e556 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -50,7 +50,7 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-${{ steps.get-date.outputs.today }}-conda-${{ env.CACHE_NUMBER }} - - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 + - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: miniforge-version: latest use-mamba: true diff --git a/.github/workflows/delete_untagged_containers.yml b/.github/workflows/delete_untagged_containers.yml index e37a0012c1c4..5b4faeb90ed9 100644 --- a/.github/workflows/delete_untagged_containers.yml +++ b/.github/workflows/delete_untagged_containers.yml @@ -19,7 +19,7 @@ jobs: if: github.repository == 'OSGeo/gdal' steps: - name: Delete all containers from gdal-deps without tags - uses: Chizkiyahu/delete-untagged-ghcr-action@b302990b6c629f3b272a31f3c3a268e1f7d0ffae # v4.0.1 + uses: Chizkiyahu/delete-untagged-ghcr-action@1c77a06b7f80ad4beb82b85919003831d47277d1 # v5.0.0 with: token: ${{ secrets.PAT_TOKEN_DELETE_UNTAGGED_CONTAINERS }} repository_owner: ${{ github.repository_owner }} diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 31f680caaa15..41cde23f8338 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 + - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: channels: conda-forge auto-update-conda: true diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index b1342e321fab..522cdbad400b 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 with: sarif_file: results.sarif diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 737a161f96b7..1c623cdc4bde 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -4712,6 +4712,19 @@ static GDALDatasetH GDALWarpCreateOutput( aosCreateOptions.FetchNameValue("PHOTOMETRIC") == nullptr) { aosCreateOptions.SetNameValue("PHOTOMETRIC", "RGB"); + + // Preserve potential ALPHA=PREMULTIPLIED from source alpha band + const char *pszAlpha; + if (aosCreateOptions.FetchNameValue("ALPHA") == nullptr && + apeColorInterpretations.size() == 4 && + apeColorInterpretations[3] == GCI_AlphaBand && + GDALGetRasterCount(pahSrcDS[0]) == 4 && + (pszAlpha = + GDALGetMetadataItem(GDALGetRasterBand(pahSrcDS[0], 4), + "ALPHA", "IMAGE_STRUCTURE"))) + { + aosCreateOptions.SetNameValue("ALPHA", pszAlpha); + } } /* The GTiff driver now supports writing band color interpretation */ diff --git a/autotest/gcore/cog.py b/autotest/gcore/cog.py index fe334a67ff72..8056e244caed 100755 --- a/autotest/gcore/cog.py +++ b/autotest/gcore/cog.py @@ -1961,3 +1961,32 @@ def test_cog_write_check_golden_file(tmp_path, src_filename, creation_options): ) assert os.stat(src_filename).st_size == os.stat(out_filename).st_size assert open(src_filename, "rb").read() == open(out_filename, "rb").read() + + +############################################################################### + + +def test_cog_preserve_ALPHA_PREMULTIPLIED_on_copy(tmp_vsimem): + + src_filename = str(tmp_vsimem / "src.tif") + src_ds = gdal.GetDriverByName("GTiff").Create( + src_filename, 1, 1, 4, options=["ALPHA=PREMULTIPLIED", "PROFILE=BASELINE"] + ) + src_ds.SetGeoTransform([500000, 1, 0, 4500000, 0, -1]) + srs = osr.SpatialReference() + srs.ImportFromEPSG(32631) + src_ds.SetProjection(srs.ExportToWkt()) + + out_filename = str(tmp_vsimem / "out.tif") + gdal.GetDriverByName("COG").CreateCopy( + out_filename, + src_ds, + options=[ + "TILING_SCHEME=GoogleMapsCompatible", + ], + ) + with gdal.Open(out_filename) as ds: + assert ( + ds.GetRasterBand(4).GetMetadataItem("ALPHA", "IMAGE_STRUCTURE") + == "PREMULTIPLIED" + ) diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index 1de431a85d64..92158526b77f 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -948,7 +948,7 @@ def test_tiff_write_20(): ############################################################################### -# Test RGBA images with TIFFTAG_EXTRASAMPLES=EXTRASAMPLE_ASSOCALPHA +# Test RGBA images with TIFFTAG_EXTRASAMPLES=EXTRASAMPLE_UNASSOCALPHA def test_tiff_write_21(): @@ -11927,3 +11927,30 @@ def test_tiff_write_check_golden_file(tmp_path, src_filename, creation_options): ) assert os.stat(src_filename).st_size == os.stat(out_filename).st_size assert open(src_filename, "rb").read() == open(out_filename, "rb").read() + + +############################################################################### +# Test preserving ALPHA=PREMULTIPLIED on copy + + +def test_tiff_write_preserve_ALPHA_PREMULTIPLIED_on_copy(tmp_path): + + src_filename = str(tmp_path / "src.tif") + out_filename = str(tmp_path / "out.tif") + gdal.GetDriverByName("GTiff").Create( + src_filename, 1, 1, 4, options=["ALPHA=PREMULTIPLIED", "PROFILE=BASELINE"] + ) + assert gdal.VSIStatL(src_filename + ".aux.xml") is None + with gdal.Open(src_filename) as src_ds: + assert ( + src_ds.GetRasterBand(4).GetMetadataItem("ALPHA", "IMAGE_STRUCTURE") + == "PREMULTIPLIED" + ) + gdal.GetDriverByName("GTiff").CreateCopy( + out_filename, src_ds, options=["PROFILE=BASELINE"] + ) + with gdal.Open(out_filename) as out_ds: + assert ( + out_ds.GetRasterBand(4).GetMetadataItem("ALPHA", "IMAGE_STRUCTURE") + == "PREMULTIPLIED" + ) diff --git a/autotest/gdrivers/avif.py b/autotest/gdrivers/avif.py index 2f92941beda3..4a382606fbe8 100644 --- a/autotest/gdrivers/avif.py +++ b/autotest/gdrivers/avif.py @@ -282,6 +282,9 @@ def test_avif_geoheif_wkt2(): assert ds.GetGeoTransform() == pytest.approx( [691000.0, 0.1, 0.0, 6090000.0, 0.0, -0.1] ) + assert ds.GetSpatialRef() is not None + assert ds.GetSpatialRef().GetAuthorityName(None) == "EPSG" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "28355" assert ds.GetGCPCount() == 1 gcp = ds.GetGCPs()[0] assert ( @@ -313,6 +316,10 @@ def test_avif_geoheif_uri(): assert ds.GetGeoTransform() == pytest.approx( [691051.2, 0.1, 0.0, 6090000.0, 0.0, -0.1] ) + assert ds.GetSpatialRef() is not None + assert ds.GetSpatialRef().GetAuthorityName(None) == "EPSG" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "32755" + assert ds.GetGCPCount() == 1 gcp = ds.GetGCPs()[0] assert ( @@ -340,6 +347,10 @@ def test_avif_geoheif_curie(): == 'CCBY "Jacobs Group (Australia) Pty Ltd and Australian Capital Territory"' ) assert ds.GetMetadataItem("TAGS", "DESCRIPTION_en-AU") == "copyright" + assert ds.GetSpatialRef() is not None + assert ds.GetSpatialRef().GetAuthorityName(None) == "EPSG" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "32755" + assert ds.GetGeoTransform() is not None assert ds.GetGeoTransform() == pytest.approx( [691051.2, 0.1, 0.0, 6090000.0, 0.0, -0.1] diff --git a/autotest/gdrivers/avif_heif.py b/autotest/gdrivers/avif_heif.py index 6ae194c919de..dc04b3754c81 100755 --- a/autotest/gdrivers/avif_heif.py +++ b/autotest/gdrivers/avif_heif.py @@ -61,9 +61,9 @@ def _has_geoheif_support(): assert ds.GetGeoTransform() == pytest.approx( [691000.0, 0.1, 0.0, 6090000.0, 0.0, -0.1] ) - print() - print("GCPs from avif_heif: ", ds.GetGCPCount()) - print() + assert ds.GetSpatialRef() is not None + assert ds.GetSpatialRef().GetAuthorityName(None) == "EPSG" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "28355" assert ds.GetGCPCount() == 1 gcp = ds.GetGCPs()[0] assert ( diff --git a/autotest/gdrivers/data/wmts/WMTSCapabilities_THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018.xml b/autotest/gdrivers/data/wmts/WMTSCapabilities_THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018.xml new file mode 100644 index 000000000000..3dceeaa00d2b --- /dev/null +++ b/autotest/gdrivers/data/wmts/WMTSCapabilities_THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018.xml @@ -0,0 +1,102 @@ + + + + + THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018 + OGC WMTS + 1.0.0 + + + + + + + + + RESTful + + + + + + + + + + + + + + RESTful + + + + + + + + + + + + THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018 + THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018 + + -179.9999997 -65.0006576 + 179.9998849 65.0007535 + + + -179.9999997 -65.0006576 + 179.9998849 65.0007535 + + + + image/png + + default028mm + + + + + default + The tile matrix set that has scale values calculated based on the dpi defined by OGC specification (dpi assumes 0.28mm as the physical distance of a pixel). + default028mm + urn:ogc:def:crs:EPSG::104905 + 02.7922763629807472E+08-180.0 90.02562562.01.0 +11.3961381814903736E+08-180.0 90.02562564.02.0 +26.9806909074518681E+07-180.0 90.02562568.04.0 +33.4903454537259340E+07-180.0 90.025625616.08.0 +41.7451727268629670E+07-180.0 90.025625632.016.0 +58.7258636343148351E+06-180.0 90.025625664.032.0 +64.3629318171574175E+06-180.0 90.0256256128.064.0 +72.1814659085787088E+06-180.0 90.0256256256.0128.0 +81.0907329542893544E+06-180.0 90.0256256512.0256.0 +95.4536647714467719E+05-180.0 90.02562561024.0512.0 + + + + + + + diff --git a/autotest/gdrivers/heif.py b/autotest/gdrivers/heif.py index 5b246069ac62..bdc30d604b6b 100644 --- a/autotest/gdrivers/heif.py +++ b/autotest/gdrivers/heif.py @@ -706,6 +706,9 @@ def test_heif_geoheif_curie(): == 'CCBY "Jacobs Group (Australia) Pty Ltd and Australian Capital Territory"' ) assert ds.GetMetadataItem("TAGS", "DESCRIPTION_en-AU") == "copyright" + assert ds.GetSpatialRef() is not None + assert ds.GetSpatialRef().GetAuthorityName(None) == "EPSG" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "32755" assert ds.GetGeoTransform() is not None assert ds.GetGeoTransform() == pytest.approx( [691051.2, 0.1, 0.0, 6090000.0, 0.0, -0.1] @@ -719,3 +722,37 @@ def test_heif_geoheif_curie(): and gcp.GCPY == pytest.approx(6090000.0, abs=1e-5) and gcp.GCPZ == pytest.approx(0, abs=1e-5) ) + + +@pytest.mark.skipif( + not _has_geoheif_support(), + reason="this libheif does not support opaque properties like geoheif", +) +def test_heif_geoheif_curie_order(): + ds = gdal.Open("data/heif/geo_curi.heif") + assert ds + assert ds.RasterCount == 3 + assert ds.RasterXSize == 256 + assert ds.RasterYSize == 64 + assert ds.GetMetadataItem("NAME", "DESCRIPTION_en-AU") == "Copyright Statement" + assert ( + ds.GetMetadataItem("DESCRIPTION", "DESCRIPTION_en-AU") + == 'CCBY "Jacobs Group (Australia) Pty Ltd and Australian Capital Territory"' + ) + assert ds.GetMetadataItem("TAGS", "DESCRIPTION_en-AU") == "copyright" + assert ds.GetGeoTransform() is not None + assert ds.GetGeoTransform() == pytest.approx( + [691051.2, 0.1, 0.0, 6090000.0, 0.0, -0.1] + ) + assert ds.GetSpatialRef() is not None + assert ds.GetSpatialRef().GetAuthorityName(None) == "EPSG" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "32755" + assert ds.GetGCPCount() == 1 + gcp = ds.GetGCPs()[0] + assert ( + gcp.GCPPixel == pytest.approx(0, abs=1e-5) + and gcp.GCPLine == pytest.approx(0, abs=1e-5) + and gcp.GCPX == pytest.approx(691051.2, abs=1e-5) + and gcp.GCPY == pytest.approx(6090000.0, abs=1e-5) + and gcp.GCPZ == pytest.approx(0, abs=1e-5) + ) diff --git a/autotest/gdrivers/netcdf_multidim.py b/autotest/gdrivers/netcdf_multidim.py index 9ee29822c564..9a953fe91974 100755 --- a/autotest/gdrivers/netcdf_multidim.py +++ b/autotest/gdrivers/netcdf_multidim.py @@ -4156,3 +4156,30 @@ def check_metadata(): assert os.path.exists(pam_filename) os.unlink(pam_filename) + + +############################################################################### + + +@gdaltest.enable_exceptions() +@pytest.mark.require_proj(9) +def test_netcdf_multidim_WGS84_and_EGM96_height(tmp_path): + + tmp_filename = str(tmp_path / "out.nc") + with gdal.GetDriverByName("netCDF").Create(tmp_filename, 3, 3) as ds: + srs = osr.SpatialReference() + srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + srs.ImportFromEPSG(9707) + ds.SetSpatialRef(srs) + ds.SetGeoTransform([2, 1, 0, 49, 0, -1]) + with gdal.Open(tmp_filename) as ds: + srs = ds.GetSpatialRef() + assert srs.GetAuthorityCode(None) == "9707" + # We currently report a 3rd axis, which is dubious + assert srs.GetDataAxisToSRSAxisMapping()[0:2] == [2, 1] + with gdal.OpenEx(tmp_filename, gdal.OF_MULTIDIM_RASTER) as ds: + rg = ds.GetRootGroup() + ar = rg.OpenMDArray("Band1") + srs = ar.GetSpatialRef() + assert srs.GetAuthorityCode(None) == "9707" + assert srs.GetDataAxisToSRSAxisMapping() == [1, 2] diff --git a/autotest/gdrivers/wmts.py b/autotest/gdrivers/wmts.py index 89d96baf2b95..ee8ffa7e1a36 100755 --- a/autotest/gdrivers/wmts.py +++ b/autotest/gdrivers/wmts.py @@ -2001,3 +2001,32 @@ def test_wmts_force_opening_no_match(): drv = gdal.IdentifyDriverEx("data/byte.tif", allowed_drivers=["WMTS"]) assert drv is None + + +############################################################################### +# Test bug fix for https://github.com/OSGeo/gdal/issues/11387 + + +@pytest.mark.require_proj(9) +@gdaltest.enable_exceptions() +def test_wmts_read_esri_code_disguised_as_epsg_and_wrong_axis_order(): + + with gdaltest.error_handler(): + with gdal.Open( + "data/wmts/WMTSCapabilities_THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018.xml" + ) as ds: + assert gdal.GetLastErrorMsg().startswith( + "Auto-correcting wrongly swapped TileMatrix.TopLeftCorner coordinates" + ) + assert ds.GetSpatialRef().GetAuthorityName(None) == "ESRI" + assert ds.GetSpatialRef().GetAuthorityCode(None) == "104905" + assert ds.GetGeoTransform() == pytest.approx( + ( + -180.0, + 0.0013717509172233527, + 0.0, + 65.00121128452162, + 0.0, + -0.0013717509172233527, + ) + ) diff --git a/autotest/gdrivers/zarr_driver.py b/autotest/gdrivers/zarr_driver.py index 79217d38542d..caed01d26f2e 100644 --- a/autotest/gdrivers/zarr_driver.py +++ b/autotest/gdrivers/zarr_driver.py @@ -818,15 +818,21 @@ def test_zarr_read_crs(crs_member): assert rg ar = rg.OpenMDArray(rg.GetMDArrayNames()[0]) srs = ar.GetSpatialRef() - if ( - not (osr.GetPROJVersionMajor() > 6 or osr.GetPROJVersionMinor() >= 2) - and crs_member == "projjson" - ): - assert srs is None - else: - assert srs is not None - assert srs.GetAuthorityCode(None) == "4326" - assert len(ar.GetAttributes()) == 0 + assert srs is not None + assert srs.GetAuthorityCode(None) == "4326" + # Mapping is 1, 2 since the slowest varying axis in multidim + # mode is the lines, which matches latitude as the first axis of the CRS. + assert srs.GetDataAxisToSRSAxisMapping() == [1, 2] + assert len(ar.GetAttributes()) == 0 + + # Open as classic CRS + ds = gdal.Open("/vsimem/test.zarr") + srs = ds.GetSpatialRef() + assert srs.GetAuthorityCode(None) == "4326" + # Inverted mapping in classic raster mode compared to multidim mode, + # because the first "axis" in our data model is columns. + assert srs.GetDataAxisToSRSAxisMapping() == [2, 1] + finally: gdal.RmdirRecursive("/vsimem/test.zarr") @@ -3469,7 +3475,8 @@ def set_crs(): assert ar crs = osr.SpatialReference() crs.ImportFromEPSG(4326) - crs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + # lat first + crs.SetDataAxisToSRSAxisMapping([1, 2]) crs.SetCoordinateEpoch(2021.2) assert ar.SetSpatialRef(crs) == gdal.CE_None @@ -3496,7 +3503,7 @@ def check_crs(): crs = ar.GetSpatialRef() assert crs is not None assert crs.GetAuthorityCode(None) == "4326" - assert crs.GetDataAxisToSRSAxisMapping() == [2, 1] + assert crs.GetDataAxisToSRSAxisMapping() == [1, 2] assert crs.GetCoordinateEpoch() == 2021.2 check_crs() @@ -3505,6 +3512,8 @@ def check_crs_classic_dataset(): ds = gdal.Open("/vsimem/test.zarr") crs = ds.GetSpatialRef() assert crs is not None + assert crs.GetAuthorityCode(None) == "4326" + assert crs.GetDataAxisToSRSAxisMapping() == [2, 1] check_crs_classic_dataset() @@ -5505,10 +5514,9 @@ def test_zarr_read_cf1(): ds = gdal.Open("data/zarr/byte_cf1.zarr") assert ds - assert ( - ds.GetSpatialRef().ExportToProj4() - == "+proj=utm +zone=11 +ellps=clrk66 +units=m +no_defs" - ) + srs = ds.GetSpatialRef() + assert srs.ExportToProj4() == "+proj=utm +zone=11 +ellps=clrk66 +units=m +no_defs" + assert srs.GetDataAxisToSRSAxisMapping() == [1, 2] ############################################################################### @@ -5525,6 +5533,40 @@ def test_zarr_read_cf1_zarrv3(): ) +############################################################################### + + +@gdaltest.enable_exceptions() +@pytest.mark.require_proj(9) +def test_zarr_write_WGS84_and_EGM96_height(tmp_vsimem): + + tmp_filename = str(tmp_vsimem / "out.zarr") + with gdal.GetDriverByName("Zarr").Create(tmp_filename, 1, 1) as ds: + srs = osr.SpatialReference() + srs.ImportFromEPSG(9707) + ds.SetSpatialRef(srs) + with gdal.Open(tmp_filename) as ds: + srs = ds.GetSpatialRef() + assert srs.GetAuthorityCode(None) == "9707" + assert srs.GetDataAxisToSRSAxisMapping() == [2, 1] + + +############################################################################### + + +@gdaltest.enable_exceptions() +def test_zarr_write_UTM31N_and_EGM96_height(tmp_vsimem): + + tmp_filename = str(tmp_vsimem / "out.zarr") + with gdal.GetDriverByName("Zarr").Create(tmp_filename, 1, 1) as ds: + srs = osr.SpatialReference() + srs.SetFromUserInput("EPSG:32631+5773") + ds.SetSpatialRef(srs) + with gdal.Open(tmp_filename) as ds: + srs = ds.GetSpatialRef() + assert srs.GetDataAxisToSRSAxisMapping() == [1, 2] + + ############################################################################### # Test bug fix for https://github.com/OSGeo/gdal/issues/11016 diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000001.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000001.gdbtable new file mode 100644 index 000000000000..b8df547eda91 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000001.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000001.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000001.gdbtablx new file mode 100644 index 000000000000..8327f75cb45c Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000001.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000002.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000002.gdbtable new file mode 100644 index 000000000000..06f10eaf022b Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000002.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000002.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000002.gdbtablx new file mode 100644 index 000000000000..b17fb6b96e25 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000002.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000003.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000003.gdbtable new file mode 100644 index 000000000000..00c610703fc6 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000003.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000003.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000003.gdbtablx new file mode 100644 index 000000000000..aa8412769fe3 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000003.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000004.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000004.gdbtable new file mode 100644 index 000000000000..9d1da5583650 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000004.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000004.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000004.gdbtablx new file mode 100644 index 000000000000..6a69871ab4e6 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000004.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000005.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000005.gdbtable new file mode 100644 index 000000000000..75dac36be0a4 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000005.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000005.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000005.gdbtablx new file mode 100644 index 000000000000..a9e25586862f Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000005.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000006.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000006.gdbtable new file mode 100644 index 000000000000..87825c3173e2 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000006.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000006.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000006.gdbtablx new file mode 100644 index 000000000000..ab7baaec612b Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000006.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000007.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000007.gdbtable new file mode 100644 index 000000000000..29fcd156792c Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000007.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000007.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000007.gdbtablx new file mode 100644 index 000000000000..f639905039c5 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000007.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbindexes b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbindexes new file mode 100644 index 000000000000..6b351b6ac2bb Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbindexes differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbtable b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbtable new file mode 100644 index 000000000000..eafc4284e360 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbtable differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbtablx b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbtablx new file mode 100644 index 000000000000..2cae1126c73f Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/a00000009.gdbtablx differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/gdb b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/gdb new file mode 100644 index 000000000000..506f9c628294 Binary files /dev/null and b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/gdb differ diff --git a/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/timestamps b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/timestamps new file mode 100644 index 000000000000..05d2b9440ec0 --- /dev/null +++ b/autotest/ogr/data/filegdb/corrupted_gdbindexes.gdb/timestamps @@ -0,0 +1 @@ +ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ \ No newline at end of file diff --git a/autotest/ogr/ogr_geojson.py b/autotest/ogr/ogr_geojson.py index 71ce18f27424..2850abb0be6d 100755 --- a/autotest/ogr/ogr_geojson.py +++ b/autotest/ogr/ogr_geojson.py @@ -4208,6 +4208,131 @@ def test_ogr_geojson_read_from_http(): webserver.server_stop(webserver_process, webserver_port) +############################################################################### +# Test reading http:// resource with GDAL_HTTP_HEADERS config option set + + +@pytest.mark.require_curl() +def test_ogr_geojson_read_from_http_with_GDAL_HTTP_HEADERS(): + + import webserver + + (webserver_process, webserver_port) = webserver.launch( + handler=webserver.DispatcherHttpHandler + ) + if webserver_port == 0: + pytest.skip() + + response = """{"type": "FeatureCollection", "features":[ + {"type": "Feature", "geometry": {"type":"Point","coordinates":[1,2]}, "properties": null}]}""" + + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/foo", + 200, + {}, + response, + expected_headers={ + "foo": "bar", + "bar": "baz", + "Accept": "text/plain, application/json", + }, + ) + + try: + with webserver.install_http_handler(handler): + with gdaltest.config_option("GDAL_HTTP_HEADERS", "foo: bar\r\nbar: baz"): + ds = ogr.Open("http://localhost:%d/foo" % webserver_port) + assert ds is not None + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + finally: + webserver.server_stop(webserver_process, webserver_port) + + +############################################################################### +# Test reading http:// resource with GDAL_HTTP_HEADERS config option set + + +@pytest.mark.require_curl() +def test_ogr_geojson_read_from_http_with_GDAL_HTTP_HEADERS_Accept(): + + import webserver + + (webserver_process, webserver_port) = webserver.launch( + handler=webserver.DispatcherHttpHandler + ) + if webserver_port == 0: + pytest.skip() + + response = """{"type": "FeatureCollection", "features":[ + {"type": "Feature", "geometry": {"type":"Point","coordinates":[1,2]}, "properties": null}]}""" + + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/foo", + 200, + {}, + response, + expected_headers={"Accept": "application/json, foo/bar"}, + ) + + try: + with webserver.install_http_handler(handler): + with gdaltest.config_option( + "GDAL_HTTP_HEADERS", "Accept: application/json, foo/bar" + ): + ds = ogr.Open("http://localhost:%d/foo" % webserver_port) + assert ds is not None + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + finally: + webserver.server_stop(webserver_process, webserver_port) + + +############################################################################### +# Test reading http:// resource with GDAL_HTTP_HEADERS config option set + + +@pytest.mark.require_curl() +def test_ogr_geojson_read_from_http_with_GDAL_HTTP_HEADERS_overriding_Accept(): + + import webserver + + (webserver_process, webserver_port) = webserver.launch( + handler=webserver.DispatcherHttpHandler + ) + if webserver_port == 0: + pytest.skip() + + response = """{"type": "FeatureCollection", "features":[ + {"type": "Feature", "geometry": {"type":"Point","coordinates":[1,2]}, "properties": null}]}""" + + handler = webserver.SequentialHandler() + handler.add( + "GET", + "/foo", + 200, + {}, + response, + expected_headers={"foo": "bar", "bar": "baz", "Accept": "application/json"}, + ) + + try: + with webserver.install_http_handler(handler): + with gdaltest.config_option( + "GDAL_HTTP_HEADERS", "foo: bar,bar: baz,Accept: application/json" + ): + ds = ogr.Open("http://localhost:%d/foo" % webserver_port) + assert ds is not None + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + finally: + webserver.server_stop(webserver_process, webserver_port) + + ############################################################################### # Test ogr2ogr -nln with a input dataset being a GeoJSON file with a name diff --git a/autotest/ogr/ogr_mvt.py b/autotest/ogr/ogr_mvt.py index 3a6ad1dde9f8..ce652b661c0a 100755 --- a/autotest/ogr/ogr_mvt.py +++ b/autotest/ogr/ogr_mvt.py @@ -1220,7 +1220,9 @@ def test_ogr_mvt_write_mbtiles(): @pytest.mark.require_driver("SQLite") @pytest.mark.require_geos -def test_ogr_mvt_write_limitations_max_size(): +@pytest.mark.parametrize("implicit_limitation", [True, False]) +@gdaltest.enable_exceptions() +def test_ogr_mvt_write_limitations_max_size(implicit_limitation, tmp_path): src_ds = gdal.GetDriverByName("Memory").Create("", 0, 0, 0, gdal.GDT_Unknown) lyr = src_ds.CreateLayer("mylayer") @@ -1244,10 +1246,22 @@ def test_ogr_mvt_write_limitations_max_size(): out_ds = gdal.VectorTranslate( "/vsimem/out.mbtiles", src_ds, - datasetCreationOptions=["MAX_SIZE=100", "SIMPLIFICATION=1"], + datasetCreationOptions=[ + "@MAX_SIZE_FOR_TEST=101" if implicit_limitation else "MAX_SIZE=101", + "SIMPLIFICATION=1", + ], ) assert out_ds is not None - out_ds = None + gdal.ErrorReset() + with gdal.quiet_errors(): + out_ds.Close() + if implicit_limitation: + assert ( + gdal.GetLastErrorMsg() + == "At least one tile exceeded the default maximum tile size of 101 bytes and was encoded at lower resolution" + ) + else: + assert gdal.GetLastErrorMsg() == "" out_ds = ogr.Open("/vsimem/out.mbtiles") assert out_ds is not None @@ -1441,7 +1455,9 @@ def test_ogr_mvt_write_limitations_max_size_polygon(): @pytest.mark.require_driver("SQLite") @pytest.mark.require_geos -def test_ogr_mvt_write_limitations_max_features(): +@pytest.mark.parametrize("implicit_limitation", [True, False]) +@gdaltest.enable_exceptions() +def test_ogr_mvt_write_limitations_max_features(implicit_limitation): src_ds = gdal.GetDriverByName("Memory").Create("", 0, 0, 0, gdal.GDT_Unknown) lyr = src_ds.CreateLayer("mylayer") @@ -1462,10 +1478,21 @@ def test_ogr_mvt_write_limitations_max_features(): "/vsimem/out.mbtiles", src_ds, format="MVT", - datasetCreationOptions=["MAX_FEATURES=1"], + datasetCreationOptions=[ + "@MAX_FEATURES_FOR_TEST=1" if implicit_limitation else "MAX_FEATURES=1" + ], ) assert out_ds is not None - out_ds = None + gdal.ErrorReset() + with gdal.quiet_errors(): + out_ds.Close() + if implicit_limitation: + assert ( + gdal.GetLastErrorMsg() + == "At least one tile exceeded the default maximum number of features per tile (1) and was truncated to satisfy it." + ) + else: + assert gdal.GetLastErrorMsg() == "" out_ds = ogr.Open("/vsimem/out.mbtiles") assert out_ds is not None diff --git a/autotest/ogr/ogr_openfilegdb.py b/autotest/ogr/ogr_openfilegdb.py index da4e529509f2..e93bb7789f95 100755 --- a/autotest/ogr/ogr_openfilegdb.py +++ b/autotest/ogr/ogr_openfilegdb.py @@ -2942,3 +2942,19 @@ def test_ogr_openfilegdb_arc_interior_point_bug_line(): f, "MULTILINESTRING ((37252520.1717 7431529.9154,38549084.9654 758964.7573))", ) + + +############################################################################### +# Test https://github.com/OSGeo/gdal/issues/11295 where .gdbindexes contains +# entry with unusual (corrupted ? disabled?) entries + + +def test_ogr_openfilegdb_weird_gdbindexes(): + + # File a00000009.gdbindexes has been replaced by file a00000029.gdbindexes + # from attachment of https://github.com/OSGeo/gdal/issues/11295#issuecomment-2491158506 + with ogr.Open("data/filegdb/corrupted_gdbindexes.gdb") as ds: + lyr = ds.GetLayer(0) + lyr.SetAttributeFilter("id = '1'") + f = lyr.GetNextFeature() + assert f diff --git a/autotest/ogr/ogr_pg.py b/autotest/ogr/ogr_pg.py index ec066ab30171..3a0c24c2bb88 100755 --- a/autotest/ogr/ogr_pg.py +++ b/autotest/ogr/ogr_pg.py @@ -6203,6 +6203,7 @@ def test_ogr_pg_ogr2ogr_with_multiple_dotted_table_name(pg_ds): ############################################################################### # Test scenario of https://lists.osgeo.org/pipermail/gdal-dev/2024-October/059608.html +# and bugfix of https://github.com/OSGeo/gdal/issues/11386 @only_without_postgis @@ -6219,6 +6220,7 @@ def test_ogr_pg_empty_search_path(pg_ds): with pg_ds.ExecuteSQL("SELECT CURRENT_USER") as lyr: f = lyr.GetNextFeature() current_user = f.GetField(0) + pg_ds.ExecuteSQL(f"ALTER ROLE {current_user} SET search_path = ''") try: ds = reconnect(pg_ds, update=1) @@ -6226,7 +6228,25 @@ def test_ogr_pg_empty_search_path(pg_ds): with ds.ExecuteSQL("SHOW search_path") as sql_lyr: f = sql_lyr.GetNextFeature() new_search_path = f.GetField(0) - assert new_search_path == "test_ogr_pg_empty_search_path_no_postgis, public" + assert ( + new_search_path + == 'test_ogr_pg_empty_search_path_no_postgis, "", public' + ) + + finally: + ds.ExecuteSQL(f"ALTER ROLE {current_user} SET search_path = {old_search_path}") + + pg_ds.ExecuteSQL(f"ALTER ROLE {current_user} SET search_path = '', '$user'") + try: + ds = reconnect(pg_ds, update=1) + + with ds.ExecuteSQL("SHOW search_path") as sql_lyr: + f = sql_lyr.GetNextFeature() + new_search_path = f.GetField(0) + assert ( + new_search_path + == 'test_ogr_pg_empty_search_path_no_postgis, "", "$user", public' + ) finally: ds.ExecuteSQL(f"ALTER ROLE {current_user} SET search_path = {old_search_path}") diff --git a/autotest/osr/osr_epsg.py b/autotest/osr/osr_epsg.py index 0a982c6fe9c3..ea9a3585becc 100755 --- a/autotest/osr/osr_epsg.py +++ b/autotest/osr/osr_epsg.py @@ -544,3 +544,31 @@ def test_osr_epsg_EPSGTreatsAsLatLong_for_CompoundCRS(): srs = osr.SpatialReference() srs.ImportFromEPSG(6697) assert srs.EPSGTreatsAsLatLong() == 1 + + +############################################################################### +# Test importing a ESRI code as a EPSG code + + +@pytest.mark.require_proj(9) +def test_osr_epsg_import_esri_code(): + + srs = osr.SpatialReference() + with gdal.quiet_errors(): + srs.ImportFromEPSG(104905) + + assert srs.GetAuthorityName(None) == "ESRI" + assert srs.GetAuthorityCode(None) == "104905" + + +############################################################################### +# Test importing a non-existent ESRI code presented as a EPSG code + + +def test_osr_epsg_import_invalid_code_that_might_have_been_esri(): + + srs = osr.SpatialReference() + with pytest.raises( + Exception, match="PROJ: proj_create_from_database: crs not found" + ): + srs.ImportFromEPSG(987654) diff --git a/autotest/pyscripts/test_ogr_layer_algebra.py b/autotest/pyscripts/test_ogr_layer_algebra.py index 2b83faa22288..af93265fde84 100644 --- a/autotest/pyscripts/test_ogr_layer_algebra.py +++ b/autotest/pyscripts/test_ogr_layer_algebra.py @@ -11,6 +11,7 @@ # SPDX-License-Identifier: MIT ############################################################################### +import gdaltest import ogrtest import pytest import test_py_scripts @@ -37,6 +38,9 @@ def script_path(): def test_ogr_layer_algebra_help(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "ogr_layer_algebra", "--help" ) @@ -48,6 +52,9 @@ def test_ogr_layer_algebra_help(script_path): def test_ogr_layer_algebra_version(script_path): + if gdaltest.is_travis_branch("sanitize"): + pytest.skip("fails on sanitize for unknown reason") + assert "ERROR" not in test_py_scripts.run_py_script( script_path, "ogr_layer_algebra", "--version" ) diff --git a/doc/images/community/survey_2024/documentation_needs.svg b/doc/images/community/survey_2024/documentation_needs.svg new file mode 100644 index 000000000000..cb05350fb5d0 --- /dev/null +++ b/doc/images/community/survey_2024/documentation_needs.svg @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Installation +Python +Other +Format options +Data format peculiarities +Developer API usage +Workflows +Examples + + + + + + + + + + + +0 +100 +200 +Number of respondents +Topic +Where does GDAL need more documentation? + + diff --git a/doc/images/community/survey_2024/easy_to_install_gdal.svg b/doc/images/community/survey_2024/easy_to_install_gdal.svg new file mode 100644 index 000000000000..4d5294ea339a --- /dev/null +++ b/doc/images/community/survey_2024/easy_to_install_gdal.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +0 +100 +200 +300 + + + + + + + +Yes +No +Yes, but not with the options I need +It is easy to install GDAL +Respondents + + diff --git a/doc/images/community/survey_2024/easy_to_install_gdal_os.svg b/doc/images/community/survey_2024/easy_to_install_gdal_os.svg new file mode 100644 index 000000000000..979de09057f9 --- /dev/null +++ b/doc/images/community/survey_2024/easy_to_install_gdal_os.svg @@ -0,0 +1,153 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Linux + + + + + + + + + + +OSX + + + + + + + + + + +Windows + + + + + +No +Yes +Yes, but not with +the options I need + + + +No +Yes +Yes, but not with +the options I need + + + +No +Yes +Yes, but not with +the options I need +0 +50 +100 +150 + + + + +Operating System +Respondents +Is it easy to install GDAL? + + diff --git a/doc/images/community/survey_2024/gdal_challenge.svg b/doc/images/community/survey_2024/gdal_challenge.svg new file mode 100644 index 000000000000..8a1b0989fe7b --- /dev/null +++ b/doc/images/community/survey_2024/gdal_challenge.svg @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Finding examples +Understanding features +Installation +Constructing workflows +Other +Coordinate systems +Formats + + + + + + + + + + + + + +0 +25 +50 +75 +100 +125 +Number of respondents +Topic +My most difficult GDAL usage challenge is... + + diff --git a/doc/images/community/survey_2024/gdal_easier_to_use.svg b/doc/images/community/survey_2024/gdal_easier_to_use.svg new file mode 100644 index 000000000000..0a1d84f57e84 --- /dev/null +++ b/doc/images/community/survey_2024/gdal_easier_to_use.svg @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +cli +doc +examples +gui +install +mac +options +python +tutorial +windows + + + + + + + + + + + + + +0 +10 +20 +Percent of responses +Keyword +What could make GDAL easier to use? + + diff --git a/doc/images/community/survey_2024/gdal_help_source.svg b/doc/images/community/survey_2024/gdal_help_source.svg new file mode 100644 index 000000000000..16226095cf49 --- /dev/null +++ b/doc/images/community/survey_2024/gdal_help_source.svg @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +gdal.org +Search engine +StackOverflow +Chatbot +GitHub +Source code +Mailing list +GDAL/OGR Cookbook +Python/R package docstrings +Frank +Telegram group + + + + + + + + + + + + + + +0 +100 +200 +Number of Respondents +Source +The first place I look for GDAL help is... + + diff --git a/doc/images/community/survey_2024/local_or_cloud_read.svg b/doc/images/community/survey_2024/local_or_cloud_read.svg new file mode 100644 index 000000000000..8f193bdb3276 --- /dev/null +++ b/doc/images/community/survey_2024/local_or_cloud_read.svg @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Local file systems +Cloud (s3, gcs, az) +Network file systems +Memory +HPC (Lustre, HDFS, IPFS, etc) +Other + + + + + + + + + + + +0 +100 +200 +300 +400 +Number of Respondents +Source +I read most data with GDAL from... + + diff --git a/doc/images/community/survey_2024/local_or_cloud_write.svg b/doc/images/community/survey_2024/local_or_cloud_write.svg new file mode 100644 index 000000000000..cb90b03702a2 --- /dev/null +++ b/doc/images/community/survey_2024/local_or_cloud_write.svg @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Local file systems +Network file systems +Cloud (s3, gcs, az) +Memory +Database +HPC (Lustre, HDFS, IPFS, etc) +Other + + + + + + + + + + + + +0 +100 +200 +300 +400 +Number of Respondents +Location +I write most data with GDAL to... + + diff --git a/doc/images/community/survey_2024/maintenance_program_activities.svg b/doc/images/community/survey_2024/maintenance_program_activities.svg new file mode 100644 index 000000000000..3f9578e8dbd6 --- /dev/null +++ b/doc/images/community/survey_2024/maintenance_program_activities.svg @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +VSI enhancements +Python bindings improvements +Documentation improvements +Dependency enhancements +(PROJ, GEOS, libgeotiff, libtiff) +Build system refactor +(CMake, plugin loading, etc. + + + + + + + + + + +0 +10 +20 +30 +40 +Percent of responses +Activity +Which maintenance program-funded activity +improved your usability of GDAL? + + diff --git a/doc/images/community/survey_2024/maintenance_program_areas_of_focus.svg b/doc/images/community/survey_2024/maintenance_program_areas_of_focus.svg new file mode 100644 index 000000000000..18f9a45d8654 --- /dev/null +++ b/doc/images/community/survey_2024/maintenance_program_areas_of_focus.svg @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Performance +Format compatibility +Integration with other software +Command line +Command line compatibility +VSI features +Backward compatibility +More formats +Static builds +Compiler compatibility +Deprecation of unused drivers + + + + + + + + + + + + + + + +0 +1 +2 +3 +Average priority +Area of focus +Maintenance program priority rankings + + diff --git a/doc/images/community/survey_2024/operating_system.svg b/doc/images/community/survey_2024/operating_system.svg new file mode 100644 index 000000000000..b5398e97b7e6 --- /dev/null +++ b/doc/images/community/survey_2024/operating_system.svg @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Linux +Windows +OSX +Other +FreeBSD + + + + + + + + + +0 +100 +200 +300 +Number of Respondents +Operating System +The operating system I execute GDAL on most is... + + diff --git a/doc/images/community/survey_2024/raster_data_formats.svg b/doc/images/community/survey_2024/raster_data_formats.svg new file mode 100644 index 000000000000..a7f2c1a031a2 --- /dev/null +++ b/doc/images/community/survey_2024/raster_data_formats.svg @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +GeoTIFF / COG +HDF/NetCDF +Other +GeoPackage +Rendered tilesets +JPEG/JPEG2000 +ECW +ENVI +KEA + + + + + + + + + + + + +0 +200 +400 +Number of Respondents +Format +Most common raster format usage + + diff --git a/doc/images/community/survey_2024/vector_data_formats.svg b/doc/images/community/survey_2024/vector_data_formats.svg new file mode 100644 index 000000000000..1cc610d52b58 --- /dev/null +++ b/doc/images/community/survey_2024/vector_data_formats.svg @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +GeoPackage +Shapefile +GeoJSON +PostGIS +(Geo)Parquet +Other +Esri FileGeodatabase +FlatGeobuf +GML + + + + + + + + + + + + + + +0 +50 +100 +150 +200 +Number of Respondents +Format +Most common vector format usage + + diff --git a/doc/images/community/survey_2024/way_gdal_used.svg b/doc/images/community/survey_2024/way_gdal_used.svg new file mode 100644 index 000000000000..926928f77f18 --- /dev/null +++ b/doc/images/community/survey_2024/way_gdal_used.svg @@ -0,0 +1,96 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Command line +GDAL Python bindings + (from osgeo import gdal) +Python packages + (rasterio, fiona, geopandas, rioxarray, etc.) +R packages + (terra, gdalraster, stars, etc.) +Other +PostGIS +C/C++ +QGIS +Java bindings +C# bindings + + + + + + + + + + + + + + + +0 +50 +100 +150 +200 +Number of Respondents +Method +I use GDAL most often via... + + diff --git a/doc/images/community/survey_2024/where_gdal_obtained_linux.svg b/doc/images/community/survey_2024/where_gdal_obtained_linux.svg new file mode 100644 index 000000000000..2c97599f8ec3 --- /dev/null +++ b/doc/images/community/survey_2024/where_gdal_obtained_linux.svg @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Standard unix distribution +Built from source +Homebrew +Docker images +OSGeo4W +QGIS +UbuntuGIS +Other +conda-forge / Anaconda +NuGet + + + + + + + + + + + + + +0 +50 +100 +Number of Respondents +Source +Primary operating system: Linux +I obtain GDAL from... + + diff --git a/doc/images/community/survey_2024/where_gdal_obtained_osx.svg b/doc/images/community/survey_2024/where_gdal_obtained_osx.svg new file mode 100644 index 000000000000..dc4940ebe5da --- /dev/null +++ b/doc/images/community/survey_2024/where_gdal_obtained_osx.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Homebrew +Standard unix distribution +Built from source +QGIS +Other + + + + + + + + + +0 +20 +40 +60 +Number of Respondents +Source +Primary operating system: OSX +I obtain GDAL from... + + diff --git a/doc/images/community/survey_2024/where_gdal_obtained_windows.svg b/doc/images/community/survey_2024/where_gdal_obtained_windows.svg new file mode 100644 index 000000000000..6410ae7be247 --- /dev/null +++ b/doc/images/community/survey_2024/where_gdal_obtained_windows.svg @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +OSGeo4W +Homebrew +QGIS +GISInternals +Built from source +Other +conda-forge / Anaconda +Christoph Gohlke +Commercial software +CRAN +NuGet +Standard unix distribution + + + + + + + + + + + + + + + + +0 +25 +50 +75 +Number of Respondents +Source +Primary operating system: Windows +I obtain GDAL from... + + diff --git a/doc/images/community/survey_2024/years_experience.svg b/doc/images/community/survey_2024/years_experience.svg new file mode 100644 index 000000000000..814071492293 --- /dev/null +++ b/doc/images/community/survey_2024/years_experience.svg @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +0 +100 +200 + + + + + + + +1 - 4 years +5 - 9 years +10 - 20+ years +I'm Frank Warmerdam +Level of experience +Number of respondents +Survey Respondents - Level of Experience + + diff --git a/doc/source/community/index.rst b/doc/source/community/index.rst index 2ebfa71939c3..bf72f2b13fe7 100644 --- a/doc/source/community/index.rst +++ b/doc/source/community/index.rst @@ -131,6 +131,17 @@ Past members: Note that discussion of proposals to the PSC take place on gdal-dev, and input from all subscribers is welcome. A :ref:`list of past RFC ` is available for review. +User survey ++++++++++++ + +Results of the :ref:`survey_2024` are available for review. + + +.. toctree:: + :hidden: + + user_survey_2024 + GDAL Service Providers ---------------------- diff --git a/doc/source/community/user_survey_2024.rst b/doc/source/community/user_survey_2024.rst new file mode 100644 index 000000000000..61e4adc6479b --- /dev/null +++ b/doc/source/community/user_survey_2024.rst @@ -0,0 +1,146 @@ +.. _survey_2024: + +2024 GDAL user survey +===================== + +In October 2024, The GDAL maintenance program created an open survey to collect +feedback on user's experience with GDAL and the direction of the maintenance +program. The survey was publicized on gdal.org, the gdal-dev mailing list, the +project GitHub page, and social media. From October 28 to November 21, the +survey received 602 responses. + +Who responded to the survey? +---------------------------- + +Survey respondents were generally very experienced users, with 79% of users +having spent 5 or more years working with GDAL. Surprisingly, two respondents +claimed to be Frank Warmerdam, who originated the project in 1998. More than +half (52%) have built GDAL from source, 29% subscribe to the gdal-dev mailing +list, and 29% have contributed to the project by submitting bug reports or pull +requests. The high experience level of respondents reflects the challenge of +reaching users who may use GDAL less often, through other software, or are not +connected to the project community via mailing lists or social media. + +.. image:: ../../images/community/survey_2024/years_experience.svg + +Operating system +^^^^^^^^^^^^^^^^ + +Most survey respondents use GDAL on Linux, followed by Windows and OS X. +Responses to "Other" included WSL2 and iOS. + +.. image:: ../../images/community/survey_2024/operating_system.svg + +Local or cloud? +^^^^^^^^^^^^^^^ + +Most survey respondents use GDAL primarily with local file systems. + +.. image:: ../../images/community/survey_2024/local_or_cloud_read.svg + +.. image:: ../../images/community/survey_2024/local_or_cloud_write.svg + +Data formats +^^^^^^^^^^^^ + +Among raster data formats, GeoTIFF commands an overwhelming majority of GDAL +usage: + +.. image:: ../../images/community/survey_2024/raster_data_formats.svg + +The most popular vector format was GeoPackage, followed by classics such as +Shapefile, GeoJSON, and PostGIS. GeoParquet, Esri FileGeodatabase, FlatGeobuf, +and GML each earned enough votes to remain out of the "Other" category. + +.. image:: ../../images/community/survey_2024/vector_data_formats.svg + +Installing GDAL +--------------- + +Survey respondents obtain GDAL from a variety of channels, depending on the +platform. On Linux, standard system packages are the most popular solution. +OSX users rely primarily on Homebrew; most Windows users user OSGeo4W. The +popularity of Homebrew among Windows users may indicate that GDAL is being used +through the Windows Subsystem for Linux (WSL). The reported usage of OSGeo4W by +Linux users is more difficult to explain. + +.. image:: ../../images/community/survey_2024/where_gdal_obtained_linux.svg + +.. image:: ../../images/community/survey_2024/where_gdal_obtained_osx.svg + +.. image:: ../../images/community/survey_2024/where_gdal_obtained_windows.svg + +As may be expected for a group of experienced users, most respondents reported +that GDAL is easy to install with the options they need. Still, installation +remains a difficulty for many users. + +.. image:: ../../images/community/survey_2024/easy_to_install_gdal.svg + +Installation difficulties were not associated with a particular operating +system. + +.. image:: ../../images/community/survey_2024/easy_to_install_gdal_os.svg + +How is GDAL used? +----------------- + +The greatest number of respondents reported using GDAL from Python, with a +roughly 50/50 split between the GDAL Python bindings and higher-level packages +such as shapely, rasterio, and geopandas. After Python, the greatest number of +respondents reported using the command line interface, followed by smaller +number of users working in R, PostGIS, and QGIS. + +.. image:: ../../images/community/survey_2024/way_gdal_used.svg + +Getting help with GDAL +---------------------- + +Most users use gdal.org (directly or via a search engine) as their starting +point when trying to get help with GDAL. + +.. image:: ../../images/community/survey_2024/gdal_help_source.svg + +Difficulties using GDAL +----------------------- + +Users did not identify a single area as a source of their challenges with GDAL. +However, the top responses of "finding examples" and "understanding features" +point to a shortage of documentation. + +.. image:: ../../images/community/survey_2024/gdal_challenge.svg + +Consistent with the above, respondents reported "examples", "workflows", and +"API usage" as high priorities for documentation efforts. + +.. image:: ../../images/community/survey_2024/documentation_needs.svg + +And "examples" and "doc" rank highly among open-ended responses to +"what could make GDAL easier to use?" + +.. image:: ../../images/community/survey_2024/gdal_easier_to_use.svg + +Maintenance program activities +------------------------------ + +Among activities undertaken by the maintenance program so far, respondents found +the most value in enhancements to GDAL's dependencies (such as PROJ, GEOS, and +libtiff), its Python bindings, and documentation. + +.. image:: ../../images/community/survey_2024/maintenance_program_activities.svg + +Asked about a variety of tasks the maintenance program could take on beyond +those listed above, respondents showed some enthusiasm for almost everything! +Still, high priorities were given to performance, improving format +capabilities, and improving the command line interface while preserving +backward compatibility. + +.. image:: ../../images/community/survey_2024/maintenance_program_areas_of_focus.svg + +Next steps +---------- + +The maintenance program will use these results to inform work over the coming +year. Some work has already been performed to `develop an improved +command-line interface `__ and `add a +mechanism for usage examples to be cross-referenced in the +documentation `__. diff --git a/doc/source/user/ogr_feature_style.rst b/doc/source/user/ogr_feature_style.rst index 2f8d72e8a9b9..25d9d8db2b26 100644 --- a/doc/source/user/ogr_feature_style.rst +++ b/doc/source/user/ogr_feature_style.rst @@ -293,7 +293,7 @@ representation. Note again that all parameters are optional: list of ids so that an application never has to rely on understanding system-specific ids. - Here is the current list of OGR pen ids (this could grow time): + Here is the current list of OGR pen ids (this could grow in time): - ogr-pen-0: solid (the default when no id is provided) - ogr-pen-1: null pen (invisible) diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index bac9d8eb3b92..bd4b1c9f421b 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -7055,10 +7055,10 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename, CPLMalloc(nExtraSamples * sizeof(uint16_t))); memcpy(pasNewExtraSamples, extraSamples, nExtraSamples * sizeof(uint16_t)); - uint16_t nAlpha = GTiffGetAlphaValue( - CPLGetConfigOption("GTIFF_ALPHA", - CSLFetchNameValue(papszOptions, "ALPHA")), - DEFAULT_ALPHA_TYPE); + const char *pszAlpha = CPLGetConfigOption( + "GTIFF_ALPHA", CSLFetchNameValue(papszOptions, "ALPHA")); + const uint16_t nAlpha = + GTiffGetAlphaValue(pszAlpha, DEFAULT_ALPHA_TYPE); const int nBaseSamples = l_nBands - nExtraSamples; for (int iExtraBand = nBaseSamples + 1; iExtraBand <= l_nBands; iExtraBand++) @@ -7067,6 +7067,16 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename, GCI_AlphaBand) { pasNewExtraSamples[iExtraBand - nBaseSamples - 1] = nAlpha; + if (!pszAlpha) + { + // Use the ALPHA metadata item from the source band, when + // present, if no explicit ALPHA creation option + pasNewExtraSamples[iExtraBand - nBaseSamples - 1] = + GTiffGetAlphaValue( + poSrcDS->GetRasterBand(iExtraBand) + ->GetMetadataItem("ALPHA", "IMAGE_STRUCTURE"), + nAlpha); + } } } TIFFSetField(l_hTIFF, TIFFTAG_EXTRASAMPLES, nExtraSamples, diff --git a/frmts/gtiff/gtiffrasterband.cpp b/frmts/gtiff/gtiffrasterband.cpp index cee100297af9..cbb2a451ccda 100644 --- a/frmts/gtiff/gtiffrasterband.cpp +++ b/frmts/gtiff/gtiffrasterband.cpp @@ -166,7 +166,12 @@ GTiffRasterBand::GTiffRasterBand(GTiffDataset *poDSIn, int nBandIn) if (nBand > nBaseSamples && nBand - nBaseSamples - 1 < count && (v[nBand - nBaseSamples - 1] == EXTRASAMPLE_ASSOCALPHA || v[nBand - nBaseSamples - 1] == EXTRASAMPLE_UNASSALPHA)) + { + if (v[nBand - nBaseSamples - 1] == EXTRASAMPLE_ASSOCALPHA) + m_oGTiffMDMD.SetMetadataItem("ALPHA", "PREMULTIPLIED", + "IMAGE_STRUCTURE"); m_eBandInterp = GCI_AlphaBand; + } else m_eBandInterp = GCI_Undefined; } diff --git a/frmts/heif/heifdataset.cpp b/frmts/heif/heifdataset.cpp index c9f215d77e7a..37238b265e45 100644 --- a/frmts/heif/heifdataset.cpp +++ b/frmts/heif/heifdataset.cpp @@ -285,6 +285,7 @@ bool GDALHEIFDataset::Init(GDALOpenInfo *poOpenInfo) void GDALHEIFDataset::ReadMetadata() { #if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 19, 0) + processProperties(); ReadUserDescription(); #endif const int nMDBlocks = heif_image_handle_get_number_of_metadata_blocks( @@ -412,6 +413,79 @@ void GDALHEIFDataset::ReadMetadata() } #if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 19, 0) +static bool GetPropertyData(heif_context *m_hCtxt, heif_item_id item_id, + heif_property_id prop_id, + std::shared_ptr> &data) +{ + size_t size; + heif_error err = + heif_item_get_property_raw_size(m_hCtxt, item_id, prop_id, &size); + if (err.code != 0) + { + return false; + } + if (size == 0) + { + return false; + } + data = std::make_shared>(size); + err = heif_item_get_property_raw_data(m_hCtxt, item_id, prop_id, + data->data()); + if (err.code != 0) + { + return false; + } + return true; +} + +void GDALHEIFDataset::processProperties() +{ + constexpr heif_item_property_type TIEP_4CC = + (heif_item_property_type)heif_fourcc('t', 'i', 'e', 'p'); + constexpr heif_item_property_type MTXF_4CC = + (heif_item_property_type)heif_fourcc('m', 't', 'x', 'f'); + constexpr heif_item_property_type MCRS_4CC = + (heif_item_property_type)heif_fourcc('m', 'c', 'r', 's'); + constexpr int MAX_PROPERTIES_REQUIRED = 50; + heif_property_id prop_ids[MAX_PROPERTIES_REQUIRED]; + heif_item_id item_id = heif_image_handle_get_item_id(m_hImageHandle); + int num_props = heif_item_get_properties_of_type( + m_hCtxt, item_id, heif_item_property_type_invalid, &prop_ids[0], + MAX_PROPERTIES_REQUIRED); + for (int i = 0; i < num_props; i++) + { + heif_item_property_type prop_type = + heif_item_get_property_type(m_hCtxt, item_id, prop_ids[i]); + if (prop_type == TIEP_4CC) + { + std::shared_ptr> data; + if (!GetPropertyData(m_hCtxt, item_id, prop_ids[i], data)) + { + continue; + } + geoHEIF.addGCPs(data->data(), data->size()); + } + else if (prop_type == MTXF_4CC) + { + std::shared_ptr> data; + if (!GetPropertyData(m_hCtxt, item_id, prop_ids[i], data)) + { + continue; + } + geoHEIF.setModelTransformation(data->data(), data->size()); + } + else if (prop_type == MCRS_4CC) + { + std::shared_ptr> data; + if (!GetPropertyData(m_hCtxt, item_id, prop_ids[i], data)) + { + continue; + } + geoHEIF.extractSRS(data->data(), data->size()); + } + } +} + /************************************************************************/ /* ReadUserDescription() */ /************************************************************************/ @@ -824,32 +898,7 @@ CPLErr GDALHEIFRasterBand::IReadBlock(int, int nBlockYOff, void *pImage) CPLErr GDALHEIFDataset::GetGeoTransform(double *padfTransform) { - constexpr int MAX_PROPERTIES_REQUIRED = 10; - heif_property_id prop_ids[MAX_PROPERTIES_REQUIRED]; - heif_item_id item_id = heif_image_handle_get_item_id(m_hImageHandle); - int num_props = heif_item_get_properties_of_type( - m_hCtxt, item_id, - (heif_item_property_type)heif_fourcc('m', 't', 'x', 'f'), &prop_ids[0], - MAX_PROPERTIES_REQUIRED); - - for (int i = 0; i < num_props; i++) - { - size_t size; - heif_error err = heif_item_get_property_raw_size(m_hCtxt, item_id, - prop_ids[i], &size); - if (err.code != 0) - { - continue; - } - auto data = std::make_shared>(size); - heif_item_get_property_raw_data(m_hCtxt, item_id, prop_ids[i], - data->data()); - geoHEIF.setModelTransformation(data->data(), data->size()); - geoHEIF.GetGeoTransform(padfTransform); - return CE_None; - } - - return CE_Failure; + return geoHEIF.GetGeoTransform(padfTransform); } /************************************************************************/ @@ -857,70 +906,11 @@ CPLErr GDALHEIFDataset::GetGeoTransform(double *padfTransform) /************************************************************************/ const OGRSpatialReference *GDALHEIFDataset::GetSpatialRef() const { - if (geoHEIF.has_SRS()) - return geoHEIF.GetSpatialRef(); - - constexpr int MAX_PROPERTIES_REQUIRED = 10; - heif_property_id prop_ids[MAX_PROPERTIES_REQUIRED]; - heif_item_id item_id = heif_image_handle_get_item_id(m_hImageHandle); - int num_props = heif_item_get_properties_of_type( - m_hCtxt, item_id, - (heif_item_property_type)heif_fourcc('m', 'c', 'r', 's'), &prop_ids[0], - MAX_PROPERTIES_REQUIRED); - - for (int i = 0; i < num_props; i++) - { - size_t size; - heif_error err = heif_item_get_property_raw_size(m_hCtxt, item_id, - prop_ids[i], &size); - if (err.code != 0) - { - continue; - } - if (size == 0) - { - continue; - } - auto data = std::make_shared>(size); - err = heif_item_get_property_raw_data(m_hCtxt, item_id, prop_ids[i], - data->data()); - if (err.code != 0) - { - continue; - } - geoHEIF.extractSRS(data->data(), data->size()); - return geoHEIF.GetSpatialRef(); - } - return nullptr; + return geoHEIF.GetSpatialRef(); } int GDALHEIFDataset::GetGCPCount() { - if (geoHEIF.has_GCPs()) - { - return geoHEIF.GetGCPCount(); - } - // Get the GCPs if we can - constexpr int MAX_PROPERTIES_REQUIRED = 10; - heif_property_id prop_ids[MAX_PROPERTIES_REQUIRED]; - heif_item_id item_id = heif_image_handle_get_item_id(m_hImageHandle); - int num_props = heif_item_get_properties_of_type( - m_hCtxt, item_id, - (heif_item_property_type)heif_fourcc('t', 'i', 'e', 'p'), &prop_ids[0], - MAX_PROPERTIES_REQUIRED); - for (int i = 0; i < num_props; i++) - { - size_t size; - heif_item_get_property_raw_size(m_hCtxt, item_id, prop_ids[i], &size); - auto data = std::make_shared>(size); - heif_error err = heif_item_get_property_raw_data( - m_hCtxt, item_id, prop_ids[i], data->data()); - if ((err.code != 0)) - { - continue; - } - geoHEIF.addGCPs(data->data(), data->size()); - } return geoHEIF.GetGCPCount(); } @@ -933,41 +923,6 @@ const OGRSpatialReference *GDALHEIFDataset::GetGCPSpatialRef() const { return this->GetSpatialRef(); } - -void GDALHEIFDataset::ExtractUserDescription(const uint8_t *payload, - size_t length) -{ - // Match version - if (payload[0] == 0x00) - { - std::stringstream ss(std::string(payload + 4, payload + length)); - std::string lang; - std::getline(ss, lang, '\0'); - std::string name; - std::getline(ss, name, '\0'); - std::string description; - std::getline(ss, description, '\0'); - std::string tags; - std::getline(ss, tags, '\0'); - std::string domain = "DESCRIPTION"; - if (!lang.empty()) - { - domain += "_"; - domain += lang; - } - GDALDataset::SetMetadataItem("NAME", name.c_str(), domain.c_str()); - GDALDataset::SetMetadataItem("DESCRIPTION", description.c_str(), - domain.c_str()); - if (!tags.empty()) - { - GDALDataset::SetMetadataItem("TAGS", tags.c_str(), domain.c_str()); - } - } - else - { - CPLDebug("HEIF", "Unsupported udes version %d", payload[0]); - } -} #endif /************************************************************************/ diff --git a/frmts/heif/heifdataset.h b/frmts/heif/heifdataset.h index 950397e935f4..d4f5205df10c 100644 --- a/frmts/heif/heifdataset.h +++ b/frmts/heif/heifdataset.h @@ -44,6 +44,7 @@ class GDALHEIFDataset final : public GDALPamDataset #endif #if LIBHEIF_NUMERIC_VERSION >= BUILD_LIBHEIF_VERSION(1, 19, 0) + void processProperties(); gdal::GeoHEIF geoHEIF{}; #endif @@ -62,7 +63,6 @@ class GDALHEIFDataset final : public GDALPamDataset bool Init(GDALOpenInfo *poOpenInfo); void ReadMetadata(); void OpenThumbnails(); - void ExtractUserDescription(const uint8_t *payload, size_t length); #ifdef HAS_CUSTOM_FILE_WRITER static heif_error VFS_WriterCallback(struct heif_context *ctx, diff --git a/frmts/http/httpdriver.cpp b/frmts/http/httpdriver.cpp index a7563c9063dc..3b15354e9543 100644 --- a/frmts/http/httpdriver.cpp +++ b/frmts/http/httpdriver.cpp @@ -11,6 +11,7 @@ * SPDX-License-Identifier: MIT ****************************************************************************/ +#include "cpl_error_internal.h" #include "cpl_string.h" #include "cpl_http.h" #include "gdal_frmts.h" @@ -133,11 +134,27 @@ static GDALDataset *HTTPOpen(GDALOpenInfo *poOpenInfo) /* Try opening this result as a gdaldataset. */ /* -------------------------------------------------------------------- */ /* suppress errors as not all drivers support /vsimem */ - CPLPushErrorHandler(CPLQuietErrorHandler); - GDALDataset *poDS = (GDALDataset *)GDALOpenEx( - osResultFilename, poOpenInfo->nOpenFlags & ~GDAL_OF_SHARED, - poOpenInfo->papszAllowedDrivers, poOpenInfo->papszOpenOptions, nullptr); - CPLPopErrorHandler(); + + GDALDataset *poDS; + std::vector aoErrors; + { + CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); + CPLInstallErrorHandlerAccumulator(aoErrors); + poDS = GDALDataset::Open(osResultFilename, + poOpenInfo->nOpenFlags & ~GDAL_OF_SHARED, + poOpenInfo->papszAllowedDrivers, + poOpenInfo->papszOpenOptions, nullptr); + CPLUninstallErrorHandlerAccumulator(); + } + + // Re-emit silenced errors if open was successful + if (poDS) + { + for (const auto &oError : aoErrors) + { + CPLError(oError.type, oError.no, "%s", oError.msg.c_str()); + } + } // The JP2OpenJPEG driver may need to reopen the file, hence this special // behavior @@ -171,10 +188,10 @@ static GDALDataset *HTTPOpen(GDALOpenInfo *poOpenInfo) } else { - poDS = (GDALDataset *)GDALOpenEx( - osTempFilename, poOpenInfo->nOpenFlags & ~GDAL_OF_SHARED, - poOpenInfo->papszAllowedDrivers, poOpenInfo->papszOpenOptions, - nullptr); + poDS = GDALDataset::Open(osTempFilename, + poOpenInfo->nOpenFlags & ~GDAL_OF_SHARED, + poOpenInfo->papszAllowedDrivers, + poOpenInfo->papszOpenOptions, nullptr); if (VSIUnlink(osTempFilename) != 0 && poDS != nullptr) poDS->MarkSuppressOnClose(); /* VSIUnlink() may not work on windows */ diff --git a/frmts/netcdf/netcdfmultidim.cpp b/frmts/netcdf/netcdfmultidim.cpp index f30513602006..9748c4530a37 100644 --- a/frmts/netcdf/netcdfmultidim.cpp +++ b/frmts/netcdf/netcdfmultidim.cpp @@ -2646,8 +2646,9 @@ std::shared_ptr netCDFVariable::GetSpatialRef() const m_poSRS.reset(poSRS->Clone()); if (iDimX > 0 && iDimY > 0) { - if (m_poSRS->GetDataAxisToSRSAxisMapping() == - std::vector{2, 1}) + const auto &oMapping = m_poSRS->GetDataAxisToSRSAxisMapping(); + if (oMapping == std::vector{2, 1} || + oMapping == std::vector{2, 1, 3}) m_poSRS->SetDataAxisToSRSAxisMapping({iDimY, iDimX}); else m_poSRS->SetDataAxisToSRSAxisMapping({iDimX, iDimY}); diff --git a/frmts/wmts/wmtsdataset.cpp b/frmts/wmts/wmtsdataset.cpp index 09affe85072b..ab4d4eb893dc 100644 --- a/frmts/wmts/wmtsdataset.cpp +++ b/frmts/wmts/wmtsdataset.cpp @@ -158,7 +158,8 @@ class WMTSDataset final : public GDALPamDataset const char *pszOperation); static int ReadTMS(CPLXMLNode *psContents, const CPLString &osIdentifier, const CPLString &osMaxTileMatrixIdentifier, - int nMaxZoomLevel, WMTSTileMatrixSet &oTMS); + int nMaxZoomLevel, WMTSTileMatrixSet &oTMS, + bool &bHasWarnedAutoSwap); static int ReadTMLimits( CPLXMLNode *psTMSLimits, std::map &aoMapTileMatrixLimits); @@ -599,10 +600,9 @@ CPLString WMTSDataset::FixCRSName(const char *pszCRS) int WMTSDataset::ReadTMS(CPLXMLNode *psContents, const CPLString &osIdentifier, const CPLString &osMaxTileMatrixIdentifier, - int nMaxZoomLevel, WMTSTileMatrixSet &oTMS) + int nMaxZoomLevel, WMTSTileMatrixSet &oTMS, + bool &bHasWarnedAutoSwap) { - bool bHasWarnedAutoSwap = false; - for (CPLXMLNode *psIter = psContents->psChild; psIter != nullptr; psIter = psIter->psNext) { @@ -629,9 +629,10 @@ int WMTSDataset::ReadTMS(CPLXMLNode *psContents, const CPLString &osIdentifier, pszSupportedCRS); return FALSE; } - int bSwap = !STARTS_WITH_CI(pszSupportedCRS, "EPSG:") && - (oTMS.oSRS.EPSGTreatsAsLatLong() || - oTMS.oSRS.EPSGTreatsAsNorthingEasting()); + const bool bSwap = + !STARTS_WITH_CI(pszSupportedCRS, "EPSG:") && + (CPL_TO_BOOL(oTMS.oSRS.EPSGTreatsAsLatLong()) || + CPL_TO_BOOL(oTMS.oSRS.EPSGTreatsAsNorthingEasting())); CPLXMLNode *psBB = CPLGetXMLNode(psIter, "BoundingBox"); oTMS.bBoundingBoxValid = false; if (psBB != nullptr) @@ -740,16 +741,21 @@ int WMTSDataset::ReadTMS(CPLXMLNode *psContents, const CPLString &osIdentifier, } // Hack for http://osm.geobretagne.fr/gwc01/service/wmts?request=getcapabilities - if (STARTS_WITH_CI(l_pszIdentifier, "EPSG:4326:") && - oTM.dfTLY == -180.0) + // or https://trek.nasa.gov/tiles/Mars/EQ/THEMIS_NightIR_ControlledMosaics_100m_v2_oct2018/1.0.0/WMTSCapabilities.xml + if (oTM.dfTLY == -180.0 && + (STARTS_WITH_CI(l_pszIdentifier, "EPSG:4326:") || + (oTMS.oSRS.IsGeographic() && oTM.dfTLX == 90))) { if (!bHasWarnedAutoSwap) { bHasWarnedAutoSwap = true; CPLError(CE_Warning, CPLE_AppDefined, "Auto-correcting wrongly swapped " - "TileMatrix.TopLeftCorner coordinates. This " - "should be reported to the server administrator."); + "TileMatrix.TopLeftCorner coordinates. " + "They should be in latitude, longitude order " + "but are presented in longitude, latitude order. " + "This should be reported to the server " + "administrator."); } std::swap(oTM.dfTLX, oTM.dfTLY); } @@ -1246,6 +1252,8 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) std::map aoMapBoundingBox; std::map aoMapTileMatrixLimits; std::map aoMapDimensions; + bool bHasWarnedAutoSwap = false; + bool bHasWarnedAutoSwapBoundingBox = false; // Collect TileMatrixSet identifiers std::set oSetTMSIdentifiers; @@ -1431,7 +1439,8 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) // 13-082_WMTS_Simple_Profile/schemas/wmts/1.0/profiles/WMTSSimple/examples/wmtsGetCapabilities_response_OSM.xml WMTSTileMatrixSet oTMS; if (ReadTMS(psContents, osSingleTileMatrixSet, - CPLString(), -1, oTMS)) + CPLString(), -1, oTMS, + bHasWarnedAutoSwap)) { osCRS = oTMS.osSRS; } @@ -1448,9 +1457,10 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) !osUpperCorner.empty() && oSRS.SetFromUserInput(FixCRSName(osCRS)) == OGRERR_NONE) { - int bSwap = !STARTS_WITH_CI(osCRS, "EPSG:") && - (oSRS.EPSGTreatsAsLatLong() || - oSRS.EPSGTreatsAsNorthingEasting()); + const bool bSwap = + !STARTS_WITH_CI(osCRS, "EPSG:") && + (CPL_TO_BOOL(oSRS.EPSGTreatsAsLatLong()) || + CPL_TO_BOOL(oSRS.EPSGTreatsAsNorthingEasting())); char **papszLC = CSLTokenizeString(osLowerCorner); char **papszUC = CSLTokenizeString(osUpperCorner); if (CSLCount(papszLC) == 2 && CSLCount(papszUC) == 2) @@ -1460,6 +1470,30 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) sEnvelope.MinY = CPLAtof(papszLC[(bSwap) ? 0 : 1]); sEnvelope.MaxX = CPLAtof(papszUC[(bSwap) ? 1 : 0]); sEnvelope.MaxY = CPLAtof(papszUC[(bSwap) ? 0 : 1]); + + if (bSwap && oSRS.IsGeographic() && + (std::fabs(sEnvelope.MinY) > 90 || + std::fabs(sEnvelope.MaxY) > 90)) + { + if (!bHasWarnedAutoSwapBoundingBox) + { + bHasWarnedAutoSwapBoundingBox = true; + CPLError( + CE_Warning, CPLE_AppDefined, + "Auto-correcting wrongly swapped " + "ows:%s coordinates. " + "They should be in latitude, longitude " + "order " + "but are presented in longitude, latitude " + "order. " + "This should be reported to the server " + "administrator.", + psSubIter->pszValue); + } + std::swap(sEnvelope.MinX, sEnvelope.MinY); + std::swap(sEnvelope.MaxX, sEnvelope.MaxY); + } + aoMapBoundingBox[osCRS] = sEnvelope; } CSLDestroy(papszLC); @@ -1568,7 +1602,7 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) WMTSTileMatrixSet oTMS; if (!ReadTMS(psContents, osSelectTMS, osMaxTileMatrixIdentifier, - nUserMaxZoomLevel, oTMS)) + nUserMaxZoomLevel, oTMS, bHasWarnedAutoSwap)) { CPLDestroyXMLNode(psXML); delete poDS; @@ -2303,7 +2337,10 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) nTileY, (bExtendBeyondDateLine) ? nSizeX1 : nSizeX, nSizeY, oTM.nTileWidth, oTM.nTileHeight, nBands, GDALGetDataTypeName(eDataType), osOtherXML.c_str())); - GDALDataset *poWMSDS = (GDALDataset *)GDALOpenEx( + const auto eLastErrorType = CPLGetLastErrorType(); + const auto eLastErrorNum = CPLGetLastErrorNo(); + const std::string osLastErrorMsg = CPLGetLastErrorMsg(); + GDALDataset *poWMSDS = GDALDataset::Open( osStr, GDAL_OF_RASTER | GDAL_OF_SHARED | GDAL_OF_VERBOSE_ERROR, nullptr, nullptr, nullptr); if (poWMSDS == nullptr) @@ -2312,6 +2349,11 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) delete poDS; return nullptr; } + // Restore error state to what it was prior to WMS dataset opening + // if WMS dataset opening did not cause any new error to be emitted + if (CPLGetLastErrorType() == CE_None) + CPLErrorSetState(eLastErrorType, eLastErrorNum, + osLastErrorMsg.c_str()); VRTDatasetH hVRTDS = VRTCreate(nRasterXSize, nRasterYSize); for (int iBand = 1; iBand <= nBands; iBand++) diff --git a/frmts/zarr/zarr_array.cpp b/frmts/zarr/zarr_array.cpp index 6d12705201f5..5f038d8945db 100644 --- a/frmts/zarr/zarr_array.cpp +++ b/frmts/zarr/zarr_array.cpp @@ -2724,6 +2724,7 @@ void ZarrArray::ParseSpecialAttributes( if (item.IsValid()) { poSRS = std::make_shared(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); if (poSRS->SetFromUserInput( item.ToString().c_str(), OGRSpatialReference:: @@ -2748,6 +2749,7 @@ void ZarrArray::ParseSpecialAttributes( if (gridMappingArray) { poSRS = std::make_shared(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); CPLStringList aosKeyValues; for (const auto &poAttr : gridMappingArray->GetAttributes()) { @@ -2798,10 +2800,12 @@ void ZarrArray::ParseSpecialAttributes( } if (iDimX > 0 && iDimY > 0) { - if (poSRS->GetDataAxisToSRSAxisMapping() == std::vector{2, 1}) + const auto &oMapping = poSRS->GetDataAxisToSRSAxisMapping(); + if (oMapping == std::vector{2, 1} || + oMapping == std::vector{2, 1, 3}) poSRS->SetDataAxisToSRSAxisMapping({iDimY, iDimX}); - else if (poSRS->GetDataAxisToSRSAxisMapping() == - std::vector{1, 2}) + else if (oMapping == std::vector{1, 2} || + oMapping == std::vector{1, 2, 3}) poSRS->SetDataAxisToSRSAxisMapping({iDimX, iDimY}); } diff --git a/gcore/gdaldrivermanager.cpp b/gcore/gdaldrivermanager.cpp index 80cdf6d3bd41..98aabe187a09 100644 --- a/gcore/gdaldrivermanager.cpp +++ b/gcore/gdaldrivermanager.cpp @@ -1672,6 +1672,19 @@ void GDALDriverManager::DeclareDeferredPluginDriver( if (osFullPath.empty()) { + // Do not try to re-register a non-existent deferred plugin + // This would cause memory leaks in case of repeated calls to GDALAllRegister() + // Cf https://github.com/rasterio/rasterio/issues/3250 + for (const auto &poDriver : m_aoHiddenDrivers) + { + if (EQUAL(poDriver->GetDescription(), + poProxyDriver->GetDescription())) + { + delete poProxyDriver; + return; + } + } + CPLDebug("GDAL", "Proxy driver %s *not* registered due to %s not being found", poProxyDriver->GetDescription(), pszPluginFileName); diff --git a/gcore/geoheif.cpp b/gcore/geoheif.cpp index 1bd139a7ab1f..878ca0bb033f 100644 --- a/gcore/geoheif.cpp +++ b/gcore/geoheif.cpp @@ -86,12 +86,27 @@ void GeoHEIF::setModelTransformation(const uint8_t *payload, size_t length) CPLErr GeoHEIF::GetGeoTransform(double *padfTransform) const { - padfTransform[1] = modelTransform[1]; - padfTransform[2] = modelTransform[2]; - padfTransform[0] = modelTransform[0]; - padfTransform[4] = modelTransform[4]; - padfTransform[5] = modelTransform[5]; - padfTransform[3] = modelTransform[3]; + std::vector axes = + has_SRS() ? m_oSRS.GetDataAxisToSRSAxisMapping() : std::vector(); + + if (axes.size() && axes[0] != 1) + { + padfTransform[1] = modelTransform[4]; + padfTransform[2] = modelTransform[5]; + padfTransform[0] = modelTransform[3]; + padfTransform[4] = modelTransform[1]; + padfTransform[5] = modelTransform[2]; + padfTransform[3] = modelTransform[0]; + } + else + { + padfTransform[1] = modelTransform[1]; + padfTransform[2] = modelTransform[2]; + padfTransform[0] = modelTransform[0]; + padfTransform[4] = modelTransform[4]; + padfTransform[5] = modelTransform[5]; + padfTransform[3] = modelTransform[3]; + } return CE_None; } @@ -124,12 +139,14 @@ void GeoHEIF::extractSRS(const uint8_t *payload, size_t length) const } else if (crsEncoding == "curi") { - if ((crs.at(0) != '[') || (crs.at(crs.length() - 1) != ']')) + // null terminated string in the form "[EPSG:4326]" + if ((crs.at(0) != '[') || (crs.at(crs.length() - 2) != ']') || + (crs.at(crs.length() - 1) != '\0')) { CPLDebug("GeoHEIF", "CRS CURIE is not a safe CURIE"); return; } - std::string curie = crs.substr(1, crs.length() - 2); + std::string curie = crs.substr(1, crs.length() - 3); size_t separatorPos = curie.find(':'); if (separatorPos == std::string::npos) { @@ -150,6 +167,7 @@ void GeoHEIF::extractSRS(const uint8_t *payload, size_t length) const CPLDebug("GeoHEIF", "CRS encoding is not supported"); return; } + m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); } void GeoHEIF::addGCPs(const uint8_t *data, size_t length) diff --git a/ogr/ogrcompoundcurve.cpp b/ogr/ogrcompoundcurve.cpp index 54f0fc0c74db..35f18d5618d1 100644 --- a/ogr/ogrcompoundcurve.cpp +++ b/ogr/ogrcompoundcurve.cpp @@ -68,7 +68,16 @@ OGRCompoundCurve &OGRCompoundCurve::operator=(const OGRCompoundCurve &other) OGRCompoundCurve *OGRCompoundCurve::clone() const { - return new (std::nothrow) OGRCompoundCurve(*this); + auto ret = new (std::nothrow) OGRCompoundCurve(*this); + if (ret) + { + if (ret->WkbSize() != WkbSize()) + { + delete ret; + ret = nullptr; + } + } + return ret; } /************************************************************************/ diff --git a/ogr/ogrgeometry.cpp b/ogr/ogrgeometry.cpp index e7a1cb4c5025..e512f1df2473 100644 --- a/ogr/ogrgeometry.cpp +++ b/ogr/ogrgeometry.cpp @@ -7173,14 +7173,16 @@ char *OGRGeometryToHexEWKB(OGRGeometry *poGeometry, int nSRSId, // When converting to hex, each byte takes 2 hex characters. In addition // we add in 8 characters to represent the SRID integer in hex, and // one for a null terminator. - - const size_t nTextSize = nWkbSize * 2 + 8 + 1; - if (nTextSize > static_cast(std::numeric_limits::max())) + // The limit of INT_MAX = 2 GB is a bit artificial, but at time of writing + // (2024), PostgreSQL by default cannot handle objects larger than 1 GB: + // https://github.com/postgres/postgres/blob/5d39becf8ba0080c98fee4b63575552f6800b012/src/include/utils/memutils.h#L40 + if (nWkbSize > + static_cast(std::numeric_limits::max() - 8 - 1) / 2) { - // FIXME: artificial limitation CPLFree(pabyWKB); return CPLStrdup(""); } + const size_t nTextSize = nWkbSize * 2 + 8 + 1; char *pszTextBuf = static_cast(VSI_MALLOC_VERBOSE(nTextSize)); if (pszTextBuf == nullptr) { @@ -7230,11 +7232,15 @@ char *OGRGeometryToHexEWKB(OGRGeometry *poGeometry, int nSRSId, // Copy the rest of the data over - subtract // 5 since we already copied 5 bytes above. pszHex = CPLBinaryToHex(static_cast(nWkbSize - 5), pabyWKB + 5); + CPLFree(pabyWKB); + if (!pszHex || pszHex[0] == 0) + { + CPLFree(pszTextBuf); + return pszHex; + } strcpy(pszTextBufCurrent, pszHex); CPLFree(pszHex); - CPLFree(pabyWKB); - return pszTextBuf; } diff --git a/ogr/ogrgeometrycollection.cpp b/ogr/ogrgeometrycollection.cpp index f1beb953ad96..0875ee7f76f2 100644 --- a/ogr/ogrgeometrycollection.cpp +++ b/ogr/ogrgeometrycollection.cpp @@ -184,7 +184,16 @@ void OGRGeometryCollection::empty() OGRGeometryCollection *OGRGeometryCollection::clone() const { - return new (std::nothrow) OGRGeometryCollection(*this); + auto ret = new (std::nothrow) OGRGeometryCollection(*this); + if (ret) + { + if (ret->WkbSize() != WkbSize()) + { + delete ret; + ret = nullptr; + } + } + return ret; } /************************************************************************/ diff --git a/ogr/ogrlinestring.cpp b/ogr/ogrlinestring.cpp index 1fb68d0c9894..9c8aa7a23850 100644 --- a/ogr/ogrlinestring.cpp +++ b/ogr/ogrlinestring.cpp @@ -3072,7 +3072,16 @@ OGRLinearRing *OGRLineString::CastToLinearRing(OGRLineString *poLS) OGRLineString *OGRLineString::clone() const { - return new (std::nothrow) OGRLineString(*this); + auto ret = new (std::nothrow) OGRLineString(*this); + if (ret) + { + if (ret->getNumPoints() != getNumPoints()) + { + delete ret; + ret = nullptr; + } + } + return ret; } //! @cond Doxygen_Suppress diff --git a/ogr/ogrmulticurve.cpp b/ogr/ogrmulticurve.cpp index 383a8d4f6686..b86d737454b7 100644 --- a/ogr/ogrmulticurve.cpp +++ b/ogr/ogrmulticurve.cpp @@ -64,7 +64,16 @@ OGRMultiCurve &OGRMultiCurve::operator=(const OGRMultiCurve &other) OGRMultiCurve *OGRMultiCurve::clone() const { - return new (std::nothrow) OGRMultiCurve(*this); + auto ret = new (std::nothrow) OGRMultiCurve(*this); + if (ret) + { + if (ret->WkbSize() != WkbSize()) + { + delete ret; + ret = nullptr; + } + } + return ret; } /************************************************************************/ diff --git a/ogr/ogrmultilinestring.cpp b/ogr/ogrmultilinestring.cpp index 2961e4dc67c1..e3c41c3cf448 100644 --- a/ogr/ogrmultilinestring.cpp +++ b/ogr/ogrmultilinestring.cpp @@ -65,7 +65,16 @@ OGRMultiLineString::operator=(const OGRMultiLineString &other) OGRMultiLineString *OGRMultiLineString::clone() const { - return new (std::nothrow) OGRMultiLineString(*this); + auto ret = new (std::nothrow) OGRMultiLineString(*this); + if (ret) + { + if (ret->WkbSize() != WkbSize()) + { + delete ret; + ret = nullptr; + } + } + return ret; } /************************************************************************/ diff --git a/ogr/ogrmultipolygon.cpp b/ogr/ogrmultipolygon.cpp index d22d27e277a0..a1bed3208c3f 100644 --- a/ogr/ogrmultipolygon.cpp +++ b/ogr/ogrmultipolygon.cpp @@ -62,7 +62,16 @@ OGRMultiPolygon &OGRMultiPolygon::operator=(const OGRMultiPolygon &other) OGRMultiPolygon *OGRMultiPolygon::clone() const { - return new (std::nothrow) OGRMultiPolygon(*this); + auto ret = new (std::nothrow) OGRMultiPolygon(*this); + if (ret) + { + if (ret->WkbSize() != WkbSize()) + { + delete ret; + ret = nullptr; + } + } + return ret; } /************************************************************************/ diff --git a/ogr/ogrmultisurface.cpp b/ogr/ogrmultisurface.cpp index 6502d41157b4..d5830342e97a 100644 --- a/ogr/ogrmultisurface.cpp +++ b/ogr/ogrmultisurface.cpp @@ -65,7 +65,16 @@ OGRMultiSurface &OGRMultiSurface::operator=(const OGRMultiSurface &other) OGRMultiSurface *OGRMultiSurface::clone() const { - return new (std::nothrow) OGRMultiSurface(*this); + auto ret = new (std::nothrow) OGRMultiSurface(*this); + if (ret) + { + if (ret->WkbSize() != WkbSize()) + { + delete ret; + ret = nullptr; + } + } + return ret; } /************************************************************************/ diff --git a/ogr/ogrpolygon.cpp b/ogr/ogrpolygon.cpp index b654a9584443..1daceec6e6c4 100644 --- a/ogr/ogrpolygon.cpp +++ b/ogr/ogrpolygon.cpp @@ -71,7 +71,16 @@ OGRPolygon &OGRPolygon::operator=(const OGRPolygon &other) OGRPolygon *OGRPolygon::clone() const { - return new (std::nothrow) OGRPolygon(*this); + auto ret = new (std::nothrow) OGRPolygon(*this); + if (ret) + { + if (ret->WkbSize() != WkbSize()) + { + delete ret; + ret = nullptr; + } + } + return ret; } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp index 7208b0d27c68..17f38a92e756 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp @@ -811,27 +811,9 @@ int OGRGeoJSONDataSource::ReadFromService(GDALOpenInfo *poOpenInfo, /* -------------------------------------------------------------------- */ /* Fetch the GeoJSON result. */ /* -------------------------------------------------------------------- */ - char *papsOptions[] = { - const_cast("HEADERS=Accept: text/plain, application/json"), - nullptr}; - - CPLHTTPResult *pResult = CPLHTTPFetch(pszSource, papsOptions); - - /* -------------------------------------------------------------------- */ - /* Try to handle CURL/HTTP errors. */ - /* -------------------------------------------------------------------- */ - if (nullptr == pResult || 0 == pResult->nDataLen || - 0 != CPLGetLastErrorNo()) - { - CPLHTTPDestroyResult(pResult); - return FALSE; - } - - if (0 != pResult->nStatus) + CPLHTTPResult *pResult = GeoJSONHTTPFetchWithContentTypeHeader(pszSource); + if (!pResult) { - CPLError(CE_Failure, CPLE_AppDefined, "Curl reports error: %d: %s", - pResult->nStatus, pResult->pszErrBuf); - CPLHTTPDestroyResult(pResult); return FALSE; } diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp index 9667db12b3dc..b41e6c3ab86b 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp @@ -837,26 +837,11 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, } else { - const char *const papsOptions[] = { - "HEADERS=Accept: text/plain, application/json", nullptr}; - CPLHTTPResult *pResult = - CPLHTTPFetch(pszUnprefixedFilename, papsOptions); - - if (nullptr == pResult || 0 == pResult->nDataLen || - 0 != CPLGetLastErrorNo()) - { - CPLHTTPDestroyResult(pResult); - return false; - } - - if (0 != pResult->nStatus) + GeoJSONHTTPFetchWithContentTypeHeader(pszUnprefixedFilename); + if (!pResult) { - CPLError(CE_Failure, CPLE_AppDefined, - "Curl reports error: %d: %s", pResult->nStatus, - pResult->pszErrBuf); - CPLHTTPDestroyResult(pResult); - return false; + return FALSE; } m_osTmpFile = VSIMemGenerateHiddenFilename("geojsonseq"); diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp index 7a11bc180ecb..debdda13973e 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp @@ -1048,3 +1048,77 @@ OGRFieldType GeoJSONStringPropertyToFieldType(json_object *poObject, } return OFTString; } + +/************************************************************************/ +/* GeoJSONHTTPFetchWithContentTypeHeader() */ +/************************************************************************/ + +CPLHTTPResult *GeoJSONHTTPFetchWithContentTypeHeader(const char *pszURL) +{ + std::string osHeaders; + const char *pszGDAL_HTTP_HEADERS = + CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr); + bool bFoundAcceptHeader = false; + if (pszGDAL_HTTP_HEADERS) + { + bool bHeadersDone = false; + // Compatibility hack for "HEADERS=Accept: text/plain, application/json" + if (strstr(pszGDAL_HTTP_HEADERS, "\r\n") == nullptr) + { + const char *pszComma = strchr(pszGDAL_HTTP_HEADERS, ','); + if (pszComma != nullptr && strchr(pszComma, ':') == nullptr) + { + osHeaders = pszGDAL_HTTP_HEADERS; + bFoundAcceptHeader = + STARTS_WITH_CI(pszGDAL_HTTP_HEADERS, "Accept:"); + bHeadersDone = true; + } + } + if (!bHeadersDone) + { + // We accept both raw headers with \r\n as a separator, or as + // a comma separated list of foo: bar values. + const CPLStringList aosTokens( + strstr(pszGDAL_HTTP_HEADERS, "\r\n") + ? CSLTokenizeString2(pszGDAL_HTTP_HEADERS, "\r\n", 0) + : CSLTokenizeString2(pszGDAL_HTTP_HEADERS, ",", + CSLT_HONOURSTRINGS)); + for (int i = 0; i < aosTokens.size(); ++i) + { + if (!osHeaders.empty()) + osHeaders += "\r\n"; + if (!bFoundAcceptHeader) + bFoundAcceptHeader = + STARTS_WITH_CI(aosTokens[i], "Accept:"); + osHeaders += aosTokens[i]; + } + } + } + if (!bFoundAcceptHeader) + { + if (!osHeaders.empty()) + osHeaders += "\r\n"; + osHeaders += "Accept: text/plain, application/json"; + } + + CPLStringList aosOptions; + aosOptions.SetNameValue("HEADERS", osHeaders.c_str()); + CPLHTTPResult *pResult = CPLHTTPFetch(pszURL, aosOptions.List()); + + if (nullptr == pResult || 0 == pResult->nDataLen || + 0 != CPLGetLastErrorNo()) + { + CPLHTTPDestroyResult(pResult); + return nullptr; + } + + if (0 != pResult->nStatus) + { + CPLError(CE_Failure, CPLE_AppDefined, "Curl reports error: %d: %s", + pResult->nStatus, pResult->pszErrBuf); + CPLHTTPDestroyResult(pResult); + return nullptr; + } + + return pResult; +} diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h index ff1cce341f64..0b0859d75c64 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h @@ -15,6 +15,7 @@ #include "ogr_core.h" #include "cpl_json_header.h" +#include "cpl_http.h" #include "cpl_vsi.h" #include "gdal_priv.h" @@ -55,4 +56,10 @@ bool JSONFGIsObject(const char *pszText, GDALOpenInfo *poOpenInfo); OGRFieldType GeoJSONStringPropertyToFieldType(json_object *poObject, int &nTZFlag); +/************************************************************************/ +/* GeoJSONHTTPFetchWithContentTypeHeader */ +/************************************************************************/ + +CPLHTTPResult *GeoJSONHTTPFetchWithContentTypeHeader(const char *pszURL); + #endif // OGR_GEOJSONUTILS_H_INCLUDED diff --git a/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp b/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp index 5b2cfbfd5069..9ab77fc4d3ce 100644 --- a/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp +++ b/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp @@ -3301,6 +3301,8 @@ class OGRMVTWriterDataset final : public GDALDataset CPLString m_osType{"overlay"}; sqlite3 *m_hDBMBTILES = nullptr; OGREnvelope m_oEnvelope; + bool m_bMaxTileSizeOptSpecified = false; + bool m_bMaxFeaturesOptSpecified = false; unsigned m_nMaxTileSize = 500000; unsigned m_nMaxFeatures = 200000; std::map m_oMapLayerNameToDesc; @@ -4972,11 +4974,28 @@ std::string OGRMVTWriterDataset::EncodeTile( const double dfCompressionRatio = static_cast(nSizeAfter) / nSizeBefore; + const bool bTooManyFeatures = nFeaturesInTile >= m_nMaxFeatures; + if (bTooManyFeatures && !m_bMaxFeaturesOptSpecified) + { + m_bMaxFeaturesOptSpecified = true; + CPLError(CE_Warning, CPLE_AppDefined, + "At least one tile exceeded the default maximum number of " + "features per tile (%u) and was truncated to satisfy it.", + m_nMaxFeatures); + } + // If the tile size is above the allowed values or there are too many // features, then sort by descending area / length until we get to the // limit. bool bTooBigTile = oTileBuffer.size() > m_nMaxTileSize; - const bool bTooManyFeatures = nFeaturesInTile >= m_nMaxFeatures; + if (bTooBigTile && !m_bMaxTileSizeOptSpecified) + { + m_bMaxTileSizeOptSpecified = true; + CPLError(CE_Warning, CPLE_AppDefined, + "At least one tile exceeded the default maximum tile size of " + "%u bytes and was encoded at lower resolution", + m_nMaxTileSize); + } GUInt32 nExtent = m_nExtent; while (bTooBigTile && !bTooManyFeatures && nExtent >= 256) @@ -6087,14 +6106,32 @@ GDALDataset *OGRMVTWriterDataset::Create(const char *pszFilename, int nXSize, poDS->m_nBuffer = static_cast(atoi(CSLFetchNameValueDef( papszOptions, "BUFFER", CPLSPrintf("%u", 5 * poDS->m_nExtent / 256)))); - poDS->m_nMaxTileSize = - std::max(100U, static_cast(atoi(CSLFetchNameValueDef( - papszOptions, "MAX_SIZE", - CPLSPrintf("%u", poDS->m_nMaxTileSize))))); - poDS->m_nMaxFeatures = - std::max(1U, static_cast(atoi(CSLFetchNameValueDef( - papszOptions, "MAX_FEATURES", - CPLSPrintf("%u", poDS->m_nMaxFeatures))))); + { + const char *pszMaxSize = CSLFetchNameValue(papszOptions, "MAX_SIZE"); + poDS->m_bMaxTileSizeOptSpecified = pszMaxSize != nullptr; + // This is used by unit tests + pszMaxSize = CSLFetchNameValueDef(papszOptions, "@MAX_SIZE_FOR_TEST", + pszMaxSize); + if (pszMaxSize) + { + poDS->m_nMaxTileSize = + std::max(100U, static_cast(atoi(pszMaxSize))); + } + } + + { + const char *pszMaxFeatures = + CSLFetchNameValue(papszOptions, "MAX_FEATURES"); + poDS->m_bMaxFeaturesOptSpecified = pszMaxFeatures != nullptr; + pszMaxFeatures = CSLFetchNameValueDef( + // This is used by unit tests + papszOptions, "@MAX_FEATURES_FOR_TEST", pszMaxFeatures); + if (pszMaxFeatures) + { + poDS->m_nMaxFeatures = + std::max(1U, static_cast(atoi(pszMaxFeatures))); + } + } poDS->m_osName = CSLFetchNameValueDef(papszOptions, "NAME", CPLGetBasename(pszFilename)); diff --git a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp index ca13652339e5..49596c6b2fe7 100644 --- a/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/filegdbtable.cpp @@ -2537,6 +2537,7 @@ int FileGDBTable::GetIndexCount() CPLDebug("OpenFileGDB", ".gdbindexes v9 not handled yet"); return 0; } + returnErrorAndCleanupIf(nIndexCount >= static_cast(GetFieldCount() + 1) * 10, VSIFree(pabyIdx)); @@ -2547,7 +2548,7 @@ int FileGDBTable::GetIndexCount() returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < sizeof(GUInt32), VSIFree(pabyIdx)); - GUInt32 nIdxNameCarCount = GetUInt32(pabyCur, 0); + const GUInt32 nIdxNameCarCount = GetUInt32(pabyCur, 0); pabyCur += sizeof(GUInt32); returnErrorAndCleanupIf(nIdxNameCarCount > 1024, VSIFree(pabyIdx)); returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < @@ -2557,13 +2558,56 @@ int FileGDBTable::GetIndexCount() ReadUTF16String(pabyCur, nIdxNameCarCount)); pabyCur += 2 * nIdxNameCarCount; + // 4 "magic fields" + returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < + sizeof(GUInt16) + sizeof(GUInt32) + + sizeof(GUInt16) + sizeof(GUInt32), + VSIFree(pabyIdx)); + // const GUInt16 nMagic1 = GetUInt16(pabyCur, 0); + const GUInt32 nMagic2 = GetUInt32(pabyCur + sizeof(GUInt16), 0); + const GUInt16 nMagic3 = + GetUInt16(pabyCur + sizeof(GUInt16) + sizeof(GUInt32), 0); + if (!((nMagic2 == 2 && nMagic3 == 0) || + (nMagic2 == 4 && nMagic3 == 0) || + (nMagic2 == 16 && nMagic3 == 65535))) + { + // Cf files a00000029.gdbindexes, a000000ea.gdbindexes, a000000ed.gdbindexes, + // a000000f8.gdbindexes, a000000fb.gdbindexes, a00000103.gdbindexes + // from https://github.com/OSGeo/gdal/issues/11295#issuecomment-2491158506 + CPLDebug("OpenFileGDB", "Reading %s", pszIndexesName); + CPLDebug( + "OpenFileGDB", + "Strange (deleted?) index descriptor at index %u of name %s", i, + osIndexName.c_str()); + + // Skip magic fields + pabyCur += sizeof(GUInt16); + + const GUInt32 nColNameCarCount = nMagic2; + pabyCur += sizeof(GUInt32); + returnErrorAndCleanupIf(nColNameCarCount > 1024, VSIFree(pabyIdx)); + returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < + 2 * nColNameCarCount, + VSIFree(pabyIdx)); + pabyCur += 2 * nColNameCarCount; + + // Skip magic field + returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < + sizeof(GUInt16), + VSIFree(pabyIdx)); + pabyCur += sizeof(GUInt16); + + continue; + } + // Skip magic fields - pabyCur += 2 + 4 + 2 + 4; + pabyCur += sizeof(GUInt16) + sizeof(GUInt32) + sizeof(GUInt16) + + sizeof(GUInt32); returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < sizeof(GUInt32), VSIFree(pabyIdx)); - GUInt32 nColNameCarCount = GetUInt32(pabyCur, 0); + const GUInt32 nColNameCarCount = GetUInt32(pabyCur, 0); pabyCur += sizeof(GUInt32); returnErrorAndCleanupIf(nColNameCarCount > 1024, VSIFree(pabyIdx)); returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < @@ -2574,7 +2618,10 @@ int FileGDBTable::GetIndexCount() pabyCur += 2 * nColNameCarCount; // Skip magic field - pabyCur += 2; + returnErrorAndCleanupIf(static_cast(pabyEnd - pabyCur) < + sizeof(GUInt16), + VSIFree(pabyIdx)); + pabyCur += sizeof(GUInt16); auto poIndex = std::make_unique(); poIndex->m_osIndexName = osIndexName; diff --git a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp index 9be77144bbe7..9034544467f6 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgdatasource.cpp @@ -649,12 +649,16 @@ int OGRPGDataSource::Open(const char *pszNewName, int bUpdate, int bTestOpen, osSearchPath.find(osPostgisSchema) == std::string::npos)) { std::string osNewSearchPath; - if (osActiveSchema != "public") + if (!osActiveSchema.empty() && osActiveSchema != "public") { osNewSearchPath += OGRPGEscapeString(hPGConn, osActiveSchema.c_str()); } - if (!osSearchPath.empty() && osSearchPath != "\"\"") + // SHOW search_path reports "", but SET search_path doesn't accept it! + // It must be transformed to '' + if (STARTS_WITH(osSearchPath.c_str(), "\"\"")) + osSearchPath = "''" + osSearchPath.substr(2); + if (!osSearchPath.empty()) { if (!osNewSearchPath.empty()) osNewSearchPath += ','; @@ -668,34 +672,38 @@ int OGRPGDataSource::Open(const char *pszNewName, int bUpdate, int bTestOpen, osNewSearchPath += OGRPGEscapeString(hPGConn, osPostgisSchema.c_str()); } - CPLDebug("PG", "Modifying search_path from %s to %s", - osSearchPath.c_str(), osNewSearchPath.c_str()); + if (osNewSearchPath != osSearchPath) + { + CPLDebug("PG", "Modifying search_path from %s to %s", + osSearchPath.c_str(), osNewSearchPath.c_str()); - std::string osCommand = "SET search_path=" + osNewSearchPath; - PGresult *hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); + std::string osCommand = "SET search_path=" + osNewSearchPath; + PGresult *hResult = OGRPG_PQexec(hPGConn, osCommand.c_str()); - if (!hResult || PQresultStatus(hResult) != PGRES_COMMAND_OK) - { - OGRPGClearResult(hResult); - CPLDebug("PG", "Command \"%s\" failed. Trying without 'public'.", - osCommand.c_str()); - osCommand = CPLSPrintf( - "SET search_path=%s", - OGRPGEscapeString(hPGConn, osActiveSchema.c_str()).c_str()); - PGresult *hResult2 = OGRPG_PQexec(hPGConn, osCommand.c_str()); - - if (!hResult2 || PQresultStatus(hResult2) != PGRES_COMMAND_OK) + if (!hResult || PQresultStatus(hResult) != PGRES_COMMAND_OK) { - OGRPGClearResult(hResult2); + OGRPGClearResult(hResult); + CPLDebug("PG", + "Command \"%s\" failed. Trying without 'public'.", + osCommand.c_str()); + osCommand = CPLSPrintf( + "SET search_path=%s", + OGRPGEscapeString(hPGConn, osActiveSchema.c_str()).c_str()); + PGresult *hResult2 = OGRPG_PQexec(hPGConn, osCommand.c_str()); + + if (!hResult2 || PQresultStatus(hResult2) != PGRES_COMMAND_OK) + { + OGRPGClearResult(hResult2); - CPLError(CE_Failure, CPLE_AppDefined, "%s", - PQerrorMessage(hPGConn)); + CPLError(CE_Failure, CPLE_AppDefined, "%s", + PQerrorMessage(hPGConn)); - return FALSE; + return FALSE; + } } - } - OGRPGClearResult(hResult); + OGRPGClearResult(hResult); + } } /* -------------------------------------------------------------------- */ diff --git a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp index 44f480c6d47d..b67ca7b1f006 100644 --- a/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp +++ b/ogr/ogrsf_frmts/pg/ogrpgtablelayer.cpp @@ -1942,8 +1942,7 @@ OGRErr OGRPGTableLayer::CreateFeatureViaInsert(OGRFeature *poFeature) else osCommand += ", "; - osCommand = - osCommand + + osCommand += OGRPGEscapeColumnName(poFeatureDefn->GetFieldDefn(i)->GetNameRef()); } @@ -1956,7 +1955,7 @@ OGRErr OGRPGTableLayer::CreateFeatureViaInsert(OGRFeature *poFeature) bNeedComma = FALSE; for (int i = 0; i < poFeatureDefn->GetGeomFieldCount(); i++) { - OGRPGGeomFieldDefn *poGeomFieldDefn = + const OGRPGGeomFieldDefn *poGeomFieldDefn = poFeatureDefn->GetGeomFieldDefn(i); OGRGeometry *poGeom = poFeature->GetGeomFieldRef(i); if (poGeom == nullptr) @@ -1982,10 +1981,27 @@ OGRErr OGRPGTableLayer::CreateFeatureViaInsert(OGRFeature *poFeature) char *pszHexEWKB = OGRGeometryToHexEWKB( poGeom, nSRSId, poDS->sPostGISVersion.nMajor, poDS->sPostGISVersion.nMinor); + if (!pszHexEWKB || pszHexEWKB[0] == 0) + { + return OGRERR_FAILURE; + } + osCommand += '\''; + try + { + osCommand += pszHexEWKB; + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory: too large geometry"); + CPLFree(pszHexEWKB); + return OGRERR_FAILURE; + } + osCommand += "'::"; if (poGeomFieldDefn->ePostgisType == GEOM_TYPE_GEOGRAPHY) - osCommand += CPLString().Printf("'%s'::GEOGRAPHY", pszHexEWKB); + osCommand += "GEOGRAPHY"; else - osCommand += CPLString().Printf("'%s'::GEOMETRY", pszHexEWKB); + osCommand += "GEOMETRY"; CPLFree(pszHexEWKB); } else if (!bWkbAsOid) @@ -1994,15 +2010,24 @@ OGRErr OGRPGTableLayer::CreateFeatureViaInsert(OGRFeature *poFeature) GeometryToBYTEA(poGeom, poDS->sPostGISVersion.nMajor, poDS->sPostGISVersion.nMinor); - if (pszBytea != nullptr) + if (!pszBytea) + { + return OGRERR_FAILURE; + } + osCommand += "E'"; + try { - osCommand += "E'"; osCommand += pszBytea; - osCommand += '\''; + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory: too large geometry"); CPLFree(pszBytea); + return OGRERR_FAILURE; } - else - osCommand += "''"; + osCommand += '\''; + CPLFree(pszBytea); } else if (poGeomFieldDefn->ePostgisType == GEOM_TYPE_WKB /* && bWkbAsOid */) @@ -2083,7 +2108,7 @@ OGRErr OGRPGTableLayer::CreateFeatureViaInsert(OGRFeature *poFeature) { CPLError(CE_Failure, CPLE_AppDefined, "INSERT command for new feature failed.\n%s\nCommand: %s", - PQerrorMessage(hPGConn), osCommand.c_str()); + PQerrorMessage(hPGConn), osCommand.substr(0, 1024).c_str()); if (!bHasWarnedAlreadySetFID && poFeature->GetFID() != OGRNullFID && pszFIDColumn != nullptr) @@ -2125,7 +2150,7 @@ OGRErr OGRPGTableLayer::CreateFeatureViaCopy(OGRFeature *poFeature) /* First process geometry */ for (int i = 0; i < poFeatureDefn->GetGeomFieldCount(); i++) { - OGRPGGeomFieldDefn *poGeomFieldDefn = + const OGRPGGeomFieldDefn *poGeomFieldDefn = poFeatureDefn->GetGeomFieldDefn(i); OGRGeometry *poGeom = poFeature->GetGeomFieldRef(i); @@ -2147,6 +2172,10 @@ OGRErr OGRPGTableLayer::CreateFeatureViaCopy(OGRFeature *poFeature) pszGeom = OGRGeometryToHexEWKB(poGeom, poGeomFieldDefn->nSRSId, poDS->sPostGISVersion.nMajor, poDS->sPostGISVersion.nMinor); + if (!pszGeom || pszGeom[0] == 0) + { + return OGRERR_FAILURE; + } } if (!osCommand.empty()) @@ -2154,7 +2183,17 @@ OGRErr OGRPGTableLayer::CreateFeatureViaCopy(OGRFeature *poFeature) if (pszGeom) { - osCommand += pszGeom; + try + { + osCommand += pszGeom; + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory: too large geometry"); + CPLFree(pszGeom); + return OGRERR_FAILURE; + } CPLFree(pszGeom); } else diff --git a/ogr/ogrsf_frmts/pgdump/ogrpgdumplayer.cpp b/ogr/ogrsf_frmts/pgdump/ogrpgdumplayer.cpp index bf103a7edebb..24b4c8560d9d 100644 --- a/ogr/ogrsf_frmts/pgdump/ogrpgdumplayer.cpp +++ b/ogr/ogrsf_frmts/pgdump/ogrpgdumplayer.cpp @@ -347,8 +347,21 @@ OGRErr OGRPGDumpLayer::CreateFeatureViaInsert(OGRFeature *poFeature) OGRGeometryToHexEWKB(poGeom, poGFldDefn->m_nSRSId, m_nPostGISMajor, m_nPostGISMinor); osCommand += "'"; - if (pszHex) + if (!pszHex || pszHex[0] == 0) + { + return false; + } + try + { osCommand += pszHex; + } + catch (const std::bad_alloc &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory: too large geometry"); + CPLFree(pszHex); + return false; + } osCommand += "'"; CPLFree(pszHex); } @@ -356,20 +369,31 @@ OGRErr OGRPGDumpLayer::CreateFeatureViaInsert(OGRFeature *poFeature) { poGeom->exportToWkt(&pszWKT, wkbVariantIso); - if (pszWKT != nullptr) + if (!pszWKT) + { + return false; + } + try + { + osCommand += CPLSPrintf("GeomFromEWKT('SRID=%d;", + poGFldDefn->m_nSRSId); + osCommand += pszWKT; + osCommand += "'::TEXT) "; + } + catch (const std::bad_alloc &) { - osCommand += CPLString().Printf( - "GeomFromEWKT('SRID=%d;%s'::TEXT) ", - poGFldDefn->m_nSRSId, pszWKT); + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory: too large geometry"); CPLFree(pszWKT); + return false; } - else - osCommand += "''"; + CPLFree(pszWKT); } bNeedComma = true; } } + return true; }; /* Set the FID */ @@ -382,7 +406,10 @@ OGRErr OGRPGDumpLayer::CreateFeatureViaInsert(OGRFeature *poFeature) } if (m_bGeomColumnPositionImmediate) - AddGeomFieldsValue(); + { + if (!AddGeomFieldsValue()) + return OGRERR_FAILURE; + } for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); i++) { @@ -401,7 +428,10 @@ OGRErr OGRPGDumpLayer::CreateFeatureViaInsert(OGRFeature *poFeature) } if (!m_bGeomColumnPositionImmediate) - AddGeomFieldsValue(); + { + if (!AddGeomFieldsValue()) + return OGRERR_FAILURE; + } osCommand += ")"; @@ -440,7 +470,8 @@ OGRErr OGRPGDumpLayer::CreateFeatureViaCopy(OGRFeature *poFeature) poGeometry /* && (bHasWkb || bHasPostGISGeometry || bHasPostGISGeography) */) { OGRPGDumpGeomFieldDefn *poGFldDefn = - (OGRPGDumpGeomFieldDefn *)poFeature->GetGeomFieldDefnRef(i); + cpl::down_cast( + poFeature->GetGeomFieldDefnRef(i)); poGeometry->closeRings(); poGeometry->set3D(poGFldDefn->m_nGeometryTypeFlags & @@ -455,20 +486,31 @@ OGRErr OGRPGDumpLayer::CreateFeatureViaCopy(OGRFeature *poFeature) if (!osCommand.empty()) osCommand += "\t"; - if (pszGeom) + if (!pszGeom || pszGeom[0] == 0) + { + return false; + } + try { osCommand += pszGeom; - CPLFree(pszGeom); } - else + catch (const std::bad_alloc &) { - osCommand += "\\N"; + CPLError(CE_Failure, CPLE_OutOfMemory, + "Out of memory: too large geometry"); + CPLFree(pszGeom); + return false; } + CPLFree(pszGeom); } + return true; }; if (m_bGeomColumnPositionImmediate) - AddGeomFieldsValue(); + { + if (!AddGeomFieldsValue()) + return OGRERR_FAILURE; + } OGRPGCommonAppendCopyRegularFields( osCommand, poFeature, m_pszFIDColumn, @@ -476,7 +518,10 @@ OGRErr OGRPGDumpLayer::CreateFeatureViaCopy(OGRFeature *poFeature) OGRPGDumpEscapeStringWithUserData, nullptr); if (!m_bGeomColumnPositionImmediate) - AddGeomFieldsValue(); + { + if (!AddGeomFieldsValue()) + return OGRERR_FAILURE; + } /* ------------------------------------------------------------ */ /* Execute the copy. */ diff --git a/ogr/ogrspatialreference.cpp b/ogr/ogrspatialreference.cpp index d8210d4b8ed8..ed1f70d7eb67 100644 --- a/ogr/ogrspatialreference.cpp +++ b/ogr/ogrspatialreference.cpp @@ -4569,6 +4569,14 @@ OGRErr OGRSpatialReference::importFromURNPart(const char *pszAuthority, OGRErr OGRSpatialReference::importFromURN(const char *pszURN) { + constexpr const char *EPSG_URN_CRS_PREFIX = "urn:ogc:def:crs:EPSG::"; + if (STARTS_WITH(pszURN, EPSG_URN_CRS_PREFIX) && + CPLGetValueType(pszURN + strlen(EPSG_URN_CRS_PREFIX)) == + CPL_VALUE_INTEGER) + { + return importFromEPSG(atoi(pszURN + strlen(EPSG_URN_CRS_PREFIX))); + } + TAKE_OPTIONAL_LOCK(); #if PROJ_AT_LEAST_VERSION(8, 1, 0) @@ -11923,12 +11931,56 @@ OGRErr OGRSpatialReference::importFromEPSGA(int nCode) CPLString osCode; osCode.Printf("%d", nCode); - auto obj = - proj_create_from_database(d->getPROJContext(), "EPSG", osCode.c_str(), - PJ_CATEGORY_CRS, true, nullptr); - if (!obj) + PJ *obj; + if (nCode <= 100000) { - return OGRERR_FAILURE; + obj = proj_create_from_database(d->getPROJContext(), "EPSG", + osCode.c_str(), PJ_CATEGORY_CRS, true, + nullptr); + if (!obj) + { + return OGRERR_FAILURE; + } + } + else + { + // Likely to be an ESRI CRS... + CPLErr eLastErrorType = CE_None; + CPLErrorNum eLastErrorNum = CPLE_None; + std::string osLastErrorMsg; + bool bIsESRI = false; + { + CPLErrorStateBackuper oBackuper(CPLQuietErrorHandler); + CPLErrorReset(); + obj = proj_create_from_database(d->getPROJContext(), "EPSG", + osCode.c_str(), PJ_CATEGORY_CRS, + true, nullptr); + if (!obj) + { + eLastErrorType = CPLGetLastErrorType(); + eLastErrorNum = CPLGetLastErrorNo(); + osLastErrorMsg = CPLGetLastErrorMsg(); + obj = proj_create_from_database(d->getPROJContext(), "ESRI", + osCode.c_str(), PJ_CATEGORY_CRS, + true, nullptr); + if (obj) + bIsESRI = true; + } + } + if (!obj) + { + if (eLastErrorType != CE_None) + CPLError(eLastErrorType, eLastErrorNum, "%s", + osLastErrorMsg.c_str()); + return OGRERR_FAILURE; + } + if (bIsESRI) + { + CPLError(CE_Warning, CPLE_AppDefined, + "EPSG:%d is not a valid CRS code, but ESRI:%d is. " + "Assuming ESRI:%d was meant", + nCode, nCode, nCode); + } } if (bUseNonDeprecated && proj_is_deprecated(obj)) diff --git a/port/cpl_http.cpp b/port/cpl_http.cpp index 26427cd2bc74..b5ef83003141 100644 --- a/port/cpl_http.cpp +++ b/port/cpl_http.cpp @@ -517,7 +517,6 @@ constexpr TupleEnvVarOptionName asAssocEnvVarOptionName[] = { {"GDAL_CURL_CA_BUNDLE", "CAINFO"}, {"CURL_CA_BUNDLE", "CAINFO"}, {"SSL_CERT_FILE", "CAINFO"}, - {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"}, {"GDAL_HTTP_CAPATH", "CAPATH"}, {"GDAL_HTTP_SSL_VERIFYSTATUS", "SSL_VERIFYSTATUS"}, {"GDAL_HTTP_USE_CAPI_STORE", "USE_CAPI_STORE"}, diff --git a/port/cpl_known_config_options.h b/port/cpl_known_config_options.h index 1a0336d541a9..b991d2b11400 100644 --- a/port/cpl_known_config_options.h +++ b/port/cpl_known_config_options.h @@ -284,7 +284,7 @@ constexpr static const char* const apszKnownConfigOptions[] = "GDAL_HTTP_COOKIEJAR", // from cpl_http.cpp "GDAL_HTTP_ENABLE_ADVISE_READ", // from cpl_vsil_curl.cpp "GDAL_HTTP_HEADER_FILE", // from cpl_google_cloud.cpp, cpl_http.cpp - "GDAL_HTTP_HEADERS", // from cpl_google_cloud.cpp, cpl_http.cpp + "GDAL_HTTP_HEADERS", // from cpl_google_cloud.cpp, cpl_http.cpp, ogrgeojsonutils.cpp "GDAL_HTTP_KEYPASSWD", // from cpl_http.cpp "GDAL_HTTP_LOW_SPEED_LIMIT", // from cpl_http.cpp "GDAL_HTTP_LOW_SPEED_TIME", // from cpl_http.cpp diff --git a/port/cpl_string.cpp b/port/cpl_string.cpp index f67ea095ce9d..3bf226058325 100644 --- a/port/cpl_string.cpp +++ b/port/cpl_string.cpp @@ -2785,12 +2785,19 @@ char *CPLUnescapeString(const char *pszInput, int *pnLength, int nScheme) char *CPLBinaryToHex(int nBytes, const GByte *pabyData) { - char *pszHex = static_cast(CPLMalloc(nBytes * 2 + 1)); + CPLAssert(nBytes >= 0); + char *pszHex = static_cast( + VSI_MALLOC_VERBOSE(static_cast(nBytes) * 2 + 1)); + if (!pszHex) + { + pszHex = CPLStrdup(""); + return pszHex; + } pszHex[nBytes * 2] = '\0'; constexpr char achHex[] = "0123456789ABCDEF"; - for (int i = 0; i < nBytes; ++i) + for (size_t i = 0; i < static_cast(nBytes); ++i) { const int nLow = pabyData[i] & 0x0f; const int nHigh = (pabyData[i] & 0xf0) >> 4;