diff --git a/.github/workflows/alpine_32bit/Dockerfile.ci b/.github/workflows/alpine_32bit/Dockerfile.ci index 4dee3a335fc0..bd85ef1320cb 100644 --- a/.github/workflows/alpine_32bit/Dockerfile.ci +++ b/.github/workflows/alpine_32bit/Dockerfile.ci @@ -9,6 +9,7 @@ RUN apk add \ brunsli-dev \ ccache \ cfitsio-dev \ + clang \ cmake \ curl-dev \ expat-dev \ diff --git a/.github/workflows/alpine_32bit/build.sh b/.github/workflows/alpine_32bit/build.sh index 77b1d9e93614..5424077ccf3e 100755 --- a/.github/workflows/alpine_32bit/build.sh +++ b/.github/workflows/alpine_32bit/build.sh @@ -4,6 +4,9 @@ set -e cmake ${GDAL_SOURCE_DIR:=..} \ -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_SHARED_LINKER_FLAGS="-lstdc++" \ -DUSE_CCACHE=ON \ -DCMAKE_INSTALL_PREFIX=/usr \ -DIconv_INCLUDE_DIR=/usr/include/gnu-libiconv \ diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index fab7efb174dd..6d6aaf1606e4 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -323,7 +323,7 @@ jobs: base-devel git mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-pcre mingw-w64-x86_64-xerces-c mingw-w64-x86_64-zstd mingw-w64-x86_64-libarchive mingw-w64-x86_64-geos mingw-w64-x86_64-libspatialite mingw-w64-x86_64-proj - mingw-w64-x86_64-cgal mingw-w64-x86_64-libfreexl mingw-w64-x86_64-hdf5 mingw-w64-x86_64-netcdf mingw-w64-x86_64-poppler mingw-w64-x86_64-postgresql + mingw-w64-x86_64-cgal mingw-w64-x86_64-libfreexl mingw-w64-x86_64-hdf5 mingw-w64-x86_64-netcdf mingw-w64-x86_64-poppler mingw-w64-x86_64-podofo mingw-w64-x86_64-postgresql mingw-w64-x86_64-libgeotiff mingw-w64-x86_64-libpng mingw-w64-x86_64-libtiff mingw-w64-x86_64-openjpeg2 mingw-w64-x86_64-python-pip mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-pytest mingw-w64-x86_64-python-setuptools mingw-w64-x86_64-python-lxml mingw-w64-x86_64-swig mingw-w64-x86_64-python-psutil mingw-w64-x86_64-blosc - name: Setup cache diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml index aec6b05ff504..ea2ee24b305d 100644 --- a/.github/workflows/linux_build.yml +++ b/.github/workflows/linux_build.yml @@ -71,7 +71,7 @@ jobs: build_script: build.sh os: ubuntu-22.04 - - name: Alpine, gcc 32-bit + - name: Alpine, clang 32-bit id: alpine_32bit container: alpine_32bit build_script: build.sh diff --git a/.travis.yml b/.travis.yml index 546aaf9e8546..930210c396ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,24 +26,24 @@ matrix: - BUILD_NAME=s390x - DETAILS="" - - os: linux - arch: arm64-graviton2 - virt: lxd - group: edge - compiler: gcc - language: cpp - sudo: false - dist: jammy - cache: - apt: true - directories: - - $HOME/.ccache - apt: - packages: - - ccache - env: - - BUILD_NAME=graviton2 - - DETAILS= + #- os: linux + # arch: arm64-graviton2 + # virt: lxd + # group: edge + # compiler: gcc + # language: cpp + # sudo: false + # dist: jammy + # cache: + # apt: true + # directories: + # - $HOME/.ccache + # apt: + # packages: + # - ccache + # env: + # - BUILD_NAME=graviton2 + # - DETAILS= before_install: - if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(.rst)$'; then travis_terminate 0; fi diff --git a/MIGRATION_GUIDE.TXT b/MIGRATION_GUIDE.TXT index 0031236db8b6..c7d011677e4c 100644 --- a/MIGRATION_GUIDE.TXT +++ b/MIGRATION_GUIDE.TXT @@ -1,3 +1,20 @@ +MIGRATION GUIDE FROM GDAL 3.9 to GDAL 3.10 +------------------------------------------ + +- User code using VSIFEofL() to potentially to end read loops should also test + the return code of the new VSIFError() function. Some virtual file systems + that used to report errors through VSIFEofL() now do through VSIFError(). + +- Out-of-tree implementations of VSIVirtualHandle(): + 2 new required virtual methods must be implemented: int Error(), and + void ClearErr() following POSIX semantics of ferror() and clearerr(). + This is to distinguish Read() that returns less bytes than requested because + of an error (Error() != 0) or because of end-of-file (Eof() != 0) + + The VSIFilesystemPluginCallbacksStruct structure is extended with 2 + corresponding optional (but recommended to be implemented to reliably detect + reading errors) callbacks "error" and "clear_err". + MIGRATION GUIDE FROM GDAL 3.8 to GDAL 3.9 ----------------------------------------- diff --git a/alg/gdal_rpc.cpp b/alg/gdal_rpc.cpp index c84178295091..018840a91d7a 100644 --- a/alg/gdal_rpc.cpp +++ b/alg/gdal_rpc.cpp @@ -659,12 +659,12 @@ void *GDALCreateRPCTransformerV1(GDALRPCInfoV1 *psRPCInfo, int bReversed, * Create an RPC based transformer. * * The geometric sensor model describing the physical relationship between - * image coordinates and ground coordinate is known as a Rigorous Projection + * image coordinates and ground coordinates is known as a Rigorous Projection * Model. A Rigorous Projection Model expresses the mapping of the image space * coordinates of rows and columns (r,c) onto the object space reference * surface geodetic coordinates (long, lat, height). * - * RPC supports a generic description of the Rigorous Projection Models. The + * A RPC supports a generic description of the Rigorous Projection Models. The * approximation used by GDAL (RPC00) is a set of rational polynomials * expressing the normalized row and column values, (rn , cn), as a function of * normalized geodetic latitude, longitude, and height, (P, L, H), given a @@ -675,7 +675,7 @@ void *GDALCreateRPCTransformerV1(GDALRPCInfoV1 *psRPCInfo, int bReversed, * normalized row and column values (rn, cn), and between the geodetic * latitude, longitude, and height and normalized geodetic latitude, * longitude, and height (P, L, H), is defined by a set of normalizing - * translations (offsets) and scales that ensure all values are contained i + * translations (offsets) and scales that ensure all values are contained in * the range -1 to +1. * * This function creates a GDALTransformFunc compatible transformer diff --git a/alg/gdalsimplewarp.cpp b/alg/gdalsimplewarp.cpp index 80c473525391..0346068fd6b8 100644 --- a/alg/gdalsimplewarp.cpp +++ b/alg/gdalsimplewarp.cpp @@ -211,6 +211,8 @@ static void GDALSimpleWarpRemapping(int nBandCount, GByte **papabySrcData, * Distinct values may be listed for each band separated by columns. * * + * For more advanced warping capabilities, consider using GDALWarp(). + * * @param hSrcDS the source image dataset. * @param hDstDS the destination image dataset. * @param nBandCount the number of bands to be warped. If zero, all bands @@ -225,6 +227,7 @@ static void GDALSimpleWarpRemapping(int nBandCount, GByte **papabySrcData, * @param papszWarpOptions additional options controlling the warp. * * @return TRUE if the operation completes, or FALSE if an error occurs. + * @see GDALWarp() */ int CPL_STDCALL GDALSimpleImageWarp(GDALDatasetH hSrcDS, GDALDatasetH hDstDS, diff --git a/alg/gdalwarper.cpp b/alg/gdalwarper.cpp index 47919c803965..a695d1d6ac2a 100644 --- a/alg/gdalwarper.cpp +++ b/alg/gdalwarper.cpp @@ -66,11 +66,12 @@ * implement the reprojection, and will default a variety of other * warp options. * + * Nodata values set on destination dataset are taken into account. + * * No metadata, projection info, or color tables are transferred - * to the output file. + * to the output file. Source overviews are not considered. * - * Starting with GDAL 2.0, nodata values set on destination dataset are taken - * into account. + * For more advanced warping capabilities, consider using GDALWarp(). * * @param hSrcDS the source image file. * @param pszSrcWKT the source projection. If NULL the source projection @@ -91,6 +92,7 @@ * @param psOptions warp options, normally NULL. * * @return CE_None on success or CE_Failure if something goes wrong. + * @see GDALWarp() */ CPLErr CPL_STDCALL GDALReprojectImage( diff --git a/apps/argparse/argparse.hpp b/apps/argparse/argparse.hpp index d37d1558d863..d472ba933023 100644 --- a/apps/argparse/argparse.hpp +++ b/apps/argparse/argparse.hpp @@ -1011,7 +1011,8 @@ class Argument { std::bind(is_optional, std::placeholders::_1, m_prefix_chars)); dist = static_cast(std::distance(start, end)); if (dist < num_args_min) { - throw std::runtime_error("Too few arguments"); + throw std::runtime_error("Too few arguments for '" + + std::string(m_used_name) + "'."); } } diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp index 518b11e43547..a1d6e77b4f44 100644 --- a/autotest/cpp/test_ogr.cpp +++ b/autotest/cpp/test_ogr.cpp @@ -3961,4 +3961,90 @@ TEST_F(test_ogr, OGRFeature_SerializeToBinary) } } +// Test OGRGeometry::IsRectangle() +TEST_F(test_ogr, OGRGeometry_IsRectangle) +{ + // Not a polygon + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt("POINT EMPTY", nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_FALSE(poGeom->IsRectangle()); + delete poGeom; + } + // Polygon empty + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt("POLYGON EMPTY", nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_FALSE(poGeom->IsRectangle()); + delete poGeom; + } + // Polygon with inner ring + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "POLYGON ((0 0,0 1,1 1,1 0,0 0),(0.2 0.2,0.2 0.8,0.8 0.8,0.8 " + "0.2,0.2 0.2))", + nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_FALSE(poGeom->IsRectangle()); + delete poGeom; + } + // Polygon with 3 points + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt("POLYGON ((0 0,0 1,1 1))", nullptr, + &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_FALSE(poGeom->IsRectangle()); + delete poGeom; + } + // Polygon with 6 points + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "POLYGON ((0 0,0.1 0,0.2 0,0.3 0,1 1,0 0))", nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_FALSE(poGeom->IsRectangle()); + delete poGeom; + } + // Polygon with 5 points, but last one not matching first (invalid) + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "POLYGON ((0 0,0 1,1 1,1 0,-999 -999))", nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_FALSE(poGeom->IsRectangle()); + delete poGeom; + } + // Polygon with 5 points, but not rectangle + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt("POLYGON ((0 0,0 1.1,1 1,1 0,0 0))", + nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_FALSE(poGeom->IsRectangle()); + delete poGeom; + } + // Rectangle (type 1) + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt("POLYGON ((0 0,0 1,1 1,1 0,0 0))", + nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_TRUE(poGeom->IsRectangle()); + delete poGeom; + } + // Rectangle2(type 1) + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt("POLYGON ((0 0,1 0,1 1,0 1,0 0))", + nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + EXPECT_TRUE(poGeom->IsRectangle()); + delete poGeom; + } +} + } // namespace diff --git a/autotest/gcore/basic_test.py b/autotest/gcore/basic_test.py index 63498fdf1581..2d2e2a91eb61 100755 --- a/autotest/gcore/basic_test.py +++ b/autotest/gcore/basic_test.py @@ -331,6 +331,22 @@ def test_basic_test_11(): with pytest.raises(Exception): gdal.OpenEx("non existing") + try: + with gdal.ExceptionMgr(useExceptions=True): + try: + gdal.OpenEx("non existing") + except Exception: + pass + except Exception: + pytest.fails("Exception thrown whereas it should not have") + + with gdal.ExceptionMgr(useExceptions=True): + try: + gdal.OpenEx("non existing") + except Exception: + pass + gdal.Open("data/byte.tif") + ############################################################################### # Test GDAL layer API diff --git a/autotest/gcore/mask.py b/autotest/gcore/mask.py index 2ff7fd5073cb..180521e11b65 100755 --- a/autotest/gcore/mask.py +++ b/autotest/gcore/mask.py @@ -996,7 +996,10 @@ def test_mask_27(): @pytest.mark.parametrize("dt", [gdal.GDT_Byte, gdal.GDT_Int64, gdal.GDT_UInt64]) -def test_mask_setting_nodata(dt): +@pytest.mark.parametrize( + "GDAL_SIMUL_MEM_ALLOC_FAILURE_NODATA_MASK_BAND", [None, "YES", "ALWAYS"] +) +def test_mask_setting_nodata(dt, GDAL_SIMUL_MEM_ALLOC_FAILURE_NODATA_MASK_BAND): def set_nodata_value(ds, val): if dt == gdal.GDT_Byte: ds.GetRasterBand(1).SetNoDataValue(val) @@ -1005,15 +1008,41 @@ def set_nodata_value(ds, val): else: ds.GetRasterBand(1).SetNoDataValueAsUInt64(val) - ds = gdal.GetDriverByName("MEM").Create("", 1, 1, 1, dt) - assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 255) - assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 255) - set_nodata_value(ds, 0) - assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 0) - assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 0) - set_nodata_value(ds, 1) - assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 255) - set_nodata_value(ds, 0) - assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 0) - ds.GetRasterBand(1).DeleteNoDataValue() - assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 255) + def test(): + ds = gdal.GetDriverByName("MEM").Create("__debug__", 1, 1, 1, dt) + assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 255) + assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 255) + set_nodata_value(ds, 0) + got = ds.GetRasterBand(1).GetMaskBand().ReadRaster() + if ( + GDAL_SIMUL_MEM_ALLOC_FAILURE_NODATA_MASK_BAND == "ALWAYS" + and dt != gdal.GDT_Byte + ): + assert got is None + assert gdal.GetLastErrorType() == gdal.CE_Failure + else: + if ( + GDAL_SIMUL_MEM_ALLOC_FAILURE_NODATA_MASK_BAND == "YES" + and dt != gdal.GDT_Byte + ): + assert gdal.GetLastErrorType() == gdal.CE_Warning + assert got == struct.pack("B", 0) + assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 0) + set_nodata_value(ds, 1) + assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack( + "B", 255 + ) + set_nodata_value(ds, 0) + assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 0) + + ds.GetRasterBand(1).DeleteNoDataValue() + assert ds.GetRasterBand(1).GetMaskBand().ReadRaster() == struct.pack("B", 255) + + if GDAL_SIMUL_MEM_ALLOC_FAILURE_NODATA_MASK_BAND: + with gdal.quiet_errors(), gdal.config_option( + "GDAL_SIMUL_MEM_ALLOC_FAILURE_NODATA_MASK_BAND", + GDAL_SIMUL_MEM_ALLOC_FAILURE_NODATA_MASK_BAND, + ): + test() + else: + test() diff --git a/autotest/gcore/vsiadls.py b/autotest/gcore/vsiadls.py index bceeed06175d..ee6995d08a7b 100755 --- a/autotest/gcore/vsiadls.py +++ b/autotest/gcore/vsiadls.py @@ -56,7 +56,19 @@ def open_for_read(uri): @pytest.fixture(autouse=True, scope="module") def startup_and_cleanup(): - with gdaltest.config_option("CPL_AZURE_VM_API_ROOT_URL", "disabled"): + with gdaltest.config_options( + { + "AZURE_STORAGE_CONNECTION_STRING": None, + "AZURE_STORAGE_ACCOUNT": None, + "AZURE_STORAGE_ACCESS_KEY": None, + "AZURE_STORAGE_SAS_TOKEN": None, + "AZURE_NO_SIGN_REQUEST": None, + "AZURE_CONFIG_DIR": "", + "AZURE_STORAGE_ACCESS_TOKEN": "", + "AZURE_FEDERATED_TOKEN_FILE": "", + "CPL_AZURE_VM_API_ROOT_URL": "disabled", + } + ): assert gdal.GetSignedURL("/vsiadls/foo/bar") is None gdaltest.webserver_process = None diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index 00862d010a32..2480571cede6 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -582,9 +582,16 @@ def test_vsicurl_test_retry(server): ) data_len = 0 if f: - data_len = len(gdal.VSIFReadL(1, 1, f)) - gdal.VSIFCloseL(f) - assert data_len == 0 + try: + data_len = len(gdal.VSIFReadL(1, 1, f)) + assert data_len == 0 + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 1 + gdal.VSIFClearErrL(f) + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 0 + finally: + gdal.VSIFCloseL(f) gdal.VSICurlClearCache() @@ -601,13 +608,17 @@ def test_vsicurl_test_retry(server): "rb", ) assert f is not None - gdal.ErrorReset() - with gdal.quiet_errors(): - data = gdal.VSIFReadL(1, 3, f).decode("ascii") - error_msg = gdal.GetLastErrorMsg() - gdal.VSIFCloseL(f) - assert data == "foo" - assert "429" in error_msg + try: + gdal.ErrorReset() + with gdal.quiet_errors(): + data = gdal.VSIFReadL(1, 3, f).decode("ascii") + assert data == "foo" + error_msg = gdal.GetLastErrorMsg() + assert "429" in error_msg + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 0 + finally: + gdal.VSIFCloseL(f) ############################################################################### diff --git a/autotest/gcore/vsifile.py b/autotest/gcore/vsifile.py index f10d39d5cbb3..3dfdfcf377ee 100755 --- a/autotest/gcore/vsifile.py +++ b/autotest/gcore/vsifile.py @@ -37,7 +37,7 @@ import pytest from lxml import etree -from osgeo import gdal +from osgeo import gdal, ogr ############################################################################### @@ -85,18 +85,32 @@ def vsifile_generic(filename, options=[]): assert start_time == pytest.approx(statBuf.mtime, abs=2) fp = gdal.VSIFOpenExL(filename, "rb", False, options) - assert gdal.VSIFReadL(1, 0, fp) is None - assert gdal.VSIFReadL(0, 1, fp) is None - buf = gdal.VSIFReadL(1, 7, fp) - assert gdal.VSIFWriteL("a", 1, 1, fp) == 0 - assert gdal.VSIFTruncateL(fp, 0) != 0 - gdal.VSIFCloseL(fp) - - assert buf.decode("ascii") == "01234XX" + try: + assert fp + assert gdal.VSIFReadL(1, 0, fp) is None + assert gdal.VSIFReadL(0, 1, fp) is None + buf = gdal.VSIFReadL(1, 7, fp) + assert gdal.VSIFEofL(fp) == 0 + assert gdal.VSIFErrorL(fp) == 0 + assert buf == b"01234XX" + + buf = gdal.VSIFReadL(1, 1, fp) + assert gdal.VSIFEofL(fp) == 1 + assert gdal.VSIFErrorL(fp) == 0 + assert buf == b"" + gdal.VSIFClearErrL(fp) + assert gdal.VSIFEofL(fp) == 0 + assert gdal.VSIFErrorL(fp) == 0 + + assert gdal.VSIFWriteL("a", 1, 1, fp) == 0 + assert gdal.VSIFTruncateL(fp, 0) != 0 + finally: + if fp: + gdal.VSIFCloseL(fp) # Test append mode on existing file fp = gdal.VSIFOpenExL(filename, "ab", False, options) - gdal.VSIFWriteL("XX", 1, 2, fp) + assert gdal.VSIFWriteL("XX", 1, 2, fp) == 2 gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( @@ -112,7 +126,7 @@ def vsifile_generic(filename, options=[]): # Test append mode on non existing file fp = gdal.VSIFOpenExL(filename, "ab", False, options) - gdal.VSIFWriteL("XX", 1, 2, fp) + assert gdal.VSIFWriteL("XX", 1, 2, fp) == 2 gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( @@ -123,6 +137,17 @@ def vsifile_generic(filename, options=[]): assert gdal.Unlink(filename) == 0 + # Test read on a file opened in write-only mode + fp = gdal.VSIFOpenExL(filename, "wb", False, options) + try: + assert fp + assert len(gdal.VSIFReadL(1, 1, fp)) == 0 + assert gdal.VSIFErrorL(fp) == 1 + assert gdal.VSIFEofL(fp) == 0 + finally: + if fp: + gdal.VSIFCloseL(fp) + ############################################################################### # Test /vsimem @@ -271,6 +296,8 @@ def test_vsifile_vsicache_read_error(): gdal.VSIFTruncateL(f, 0) assert len(gdal.VSIFReadL(1, 5000 * 1000, f2)) == 0 + assert gdal.VSIFEofL(f2) + assert gdal.VSIFErrorL(f2) == 0 # Extend the file again gdal.VSIFTruncateL(f, 1000 * 1000) @@ -291,6 +318,8 @@ def test_vsifile_vsicache_read_error(): gdal.VSIFSeekL(f2, 0, 0) assert len(gdal.VSIFReadL(1, CHUNK_SIZE, f2)) == 10 + assert gdal.VSIFEofL(f2) + assert gdal.VSIFErrorL(f2) == 0 gdal.VSIFSeekL(f2, 100, 0) assert len(gdal.VSIFReadL(1, CHUNK_SIZE, f2)) == 0 @@ -365,6 +394,7 @@ def test_vsifile_7(): assert gdal.VSIFTellL(fp) == 0x7FFFFFFFFFFFFFFF assert not gdal.VSIFReadL(1, 1, fp) assert gdal.VSIFEofL(fp) == 1 + assert gdal.VSIFErrorL(fp) == 0 gdal.VSIFCloseL(fp) gdal.Unlink("/vsimem/vsifile_7.bin") @@ -650,32 +680,44 @@ def test_vsifile_14(): ############################################################################### -# Test issue with Eof() not detecting end of corrupted gzip stream (#6944) +# Test issue with Error() not detecting end of corrupted gzip stream (#6944) def test_vsifile_15(): fp = gdal.VSIFOpenL("/vsigzip/data/corrupted_z_buf_error.gz", "rb") assert fp is not None - file_len = 0 - while not gdal.VSIFEofL(fp): + try: + file_len = 0 + while not gdal.VSIFErrorL(fp): + with gdal.quiet_errors(): + file_len += len(gdal.VSIFReadL(1, 4, fp)) + assert file_len == 6469 + assert gdal.VSIFEofL(fp) == 0 + with gdal.quiet_errors(): file_len += len(gdal.VSIFReadL(1, 4, fp)) - assert file_len == 6469 + assert file_len == 6469 + assert gdal.VSIFErrorL(fp) == 1 + assert gdal.VSIFEofL(fp) == 0 - with gdal.quiet_errors(): - file_len += len(gdal.VSIFReadL(1, 4, fp)) - assert file_len == 6469 - - with gdal.quiet_errors(): - assert gdal.VSIFSeekL(fp, 0, 2) != 0 + with gdal.quiet_errors(): + assert gdal.VSIFSeekL(fp, 0, 2) != 0 - assert gdal.VSIFSeekL(fp, 0, 0) == 0 + assert gdal.VSIFSeekL(fp, 0, 0) == 0 + assert gdal.VSIFErrorL(fp) == 1 + assert gdal.VSIFEofL(fp) == 0 - len_read = len(gdal.VSIFReadL(1, file_len, fp)) - assert len_read == file_len + gdal.VSIFClearErrL(fp) + assert gdal.VSIFErrorL(fp) == 0 + assert gdal.VSIFEofL(fp) == 0 - gdal.VSIFCloseL(fp) + len_read = len(gdal.VSIFReadL(1, file_len, fp)) + assert len_read == file_len + assert gdal.VSIFErrorL(fp) == 0 + assert gdal.VSIFEofL(fp) == 0 + finally: + gdal.VSIFCloseL(fp) ############################################################################### @@ -1141,6 +1183,11 @@ def test_vsifile_vsitar_gz_with_tar_multiple_of_65536_bytes(): f = gdal.VSIFOpenL("/vsitar/data/tar_of_65536_bytes.tar.gz/zero.bin", "rb") assert f is not None read_bytes = gdal.VSIFReadL(1, 65024, f) + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 0 + assert gdal.VSIFReadL(1, 1, f) == b"" + assert gdal.VSIFEofL(f) == 1 + assert gdal.VSIFErrorL(f) == 0 gdal.VSIFCloseL(f) assert read_bytes == b"\x00" * 65024 gdal.Unlink("data/tar_of_65536_bytes.tar.gz.properties") @@ -1155,7 +1202,18 @@ def test_vsifile_vsizip_stored(): f = gdal.VSIFOpenL("/vsizip/data/stored.zip/foo.txt", "rb") assert f assert gdal.VSIFReadL(1, 5, f) == b"foo\n" - assert gdal.VSIFEofL(f) + assert gdal.VSIFEofL(f) == 1 + assert gdal.VSIFErrorL(f) == 0 + gdal.VSIFCloseL(f) + + f = gdal.VSIFOpenL("/vsizip/data/stored.zip/foo.txt", "rb") + assert f + assert gdal.VSIFReadL(1, 4, f) == b"foo\n" + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 0 + assert gdal.VSIFReadL(1, 1, f) == b"" + assert gdal.VSIFEofL(f) == 1 + assert gdal.VSIFErrorL(f) == 0 gdal.VSIFCloseL(f) @@ -1181,24 +1239,40 @@ def test_vsifile_vsimem_truncate_zeroize(): # Test VSICopyFile() -def test_vsifile_copyfile(): +def test_vsifile_copyfile_regular(tmp_vsimem): # Most simple invocation - dstfilename = "/vsimem/test_vsifile_copyfile.tif" + dstfilename = str(tmp_vsimem / "out.bin") assert gdal.CopyFile("data/byte.tif", dstfilename) == 0 assert gdal.VSIStatL(dstfilename).size == gdal.VSIStatL("data/byte.tif").size + +def test_vsifile_copyfile_srcfilename_none(tmp_vsimem): + # Test srcfilename passed to None - srcfilename = "/vsimem/test.bin" + srcfilename = str(tmp_vsimem / "src.bin") + dstfilename = str(tmp_vsimem / "out.bin") f = gdal.VSIFOpenL(srcfilename, "wb+") gdal.VSIFTruncateL(f, 1000 * 1000) assert gdal.CopyFile(None, dstfilename, f) == 0 gdal.VSIFCloseL(f) - gdal.Unlink(srcfilename) assert gdal.VSIStatL(dstfilename).size == 1000 * 1000 + +def test_vsifile_copyfile_srcfilename_and_srcfilehandle_none(tmp_vsimem): + + # Test srcfilename passed to None + dstfilename = str(tmp_vsimem / "out.bin") + with gdal.quiet_errors(): + assert gdal.CopyFile(None, dstfilename) != 0 + assert gdal.VSIStatL(dstfilename) is None + + +def test_vsifile_copyfile_progress(tmp_vsimem): + # Test progress callback - srcfilename = "/vsimem/test.bin" + srcfilename = str(tmp_vsimem / "src.bin") + dstfilename = str(tmp_vsimem / "out.bin") f = gdal.VSIFOpenL(srcfilename, "wb+") gdal.VSIFTruncateL(f, 1000 * 1000) gdal.VSIFCloseL(f) @@ -1213,11 +1287,14 @@ def progress(pct, msg, user_data): == 0 ) assert tab[-1] == 1.0 - gdal.Unlink(srcfilename) assert gdal.VSIStatL(dstfilename).size == 1000 * 1000 + +def test_vsifile_copyfile_progress_cancel(tmp_vsimem): + # Test progress callback in error situation - srcfilename = "/vsimem/test.bin" + srcfilename = str(tmp_vsimem / "src.bin") + dstfilename = str(tmp_vsimem / "out.bin") f = gdal.VSIFOpenL(srcfilename, "wb+") gdal.VSIFTruncateL(f, 1000 * 1000) gdal.VSIFCloseL(f) @@ -1229,15 +1306,33 @@ def progress(pct, msg, user_data): return 1 tab = [] - assert ( - gdal.CopyFile(srcfilename, dstfilename, callback=progress, callback_data=tab) - != 0 - ) + with gdal.quiet_errors(): + assert ( + gdal.CopyFile( + srcfilename, dstfilename, callback=progress, callback_data=tab + ) + != 0 + ) assert tab[-1] != 1.0 - gdal.Unlink(srcfilename) - assert gdal.VSIStatL(dstfilename).size != 1000 * 1000 + assert gdal.VSIStatL(dstfilename) is None + + +def test_vsifile_copyfile_error_on_input(tmp_vsimem): + + srcfilename = "/vsigzip/data/corrupted_z_buf_error.gz" + dstfilename = str(tmp_vsimem / "out.bin") + fp = gdal.VSIFOpenL(srcfilename, "rb") + assert fp + try: + with gdal.quiet_errors(): + assert gdal.CopyFile(None, dstfilename, fpSource=fp) != 0 + assert "error while reading source file" in gdal.GetLastErrorMsg() + assert gdal.VSIStatL(dstfilename) is None + finally: + gdal.VSIFCloseL(fp) - gdal.Unlink(dstfilename) + +############################################################################### def test_vsimem_illegal_filename(): @@ -1351,9 +1446,10 @@ def test_vsifile_CopyFileRestartable(tmp_vsimem): dstfilename = str(tmp_vsimem / "out.txt") - retcode, output_payload = gdal.CopyFileRestartable( - str(tmp_vsimem / "i_do_not_exist.txt"), dstfilename, None - ) + with gdal.quiet_errors(): + retcode, output_payload = gdal.CopyFileRestartable( + str(tmp_vsimem / "i_do_not_exist.txt"), dstfilename, None + ) assert retcode == -1 assert output_payload is None assert gdal.VSIStatL(dstfilename) is None @@ -1364,3 +1460,232 @@ def test_vsifile_CopyFileRestartable(tmp_vsimem): assert retcode == 0 assert output_payload is None assert gdal.VSIStatL(dstfilename).size == 3 + + +############################################################################### +# Test VSIFile helper class + + +def test_vsifile_class_write_ascii(tmp_path): + + fname = tmp_path / "test.txt" + + lines = ["permission is hereby granted", "free of charge", "to any person"] + + with gdaltest.vsi_open(fname, "w") as f: + assert f.tell() == 0 + + for line in lines: + f.write(line) + f.write("\n") + + with open(fname, "r") as f: + assert [line.strip() for line in f.readlines()] == lines + + +def test_vsifile_class_read_ascii(tmp_path): + + fname = str(tmp_path / "test.txt") + + lines = ["permission is hereby granted", "free of charge", "to any person"] + + with open(fname, "w", newline="\n") as f: + for line in lines: + f.write(line) + f.write("\n") + + with pytest.raises(Exception): + f.write(b"some bytes") + + # read entire file + with gdaltest.vsi_open(fname, "r") as f: + contents = f.read() + + assert type(contents) is str + + lines_in = [line.strip() for line in contents.strip().split("\n")] + assert lines_in == lines + + # read some characters + f = gdaltest.vsi_open(fname) + assert f.read(10) == "permission" + + # skip a character + f.seek(1, os.SEEK_CUR) + assert f.read(9) == "is hereby" + + # seek backwards + f.seek(-2, os.SEEK_CUR) + assert f.read(2) == "by" + + # jump to beginning + f.seek(0, os.SEEK_SET) + assert f.read(10) == "permission" + + # can't jump before the beginning + pos = f.tell() + with pytest.raises(OSError, match="negative offset"): + f.seek(-2, os.SEEK_SET) == -1 + assert pos == f.tell() + + # jump to end + f.seek(0, os.SEEK_END) + assert f.read(10) == "" + + f.seek(-7, os.SEEK_END) + assert f.read() == "person\n" + + f.close() + f.close() # no harm in closing an already-closed file + + +def test_vsifile_class_read_binary(tmp_path): + + fname = tmp_path / "test.wkb" + + g = ogr.CreateGeometryFromWkt("POINT (15 17)") + wkb = g.ExportToWkb() + + with open(fname, "wb") as f: + f.write(wkb) + + # read entire file + with gdaltest.vsi_open(fname, "rb") as f: + contents = f.read() + + assert type(contents) is bytes + + assert contents == wkb + + # read some bytes + f = gdaltest.vsi_open(fname, "rb") + assert f.read(5) == wkb[:5] + + f.seek(10, os.SEEK_SET) + assert f.read(5) == wkb[10:15] + + +def test_vsifile_class_write_binary(tmp_path): + + fname = tmp_path / "test.wkb" + + g = ogr.CreateGeometryFromWkt("POINT (15 17)") + wkb = g.ExportToWkb() + + with gdaltest.vsi_open(fname, "wb") as f: + f.write(wkb[:8]) + f.write(wkb[8:]) + + with open(fname, "rb") as f: + assert f.read() == wkb + + with gdaltest.vsi_open(fname, "rb") as f: + with pytest.raises(OSError, match="Expected to write"): + f.write(wkb) + + +def random_lines(): + import random + import string + + lines = [] + for i in range(50): + lines.append( + "".join([random.choice(string.ascii_letters) for j in range(20 + 3 * i)]) + ) + lines.append(" ") + lines.append("") + lines.append("theend") + lines.append("") + + return lines + + +@pytest.mark.parametrize("terminating_newline", (True, False)) +def test_vsifile_class_line_iteration(tmp_path, terminating_newline): + + fname = str(tmp_path / "test.txt") + + lines_out = random_lines() + + with open(fname, "w") as f: + for line in lines_out: + f.write(line) + f.write("\n") + + if not terminating_newline: + f.write("lastline") + lines_out.append("lastline") + + with gdaltest.vsi_open(fname) as f: + lines_in = [line for line in f] + + assert lines_in == lines_out + + +def test_vsifile_class_binary_line_iteration(tmp_path): + + fname = str(tmp_path / "test.txt") + + lines_out = [x.encode() for x in random_lines()] + + with open(fname, "wb") as f: + for line in lines_out: + f.write(line) + f.write(b"\n") + + with gdaltest.vsi_open(fname, "rb") as f: + lines_in = [line for line in f] + + assert lines_in == lines_out + + +def test_vsifile_class_zipped_csv_reader(tmp_path): + + test_csv = str(tmp_path / "input.csv") + test_zip = str(tmp_path / "input.zip") + + import csv + import shutil + import zipfile + + shutil.copy("../ogr/data/prime_meridian.csv", test_csv) + + with zipfile.ZipFile(test_zip, "w") as zf: + zf.write(test_csv, arcname="input.csv") + + with gdaltest.vsi_open(f"/vsizip/{test_zip}/input.csv") as f: + records = [x for x in csv.DictReader(f)] + + assert len(records) == 4 + assert ( + records[2]["INFORMATION_SOURCE"] + == "Institut Geographique National (IGN), Paris" + ) + + +def test_vsifile_class_file_does_not_exist(tmp_path): + + with pytest.raises(OSError, match="No such file or directory"): + gdaltest.vsi_open(tmp_path / "does_not_exist.txt") + + +def test_vsifile_class_read_from_closed_file(tmp_path): + + with gdaltest.vsi_open(tmp_path / "out.txt", "w") as f: + f.write("abc") + + with pytest.raises(ValueError, match="closed file"): + f.seek(0) + + +def test_vsifile_class_append(tmp_vsimem): + + fname = tmp_vsimem / "out.txt" + + with gdaltest.vsi_open(fname, "w") as f: + f.write("abc") + with gdaltest.vsi_open(fname, "a") as f: + f.write("def") + with gdaltest.vsi_open(fname) as f: + assert f.read() == "abcdef" diff --git a/autotest/gcore/vsizip.py b/autotest/gcore/vsizip.py index c10be914dfeb..d28765150af6 100755 --- a/autotest/gcore/vsizip.py +++ b/autotest/gcore/vsizip.py @@ -737,10 +737,16 @@ def test_vsizip_deflate64(): assert f try: data = gdal.VSIFReadL(1, size, f) + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 0 assert len(data) == size assert len(gdal.VSIFReadL(1, 1, f)) == 0 + assert gdal.VSIFEofL(f) == 1 + assert gdal.VSIFErrorL(f) == 0 assert gdal.VSIFSeekL(f, 0, 0) == 0 data2 = gdal.VSIFReadL(1, size, f) + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 0 len_data2 = len(data2) assert len_data2 == size assert data2 == data @@ -752,6 +758,11 @@ def test_vsizip_deflate64(): ]: assert gdal.VSIFSeekL(f, pos, 0) == 0 data2 = gdal.VSIFReadL(1, nread, f) + if pos + nread > size: + assert gdal.VSIFEofL(f) == 1 + else: + assert gdal.VSIFEofL(f) == 0 + assert gdal.VSIFErrorL(f) == 0, (pos, nread) len_data2 = len(data2) assert len_data2 == min(nread, size - pos), (pos, nread) assert data2 == data[pos : pos + len_data2], (pos, nread) diff --git a/autotest/gdrivers/data/byte_nodata_0.tif b/autotest/gdrivers/data/byte_nodata_0.tif new file mode 100644 index 000000000000..c10d7f2aabd7 Binary files /dev/null and b/autotest/gdrivers/data/byte_nodata_0.tif differ diff --git a/autotest/gdrivers/data/s102/test_s102_v2.1.h5 b/autotest/gdrivers/data/s102/test_s102_v2.1.h5 index ea3bfdc44d1c..3d6c40ba2bb9 100644 Binary files a/autotest/gdrivers/data/s102/test_s102_v2.1.h5 and b/autotest/gdrivers/data/s102/test_s102_v2.1.h5 differ diff --git a/autotest/gdrivers/data/s102/test_s102_v2.2.h5 b/autotest/gdrivers/data/s102/test_s102_v2.2.h5 index 3459de09da0f..3af45a901e8e 100644 Binary files a/autotest/gdrivers/data/s102/test_s102_v2.2.h5 and b/autotest/gdrivers/data/s102/test_s102_v2.2.h5 differ diff --git a/autotest/gdrivers/data/stacit/overlapping_sources_with_nodata.json b/autotest/gdrivers/data/stacit/overlapping_sources_with_nodata.json new file mode 100644 index 000000000000..6986e6945141 --- /dev/null +++ b/autotest/gdrivers/data/stacit/overlapping_sources_with_nodata.json @@ -0,0 +1,103 @@ +{ + "type": "FeatureCollection", + "stac_version": "1.0.0-beta.2", + "stac_extensions": [], + "features": [ + { + "type": "Feature", + "stac_version": "1.0.0-beta.2", + "stac_extensions": [ + "eo", + "proj" + ], + "id": "byte", + "geometry": null, + "properties": { + "datetime": "2021-07-25T00:00:00Z", + "proj:epsg": 26711, + }, + "collection": "my_collection", + "assets": { + "B01": { + "title": "Band 1 (coastal)", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal", + "center_wavelength": 0.4439, + "full_width_half_max": 0.027 + } + ], + "href": "data/byte_nodata_0.tif", + "proj:bbox": [ + 440720.000, 3750120.000, + 441920.000, 3751320.000 + ], + "proj:transform": [ + 60, + 0, + 440720, + 0, + -60, + 3751320, + 0, + 0, + 1 + ] + } + } + }, + { + "type": "Feature", + "stac_version": "1.0.0-beta.2", + "stac_extensions": [ + "eo", + "proj" + ], + "id": "under", + "geometry": null, + "properties": { + "datetime": "2021-07-19T10:57:30Z", + "proj:epsg": 26711, + }, + "collection": "my_collection", + "assets": { + "B01": { + "title": "Band 1 (coastal)", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data" + ], + "eo:bands": [ + { + "name": "B01", + "common_name": "coastal", + "center_wavelength": 0.4439, + "full_width_half_max": 0.027 + } + ], + "href": "data/byte.tif", + "proj:bbox": [ + 440720.000, 3750120.000, + 441920.000, 3751320.000 + ], + "proj:transform": [ + 60, + 0, + 440720, + 0, + -60, + 3751320, + 0, + 0, + 1 + ] + } + } + } + ] +} diff --git a/autotest/gdrivers/esric.py b/autotest/gdrivers/esric.py index b27f9186f407..93ec860a4416 100755 --- a/autotest/gdrivers/esric.py +++ b/autotest/gdrivers/esric.py @@ -181,3 +181,20 @@ def test_tpkx_4(tpkx_ds): cs = l1b2.Checksum() expectedcs = 53503 assert cs == expectedcs, "wrong data checksum" + + +############################################################################### +# Open a tpkx dataset where we need to ingest more bytes + + +def test_tpkx_ingest_more_bytes(tmp_vsimem): + filename = str(tmp_vsimem / "root.json") + f = gdal.VSIFOpenL("/vsizip/{data/esric/Usa.tpkx}/root.json", "rb") + assert f + data = gdal.VSIFReadL(1, 10000, f) + gdal.VSIFCloseL(f) + # Append spaces at the beginning of the root.json file to test we try + # to ingest more bytes + data = b"{" + (b" " * 900) + data[1:] + gdal.FileFromMemBuffer(filename, data) + gdal.Open(filename) diff --git a/autotest/gdrivers/stacit.py b/autotest/gdrivers/stacit.py index c0e5054d751c..6b0da1308bf3 100755 --- a/autotest/gdrivers/stacit.py +++ b/autotest/gdrivers/stacit.py @@ -156,7 +156,7 @@ def test_stacit_overlapping_sources(): # Check that the source covered by another one is not listed vrt = ds.GetMetadata("xml:VRT")[0] - placement_vrt = """ + only_one_simple_source = """ Gray data/byte.tif @@ -166,4 +166,78 @@ def test_stacit_overlapping_sources(): """ # print(vrt) - assert placement_vrt in vrt + assert only_one_simple_source in vrt + + ds = gdal.OpenEx( + "data/stacit/overlapping_sources.json", + open_options=["OVERLAP_STRATEGY=REMOVE_IF_NO_NODATA"], + ) + assert ds is not None + vrt = ds.GetMetadata("xml:VRT")[0] + assert only_one_simple_source in vrt + + ds = gdal.OpenEx( + "data/stacit/overlapping_sources.json", + open_options=["OVERLAP_STRATEGY=USE_MOST_RECENT"], + ) + assert ds is not None + vrt = ds.GetMetadata("xml:VRT")[0] + assert only_one_simple_source in vrt + + ds = gdal.OpenEx( + "data/stacit/overlapping_sources.json", + open_options=["OVERLAP_STRATEGY=USE_ALL"], + ) + assert ds is not None + assert len(ds.GetFileList()) == 4 + vrt = ds.GetMetadata("xml:VRT")[0] + + +@pytest.mark.require_geos +def test_stacit_overlapping_sources_with_nodata(): + + ds = gdal.Open("data/stacit/overlapping_sources_with_nodata.json") + assert ds is not None + assert len(ds.GetFileList()) == 3 + vrt = ds.GetMetadata("xml:VRT")[0] + # print(vrt) + two_sources = """ + data/byte.tif + 1 + + + 0 + + + data/byte_nodata_0.tif + 1 + + + 0 + """ + assert two_sources in vrt + + ds = gdal.OpenEx( + "data/stacit/overlapping_sources_with_nodata.json", + open_options=["OVERLAP_STRATEGY=REMOVE_IF_NO_NODATA"], + ) + assert ds is not None + vrt = ds.GetMetadata("xml:VRT")[0] + assert len(ds.GetFileList()) == 3 + assert two_sources in vrt + + ds = gdal.OpenEx( + "data/stacit/overlapping_sources_with_nodata.json", + open_options=["OVERLAP_STRATEGY=USE_MOST_RECENT"], + ) + assert ds is not None + assert len(ds.GetFileList()) == 2 + + ds = gdal.OpenEx( + "data/stacit/overlapping_sources_with_nodata.json", + open_options=["OVERLAP_STRATEGY=USE_ALL"], + ) + assert ds is not None + vrt = ds.GetMetadata("xml:VRT")[0] + assert len(ds.GetFileList()) == 3 + assert two_sources in vrt diff --git a/autotest/gdrivers/tiledb_multidim.py b/autotest/gdrivers/tiledb_multidim.py index 68a7c436351c..7b3b87f7da1c 100755 --- a/autotest/gdrivers/tiledb_multidim.py +++ b/autotest/gdrivers/tiledb_multidim.py @@ -685,8 +685,8 @@ def test(): filename, "data/small_world.tif", format="TileDB", - creationOptions=["INTERLEAVE=ATTRIBUTES"], - ) + creationOptions=["INTERLEAVE=ATTRIBUTES", "CREATE_GROUP=NO"], + ), ds = gdal.OpenEx(filename, gdal.OF_MULTIDIM_RASTER) rg = ds.GetRootGroup() assert rg.GetMDArrayNames() == [ diff --git a/autotest/gdrivers/tiledb_write.py b/autotest/gdrivers/tiledb_write.py index 1bfa45155c81..8fc07d0e5f6f 100755 --- a/autotest/gdrivers/tiledb_write.py +++ b/autotest/gdrivers/tiledb_write.py @@ -30,6 +30,7 @@ ############################################################################### import math +import os import gdaltest import pytest @@ -45,7 +46,7 @@ def test_tiledb_write_complex(tmp_path, mode): src_ds = gdal.Open("../gcore/data/cfloat64.tif") - options = ["INTERLEAVE=%s" % (mode)] + options = ["INTERLEAVE=%s" % (mode), "CREATE_GROUP=NO"] dsname = str(tmp_path / "tiledb_complex64") new_ds = gdaltest.tiledb_drv.CreateCopy(dsname, src_ds, options=options) @@ -56,6 +57,11 @@ def test_tiledb_write_complex(tmp_path, mode): bnd = new_ds.GetRasterBand(1) assert bnd.Checksum() == 5028, "Did not get expected checksum on still-open file" + with pytest.raises( + Exception, match="only supported for datasets created with CREATE_GROUP=YES" + ): + new_ds.BuildOverviews("NEAR", [2]) + bnd = None new_ds = None @@ -430,10 +436,208 @@ def test_tiledb_write_nodata_not_identical_all_bands(tmp_path): def test_tiledb_write_nodata_error_after_rasterio(tmp_path): dsname = str(tmp_path / "test_tiledb_write_nodata_error_after_rasterio.tiledb") - ds = gdal.GetDriverByName("TileDB").Create(dsname, 1, 1) + ds = gdal.GetDriverByName("TileDB").Create( + dsname, 1, 1, options=["CREATE_GROUP=NO"] + ) ds.GetRasterBand(1).Fill(0) ds.GetRasterBand(1).FlushCache() with pytest.raises( Exception, match="cannot be called after pixel values have been set" ): ds.GetRasterBand(1).SetNoDataValue(1) + + +def test_tiledb_write_create_group(tmp_path): + + # Create dataset and add a overview level + dsname = str(tmp_path / "test_tiledb_write_create_group.tiledb") + ds = gdal.GetDriverByName("TileDB").Create(dsname, 1, 2) + ds.Close() + + # Check that it resulted in an auxiliary dataset + ds = gdal.OpenEx(dsname, gdal.OF_MULTIDIM_RASTER) + assert set(ds.GetRootGroup().GetMDArrayNames()) == set( + [ + "l_0", + ] + ) + ds.Close() + + ds = gdal.Open(dsname) + assert ds.RasterXSize == 1 + assert ds.RasterYSize == 2 + + +def test_tiledb_write_overviews(tmp_path): + + # This dataset name must be kept short, otherwise strange I/O errors will + # occur on Windows ! + dsname = str(tmp_path / "test.tiledb") + + src_ds = gdal.Open("data/rgbsmall.tif") + src_ds = gdal.Translate("", src_ds, format="MEM") + src_ds.GetRasterBand(1).SetNoDataValue(254) + + # Create dataset and add a overview level + ds = gdal.GetDriverByName("TileDB").CreateCopy(dsname, src_ds) + ds.BuildOverviews("NEAR", [2]) + assert ds.GetRasterBand(1).GetOverviewCount() == 1 + assert ds.GetRasterBand(1).GetOverview(-1) is None + assert ds.GetRasterBand(1).GetOverview(1) is None + ref_ds = gdal.Translate("", src_ds, format="MEM") + ref_ds.BuildOverviews("NEAR", [2]) + assert [ + ds.GetRasterBand(i + 1).GetOverview(0).Checksum() for i in range(ds.RasterCount) + ] == [ + ref_ds.GetRasterBand(i + 1).GetOverview(0).Checksum() + for i in range(ref_ds.RasterCount) + ] + ds.Close() + + # Check that it resulted in an auxiliary dataset + ds = gdal.OpenEx(dsname, gdal.OF_MULTIDIM_RASTER) + assert set(ds.GetRootGroup().GetMDArrayNames()) == set( + [ + "l_0", + "l_1", + ] + ) + ds.Close() + ds = gdal.Open(dsname + "/l_1") + assert ds.RasterXSize == src_ds.RasterXSize // 2 + assert ds.RasterYSize == src_ds.RasterYSize // 2 + assert ds.RasterCount == src_ds.RasterCount + assert ds.GetRasterBand(1).GetNoDataValue() == 254 + assert ds.GetRasterBand(1).GetMetadataItem("RESAMPLING") == "NEAREST" + assert ds.GetGeoTransform()[0] == src_ds.GetGeoTransform()[0] + assert ds.GetGeoTransform()[1] == src_ds.GetGeoTransform()[1] * 2 + assert ds.GetGeoTransform()[2] == src_ds.GetGeoTransform()[2] * 2 + assert ds.GetGeoTransform()[3] == src_ds.GetGeoTransform()[3] + assert ds.GetGeoTransform()[4] == src_ds.GetGeoTransform()[4] * 2 + assert ds.GetGeoTransform()[5] == src_ds.GetGeoTransform()[5] * 2 + ds.Close() + + # Check we can access the overview after re-opening + ds = gdal.Open(dsname) + assert ds.GetRasterBand(1).GetOverviewCount() == 1 + assert ds.GetRasterBand(1).GetOverview(-1) is None + assert ds.GetRasterBand(1).GetOverview(1) is None + assert [ + ds.GetRasterBand(i + 1).GetOverview(0).Checksum() for i in range(ds.RasterCount) + ] == [ + ref_ds.GetRasterBand(i + 1).GetOverview(0).Checksum() + for i in range(ref_ds.RasterCount) + ] + with pytest.raises( + Exception, match="Cannot delete overviews in TileDB format in read-only mode" + ): + ds.BuildOverviews("", []) + with pytest.raises( + Exception, match="Cannot create overviews in TileDB format in read-only mode" + ): + ds.BuildOverviews("NEAR", [2]) + ds.Close() + + # Update existing overview and change to AVERAGE resampling + ds = gdal.Open(dsname, gdal.GA_Update) + ds.BuildOverviews("AVERAGE", [2]) + ds.Close() + + ds = gdal.Open(dsname) + ref_ds = gdal.Translate("", src_ds, format="MEM") + ref_ds.BuildOverviews("AVERAGE", [2]) + assert ds.GetRasterBand(1).GetOverview(0).GetMetadataItem("RESAMPLING") == "AVERAGE" + assert [ + ds.GetRasterBand(i + 1).GetOverview(0).Checksum() for i in range(ds.RasterCount) + ] == [ + ref_ds.GetRasterBand(i + 1).GetOverview(0).Checksum() + for i in range(ref_ds.RasterCount) + ] + ds.Close() + + # Clear overviews + ds = gdal.Open(dsname, gdal.GA_Update) + ds.BuildOverviews(None, []) + assert ds.GetRasterBand(1).GetOverviewCount() == 0 + ds.Close() + + # Check there are no more overviews after reopening + assert not os.path.exists(dsname + "/l_1") + + ds = gdal.OpenEx(dsname, gdal.OF_MULTIDIM_RASTER) + assert ds.GetRootGroup().GetMDArrayNames() == ["l_0"] + ds.Close() + + ds = gdal.Open(dsname) + assert ds.GetRasterBand(1).GetOverviewCount() == 0 + ds.Close() + + # Try accessing an overview band after clearing overviews + # (the GDAL API doesn't really promise this is safe to do in general, but + # this is implemented in this driver) + ds = gdal.Open(dsname, gdal.GA_Update) + ds.BuildOverviews("NEAR", [2]) + ovr_band = ds.GetRasterBand(1).GetOverview(0) + ds.BuildOverviews(None, []) + with pytest.raises(Exception, match="Dataset has been closed"): + ovr_band.ReadRaster() + with pytest.raises(Exception, match="Dataset has been closed"): + ovr_band.GetDataset().ReadRaster() + ds.Close() + + # Test adding overviews in 2 steps + ds = gdal.Open(dsname, gdal.GA_Update) + ds.BuildOverviews("NEAR", [2]) + ds.Close() + ds = gdal.Open(dsname, gdal.GA_Update) + ds.BuildOverviews("NEAR", [4]) + ref_ds = gdal.Translate("", src_ds, format="MEM") + ref_ds.BuildOverviews("NEAR", [2, 4]) + assert ds.GetRasterBand(1).GetOverviewCount() == 2 + assert [ + ds.GetRasterBand(i + 1).GetOverview(0).Checksum() for i in range(ds.RasterCount) + ] == [ + ref_ds.GetRasterBand(i + 1).GetOverview(0).Checksum() + for i in range(ref_ds.RasterCount) + ] + assert [ + ds.GetRasterBand(i + 1).GetOverview(1).Checksum() for i in range(ds.RasterCount) + ] == [ + ref_ds.GetRasterBand(i + 1).GetOverview(1).Checksum() + for i in range(ref_ds.RasterCount) + ] + ds.Close() + + +def test_tiledb_write_overviews_as_geotiff(tmp_path): + + # We don't want to promote that since GDAL 3.10 because we have now native + # support for overviews, but GeoTIFF side-car .ovr used to work until now + # due to base PAM mechanisms. So test this + + dsname = str(tmp_path / "test.tiledb") + + src_ds = gdal.Open("data/rgbsmall.tif") + src_ds = gdal.Translate("", src_ds, format="MEM") + src_ds.GetRasterBand(1).SetNoDataValue(0) + gdal.GetDriverByName("TileDB").CreateCopy( + dsname, src_ds, options=["CREATE_GROUP=NO"] + ) + + ds = gdal.Open(dsname) + with gdal.config_option("TILEDB_GEOTIFF_OVERVIEWS", "YES"): + ds.BuildOverviews("NEAR", [2]) + ds.Close() + + assert os.path.exists(str(tmp_path / "test.tiledb" / "test.tdb_0.ovr")) + + ds = gdal.Open(dsname) + assert ds.GetRasterBand(1).GetOverviewCount() == 1 + assert ds.GetRasterBand(1).GetOverview(0) is not None + ds.Close() + + # If there are GeoTIFF .ovr, handle them through PAM even in update mode. + ds = gdal.Open(dsname, gdal.GA_Update) + ds.BuildOverviews(None, []) + assert ds.GetRasterBand(1).GetOverviewCount() == 0 + ds.Close() diff --git a/autotest/ogr/data/kml/point_with_external_style.kml b/autotest/ogr/data/kml/point_with_external_style.kml new file mode 100644 index 000000000000..aee7354616d2 --- /dev/null +++ b/autotest/ogr/data/kml/point_with_external_style.kml @@ -0,0 +1,13 @@ + + + + + point + + my point + style_of_point_with_external_style/style.kml#myStyle + 2,49,0 + + + + diff --git a/autotest/ogr/data/kml/style_of_point_with_external_style/style.kml b/autotest/ogr/data/kml/style_of_point_with_external_style/style.kml new file mode 100644 index 000000000000..8131fdee5e29 --- /dev/null +++ b/autotest/ogr/data/kml/style_of_point_with_external_style/style.kml @@ -0,0 +1,14 @@ + + + + + diff --git a/autotest/ogr/data/parquet/test_with_fid_and_geometry_bbox.parquet b/autotest/ogr/data/parquet/test_with_fid_and_geometry_bbox.parquet new file mode 100644 index 000000000000..79f577585eae Binary files /dev/null and b/autotest/ogr/data/parquet/test_with_fid_and_geometry_bbox.parquet differ diff --git a/autotest/ogr/ogr_esrijson.py b/autotest/ogr/ogr_esrijson.py index 0f5eda1c2e48..83c1e42319dc 100755 --- a/autotest/ogr/ogr_esrijson.py +++ b/autotest/ogr/ogr_esrijson.py @@ -101,6 +101,11 @@ def test_ogr_esrijson_read_point(): rc = validate_layer(lyr, "esripoint", 1, ogr.wkbPoint, 7, extent) assert rc + layer_defn = lyr.GetLayerDefn() + fld_defn = layer_defn.GetFieldDefn(layer_defn.GetFieldIndex("fooDate")) + assert fld_defn.GetType() == ogr.OFTDateTime + assert fld_defn.GetWidth() == 0 + ref = lyr.GetSpatialRef() gcs = int(ref.GetAuthorityCode("GEOGCS")) diff --git a/autotest/ogr/ogr_libkml.py b/autotest/ogr/ogr_libkml.py index 5576ee4fa71a..1c616123436e 100755 --- a/autotest/ogr/ogr_libkml.py +++ b/autotest/ogr/ogr_libkml.py @@ -707,10 +707,8 @@ def test_ogr_libkml_camera(tmp_vsimem): ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_camera.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(tmp_vsimem / "ogr_libkml_camera.kml", "rb") as f: + data = f.read().decode("ascii") assert not ( data.find("") == -1 @@ -771,10 +769,8 @@ def test_ogr_libkml_write_layer_lookat(tmp_vsimem): ds.CreateLayer("test2", options=options) ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_write_layer_lookat.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(tmp_vsimem / "ogr_libkml_write_layer_lookat.kml", "rb") as f: + data = f.read().decode("ascii") assert not ( data.find("") == -1 @@ -816,10 +812,8 @@ def test_ogr_libkml_write_layer_camera(tmp_vsimem): ds.CreateLayer("test", options=options) ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_write_layer_camera.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(tmp_vsimem / "ogr_libkml_write_layer_camera.kml", "rb") as f: + data = f.read().decode("ascii") assert not ( data.find("") == -1 @@ -884,10 +878,8 @@ def test_ogr_libkml_write_snippet(tmp_vsimem): lyr.CreateFeature(feat) ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_write_snippet.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(tmp_vsimem / "ogr_libkml_write_snippet.kml", "rb") as f: + data = f.read().decode("ascii") assert data.find("test_snippet") != -1 @@ -920,10 +912,8 @@ def test_ogr_libkml_write_atom_author(tmp_vsimem): assert ds is not None, "Unable to create %s." % filepath ds = None - f = gdal.VSIFOpenL(filepath, "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(filepath, "rb") as f: + data = f.read().decode("ascii") assert not ( data.find( @@ -949,10 +939,8 @@ def test_ogr_libkml_write_atom_link(tmp_vsimem): assert ds is not None, "Unable to create %s." % filepath ds = None - f = gdal.VSIFOpenL(filepath, "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(filepath, "rb") as f: + data = f.read().decode("ascii") assert not ( data.find( @@ -976,10 +964,8 @@ def test_ogr_libkml_write_phonenumber(tmp_vsimem): assert ds is not None, "Unable to create %s." % filepath ds = None - f = gdal.VSIFOpenL(filepath, "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(filepath, "rb") as f: + data = f.read().decode("ascii") assert data.find("tel:911") != -1 @@ -1013,10 +999,8 @@ def test_ogr_libkml_write_region(tmp_vsimem): ) ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_write_region.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(tmp_vsimem / "ogr_libkml_write_region.kml", "rb") as f: + data = f.read().decode("ascii") assert not ( data.find("49") == -1 @@ -1071,10 +1055,10 @@ def test_ogr_libkml_write_screenoverlay(tmp_vsimem): ) ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_write_screenoverlay.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open( + tmp_vsimem / "ogr_libkml_write_screenoverlay.kml", "rb" + ) as f: + data = f.read().decode("ascii") assert not ( data.find("http://foo") == -1 @@ -1135,10 +1119,8 @@ def test_ogr_libkml_write_model(tmp_vsimem): ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_write_model.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open(tmp_vsimem / "ogr_libkml_write_model.kml", "rb") as f: + data = f.read().decode("ascii") assert not ( data.find("2") == -1 @@ -1274,10 +1256,10 @@ def test_ogr_libkml_read_write_style(tmp_vsimem): ds = None src_ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_read_write_style_write.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open( + tmp_vsimem / "ogr_libkml_read_write_style_write.kml", "rb" + ) as f: + data = f.read().decode("ascii") lines = [l.strip() for l in data.split("\n")] lines_got = lines[ @@ -1308,10 +1290,10 @@ def test_ogr_libkml_read_write_style(tmp_vsimem): ds = None src_ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_read_write_style_write.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open( + tmp_vsimem / "ogr_libkml_read_write_style_write.kml", "rb" + ) as f: + data = f.read().decode("ascii") lines = [l.strip() for l in data.split("\n")] lines_got = lines[ @@ -1346,10 +1328,12 @@ def test_ogr_libkml_read_write_style(tmp_vsimem): assert feat.GetStyleString() == style_string ds = None - f = gdal.VSIFOpenL(tmp_vsimem / "ogr_libkml_read_write_style_write.kml", "rb") - data = gdal.VSIFReadL(1, 2048, f) - data = data.decode("ascii") - gdal.VSIFCloseL(f) + with gdaltest.vsi_open( + tmp_vsimem / "ogr_libkml_read_write_style_write.kml", "rb" + ) as f: + data = f.read().decode("ascii") + + assert "#unknown_style" in data expected_style = """