From e603e28ba526a99599794669606d86d8c1c562af Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 29 Aug 2024 02:18:57 +0200 Subject: [PATCH 001/710] Add RFC101 text: Raster dataset read-only thread-safety --- doc/source/development/rfc/index.rst | 1 + .../rfc101_raster_dataset_threadsafety.rst | 484 ++++++++++++++++++ doc/source/spelling_wordlist.txt | 4 + 3 files changed, 489 insertions(+) create mode 100644 doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst diff --git a/doc/source/development/rfc/index.rst b/doc/source/development/rfc/index.rst index f0763fe8d0ca..73f24bcf634e 100644 --- a/doc/source/development/rfc/index.rst +++ b/doc/source/development/rfc/index.rst @@ -106,3 +106,4 @@ RFC list rfc97_feature_and_fielddefn_sealing rfc98_build_requirements_gdal_3_9 rfc99_geometry_coordinate_precision + rfc101_raster_dataset_threadsafety diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst new file mode 100644 index 000000000000..73b377983556 --- /dev/null +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -0,0 +1,484 @@ +.. _rfc-101: + +=================================================================== +RFC 101: Raster dataset read-only thread-safety +=================================================================== + +============== ============================================= +Author: Even Rouault +Contact: even.rouault @ spatialys.com +Started: 2024-Aug-29 +Status: Draft +Target: GDAL 3.10 +============== ============================================= + +Summary +------- + +This RFC enables users to get instances of :cpp:class:`GDALDataset` +(and their related objects such as :cpp:class:`GDALRasterBand`) that are +thread-safe for read-only raster operations, that is such instances can +be safely used from multiple threads without locking. + +Terminology +----------- + +The exact meaning of the terms ``thread-safe`` or ``re-entrant`` is not fully +standardized. We will use here the `QT definitions `__. +In particular, a C function or C++ method is said to be re-entrant if it can +be called simultaneously from multiple threads, *but* only if each invocation +uses its own data/instance. On the contrary, it is thread-safe is if can be +called on the same data/instance (so thread-safe is stronger than re-entrant) + +Motivation +---------- + +A number of raster algorithms can be designed to read chunks of a raster in +an independent and concurrent way, with resulting speed-ups when using +multi-threading. Currently, given a GDALDataset instance is not thread-safe, +this requires either to deal with I/O in a single thread, or through a mutex +to protect against concurrent use, or one needs to open a separate GDALDataset +for each worker thread. Both approaches can complicate the writing of such +algorithms. The enhancement of this RFC aims at providing a special GDALDataset +instance that can be used safely from multiple threads. Internally, it does use +one GDALDataset per thread, but hides this implementation detail to the user. + +C and C++ API extensions +------------------------ + +A new ``GDAL_OF_THREAD_SAFE`` opening flag is added to be specified to +:cpp:func:`GDALOpenEx` / :cpp:func:`GDALDataset::Open`. This flag is for now +mutually exclusive with ``GDAL_OF_VECTOR``, ``GDAL_OF_MULTIDIM_RASTER`` and +``GDAL_OF_UPDATE``. That is this flag is only meant for read-only raster +operations (``GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE``). + +To know if a given dataset can be used in a thread-safe way, the following +C++ method is added to the GDALDataset class: + +.. code-block:: c++ + + /** Return whether this dataset, and its related objects (typically raster + * bands), can be called for the intended scope. + * + * Note that in the current implementation, nScopeFlags should be set to + * GDAL_OF_RASTER, as thread-safety is limited to read-only operations and + * excludes operations on vector layers (OGRLayer) or multidimensional API + * (GDALGroup, GDALMDArray, etc.) + * + * This is the same as the C function GDALDatasetIsThreadSafe(). + * + * @since 3.10 + */ + bool IsThreadSafe(int nScopeFlags) const; + + +The corresponding C function is added: + +.. code-block:: c + + bool GDALDatasetIsThreadSafe(GDALDatasetH hDS, int nScopeFlags, + CSLConstList papszOptions); + + +A new C++ function, GDALCreateThreadSafeDataset, is added with two forms: + +.. code-block:: c++ + + std::unique_ptr GDALCreateThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags); + + std::unique_ptr GDALCreateThreadSafeDataset(GDALDataset* poDS, int nScopeFlags); + +This function accepts a (generally non thread-safe) source dataset and return +a new dataset that is a thread-safe wrapper around it. +The nScopeFlags argument must be compulsory set to GDAL_OF_RASTER to express that +the intended scope is read-only raster operations (other values will result in +an error and a NULL returned dataset). +This function is used internally by GDALOpenEx() when the GDAL_OF_THREAD_SAFE +flag is passed to wrap the dataset returned by the driver. +The first form takes ownership of the source dataset. The second form does not, +but references it internally, and assumes that its lifetime will be longer than +the lifetime of the returned thread-safe dataset. Note that the second form does +increase the reference count on the passed dataset while it is being used, so +patterns like the following one are valid: + +.. code-block:: c++ + + auto poDS = GDALDataset::Open(...); + auto poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); + poDS->ReleaseRef(); + // ... do something with poThreadSafeDS ... + poThreadSafeDS.reset(); // optional + + +The corresponding C function for the second form is added: + +.. code-block:: c + + GDALDatasetH GDALCreateThreadSafeDataset(GDALDatasetH hDS, int nScopeFlags, CSLConstList papszOptions); + + +Usage examples +-------------- + +Example of a function processing a whole dataset passed as a filename: + +.. code-block:: c++ + + void foo(const char* pszFilename) + { + auto poDS = std::unique_ptr(GDALDataset::Open( + pszFilename, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE | GDAL_OF_VERBOSE_ERROR)); + if( !poDS ) + { + return; + } + + // TODO: spawn threads using poDS + } + + +Example of a function processing a whole dataset passed as an object: + +.. code-block:: c++ + + void foo(GDALDataset* poDS) + { + std::unique_ptr poThreadSafeDSUniquePtr; // keep in that scope + GDALDataset* poThreadSafeDS; + if( poDS->IsThreadSafe(GDAL_OF_RASTER) ) + { + poThreadSafeDS = poDS; + } + else + { + poThreadSafeDSUniquePtr = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); + // TODO: check poThreadSafeDSUniquePtr is not null. + poThreadSafeDS = poThreadSafeDSUniquePtr.get(); + } + + if( poThreadSafeDS ) + { + // TODO: spawn threads using poThreadSafeDS + } + else + { + // TODO: Serial version of the algorithm. It can happen if + // poDS is a on-the-fly dataset. + } + } + + +Example of a function processing a single band passed as an object: + +.. code-block:: c++ + + void foo(GDALRasterBand* poBand) + { + std::unique_ptr poThreadSafeDSUniquePtr; // keep in that scope + + GDALRasterBand* poThreadSafeBand = nullptr; + GDALDataset* poDS = poBand->GetDataset(); + // Check that poBand has a matching owing dataset + if( poDS && poDS->GetRasterBand(poBand->GetBand()) == poBand ) + { + GDALDataset* poThreadSafeDS; + if( poDS->IsThreadSafe(GDAL_OF_RASTER) ) + { + poThreadSafeDS = poDS; + } + else + { + poThreadSafeDSUniquePtr = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); + poThreadSafeDS = poThreadSafeDSUniquePtr.get(); + } + + if( poThreadSafeDS ) + poThreadSafeBand = poThreadSafeDS->GetBand(poBand->GetBand()); + } + + if( poThreadSafeBand ) + { + // TODO: spawn threads using poThreadSafeBand + } + else + { + // TODO: Serial version of the algorithm. It can happen if + // poBand is a on-the-fly band, or a "detached" band, such as a + // mask band, or an overview band as returned by some drivers. + } + } + + +SWIG bindings +------------- + +The new C macro and functions are bound to SWIG as: + +- ``gdal.OF_THREAD_SAFE`` +- :py:func:`Dataset.IsThreadSafe(nScopeFlags)` +- :py:func:`Dataset.CreateThreadSafeDataset(nScopeFlags)`. The Python + implementation of this method takes care of keeping a reference on the source + dataset in the returned thread-safe dataset, so the user does not have to + care about their respective lifetimes. + +Usage and design limitations +---------------------------- + +* As implied by the RFC title, the scope of thread-safety is restricted to + **raster** and **read-only** operations. + +* For GDALDataset instances pointing to a file on the regular filesystem, the + limitation of the maximum number of file descriptor opened by a process + (1024 on most Linux systems) could be hit if working with a sufficiently large + number of worker threads and/or instances of GDALThreadSafeDataset. + +* The generic implementation of GDALCreateThreadSafeDataset assumes that the + source dataset can be re-opened by its name (GetDescription()), which is the + case for datasets opened by GDALOpenEx(). A special implementation is also + made for dataset instances of the MEM driver. But, there is currently no + support for creating a thread-safe dataset wrapper on on-the-fly datasets + returned by some algorithms. + +* Inherent to the selected approach, there is a band block cache per thread, and + thus no sharing of cached blocks between threads. + However, this should not be a too severe limitation for algorithms where + threads process independent regions of the raster, hence reuse of cached blocks + would be inexistent or low. Optimal algorithms will make sure to work on + regions of interest aligned on the block size (this advice also applies for + the current approach of manually opening a dataset for each worker thread). + +* Due to implementation limitations, :cpp:func:`GDALRasterBand::GetDefaultRAT` + on a GDALThreadSafeDataset instance only works if the RAT is an instance of + :cpp:class:`GDALDefaultRasterAttributeTable`. A clean error is emitted if + this is not the case. This could potentially be extended to work with any + subclass of :cpp:class:`GDALRasterAttributeTable` but with significant + additional coding to create a thread-safe wrapper. (GDALDefaultRasterAttributeTable + is intrinsically thread-safe for read-only operations). This is not perceived + as a limitation for the intended use cases of this RFC (reading pixel values + in parallel). + +* Some drivers, like netCDF, and HDF5 in some builds, use a global lock around + each call to their APIs, due to the underlying libraries not being re-entrant. + Obviously scalability of GDALThreadSafeDataset will be limited by such global + lock. + But this is no different than the approach of opening as many dataset as + worker threads. + +Implementation details +---------------------- + +(This section is mostly of interest for developers familiar with GDAL internals +and may be skipped by users of the GDAL API) + +The gist of the implementation lies in a new file ``gcore/gdalthreadsafedataset.cpp`` +which defines several classes (internal details): + +- ``GDALThreadSafeDataset`` extending :cpp:class:`GDALProxyDataset`. + Instances of that class are returned by GDALCreateThreadSafeDataset(). + On instantiation, it creates as many GDALThreadSafeRasterBand instances as + the number of bands of the source dataset. + All virtual methods of GDALDataset are redefined by GDALProxyDataset. + GDALThreadSafeDataset overloads its ReferenceUnderlyingDataset method, so that + a thread-locale dataset is opened the first-time a thread calls a method on + the GDALThreadSafeDataset instance, cached for later use, and method call is + forwarded to it. + +- ``GDALThreadSafeRasterBand`` extending :cpp:class:`GDALProxyRasterBand`. + On instantiation, it creates child GDALThreadSafeRasterBand instances for + band mask and overviews. + Its ReferenceUnderlyingRasterBand method calls ReferenceUnderlyingDataset + on the GDALThreadSafeDataset instance to get a thread-locale dataset, fetches + the appropriate thread-locale band and forwards its the method call. + +- ``GDALThreadLocalDatasetCache``. Instances of that class use thread-local + storage. The main member of such instances is a LRU cache that maps + GDALThreadSafeDataset* instances to a thread specific GDALDataset smart pointer. + On GDALThreadSafeDataset destruction, there's code to iterate over all + alive GDALThreadLocalDatasetCache instances and evict no-longer needed entries + in them, within a per-GDALThreadLocalDatasetCache instance mutex, to avoid + issues when dealing with several instances of GDALThreadLocalDatasetCache... + Note that the existence of this mutex should not cause performance issues, since + contention on it, should be very low in real-world use cases (it could become + a bottleneck if GDALThreadSafeDataset were created and destroyed at a very + high pace) + +Two protected virtual methods are added to GDALDataset for GDALThreadSafeDataset +implementation, and may be overloaded by drivers if needed (but it is not +anticipated that drivers but the MEM driver need to do that) + +- ``bool CanBeCloned(int nScopeFlags, bool bCanShareState) const``. + This method determines if a source dataset can be "cloned" (or re-opened). + It returns true for instances returned by GDALOpenEx, for instances of the MEM + driver if ``nScopeFlags`` == ``GDAL_OF_RASTER`` (and ``bCanShareState`` is + true for instances of the MEM driver) + +- ``std::unique_ptr Clone(int nScopeFlags, bool bCanShareState) const``. + This method returns a "clone" of the dataset on which it is called, and is used + by GDALThreadSafeDataset::ReferenceUnderlyingDataset() when a thread-locale + dataset is needed. + Implementation of that method must be thread-safe. + The base implementation calls GDALOpenEx() reusing the dataset name, open flags + and open option. It is overloaded in the MEM driver to return a new instance + of MEMDataset, but sharing the memory buffers with the source dataset. + +No code in drivers, but the MEM driver, is modified by the candidate +implementation. + +A few existing non-virtual methods of GDALDataset and GDALRasterBand have been +made virtual (and overloaded by GDALProxyDataset and GDALProxyRasterBand), +to avoid modifying state on the GDALThreadSafeRasterBand instance, which +wouldn't be thread-safe. + +- :cpp:func:`GDALDataset::BlockBasedRasterIO`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::GetLockedBlockRef`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::TryGetLockedBlockRef`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::FlushBlock`: + it interacts with the block cache +- :cpp:func:`GDALRasterBand::InterpolateAtPoint`: + it uses a per-band cache +- :cpp:func:`GDALRasterBand::EnablePixelTypeSignedByteWarning`: it should + already have been made virtual for GDALProxyRasterBand needs. + +Performance +----------- + +The existing multireadtest utility that reads a dataset from multiple threads +has been extended with a -thread_safe flag to asks to use GDAL_OF_THREAD_SAFE +when opening the dataset in the main thread and use it in the worker threads, +instead of the default behavior of opening explicitly a dataset in each thread. +The thread-safe mode shows similar scalability as the default mode, sometimes +with a slightly decreased efficiency, but not in a too problematic way. + +For example on a 20x20 raster: + +.. code-block:: shell + + $ time multireadtest -t 4 -i 1000000 20x20.tif + real 0m2.084s + user 0m8.155s + sys 0m0.020s + + vs + + $ time multireadtest -thread_safe -t 4 -i 1000000 20x20.tif + real 0m2.387s + user 0m9.334s + sys 0m0.029s + + +But on a 4096x4096 raster with a number of iterations reduced to 100. the +thread_safe mode is slightly faster: + +.. code-block:: shell + + $ time multireadtest -t 4 -i 100 4096x4096.tif + real 0m4.845s + user 0m18.666s + sys 0m0.084s + + vs + + $ time multireadtest -thread_safe -t 4 -i 100 4096x4096.tif + real 0m4.255s + user 0m16.370s + sys 0m0.096s + + +A Python equivalent of multireadtest has been written. Scalability depends +on how much Python code is executed. If relatively few long-enough calls to +GDAL are done, scalability tends to be good due to the Python Global Interpreter +Lock (GIL) being dropped around them. If many short calls are done, the GIL +itself, or its repeated acquisition and release, becomes the bottleneck. This is +no different than using a GDALDataset per thread. + +Documentation +------------- + +Documentation for the new constant and functions will be added. The +:ref:`multithreading` page will be updated to reflect the new capability +introduced by this RFC. + +Backward compatibility +---------------------- + +No issue anticipated: the C and C++ API are extended. +The C++ ABI is modified due to additions of new virtual methods. + +Testing +------- + +Tests will be added for the new functionality, including stress tests to have +sufficiently high confidence in the correctness of the implementation for common +use cases. + +Risks +----- + +Like all code related to multi-threading, the C++ language and tooling offers +hardly any safety belt against thread-safety programming errors. So it cannot +be excluded that the implementation suffers from bugs in some edge scenarios, +or in the usage of some methods of GDALDataset, GDALRasterBand and related objects +(particularly existing non-virtual methods of those classes that could happen +to have a non thread-safe implementation) + +Open questions +-------------- + +The question in this section needs to be solved during the discussion phase of +the RFC. + +For the unusual situations where a dataset cannot be reopened and thus +GDALCreateThreadSafeDataset() fails, should we provide an additional ``bForce`` +argument to force it to still return a dataset, where calls to the wrapped +dataset are protected by a mutex? This would enable to always write multi-thread +safe code, even if the access to the dataset is serialized. +Similarly we could have a +``std::unique_ptr GDALCreateThreadSafeRasterBand(GDALRasterBand* poBand, int nOpenFlags, bool bForce)`` +function that would try to use GDALCreateThreadSafeDataset() internally if it +manages to identify the dataset to which the band belongs to, and otherwise would +fallback to protecting calls to the wrapped band with a mutex. + +Design discussion +----------------- + +This paragraph discusses a number of thoughts that arose during the writing of +this RFC. + +A significantly different alternative could have consisted in adding native +thread-safety in each driver. But this is not realistic for the following reasons: + +* if that was feasible, it would require considerable development effort to + rework each drivers. So realistically, only a few select drivers would be updated. + +* Even updating a reduced number of drivers would be extremely difficult, in + particular the GeoTIFF one, due to the underlying library not being reentrant, + and deferred loading strategies and many state variables being modified even + by read-only APIs. And this applies to most typical drivers. + +* Due to the inevitable locks, there would be a (small) cost bore by callers + even on single-thread uses of thread-safe native drivers. + +* Some core mechanisms, particularly around the per-band block cache structures, + are not currently thread-safe. + +A variant of the proposed implementation that did not use thread-local storage +has been initially attempted. It stored instead a +``std::map>`` on each GDALThreadSafeDataset +instance. This implementation was simpler, but unfortunately suffered from high +lock contention since a mutex had to be taken around each access to this map, +with the contention increasing with the number of concurrent threads. + +Related issues and PRs +---------------------- + +- Candidate implementation: https://github.com/OSGeo/gdal/compare/master...rouault:gdal:raster_multi_thread?expand=1 + +- https://github.com/OSGeo/gdal/issues/8448: GTiff: Allow concurrent reading of single blocks + +Voting history +-------------- + +TBD diff --git a/doc/source/spelling_wordlist.txt b/doc/source/spelling_wordlist.txt index 46bea3a8339d..fec3027e6fa6 100644 --- a/doc/source/spelling_wordlist.txt +++ b/doc/source/spelling_wordlist.txt @@ -988,6 +988,7 @@ gdalbuildvrt gdalcompare gdalconst gdaldata +GDALDataset gdaldem gdaldriver gdalenhance @@ -1017,6 +1018,7 @@ gdalrasterblock gdalrc gdalsrsinfo gdaltest +GDALThreadLocalDatasetCache gdaltindex gdaltransform gdalvirtualmem @@ -1850,6 +1852,7 @@ MultiPolygon multipolygons multiproc multiprocess +multireadtest multirecords multispectral multistring @@ -2047,6 +2050,7 @@ nRecurseDepth nRefCount nReqOrder nSampleStep +nScopeFlags nSecond nSize nSrcBufferAllocSize From 86562c6df270aa2cbd31d1cceca98385a4304a7b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 31 Aug 2024 18:14:53 +0200 Subject: [PATCH 002/710] SQLite SQL dialect: add MEDIAN, PERCENTILE, PERCENTILE_CONT and MODE ordered-set aggregate functions --- autotest/ogr/ogr_sqlite.py | 117 +++++++ doc/source/user/sql_sqlite_dialect.rst | 14 +- .../sqlite/ogrsqlitesqlfunctionscommon.cpp | 290 ++++++++++++++++++ 3 files changed, 419 insertions(+), 2 deletions(-) diff --git a/autotest/ogr/ogr_sqlite.py b/autotest/ogr/ogr_sqlite.py index 621e5c990fe9..6ae3f9fddcdc 100755 --- a/autotest/ogr/ogr_sqlite.py +++ b/autotest/ogr/ogr_sqlite.py @@ -4106,6 +4106,123 @@ def test_ogr_sqlite_stddev(): assert f.GetField(1) == pytest.approx(0.5**0.5, rel=1e-15) +@pytest.mark.parametrize( + "input_values,expected_res", + [ + ([], None), + ([1], 1), + ([2.5, None, 1], 1.75), + ([3, 2.2, 1], 2.2), + ([1, "invalid"], None), + ], +) +def test_ogr_sqlite_median(input_values, expected_res): + """Test MEDIAN""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + for v in input_values: + ds.ExecuteSQL( + "INSERT INTO test VALUES (%s)" + % ( + "NULL" + if v is None + else ("'" + v + "'") + if isinstance(v, str) + else str(v) + ) + ) + if expected_res is None and input_values: + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT MEDIAN(v) FROM test"): + pass + else: + with ds.ExecuteSQL("SELECT MEDIAN(v) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == pytest.approx(expected_res) + with ds.ExecuteSQL("SELECT PERCENTILE(v, 50) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == pytest.approx(expected_res) + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, 0.5) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == pytest.approx(expected_res) + + +def test_ogr_sqlite_percentile(): + """Test PERCENTILE""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + ds.ExecuteSQL("INSERT INTO test VALUES (5),(6),(4),(7),(3),(8),(2),(9),(1),(10)") + + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, 'invalid') FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, -0.1) FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, 100.1) FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE(v, v) FROM test"): + pass + + +def test_ogr_sqlite_percentile_cont(): + """Test PERCENTILE_CONT""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + ds.ExecuteSQL("INSERT INTO test VALUES (5),(6),(4),(7),(3),(8),(2),(9),(1),(10)") + + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, 'invalid') FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, -0.1) FROM test"): + pass + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT PERCENTILE_CONT(v, 1.1) FROM test"): + pass + + +@pytest.mark.parametrize( + "input_values,expected_res", + [ + ([], None), + ([1, 2, None, 3, 2], 2), + (["foo", "bar", "baz", "bar"], "bar"), + ([1, "foo", 2, "foo", "bar"], "foo"), + ([1, "foo", 2, "foo", 1], "foo"), + ], +) +def test_ogr_sqlite_mode(input_values, expected_res): + """Test MODE""" + + ds = ogr.Open(":memory:", update=1) + ds.ExecuteSQL("CREATE TABLE test(v)") + for v in input_values: + ds.ExecuteSQL( + "INSERT INTO test VALUES (%s)" + % ( + "NULL" + if v is None + else ("'" + v + "'") + if isinstance(v, str) + else str(v) + ) + ) + if expected_res is None and input_values: + with pytest.raises(Exception), gdaltest.error_handler(): + with ds.ExecuteSQL("SELECT MODE(v) FROM test"): + pass + else: + with ds.ExecuteSQL("SELECT MODE(v) FROM test") as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f.GetField(0) == expected_res + + def test_ogr_sqlite_run_deferred_actions_before_start_transaction(): ds = ogr.Open(":memory:", update=1) diff --git a/doc/source/user/sql_sqlite_dialect.rst b/doc/source/user/sql_sqlite_dialect.rst index 0a539e115d26..9433694b78c0 100644 --- a/doc/source/user/sql_sqlite_dialect.rst +++ b/doc/source/user/sql_sqlite_dialect.rst @@ -208,8 +208,18 @@ Statistics functions In addition to standard COUNT(), SUM(), AVG(), MIN(), MAX(), the following aggregate functions are available: -- STDDEV_POP: (GDAL >= 3.10) numerical population standard deviation. -- STDDEV_SAMP: (GDAL >= 3.10) numerical `sample standard deviation `__ +- ``STDDEV_POP(numeric_value)``: (GDAL >= 3.10) numerical population standard deviation. +- ``STDDEV_SAMP(numeric_value)``: (GDAL >= 3.10) numerical `sample standard deviation `__ + +Ordered-set aggregate functions ++++++++++++++++++++++++++++++++ + +The following aggregate functions are available. Note that they require to allocate an amount of memory proportional to the number of selected rows (for ``MEDIAN``, ``PERCENTILE`` and ``PERCENTILE_CONT``) or to the number of values (for ``MODE``). + +- ``MEDIAN(numeric_value)``: (GDAL >= 3.10) (continuous) median (equivalent to ``PERCENTILE(numeric_value, 50)``). NULL values are ignored. +- ``PERCENTILE(numeric_value, percentage)``: (GDAL >= 3.10) (continuous) percentile, with percentage between 0 and 100 (equivalent to ``PERCENTILE_CONT(numeric_value, percentage / 100)``). NULL values are ignored. +- ``PERCENTILE_CONT(numeric_value, fraction)``: (GDAL >= 3.10) (continuous) percentile, with fraction between 0 and 1. NULL values are ignored. +- ``MODE(value)``: (GDAL >= 3.10): mode, i.e. most frequent input value (strings and numeric values are supported), arbitrarily choosing the first one if there are multiple equally-frequent results. NULL values are ignored. Spatialite SQL functions ++++++++++++++++++++++++ diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp b/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp index da3f35fd4b51..bf1ca9dd1b19 100644 --- a/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp +++ b/ogr/ogrsf_frmts/sqlite/ogrsqlitesqlfunctionscommon.cpp @@ -37,6 +37,9 @@ #include "ogrsqliteregexp.cpp" /* yes the .cpp file, to make it work on Windows with load_extension('gdalXX.dll') */ +#include +#include +#include #include #include "ogr_swq.h" @@ -321,6 +324,277 @@ static void OGRSQLITE_STDDEV_SAMP_Finalize(sqlite3_context *pContext) } } +/************************************************************************/ +/* OGRSQLITE_Percentile_Step() */ +/************************************************************************/ + +// Percentile related code inspired from https://sqlite.org/src/file/ext/misc/percentile.c +// of https://www.sqlite.org/draft/percentile.html + +// Constant addd to Percentile::rPct, since rPct is initialized to 0 when unset. +constexpr double PERCENT_ADD_CONSTANT = 1; + +namespace +{ +struct Percentile +{ + double rPct; /* PERCENT_ADD_CONSTANT more than the value for P */ + std::vector *values; /* Array of Y values */ +}; +} // namespace + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void OGRSQLITE_Percentile_Step(sqlite3_context *pCtx, int argc, + sqlite3_value **argv) +{ + assert(argc == 2 || argc == 1); + + double rPct; + + if (argc == 1) + { + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 50.0; + } + else if (sqlite3_user_data(pCtx) == nullptr) + { + /* Requirement 3: P must be a number between 0 and 100 */ + const int eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1]); + if ((eType != SQLITE_INTEGER && eType != SQLITE_FLOAT) || rPct < 0.0 || + rPct > 100.0) + { + sqlite3_result_error(pCtx, + "2nd argument to percentile() is not " + "a number between 0.0 and 100.0", + -1); + return; + } + } + else + { + /* Requirement 3: P must be a number between 0 and 1 */ + const int eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1]); + if ((eType != SQLITE_INTEGER && eType != SQLITE_FLOAT) || rPct < 0.0 || + rPct > 1.0) + { + sqlite3_result_error(pCtx, + "2nd argument to percentile_cont() is not " + "a number between 0.0 and 1.0", + -1); + return; + } + rPct *= 100.0; + } + + /* Allocate the session context. */ + auto p = static_cast( + sqlite3_aggregate_context(pCtx, sizeof(Percentile))); + if (!p) + return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if (p->rPct == 0.0) + { + p->rPct = rPct + PERCENT_ADD_CONSTANT; + } + else if (p->rPct != rPct + PERCENT_ADD_CONSTANT) + { + sqlite3_result_error(pCtx, + "2nd argument to percentile() is not the " + "same for all input rows", + -1); + return; + } + + /* Ignore rows for which the value is NULL */ + const int eType = sqlite3_value_type(argv[0]); + if (eType == SQLITE_NULL) + return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if (eType != SQLITE_INTEGER && eType != SQLITE_FLOAT) + { + sqlite3_result_error(pCtx, + "1st argument to percentile() is not " + "numeric", + -1); + return; + } + + /* Ignore rows for which the value is NaN */ + const double v = sqlite3_value_double(argv[0]); + if (std::isnan(v)) + { + return; + } + + if (!p->values) + p->values = new std::vector(); + try + { + p->values->push_back(v); + } + catch (const std::exception &) + { + delete p->values; + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } +} + +/************************************************************************/ +/* OGRSQLITE_Percentile_Finalize() */ +/************************************************************************/ + +/* +** Called to compute the final output of percentile() and to clean +** up all allocated memory. +*/ +static void OGRSQLITE_Percentile_Finalize(sqlite3_context *pCtx) +{ + auto p = static_cast(sqlite3_aggregate_context(pCtx, 0)); + if (!p) + return; + if (!p->values) + return; + if (!p->values->empty()) + { + std::sort(p->values->begin(), p->values->end()); + const double ix = (p->rPct - PERCENT_ADD_CONSTANT) * + static_cast(p->values->size() - 1) * 0.01; + const size_t i1 = static_cast(ix); + const size_t i2 = + ix == static_cast(i1) || i1 == p->values->size() - 1 + ? i1 + : i1 + 1; + const double v1 = (*p->values)[i1]; + const double v2 = (*p->values)[i2]; + const double vx = v1 + (v2 - v1) * static_cast(ix - i1); + sqlite3_result_double(pCtx, vx); + } + delete p->values; + memset(p, 0, sizeof(*p)); +} + +/************************************************************************/ +/* OGRSQLITE_Mode_Step() */ +/************************************************************************/ + +namespace +{ +struct Mode +{ + std::map *numericValues; + std::map *stringValues; + double mostFrequentNumValue; + std::string *mostFrequentStr; + uint64_t mostFrequentValueCount; + bool mostFrequentValueIsStr; +}; +} // namespace + +static void OGRSQLITE_Mode_Step(sqlite3_context *pCtx, int /*argc*/, + sqlite3_value **argv) +{ + const int eType = sqlite3_value_type(argv[0]); + if (eType == SQLITE_NULL) + return; + + if (eType == SQLITE_BLOB) + { + sqlite3_result_error(pCtx, "BLOB argument not supported for mode()", + -1); + return; + } + + /* Allocate the session context. */ + auto p = static_cast(sqlite3_aggregate_context(pCtx, sizeof(Mode))); + if (!p) + return; + + try + { + if (eType == SQLITE_TEXT) + { + const char *pszStr = + reinterpret_cast(sqlite3_value_text(argv[0])); + if (!p->stringValues) + { + p->stringValues = new std::map(); + p->mostFrequentStr = new std::string(); + } + const uint64_t count = ++(*p->stringValues)[pszStr]; + if (count > p->mostFrequentValueCount) + { + p->mostFrequentValueCount = count; + p->mostFrequentValueIsStr = true; + *(p->mostFrequentStr) = pszStr; + } + } + else + { + const double v = sqlite3_value_double(argv[0]); + if (std::isnan(v)) + return; + if (!p->numericValues) + p->numericValues = new std::map(); + const uint64_t count = ++(*p->numericValues)[v]; + if (count > p->mostFrequentValueCount) + { + p->mostFrequentValueCount = count; + p->mostFrequentValueIsStr = false; + p->mostFrequentNumValue = v; + } + } + } + catch (const std::exception &) + { + delete p->stringValues; + delete p->numericValues; + delete p->mostFrequentStr; + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } +} + +/************************************************************************/ +/* OGRSQLITE_Mode_Finalize() */ +/************************************************************************/ + +static void OGRSQLITE_Mode_Finalize(sqlite3_context *pCtx) +{ + auto p = static_cast(sqlite3_aggregate_context(pCtx, 0)); + if (!p) + return; + + if (p->mostFrequentValueCount) + { + if (p->mostFrequentValueIsStr) + { + sqlite3_result_text(pCtx, p->mostFrequentStr->c_str(), -1, + SQLITE_TRANSIENT); + } + else + { + sqlite3_result_double(pCtx, p->mostFrequentNumValue); + } + } + + delete p->stringValues; + delete p->numericValues; + delete p->mostFrequentStr; + memset(p, 0, sizeof(*p)); +} + /************************************************************************/ /* OGRSQLiteRegisterSQLFunctionsCommon() */ /************************************************************************/ @@ -360,6 +634,22 @@ static OGRSQLiteExtensionData *OGRSQLiteRegisterSQLFunctionsCommon(sqlite3 *hDB) nullptr, OGRSQLITE_STDDEV_Step, OGRSQLITE_STDDEV_SAMP_Finalize); + sqlite3_create_function(hDB, "median", 1, UTF8_INNOCUOUS, nullptr, nullptr, + OGRSQLITE_Percentile_Step, + OGRSQLITE_Percentile_Finalize); + + sqlite3_create_function(hDB, "percentile", 2, UTF8_INNOCUOUS, nullptr, + nullptr, OGRSQLITE_Percentile_Step, + OGRSQLITE_Percentile_Finalize); + + sqlite3_create_function( + hDB, "percentile_cont", 2, UTF8_INNOCUOUS, + const_cast("percentile_cont"), // any non-null ptr + nullptr, OGRSQLITE_Percentile_Step, OGRSQLITE_Percentile_Finalize); + + sqlite3_create_function(hDB, "mode", 1, UTF8_INNOCUOUS, nullptr, nullptr, + OGRSQLITE_Mode_Step, OGRSQLITE_Mode_Finalize); + pData->SetRegExpCache(OGRSQLiteRegisterRegExpFunction(hDB)); return pData; From 588b0f41e72ab34fd123bb36a02d85653d150a90 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 1 Sep 2024 19:29:46 +0200 Subject: [PATCH 003/710] Doc: add instructions for Android installation Fixes #10692 --- doc/source/download.rst | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/source/download.rst b/doc/source/download.rst index 3bc444622f68..bfbdf792e545 100644 --- a/doc/source/download.rst +++ b/doc/source/download.rst @@ -90,6 +90,25 @@ GDAL packages are available on `Homebrew`_. .. _`Homebrew`: https://formulae.brew.sh/formula/gdal +Android +....... + +GDAL can be installed using :ref:`vcpkg`. You may also refer to `vcpkg Android support `__ for general instructions. + +For example to install default configuration for the ``arm64-android`` target: + +.. code-block:: shell + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh # ./bootstrap-vcpkg.bat for Windows + ./vcpkg integrate install + export ANDROID_NDK_HOME=/path/to/android_ndk_home # to adapt + ./vcpkg search gdal --featurepackages # list optional features + ./vcpkg install gdal:arm64-android # install with default configuration + ./vcpkg install gdal[poppler,netcdf]:arm64-android # install with Poppler and netdf support + + Cross-Platform Package Managers ............................... @@ -186,13 +205,15 @@ vcpkg The GDAL port in the `vcpkg `__ dependency manager is kept up to date by Microsoft team members and community contributors. You can download and install GDAL using the vcpkg as follows: -:: +.. code-block:: shell git clone https://github.com/Microsoft/vcpkg.git cd vcpkg ./bootstrap-vcpkg.sh # ./bootstrap-vcpkg.bat for Windows ./vcpkg integrate install - ./vcpkg install gdal + ./vcpkg search gdal --featurepackages # list optional features + ./vcpkg install gdal # install with default configuration + ./vcpkg install gdal[poppler,netcdf] # install with Poppler and netdf support If the version is out of date, please `create an issue or pull request `__ on the vcpkg repository. From c2fa5d11f003fce6e4f216543f7d71c92a988de3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 13:00:16 +0200 Subject: [PATCH 004/710] RFC 101 text: clarifications --- .../rfc/rfc101_raster_dataset_threadsafety.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst index 73b377983556..c4bb89dd65b7 100644 --- a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -237,7 +237,9 @@ Usage and design limitations case for datasets opened by GDALOpenEx(). A special implementation is also made for dataset instances of the MEM driver. But, there is currently no support for creating a thread-safe dataset wrapper on on-the-fly datasets - returned by some algorithms. + returned by some algorithms (e.g GDALTranslate() or GDALWarp() with VRT as + the output driver and with an empty filename, or custom GDALDataset + implementation by external code). * Inherent to the selected approach, there is a band block cache per thread, and thus no sharing of cached blocks between threads. @@ -249,7 +251,7 @@ Usage and design limitations * Due to implementation limitations, :cpp:func:`GDALRasterBand::GetDefaultRAT` on a GDALThreadSafeDataset instance only works if the RAT is an instance of - :cpp:class:`GDALDefaultRasterAttributeTable`. A clean error is emitted if + :cpp:class:`GDALDefaultRasterAttributeTable`. An error is emitted if this is not the case. This could potentially be extended to work with any subclass of :cpp:class:`GDALRasterAttributeTable` but with significant additional coding to create a thread-safe wrapper. (GDALDefaultRasterAttributeTable @@ -279,7 +281,7 @@ which defines several classes (internal details): the number of bands of the source dataset. All virtual methods of GDALDataset are redefined by GDALProxyDataset. GDALThreadSafeDataset overloads its ReferenceUnderlyingDataset method, so that - a thread-locale dataset is opened the first-time a thread calls a method on + a thread-local dataset is opened the first-time a thread calls a method on the GDALThreadSafeDataset instance, cached for later use, and method call is forwarded to it. @@ -287,8 +289,8 @@ which defines several classes (internal details): On instantiation, it creates child GDALThreadSafeRasterBand instances for band mask and overviews. Its ReferenceUnderlyingRasterBand method calls ReferenceUnderlyingDataset - on the GDALThreadSafeDataset instance to get a thread-locale dataset, fetches - the appropriate thread-locale band and forwards its the method call. + on the GDALThreadSafeDataset instance to get a thread-local dataset, fetches + the appropriate thread-local band and forwards its the method call. - ``GDALThreadLocalDatasetCache``. Instances of that class use thread-local storage. The main member of such instances is a LRU cache that maps @@ -314,7 +316,7 @@ anticipated that drivers but the MEM driver need to do that) - ``std::unique_ptr Clone(int nScopeFlags, bool bCanShareState) const``. This method returns a "clone" of the dataset on which it is called, and is used - by GDALThreadSafeDataset::ReferenceUnderlyingDataset() when a thread-locale + by GDALThreadSafeDataset::ReferenceUnderlyingDataset() when a thread-local dataset is needed. Implementation of that method must be thread-safe. The base implementation calls GDALOpenEx() reusing the dataset name, open flags From a8cc696137b8d349592680be63db3c98b06a0e1c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 13:36:39 +0200 Subject: [PATCH 005/710] RFC101 text: change signature of GDALCreateThreadSafeDataset(GDALDataset *poDS, int nScopeFlags) to return a GDALDataset* --- .../rfc101_raster_dataset_threadsafety.rst | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst index c4bb89dd65b7..82b62e96ce7a 100644 --- a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -86,10 +86,11 @@ A new C++ function, GDALCreateThreadSafeDataset, is added with two forms: std::unique_ptr GDALCreateThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags); - std::unique_ptr GDALCreateThreadSafeDataset(GDALDataset* poDS, int nScopeFlags); + GDALDataset* GDALCreateThreadSafeDataset(GDALDataset* poDS, int nScopeFlags); This function accepts a (generally non thread-safe) source dataset and return -a new dataset that is a thread-safe wrapper around it. +a new dataset that is a thread-safe wrapper around it, or the source dataset if +it is already thread-safe. The nScopeFlags argument must be compulsory set to GDAL_OF_RASTER to express that the intended scope is read-only raster operations (other values will result in an error and a NULL returned dataset). @@ -104,10 +105,18 @@ patterns like the following one are valid: .. code-block:: c++ auto poDS = GDALDataset::Open(...); - auto poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); - poDS->ReleaseRef(); - // ... do something with poThreadSafeDS ... - poThreadSafeDS.reset(); // optional + GDALDataset* poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); + poDS->ReleaseRef(); // can be done here, or any time later + if (poThreadSafeDS ) + { + // ... do something with poThreadSafeDS ... + poThreadSafeDS->ReleaseRef(); + } + + +For proper working both when a new dataset is returned or the passed one if it +is already thread-safe, :cpp:func:`GDALDataset::ReleaseRef()` (and not delete or +GDALClose()) must be called on the returned dataset. The corresponding C function for the second form is added: @@ -143,22 +152,12 @@ Example of a function processing a whole dataset passed as an object: void foo(GDALDataset* poDS) { - std::unique_ptr poThreadSafeDSUniquePtr; // keep in that scope - GDALDataset* poThreadSafeDS; - if( poDS->IsThreadSafe(GDAL_OF_RASTER) ) - { - poThreadSafeDS = poDS; - } - else - { - poThreadSafeDSUniquePtr = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); - // TODO: check poThreadSafeDSUniquePtr is not null. - poThreadSafeDS = poThreadSafeDSUniquePtr.get(); - } - + GDALDataset* poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); if( poThreadSafeDS ) { // TODO: spawn threads using poThreadSafeDS + + poThreadSafeDS->ReleaseRef(); } else { @@ -174,24 +173,13 @@ Example of a function processing a single band passed as an object: void foo(GDALRasterBand* poBand) { - std::unique_ptr poThreadSafeDSUniquePtr; // keep in that scope - + GDALDataset* poThreadSafeDS = nullptr; GDALRasterBand* poThreadSafeBand = nullptr; GDALDataset* poDS = poBand->GetDataset(); // Check that poBand has a matching owing dataset if( poDS && poDS->GetRasterBand(poBand->GetBand()) == poBand ) { - GDALDataset* poThreadSafeDS; - if( poDS->IsThreadSafe(GDAL_OF_RASTER) ) - { - poThreadSafeDS = poDS; - } - else - { - poThreadSafeDSUniquePtr = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); - poThreadSafeDS = poThreadSafeDSUniquePtr.get(); - } - + poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); if( poThreadSafeDS ) poThreadSafeBand = poThreadSafeDS->GetBand(poBand->GetBand()); } @@ -199,6 +187,8 @@ Example of a function processing a single band passed as an object: if( poThreadSafeBand ) { // TODO: spawn threads using poThreadSafeBand + + poThreadSafeDS->ReleaseRef(); } else { From 831df5a2a5afee89246d6d4ad5ebb96e19022e52 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 3 Sep 2024 16:37:28 +0200 Subject: [PATCH 006/710] GDALRegenerateOverviewsMultiBand(): make sure than when computing large reduction factors (like > 1024) on huge rasters does not lead to excessive memory requirements The new code paths are well tested by existing tests in tiff_ovr.py that set GDAL_OVR_CHUNK_MAX_SIZE --- autotest/gcore/tiff_ovr.py | 4 +- gcore/overview.cpp | 239 ++++++++++++++++++++++++++++++++++--- 2 files changed, 225 insertions(+), 18 deletions(-) diff --git a/autotest/gcore/tiff_ovr.py b/autotest/gcore/tiff_ovr.py index ba49b4b3fb45..d047ea29843e 100755 --- a/autotest/gcore/tiff_ovr.py +++ b/autotest/gcore/tiff_ovr.py @@ -2642,7 +2642,9 @@ def test_tiff_ovr_fallback_to_multiband_overview_generate(): "data/byte.tif", options="-b 1 -b 1 -b 1 -co INTERLEAVE=BAND -co TILED=YES -outsize 1024 1024", ) - with gdaltest.config_option("GDAL_OVR_CHUNK_MAX_SIZE", "1000"): + with gdaltest.config_options( + {"GDAL_OVR_CHUNK_MAX_SIZE": "1000", "GDAL_OVR_TEMP_DRIVER": "MEM"} + ): ds.BuildOverviews("NEAR", overviewlist=[2, 4, 8]) ds = None diff --git a/gcore/overview.cpp b/gcore/overview.cpp index c962aa73bfc2..df5a96d76707 100644 --- a/gcore/overview.cpp +++ b/gcore/overview.cpp @@ -5031,13 +5031,14 @@ CPLErr GDALRegenerateOverviewsMultiBand( for (int iOverview = 0; iOverview < nOverviews; ++iOverview) { - const int nDstWidth = papapoOverviewBands[0][iOverview]->GetXSize(); - const int nDstHeight = papapoOverviewBands[0][iOverview]->GetYSize(); + const auto poOvrFirstBand = papapoOverviewBands[0][iOverview]; + const int nDstWidth = poOvrFirstBand->GetXSize(); + const int nDstHeight = poOvrFirstBand->GetYSize(); for (int iBand = 1; iBand < nBands; ++iBand) { - if (papapoOverviewBands[iBand][iOverview]->GetXSize() != - nDstWidth || - papapoOverviewBands[iBand][iOverview]->GetYSize() != nDstHeight) + const auto poOvrBand = papapoOverviewBands[iBand][iOverview]; + if (poOvrBand->GetXSize() != nDstWidth || + poOvrBand->GetYSize() != nDstHeight) { CPLError( CE_Failure, CPLE_NotSupported, @@ -5045,8 +5046,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( "of the same level must have the same dimensions"); return CE_Failure; } - if (papapoOverviewBands[iBand][iOverview]->GetRasterDataType() != - eDataType) + if (poOvrBand->GetRasterDataType() != eDataType) { CPLError( CE_Failure, CPLE_NotSupported, @@ -5076,6 +5076,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( const GDALDataType eWrkDataType = GDALGetOvrWorkDataType(pszResampling, eDataType); + const int nWrkDataTypeSize = GDALGetDataTypeSizeBytes(eWrkDataType); const bool bIsMask = papoSrcBands[0]->IsMaskBand(); @@ -5116,8 +5117,8 @@ CPLErr GDALRegenerateOverviewsMultiBand( : std::unique_ptr(nullptr); // Only configurable for debug / testing - const int nChunkMaxSize = - atoi(CPLGetConfigOption("GDAL_OVR_CHUNK_MAX_SIZE", "10485760")); + const int nChunkMaxSize = std::max( + 100, atoi(CPLGetConfigOption("GDAL_OVR_CHUNK_MAX_SIZE", "10485760"))); // Second pass to do the real job. double dfCurPixelCount = 0; @@ -5127,11 +5128,6 @@ CPLErr GDALRegenerateOverviewsMultiBand( { int iSrcOverview = -1; // -1 means the source bands. - int nDstChunkXSize = 0; - int nDstChunkYSize = 0; - papapoOverviewBands[0][iOverview]->GetBlockSize(&nDstChunkXSize, - &nDstChunkYSize); - const int nDstTotalWidth = papapoOverviewBands[0][iOverview]->GetXSize(); const int nDstTotalHeight = @@ -5182,6 +5178,23 @@ CPLErr GDALRegenerateOverviewsMultiBand( if (nOvrFactor == 0) nOvrFactor = 1; + int nDstChunkXSize = 0; + int nDstChunkYSize = 0; + papapoOverviewBands[0][iOverview]->GetBlockSize(&nDstChunkXSize, + &nDstChunkYSize); + + const char *pszDST_CHUNK_X_SIZE = + CSLFetchNameValue(papszOptions, "DST_CHUNK_X_SIZE"); + const char *pszDST_CHUNK_Y_SIZE = + CSLFetchNameValue(papszOptions, "DST_CHUNK_Y_SIZE"); + if (pszDST_CHUNK_X_SIZE && pszDST_CHUNK_Y_SIZE) + { + nDstChunkXSize = std::max(1, atoi(pszDST_CHUNK_X_SIZE)); + nDstChunkYSize = std::max(1, atoi(pszDST_CHUNK_Y_SIZE)); + CPLDebug("GDAL", "Using dst chunk size %d x %d", nDstChunkXSize, + nDstChunkYSize); + } + // Try to extend the chunk size so that the memory needed to acquire // source pixels goes up to 10 MB. // This can help for drivers that support multi-threaded reading @@ -5198,8 +5211,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( nFullResXChunk + 2 * nKernelRadius * nOvrFactor; if (static_cast(nFullResXChunkQueried) * - nFullResYChunkQueried * nBands * - GDALGetDataTypeSizeBytes(eWrkDataType) > + nFullResYChunkQueried * nBands * nWrkDataTypeSize > nChunkMaxSize) { break; @@ -5214,6 +5226,199 @@ CPLErr GDALRegenerateOverviewsMultiBand( const int nFullResXChunkQueried = nFullResXChunk + 2 * nKernelRadius * nOvrFactor; + // Make sure that the RAM requirements to acquire the source data does + // not exceed nChunkMaxSize + // If so, reduce the destination chunk size, generate overviews in a + // temporary dataset, and copy that temporary dataset over the target + // overview bands (to avoid issues with lossy compression) + const auto nMemRequirement = + static_cast(nFullResXChunkQueried) * + nFullResYChunkQueried * nBands * nWrkDataTypeSize; + if (nMemRequirement > nChunkMaxSize && + !(pszDST_CHUNK_X_SIZE && pszDST_CHUNK_Y_SIZE)) + { + // Compute a smaller destination chunk size + const auto nOverShootFactor = nMemRequirement / nChunkMaxSize; + const auto nSqrtOverShootFactor = std::max( + 4, static_cast(std::ceil( + std::sqrt(static_cast(nOverShootFactor))))); + const int nReducedDstChunkXSize = std::max( + 1, static_cast(nDstChunkXSize / nSqrtOverShootFactor)); + const int nReducedDstChunkYSize = std::max( + 1, static_cast(nDstChunkYSize / nSqrtOverShootFactor)); + if (nReducedDstChunkXSize < nDstChunkXSize || + nReducedDstChunkYSize < nDstChunkYSize) + { + CPLStringList aosOptions(papszOptions); + aosOptions.SetNameValue( + "DST_CHUNK_X_SIZE", + CPLSPrintf("%d", nReducedDstChunkXSize)); + aosOptions.SetNameValue( + "DST_CHUNK_Y_SIZE", + CPLSPrintf("%d", nReducedDstChunkYSize)); + + const auto nTmpDSMemRequirement = + static_cast(nDstTotalWidth) * nDstTotalHeight * + nBands * GDALGetDataTypeSizeBytes(eDataType); + std::unique_ptr poTmpDS; + // Config option mostly/only for autotest purposes + const char *pszGDAL_OVR_TEMP_DRIVER = + CPLGetConfigOption("GDAL_OVR_TEMP_DRIVER", ""); + if ((nTmpDSMemRequirement <= nChunkMaxSize && + !EQUAL(pszGDAL_OVR_TEMP_DRIVER, "GTIFF")) || + EQUAL(pszGDAL_OVR_TEMP_DRIVER, "MEM")) + { + auto poTmpDrv = + GetGDALDriverManager()->GetDriverByName("MEM"); + if (!poTmpDrv) + { + eErr = CE_Failure; + break; + } + poTmpDS.reset(poTmpDrv->Create("", nDstTotalWidth, + nDstTotalHeight, nBands, + eDataType, nullptr)); + } + else + { + auto poTmpDrv = + GetGDALDriverManager()->GetDriverByName("GTiff"); + if (!poTmpDrv) + { + eErr = CE_Failure; + break; + } + std::string osTmpFilename; + auto poDstDS = papapoOverviewBands[0][0]->GetDataset(); + if (poDstDS) + { + osTmpFilename = poDstDS->GetDescription(); + VSIStatBufL sStatBuf; + if (!osTmpFilename.empty() && + VSIStatL(osTmpFilename.c_str(), &sStatBuf) == 0) + osTmpFilename += "_tmp_ovr.tif"; + } + if (osTmpFilename.empty()) + { + osTmpFilename = CPLGenerateTempFilename(nullptr); + osTmpFilename += ".tif"; + } + CPLDebug("GDAL", + "Creating temporary file %s of %d x %d x %d", + osTmpFilename.c_str(), nDstTotalWidth, + nDstTotalHeight, nBands); + CPLStringList aosCO; + poTmpDS.reset(poTmpDrv->Create( + osTmpFilename.c_str(), nDstTotalWidth, nDstTotalHeight, + nBands, eDataType, aosCO.List())); + if (poTmpDS) + { + poTmpDS->MarkSuppressOnClose(); + VSIUnlink(osTmpFilename.c_str()); + } + } + if (!poTmpDS) + { + eErr = CE_Failure; + break; + } + + std::vector apapoOverviewBands(nBands); + for (int i = 0; i < nBands; ++i) + { + apapoOverviewBands[i] = static_cast( + CPLMalloc(sizeof(GDALRasterBand *))); + apapoOverviewBands[i][0] = poTmpDS->GetRasterBand(i + 1); + } + + const double dfExtraPixels = + static_cast(nSrcXSize) / nToplevelSrcWidth * + papapoOverviewBands[0][iOverview]->GetXSize() * + static_cast(nSrcYSize) / nToplevelSrcHeight * + papapoOverviewBands[0][iOverview]->GetYSize(); + + void *pScaledProgressData = GDALCreateScaledProgress( + dfCurPixelCount / dfTotalPixelCount, + (dfCurPixelCount + dfExtraPixels) / dfTotalPixelCount, + pfnProgress, pProgressData); + + // Generate overviews in temporary dataset + eErr = GDALRegenerateOverviewsMultiBand( + nBands, papoSrcBands, 1, apapoOverviewBands.data(), + pszResampling, GDALScaledProgress, pScaledProgressData, + aosOptions.List()); + + GDALDestroyScaledProgress(pScaledProgressData); + + dfCurPixelCount += dfExtraPixels; + + for (int i = 0; i < nBands; ++i) + { + CPLFree(apapoOverviewBands[i]); + } + + // Copy temporary dataset to destination overview bands + + if (eErr == CE_None) + { + // Check if all papapoOverviewBands[][iOverview] bands point + // to the same dataset. If so, we can use + // GDALDatasetCopyWholeRaster() + GDALDataset *poDstOvrBandDS = + papapoOverviewBands[0][iOverview]->GetDataset(); + if (poDstOvrBandDS) + { + if (poDstOvrBandDS->GetRasterCount() != nBands || + poDstOvrBandDS->GetRasterBand(1) != + papapoOverviewBands[0][iOverview]) + { + poDstOvrBandDS = nullptr; + } + else + { + for (int i = 1; poDstOvrBandDS && i < nBands; ++i) + { + GDALDataset *poThisDstOvrBandDS = + papapoOverviewBands[i][iOverview] + ->GetDataset(); + if (poThisDstOvrBandDS == nullptr || + poThisDstOvrBandDS != poDstOvrBandDS || + poThisDstOvrBandDS->GetRasterBand(i + 1) != + papapoOverviewBands[i][iOverview]) + { + poDstOvrBandDS = nullptr; + } + } + } + } + if (poDstOvrBandDS) + { + eErr = GDALDatasetCopyWholeRaster( + GDALDataset::ToHandle(poTmpDS.get()), + GDALDataset::ToHandle(poDstOvrBandDS), nullptr, + nullptr, nullptr); + } + else + { + for (int i = 0; eErr == CE_None && i < nBands; ++i) + { + eErr = GDALRasterBandCopyWholeRaster( + GDALRasterBand::ToHandle( + poTmpDS->GetRasterBand(i + 1)), + GDALRasterBand::ToHandle( + papapoOverviewBands[i][iOverview]), + nullptr, nullptr, nullptr); + } + } + } + + if (eErr != CE_None) + break; + + continue; + } + } + // Structure describing a resampling job struct OvrJob { @@ -5415,7 +5620,7 @@ CPLErr GDALRegenerateOverviewsMultiBand( { apaChunk[iBand] = VSI_MALLOC3_VERBOSE( nFullResXChunkQueried, nFullResYChunkQueried, - GDALGetDataTypeSizeBytes(eWrkDataType)); + nWrkDataTypeSize); if (apaChunk[iBand] == nullptr) { eErr = CE_Failure; From c4c0a1c192a193ef6e50b54f8fbf023a84c9869e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 4 Sep 2024 06:18:21 +0200 Subject: [PATCH 007/710] VSIVirtualHandle: add a Interrupt() method and implement in in /vsicurl/ (and related filesystems) --- port/cpl_vsi_virtual.h | 8 ++++++++ port/cpl_vsil_curl.cpp | 28 ++++++++++++++++++---------- port/cpl_vsil_curl_class.h | 11 ++++++++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/port/cpl_vsi_virtual.h b/port/cpl_vsi_virtual.h index b223c49eb999..c3c1d16eafa8 100644 --- a/port/cpl_vsi_virtual.h +++ b/port/cpl_vsi_virtual.h @@ -139,6 +139,14 @@ struct CPL_DLL VSIVirtualHandle virtual size_t PRead(void *pBuffer, size_t nSize, vsi_l_offset nOffset) const; + /** Ask current operations to be interrupted. + * Implementations must be thread-safe, as this will typically be called + * from another thread than the active one for this file. + */ + virtual void Interrupt() + { + } + // NOTE: when adding new methods, besides the "actual" implementations, // also consider the VSICachedFile one. diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp index 2308a26876c2..f60bb060e25b 100644 --- a/port/cpl_vsil_curl.cpp +++ b/port/cpl_vsil_curl.cpp @@ -831,7 +831,8 @@ static GIntBig VSICurlGetExpiresFromS3LikeSignedURL(const char *pszURL) /* VSICURLMultiPerform() */ /************************************************************************/ -void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle) +void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle, + std::atomic *pbInterrupt) { int repeats = 0; @@ -866,6 +867,9 @@ void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle) #endif CPLMultiPerformWait(hCurlMultiHandle, repeats); + + if (pbInterrupt && *pbInterrupt) + break; } CPLHTTPRestoreSigPipeHandler(old_handler); @@ -1150,7 +1154,7 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); - VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); + VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); VSICURLResetHeaderAndWriterFunctions(hCurlHandle); @@ -1863,7 +1867,7 @@ std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FILETIME, 1); - VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle); + VSICURLMultiPerform(hCurlMultiHandle, hCurlHandle, &m_bInterrupt); VSICURLResetHeaderAndWriterFunctions(hCurlHandle); @@ -1871,10 +1875,13 @@ std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize); - if (sWriteFuncData.bInterrupted) + if (sWriteFuncData.bInterrupted || m_bInterrupt) { bInterrupted = true; + // Notify that the download of the current region is finished + currentDownload.SetData(std::string()); + CPLFree(sWriteFuncData.pBuffer); CPLFree(sWriteFuncHeaderData.pBuffer); curl_easy_cleanup(hCurlHandle); @@ -3092,8 +3099,7 @@ size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers); CURLM *hMultiHandle = poFS->GetCurlMultiHandleFor(osURL); - curl_multi_add_handle(hMultiHandle, hCurlHandle); - VSICURLMultiPerform(hMultiHandle); + VSICURLMultiPerform(hMultiHandle, hCurlHandle, &m_bInterrupt); { std::lock_guard oLock(m_oMutex); @@ -3116,9 +3122,12 @@ size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, if ((response_code != 206 && response_code != 225) || sWriteFuncData.nSize == 0) { - CPLDebug(poFS->GetDebugKey(), - "Request for %s failed with response_code=%ld", rangeStr, - response_code); + if (!m_bInterrupt) + { + CPLDebug(poFS->GetDebugKey(), + "Request for %s failed with response_code=%ld", rangeStr, + response_code); + } nRet = static_cast(-1); } else @@ -3128,7 +3137,6 @@ size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, memcpy(pBuffer, sWriteFuncData.pBuffer, nRet); } - curl_multi_remove_handle(hMultiHandle, hCurlHandle); VSICURLResetHeaderAndWriterFunctions(hCurlHandle); curl_easy_cleanup(hCurlHandle); CPLFree(sWriteFuncData.pBuffer); diff --git a/port/cpl_vsil_curl_class.h b/port/cpl_vsil_curl_class.h index 90120b069f2c..5d33d641a387 100644 --- a/port/cpl_vsil_curl_class.h +++ b/port/cpl_vsil_curl_class.h @@ -42,6 +42,7 @@ #include "cpl_curl_priv.h" #include +#include #include #include #include @@ -411,6 +412,8 @@ class VSICurlHandle : public VSIVirtualHandle bool m_bUseHead = false; bool m_bUseRedirectURLIfNoQueryStringParams = false; + mutable std::atomic m_bInterrupt = false; + // Specific to Planetary Computer signing: // https://planetarycomputer.microsoft.com/docs/concepts/sas/ mutable bool m_bPlanetaryComputerURLSigning = false; @@ -497,6 +500,11 @@ class VSICurlHandle : public VSIVirtualHandle int Flush() override; int Close() override; + void Interrupt() override + { + m_bInterrupt = true; + } + bool HasPRead() const override { return true; @@ -1188,7 +1196,8 @@ void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp, void *pReadCbkUserData); size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb, void *req); -void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr); +void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr, + std::atomic *pbInterrupt = nullptr); void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle); int VSICurlParseUnixPermissions(const char *pszPermissions); From 2ae417649a0c485d14983ab58043da8342996687 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 4 Sep 2024 06:23:04 +0200 Subject: [PATCH 008/710] Parquet: in dataset mode, make sure all files are closed before closing the GDALDataset --- ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp | 14 ++- ogr/ogrsf_frmts/arrow_common/ogr_arrow.h | 7 ++ .../arrow_common/ograrrowrandomaccessfile.h | 106 +++++++++++++++++- .../arrow_common/vsiarrowfilesystem.hpp | 62 +++++++++- ogr/ogrsf_frmts/parquet/ogr_parquet.h | 8 ++ ogr/ogrsf_frmts/parquet/ogrparquetdataset.cpp | 20 ++++ ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp | 11 +- 7 files changed, 209 insertions(+), 19 deletions(-) diff --git a/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp b/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp index 8e5dad1c0053..290ca04cc8ed 100644 --- a/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp +++ b/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp @@ -94,8 +94,8 @@ static bool IsArrowIPCStream(GDALOpenInfo *poOpenInfo) auto fp = VSIVirtualHandleUniquePtr(VSIFileFromMemBuffer( osTmpFilename.c_str(), poOpenInfo->pabyHeader, nSizeToRead, false)); - auto infile = - std::make_shared(std::move(fp)); + auto infile = std::make_shared( + osTmpFilename.c_str(), std::move(fp)); auto options = arrow::ipc::IpcReadOptions::Defaults(); auto result = arrow::ipc::RecordBatchStreamReader::Open(infile, options); @@ -113,8 +113,8 @@ static bool IsArrowIPCStream(GDALOpenInfo *poOpenInfo) return false; // Do not give ownership of poOpenInfo->fpL to infile - auto infile = - std::make_shared(poOpenInfo->fpL, false); + auto infile = std::make_shared( + poOpenInfo->pszFilename, poOpenInfo->fpL, false); auto options = arrow::ipc::IpcReadOptions::Defaults(); auto result = arrow::ipc::RecordBatchStreamReader::Open(infile, options); @@ -164,14 +164,16 @@ static GDALDataset *OGRFeatherDriverOpen(GDALOpenInfo *poOpenInfo) osFilename.c_str()); return nullptr; } - infile = std::make_shared(std::move(fp)); + infile = std::make_shared(osFilename.c_str(), + std::move(fp)); } else if (STARTS_WITH(poOpenInfo->pszFilename, "/vsi") || CPLTestBool(CPLGetConfigOption("OGR_ARROW_USE_VSI", "NO"))) { VSIVirtualHandleUniquePtr fp(poOpenInfo->fpL); poOpenInfo->fpL = nullptr; - infile = std::make_shared(std::move(fp)); + infile = std::make_shared( + poOpenInfo->pszFilename, std::move(fp)); } else { diff --git a/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h b/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h index 6179fd009b07..3f09195c1987 100644 --- a/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h +++ b/ogr/ogrsf_frmts/arrow_common/ogr_arrow.h @@ -352,6 +352,13 @@ class OGRArrowDataset CPL_NON_FINAL : public GDALPamDataset std::vector m_aosDomainNames{}; std::map m_oMapDomainNameToCol{}; + protected: + void close() + { + m_poLayer.reset(); + m_poMemoryPool.reset(); + } + public: explicit OGRArrowDataset( const std::shared_ptr &poMemoryPool); diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h b/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h index cd16ddd14868..a1ab3d4d34c6 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h @@ -36,6 +36,9 @@ #include "arrow/io/file.h" #include "arrow/io/interfaces.h" +#include +#include + /************************************************************************/ /* OGRArrowRandomAccessFile */ /************************************************************************/ @@ -43,22 +46,58 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile { int64_t m_nSize = -1; + const std::string m_osFilename; VSILFILE *m_fp; - bool m_bOwnFP; + const bool m_bOwnFP; + std::atomic m_bAskedToClosed = false; + +#ifdef OGR_ARROW_USE_PREAD + const bool m_bDebugReadAt; + const bool m_bUsePRead; +#endif OGRArrowRandomAccessFile(const OGRArrowRandomAccessFile &) = delete; OGRArrowRandomAccessFile & operator=(const OGRArrowRandomAccessFile &) = delete; public: - explicit OGRArrowRandomAccessFile(VSILFILE *fp, bool bOwnFP) - : m_fp(fp), m_bOwnFP(bOwnFP) + OGRArrowRandomAccessFile(const std::string &osFilename, VSILFILE *fp, + bool bOwnFP) + : m_osFilename(osFilename), m_fp(fp), m_bOwnFP(bOwnFP) +#ifdef OGR_ARROW_USE_PREAD + , + m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())), + // Due to the lack of caching for current /vsicurl PRead(), do not + // use the PRead() implementation on those files + m_bUsePRead(m_fp->HasPRead() && + CPLTestBool(CPLGetConfigOption( + "OGR_ARROW_USE_PREAD", + VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO"))) +#endif + { + } + + OGRArrowRandomAccessFile(const std::string &osFilename, + VSIVirtualHandleUniquePtr &&fp) + : m_osFilename(osFilename), m_fp(fp.release()), m_bOwnFP(true) +#ifdef OGR_ARROW_USE_PREAD + , + m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())), + // Due to the lack of caching for current /vsicurl PRead(), do not + // use the PRead() implementation on those files + m_bUsePRead(m_fp->HasPRead() && + CPLTestBool(CPLGetConfigOption( + "OGR_ARROW_USE_PREAD", + VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO"))) +#endif { } - explicit OGRArrowRandomAccessFile(VSIVirtualHandleUniquePtr &&fp) - : m_fp(fp.release()), m_bOwnFP(true) + void AskToClose() { + m_bAskedToClosed = true; + if (m_fp) + m_fp->Interrupt(); } ~OGRArrowRandomAccessFile() override @@ -85,11 +124,14 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile bool closed() const override { - return m_fp == nullptr; + return m_bAskedToClosed || m_fp == nullptr; } arrow::Status Seek(int64_t position) override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + if (VSIFSeekL(m_fp, static_cast(position), SEEK_SET) == 0) return arrow::Status::OK(); return arrow::Status::IOError("Error while seeking"); @@ -97,6 +139,9 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile arrow::Result Read(int64_t nbytes, void *out) override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + CPLAssert(static_cast(static_cast(nbytes)) == nbytes); return static_cast( VSIFReadL(out, 1, static_cast(nbytes), m_fp)); @@ -104,6 +149,9 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile arrow::Result> Read(int64_t nbytes) override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + // CPLDebug("ARROW", "Reading %d bytes", int(nbytes)); // Ugly hack for https://github.com/OSGeo/gdal/issues/9497 if (CPLGetConfigOption("OGR_ARROW_STOP_IO", nullptr)) @@ -122,8 +170,54 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile return buffer; } +#ifdef OGR_ARROW_USE_PREAD + using arrow::io::RandomAccessFile::ReadAt; + + arrow::Result> + ReadAt(int64_t position, int64_t nbytes) override + { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + + if (m_bUsePRead) + { + auto buffer = arrow::AllocateResizableBuffer(nbytes); + if (!buffer.ok()) + { + return buffer; + } + if (m_bDebugReadAt) + { + CPLDebug( + "ARROW", + "Start ReadAt() called on %s (this=%p) from " + "thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64, + m_osFilename.c_str(), this, CPLGetPID(), position, nbytes); + } + uint8_t *buffer_data = (*buffer)->mutable_data(); + auto nread = m_fp->PRead(buffer_data, static_cast(nbytes), + static_cast(position)); + CPL_IGNORE_RET_VAL( + (*buffer)->Resize(nread)); // shrink --> cannot fail + if (m_bDebugReadAt) + { + CPLDebug( + "ARROW", + "End ReadAt() called on %s (this=%p) from " + "thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64, + m_osFilename.c_str(), this, CPLGetPID(), position, nbytes); + } + return buffer; + } + return arrow::io::RandomAccessFile::ReadAt(position, nbytes); + } +#endif + arrow::Result GetSize() override { + if (m_bAskedToClosed) + return arrow::Status::IOError("File requested to close"); + if (m_nSize < 0) { const auto nPos = VSIFTellL(m_fp); diff --git a/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp b/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp index 6a259e5d880b..0f50d4162ebe 100644 --- a/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp +++ b/ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp @@ -33,6 +33,12 @@ #include "ograrrowrandomaccessfile.h" +#include +#include +#include +#include +#include + /************************************************************************/ /* VSIArrowFileSystem */ /************************************************************************/ @@ -42,14 +48,54 @@ class VSIArrowFileSystem final : public arrow::fs::FileSystem const std::string m_osEnvVarPrefix; const std::string m_osQueryParameters; + std::atomic m_bAskedToClosed = false; + std::mutex m_oMutex{}; + std::vector>> + m_oSetFiles{}; + public: - explicit VSIArrowFileSystem(const std::string &osEnvVarPrefix, - const std::string &osQueryParameters) + VSIArrowFileSystem(const std::string &osEnvVarPrefix, + const std::string &osQueryParameters) : m_osEnvVarPrefix(osEnvVarPrefix), m_osQueryParameters(osQueryParameters) { } + // Cf comment in OGRParquetDataset::~OGRParquetDataset() for rationale + // for this method + void AskToClose() + { + m_bAskedToClosed = true; + std::vector< + std::pair>> + oSetFiles; + { + std::lock_guard oLock(m_oMutex); + oSetFiles = m_oSetFiles; + } + for (auto &[osName, poFile] : oSetFiles) + { + bool bWarned = false; + while (!poFile.expired()) + { + if (!bWarned) + { + bWarned = true; + auto poFileLocked = poFile.lock(); + if (poFileLocked) + { + CPLDebug("PARQUET", + "Still on-going reads on %s. Waiting for it " + "to be closed.", + osName.c_str()); + poFileLocked->AskToClose(); + } + } + CPLSleep(0.01); + } + } + } + std::string type_name() const override { return "vsi" + m_osEnvVarPrefix; @@ -203,6 +249,10 @@ class VSIArrowFileSystem final : public arrow::fs::FileSystem arrow::Result> OpenInputFile(const std::string &path) override { + if (m_bAskedToClosed) + return arrow::Status::IOError( + "OpenInputFile(): file system in shutdown"); + std::string osPath(path); osPath += m_osQueryParameters; CPLDebugOnly(m_osEnvVarPrefix.c_str(), "Opening %s", osPath.c_str()); @@ -210,7 +260,13 @@ class VSIArrowFileSystem final : public arrow::fs::FileSystem if (fp == nullptr) return arrow::Status::IOError("OpenInputFile() failed for " + osPath); - return std::make_shared(std::move(fp)); + auto poFile = + std::make_shared(osPath, std::move(fp)); + { + std::lock_guard oLock(m_oMutex); + m_oSetFiles.emplace_back(path, poFile); + } + return poFile; } using arrow::fs::FileSystem::OpenOutputStream; diff --git a/ogr/ogrsf_frmts/parquet/ogr_parquet.h b/ogr/ogrsf_frmts/parquet/ogr_parquet.h index 9323f54bac2b..1b4fd97c6404 100644 --- a/ogr/ogrsf_frmts/parquet/ogr_parquet.h +++ b/ogr/ogrsf_frmts/parquet/ogr_parquet.h @@ -304,9 +304,12 @@ class OGRParquetDatasetLayer final : public OGRParquetLayerBase class OGRParquetDataset final : public OGRArrowDataset { + std::shared_ptr m_poFS{}; + public: explicit OGRParquetDataset( const std::shared_ptr &poMemoryPool); + ~OGRParquetDataset(); OGRLayer *ExecuteSQL(const char *pszSQLCommand, OGRGeometry *poSpatialFilter, @@ -314,6 +317,11 @@ class OGRParquetDataset final : public OGRArrowDataset void ReleaseResultSet(OGRLayer *poResultsSet) override; int TestCapability(const char *) override; + + void SetFileSystem(const std::shared_ptr &fs) + { + m_poFS = fs; + } }; /************************************************************************/ diff --git a/ogr/ogrsf_frmts/parquet/ogrparquetdataset.cpp b/ogr/ogrsf_frmts/parquet/ogrparquetdataset.cpp index a8d53bc139db..d0707cdacb09 100644 --- a/ogr/ogrsf_frmts/parquet/ogrparquetdataset.cpp +++ b/ogr/ogrsf_frmts/parquet/ogrparquetdataset.cpp @@ -32,6 +32,7 @@ #include "../arrow_common/ograrrowdataset.hpp" #include "../arrow_common/ograrrowlayer.hpp" +#include "../arrow_common/vsiarrowfilesystem.hpp" /************************************************************************/ /* OGRParquetDataset() */ @@ -43,6 +44,25 @@ OGRParquetDataset::OGRParquetDataset( { } +/************************************************************************/ +/* ~OGRParquetDataset() */ +/************************************************************************/ + +OGRParquetDataset::~OGRParquetDataset() +{ + // libarrow might continue to do I/O in auxiliary threads on the underlying + // files when using the arrow::dataset API even after we closed the dataset. + // This is annoying as it can cause crashes when closing GDAL, in particular + // the virtual file manager, as this could result in VSI files being + // accessed after their VSIVirtualFileSystem has been destroyed, resulting + // in crashes. The workaround is to make sure that VSIArrowFileSystem + // waits for all file handles it is aware of to have been destroyed. + close(); + auto poFS = std::dynamic_pointer_cast(m_poFS); + if (poFS) + poFS->AskToClose(); +} + /***********************************************************************/ /* ExecuteSQL() */ /***********************************************************************/ diff --git a/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp b/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp index 096d7bb9abda..f3ab16ce77c0 100644 --- a/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp +++ b/ogr/ogrsf_frmts/parquet/ogrparquetdriver.cpp @@ -51,7 +51,8 @@ static GDALDataset *OpenFromDatasetFactory( const std::string &osBasePath, const std::shared_ptr &factory, - CSLConstList papszOpenOptions) + CSLConstList papszOpenOptions, + const std::shared_ptr &fs) { std::shared_ptr dataset; PARQUET_ASSIGN_OR_THROW(dataset, factory->Finish()); @@ -65,6 +66,7 @@ static GDALDataset *OpenFromDatasetFactory( poDS.get(), CPLGetBasename(osBasePath.c_str()), bIsVSI, dataset, papszOpenOptions); poDS->SetLayer(std::move(poLayer)); + poDS->SetFileSystem(fs); return poDS.release(); } @@ -134,7 +136,7 @@ static GDALDataset *OpenParquetDatasetWithMetadata( std::make_shared(), std::move(options))); - return OpenFromDatasetFactory(osBasePath, factory, papszOpenOptions); + return OpenFromDatasetFactory(osBasePath, factory, papszOpenOptions, fs); } /************************************************************************/ @@ -182,7 +184,7 @@ OpenParquetDatasetWithoutMetadata(const std::string &osBasePathIn, std::move(options))); } - return OpenFromDatasetFactory(osBasePath, factory, papszOpenOptions); + return OpenFromDatasetFactory(osBasePath, factory, papszOpenOptions, fs); } #endif @@ -448,7 +450,8 @@ static GDALDataset *OGRParquetDriverOpen(GDALOpenInfo *poOpenInfo) if (fp == nullptr) return nullptr; } - infile = std::make_shared(std::move(fp)); + infile = std::make_shared(osFilename, + std::move(fp)); } else { From b0f02448ce504967dfdb3a0b82cebbe4d7175a3c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 4 Sep 2024 06:23:19 +0200 Subject: [PATCH 009/710] ogr2ogr: remove hack related to Parquet dataset --- apps/ogr2ogr_lib.cpp | 40 ------------------- .../arrow_common/ograrrowrandomaccessfile.h | 5 --- 2 files changed, 45 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index e42155fdfa49..2f58dc31b6b9 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -5747,46 +5747,6 @@ bool LayerTranslator::TranslateArrow( schema.release(&schema); - // Ugly hack to work around https://github.com/OSGeo/gdal/issues/9497 - // Deleting a RecordBatchReader obtained from arrow::dataset::Scanner.ToRecordBatchReader() - // is a lengthy operation since all batches are read in its destructors. - // Here we ask to our custom I/O layer to return in error to short circuit - // that lengthy operation. - if (auto poDS = psInfo->m_poSrcLayer->GetDataset()) - { - if (poDS->GetLayerCount() == 1 && poDS->GetDriver() && - EQUAL(poDS->GetDriver()->GetDescription(), "PARQUET")) - { - bool bStopIO = false; - const char *pszArrowStopIO = - CPLGetConfigOption("OGR_ARROW_STOP_IO", nullptr); - if (pszArrowStopIO && CPLTestBool(pszArrowStopIO)) - { - bStopIO = true; - } - else if (!pszArrowStopIO) - { - std::string osExePath; - osExePath.resize(1024); - if (CPLGetExecPath(osExePath.data(), - static_cast(osExePath.size()))) - { - osExePath.resize(strlen(osExePath.data())); - if (strcmp(CPLGetBasename(osExePath.data()), "ogr2ogr") == - 0) - { - bStopIO = true; - } - } - } - if (bStopIO) - { - CPLSetConfigOption("OGR_ARROW_STOP_IO", "YES"); - CPLDebug("OGR2OGR", "Forcing interruption of Parquet I/O"); - } - } - } - stream.release(&stream); return bRet; } diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h b/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h index a1ab3d4d34c6..a1c4a463459a 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h @@ -153,11 +153,6 @@ class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile return arrow::Status::IOError("File requested to close"); // CPLDebug("ARROW", "Reading %d bytes", int(nbytes)); - // Ugly hack for https://github.com/OSGeo/gdal/issues/9497 - if (CPLGetConfigOption("OGR_ARROW_STOP_IO", nullptr)) - { - return arrow::Result>(); - } auto buffer = arrow::AllocateResizableBuffer(nbytes); if (!buffer.ok()) { From 3d6c01007858d17b9fbb05b29d468f757a6846c4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 5 Sep 2024 13:35:40 +0200 Subject: [PATCH 010/710] WMTS: when reading a WMTS capabilities file, use in priority Operation.GetCapabilities.DCP.HTTP to retrieve the URL Fixes #10732 --- autotest/gdrivers/wmts.py | 1 + frmts/wmts/wmtsdataset.cpp | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/autotest/gdrivers/wmts.py b/autotest/gdrivers/wmts.py index 93e117939475..cfa9a1341577 100755 --- a/autotest/gdrivers/wmts.py +++ b/autotest/gdrivers/wmts.py @@ -880,6 +880,7 @@ def test_wmts_15(): + """, ) diff --git a/frmts/wmts/wmtsdataset.cpp b/frmts/wmts/wmtsdataset.cpp index d2245c9ca301..2ec40f084451 100644 --- a/frmts/wmts/wmtsdataset.cpp +++ b/frmts/wmts/wmtsdataset.cpp @@ -1226,20 +1226,21 @@ GDALDataset *WMTSDataset::Open(GDALOpenInfo *poOpenInfo) if (STARTS_WITH(osGetCapabilitiesURL, "/vsimem/")) { - const char *pszHref = CPLGetXMLValue( - psXML, "=Capabilities.ServiceMetadataURL.href", nullptr); - if (pszHref) - osGetCapabilitiesURL = pszHref; + osGetCapabilitiesURL = GetOperationKVPURL(psXML, "GetCapabilities"); + if (osGetCapabilitiesURL.empty()) + { + // (ERO) I'm not even sure this is correct at all... + const char *pszHref = CPLGetXMLValue( + psXML, "=Capabilities.ServiceMetadataURL.href", nullptr); + if (pszHref) + osGetCapabilitiesURL = pszHref; + } else { - osGetCapabilitiesURL = GetOperationKVPURL(psXML, "GetCapabilities"); - if (!osGetCapabilitiesURL.empty()) - { - osGetCapabilitiesURL = - CPLURLAddKVP(osGetCapabilitiesURL, "service", "WMTS"); - osGetCapabilitiesURL = CPLURLAddKVP( - osGetCapabilitiesURL, "request", "GetCapabilities"); - } + osGetCapabilitiesURL = + CPLURLAddKVP(osGetCapabilitiesURL, "service", "WMTS"); + osGetCapabilitiesURL = CPLURLAddKVP(osGetCapabilitiesURL, "request", + "GetCapabilities"); } } CPLString osCapabilitiesFilename(osGetCapabilitiesURL); From 5d87e68976dab59a62f60453be637cfdd6c16003 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 5 Sep 2024 17:44:43 +0200 Subject: [PATCH 011/710] Doc: RasterIO(): clarify that the window of interest must be fully within the raster space --- gcore/gdaldataset.cpp | 6 ++++++ gcore/gdalrasterband.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/gcore/gdaldataset.cpp b/gcore/gdaldataset.cpp index 712157b057ca..674610dd4604 100644 --- a/gcore/gdaldataset.cpp +++ b/gcore/gdaldataset.cpp @@ -2598,6 +2598,12 @@ CPLErr GDALDataset::ValidateRasterIOOrAdviseReadParameters( * buffer size (nBufXSize x nBufYSize) is different than the size of the * region being accessed (nXSize x nYSize). * + * The window of interest expressed by (nXOff, nYOff, nXSize, nYSize) should be + * fully within the raster space, that is nXOff >= 0, nYOff >= 0, + * nXOff + nXSize <= GetRasterXSize() and nYOff + nYSize <= GetRasterYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * The nPixelSpace, nLineSpace and nBandSpace parameters allow reading into or * writing from various organization of buffers. * diff --git a/gcore/gdalrasterband.cpp b/gcore/gdalrasterband.cpp index 93dbe886bed4..f1917bf19f8a 100644 --- a/gcore/gdalrasterband.cpp +++ b/gcore/gdalrasterband.cpp @@ -131,6 +131,12 @@ GDALRasterBand::~GDALRasterBand() * buffer size (nBufXSize x nBufYSize) is different than the size of the * region being accessed (nXSize x nYSize). * + * The window of interest expressed by (nXOff, nYOff, nXSize, nYSize) should be + * fully within the raster space, that is nXOff >= 0, nYOff >= 0, + * nXOff + nXSize <= GetXSize() and nYOff + nYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * The nPixelSpace and nLineSpace parameters allow reading into or * writing from unusually organized buffers. This is primarily used * for buffers containing more than one bands raster data in interleaved @@ -232,6 +238,12 @@ GDALRasterBand::~GDALRasterBand() * buffer size (nBufXSize x nBufYSize) is different than the size of the * region being accessed (nXSize x nYSize). * + * The window of interest expressed by (nXOff, nYOff, nXSize, nYSize) should be + * fully within the raster space, that is nXOff >= 0, nYOff >= 0, + * nXOff + nXSize <= GetXSize() and nYOff + nYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * The nPixelSpace and nLineSpace parameters allow reading into or * writing from unusually organized buffers. This is primarily used * for buffers containing more than one bands raster data in interleaved @@ -580,6 +592,12 @@ DEFINE_GetGDTFromCppType(std::complex, GDT_CFloat64); * be called on the same GDALRasterBand instance (or another GDALRasterBand * instance of this dataset) concurrently from several threads. * + * The window of interest expressed by (dfXOff, dfYOff, dfXSize, dfYSize) should be + * fully within the raster space, that is dfXOff >= 0, dfYOff >= 0, + * dfXOff + dfXSize <= GetXSize() and dfYOff + dfYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * @param[out] pData The buffer into which the data should be written. * This buffer must contain at least nBufXSize * * nBufYSize words of type T. It is organized in left to right, @@ -796,6 +814,12 @@ INSTANTIATE_READ_RASTER(std::complex) * be called on the same GDALRasterBand instance (or another GDALRasterBand * instance of this dataset) concurrently from several threads. * + * The window of interest expressed by (dfXOff, dfYOff, dfXSize, dfYSize) should be + * fully within the raster space, that is dfXOff >= 0, dfYOff >= 0, + * dfXOff + dfXSize <= GetXSize() and dfYOff + dfYSize <= GetYSize(). + * If reads larger than the raster space are wished, GDALTranslate() might be used. + * Or use nLineSpace and a possibly shifted pData value. + * * @param[out] vData The vector into which the data should be written. * The vector will be resized, if needed, to contain at least nBufXSize * * nBufYSize values. The values in the vector are organized in left to right, From cff8a490c5813e498268af7ba8a99d2a75d47180 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 6 Sep 2024 18:54:47 +0200 Subject: [PATCH 012/710] RFC101 text: move open question to design discussion (answering 'no') --- .../rfc101_raster_dataset_threadsafety.rst | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst index 82b62e96ce7a..79c2c29a2969 100644 --- a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -235,7 +235,7 @@ Usage and design limitations thus no sharing of cached blocks between threads. However, this should not be a too severe limitation for algorithms where threads process independent regions of the raster, hence reuse of cached blocks - would be inexistent or low. Optimal algorithms will make sure to work on + would be non-existent or low. Optimal algorithms will make sure to work on regions of interest aligned on the block size (this advice also applies for the current approach of manually opening a dataset for each worker thread). @@ -416,52 +416,50 @@ or in the usage of some methods of GDALDataset, GDALRasterBand and related objec (particularly existing non-virtual methods of those classes that could happen to have a non thread-safe implementation) -Open questions --------------- - -The question in this section needs to be solved during the discussion phase of -the RFC. - -For the unusual situations where a dataset cannot be reopened and thus -GDALCreateThreadSafeDataset() fails, should we provide an additional ``bForce`` -argument to force it to still return a dataset, where calls to the wrapped -dataset are protected by a mutex? This would enable to always write multi-thread -safe code, even if the access to the dataset is serialized. -Similarly we could have a -``std::unique_ptr GDALCreateThreadSafeRasterBand(GDALRasterBand* poBand, int nOpenFlags, bool bForce)`` -function that would try to use GDALCreateThreadSafeDataset() internally if it -manages to identify the dataset to which the band belongs to, and otherwise would -fallback to protecting calls to the wrapped band with a mutex. - Design discussion ----------------- This paragraph discusses a number of thoughts that arose during the writing of this RFC. -A significantly different alternative could have consisted in adding native -thread-safety in each driver. But this is not realistic for the following reasons: +1. A significantly different alternative could have consisted in adding native + thread-safety in each driver. But this is not realistic for the following reasons: + + * if that was feasible, it would require considerable development effort to + rework each drivers. So realistically, only a few select drivers would be updated. + + * Even updating a reduced number of drivers would be extremely difficult, in + particular the GeoTIFF one, due to the underlying library not being reentrant, + and deferred loading strategies and many state variables being modified even + by read-only APIs. And this applies to most typical drivers. + + * Due to the inevitable locks, there would be a (small) cost bore by callers + even on single-thread uses of thread-safe native drivers. -* if that was feasible, it would require considerable development effort to - rework each drivers. So realistically, only a few select drivers would be updated. + * Some core mechanisms, particularly around the per-band block cache structures, + are not currently thread-safe. -* Even updating a reduced number of drivers would be extremely difficult, in - particular the GeoTIFF one, due to the underlying library not being reentrant, - and deferred loading strategies and many state variables being modified even - by read-only APIs. And this applies to most typical drivers. +2. A variant of the proposed implementation that did not use thread-local storage + has been initially attempted. It stored instead a + ``std::map>`` on each GDALThreadSafeDataset + instance. This implementation was simpler, but unfortunately suffered from high + lock contention since a mutex had to be taken around each access to this map, + with the contention increasing with the number of concurrent threads. -* Due to the inevitable locks, there would be a (small) cost bore by callers - even on single-thread uses of thread-safe native drivers. +3. For the unusual situations where a dataset cannot be reopened and thus + GDALCreateThreadSafeDataset() fails, should we provide an additional ``bForce`` + argument to force it to still return a dataset, where calls to the wrapped + dataset are protected by a mutex? This would enable to always write multi-thread + safe code, even if the access to the dataset is serialized. + Similarly we could have a + ``std::unique_ptr GDALCreateThreadSafeRasterBand(GDALRasterBand* poBand, int nOpenFlags, bool bForce)`` + function that would try to use GDALCreateThreadSafeDataset() internally if it + manages to identify the dataset to which the band belongs to, and otherwise would + fallback to protecting calls to the wrapped band with a mutex. -* Some core mechanisms, particularly around the per-band block cache structures, - are not currently thread-safe. + Given the absence of evidence that such option is necessary, this has been excluded + from the scope of this RFC. -A variant of the proposed implementation that did not use thread-local storage -has been initially attempted. It stored instead a -``std::map>`` on each GDALThreadSafeDataset -instance. This implementation was simpler, but unfortunately suffered from high -lock contention since a mutex had to be taken around each access to this map, -with the contention increasing with the number of concurrent threads. Related issues and PRs ---------------------- From 9dcfd4b03a6c97a16b151f7d5f6e5f04180d69e3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 6 Sep 2024 19:50:07 +0200 Subject: [PATCH 013/710] RFC101: more accurate conclusion of 4096x4096 100 times experiment --- .../rfc101_raster_dataset_threadsafety.rst | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst index 79c2c29a2969..af9fdbeb3625 100644 --- a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -361,23 +361,8 @@ For example on a 20x20 raster: sys 0m0.029s -But on a 4096x4096 raster with a number of iterations reduced to 100. the -thread_safe mode is slightly faster: - -.. code-block:: shell - - $ time multireadtest -t 4 -i 100 4096x4096.tif - real 0m4.845s - user 0m18.666s - sys 0m0.084s - - vs - - $ time multireadtest -thread_safe -t 4 -i 100 4096x4096.tif - real 0m4.255s - user 0m16.370s - sys 0m0.096s - +But on a 4096x4096 raster with a number of iterations reduced to 100, the +timings between the default and thread_safe modes are very similar. A Python equivalent of multireadtest has been written. Scalability depends on how much Python code is executed. If relatively few long-enough calls to From fb67a7d65c271c2883f532b386225c6659221fa4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 7 Sep 2024 14:49:40 +0200 Subject: [PATCH 014/710] RFC101 text: rename to GDALGetThreadSafeDataset() --- .../rfc101_raster_dataset_threadsafety.rst | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst index af9fdbeb3625..dbffd042d82b 100644 --- a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -80,13 +80,13 @@ The corresponding C function is added: CSLConstList papszOptions); -A new C++ function, GDALCreateThreadSafeDataset, is added with two forms: +A new C++ function, GDALGetThreadSafeDataset, is added with two forms: .. code-block:: c++ - std::unique_ptr GDALCreateThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags); + std::unique_ptr GDALGetThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags); - GDALDataset* GDALCreateThreadSafeDataset(GDALDataset* poDS, int nScopeFlags); + GDALDataset* GDALGetThreadSafeDataset(GDALDataset* poDS, int nScopeFlags); This function accepts a (generally non thread-safe) source dataset and return a new dataset that is a thread-safe wrapper around it, or the source dataset if @@ -105,7 +105,7 @@ patterns like the following one are valid: .. code-block:: c++ auto poDS = GDALDataset::Open(...); - GDALDataset* poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); + GDALDataset* poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); poDS->ReleaseRef(); // can be done here, or any time later if (poThreadSafeDS ) { @@ -123,7 +123,7 @@ The corresponding C function for the second form is added: .. code-block:: c - GDALDatasetH GDALCreateThreadSafeDataset(GDALDatasetH hDS, int nScopeFlags, CSLConstList papszOptions); + GDALDatasetH GDALGetThreadSafeDataset(GDALDatasetH hDS, int nScopeFlags, CSLConstList papszOptions); Usage examples @@ -152,7 +152,7 @@ Example of a function processing a whole dataset passed as an object: void foo(GDALDataset* poDS) { - GDALDataset* poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); + GDALDataset* poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER); if( poThreadSafeDS ) { // TODO: spawn threads using poThreadSafeDS @@ -179,7 +179,7 @@ Example of a function processing a single band passed as an object: // Check that poBand has a matching owing dataset if( poDS && poDS->GetRasterBand(poBand->GetBand()) == poBand ) { - poThreadSafeDS = GDALCreateThreadSafeDataset(poDS, GDAL_OF_RASTER); + poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER); if( poThreadSafeDS ) poThreadSafeBand = poThreadSafeDS->GetBand(poBand->GetBand()); } @@ -206,7 +206,7 @@ The new C macro and functions are bound to SWIG as: - ``gdal.OF_THREAD_SAFE`` - :py:func:`Dataset.IsThreadSafe(nScopeFlags)` -- :py:func:`Dataset.CreateThreadSafeDataset(nScopeFlags)`. The Python +- :py:func:`Dataset.GetThreadSafeDataset(nScopeFlags)`. The Python implementation of this method takes care of keeping a reference on the source dataset in the returned thread-safe dataset, so the user does not have to care about their respective lifetimes. @@ -222,7 +222,7 @@ Usage and design limitations (1024 on most Linux systems) could be hit if working with a sufficiently large number of worker threads and/or instances of GDALThreadSafeDataset. -* The generic implementation of GDALCreateThreadSafeDataset assumes that the +* The generic implementation of GDALGetThreadSafeDataset assumes that the source dataset can be re-opened by its name (GetDescription()), which is the case for datasets opened by GDALOpenEx(). A special implementation is also made for dataset instances of the MEM driver. But, there is currently no @@ -266,7 +266,7 @@ The gist of the implementation lies in a new file ``gcore/gdalthreadsafedataset. which defines several classes (internal details): - ``GDALThreadSafeDataset`` extending :cpp:class:`GDALProxyDataset`. - Instances of that class are returned by GDALCreateThreadSafeDataset(). + Instances of that class are returned by GDALGetThreadSafeDataset(). On instantiation, it creates as many GDALThreadSafeRasterBand instances as the number of bands of the source dataset. All virtual methods of GDALDataset are redefined by GDALProxyDataset. @@ -432,13 +432,13 @@ this RFC. with the contention increasing with the number of concurrent threads. 3. For the unusual situations where a dataset cannot be reopened and thus - GDALCreateThreadSafeDataset() fails, should we provide an additional ``bForce`` + GDALGetThreadSafeDataset() fails, should we provide an additional ``bForce`` argument to force it to still return a dataset, where calls to the wrapped dataset are protected by a mutex? This would enable to always write multi-thread safe code, even if the access to the dataset is serialized. Similarly we could have a - ``std::unique_ptr GDALCreateThreadSafeRasterBand(GDALRasterBand* poBand, int nOpenFlags, bool bForce)`` - function that would try to use GDALCreateThreadSafeDataset() internally if it + ``std::unique_ptr GDALGetThreadSafeRasterBand(GDALRasterBand* poBand, int nOpenFlags, bool bForce)`` + function that would try to use GDALGetThreadSafeDataset() internally if it manages to identify the dataset to which the band belongs to, and otherwise would fallback to protecting calls to the wrapped band with a mutex. From b60fccd05aaecbd4fa689e6a63960b53035009b8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 7 Sep 2024 18:47:34 +0200 Subject: [PATCH 015/710] RFC101: points to pull request for candidate implementation --- .../development/rfc/rfc101_raster_dataset_threadsafety.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst index dbffd042d82b..641afa164175 100644 --- a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -449,7 +449,7 @@ this RFC. Related issues and PRs ---------------------- -- Candidate implementation: https://github.com/OSGeo/gdal/compare/master...rouault:gdal:raster_multi_thread?expand=1 +- Candidate implementation: https://github.com/OSGeo/gdal/pull/10746 - https://github.com/OSGeo/gdal/issues/8448: GTiff: Allow concurrent reading of single blocks From fa4c932d2a14ab861b1ab15ad7ea6cdaf7d79bf0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 16:51:12 +0200 Subject: [PATCH 016/710] ogr2ogr: implement GetInverse() in CompositeCT and AxisMappingCoordinateTransformation --- apps/ogr2ogr_lib.cpp | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 16b3539ddd42..42dea3bce9dc 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -1205,6 +1206,14 @@ class GCPCoordTransformation : public OGRCoordinateTransformation virtual OGRCoordinateTransformation *GetInverse() const override { + static std::once_flag flag; + std::call_once(flag, + []() + { + CPLDebug("OGR2OGR", + "GCPCoordTransformation::GetInverse() " + "called, but not implemented"); + }); return nullptr; } }; @@ -1292,7 +1301,21 @@ class CompositeCT : public OGRCoordinateTransformation virtual OGRCoordinateTransformation *GetInverse() const override { - return nullptr; + if (!poCT1 && !poCT2) + return nullptr; + if (!poCT2) + return poCT1->GetInverse(); + if (!poCT1) + return poCT2->GetInverse(); + auto poInvCT1 = + std::unique_ptr(poCT1->GetInverse()); + auto poInvCT2 = + std::unique_ptr(poCT2->GetInverse()); + if (!poInvCT1 || !poInvCT2) + return nullptr; + return std::make_unique(poInvCT2.release(), true, + poInvCT1.release(), true) + .release(); } }; @@ -1302,9 +1325,14 @@ class CompositeCT : public OGRCoordinateTransformation class AxisMappingCoordinateTransformation : public OGRCoordinateTransformation { - public: bool bSwapXY = false; + explicit AxisMappingCoordinateTransformation(bool bSwapXYIn) + : bSwapXY(bSwapXYIn) + { + } + + public: AxisMappingCoordinateTransformation(const std::vector &mappingIn, const std::vector &mappingOut) { @@ -1360,7 +1388,7 @@ class AxisMappingCoordinateTransformation : public OGRCoordinateTransformation virtual OGRCoordinateTransformation *GetInverse() const override { - return nullptr; + return new AxisMappingCoordinateTransformation(bSwapXY); } }; From ffb6004c25503d80d00575524c44c4adb8bccadd Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 15:40:08 +0200 Subject: [PATCH 017/710] OGRGeometryFactory::transformWithOptions(): deal with polar or anti-meridian discontinuities when going from projected to (any) geographic CRS The logic already existed but was restricted to WGS 84 without any good reason. --- autotest/cpp/test_ogr.cpp | 69 ++++++++++++++++++++ ogr/ogrgeometryfactory.cpp | 125 ++++++++++++++++++++++++------------- 2 files changed, 150 insertions(+), 44 deletions(-) diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp index e75d0b6253b9..8fe4e7469977 100644 --- a/autotest/cpp/test_ogr.cpp +++ b/autotest/cpp/test_ogr.cpp @@ -4201,4 +4201,73 @@ TEST_F(test_ogr, OGRCurve_reversePoints) } } +// Test OGRGeometryFactory::transformWithOptions() +TEST_F(test_ogr, transformWithOptions) +{ + // Projected CRS to national geographic CRS (not including poles or antimeridian) + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "LINESTRING(700000 6600000, 700001 6600001)", nullptr, &poGeom); + ASSERT_NE(poGeom, nullptr); + + OGRSpatialReference oEPSG_2154; + oEPSG_2154.importFromEPSG(2154); // "RGF93 v1 / Lambert-93" + OGRSpatialReference oEPSG_4171; + oEPSG_4171.importFromEPSG(4171); // "RGF93 v1" + oEPSG_4171.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + auto poCT = std::unique_ptr( + OGRCreateCoordinateTransformation(&oEPSG_2154, &oEPSG_4171)); + OGRGeometryFactory::TransformWithOptionsCache oCache; + poGeom = OGRGeometryFactory::transformWithOptions(poGeom, poCT.get(), + nullptr, oCache); + EXPECT_NEAR(poGeom->toLineString()->getX(0), 3, 1e-8); + EXPECT_NEAR(poGeom->toLineString()->getY(0), 46.5, 1e-8); + + delete poGeom; + } + +#ifdef HAVE_GEOS + // Projected CRS to national geographic CRS including antimeridian + { + OGRGeometry *poGeom = nullptr; + OGRGeometryFactory::createFromWkt( + "LINESTRING(657630.64 4984896.17,815261.43 4990738.26)", nullptr, + &poGeom); + ASSERT_NE(poGeom, nullptr); + + OGRSpatialReference oEPSG_6329; + oEPSG_6329.importFromEPSG(6329); // "NAD83(2011) / UTM zone 60N" + OGRSpatialReference oEPSG_6318; + oEPSG_6318.importFromEPSG(6318); // "NAD83(2011)" + oEPSG_6318.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + auto poCT = std::unique_ptr( + OGRCreateCoordinateTransformation(&oEPSG_6329, &oEPSG_6318)); + OGRGeometryFactory::TransformWithOptionsCache oCache; + poGeom = OGRGeometryFactory::transformWithOptions(poGeom, poCT.get(), + nullptr, oCache); + EXPECT_EQ(poGeom->getGeometryType(), wkbMultiLineString); + if (poGeom->getGeometryType() == wkbMultiLineString) + { + const auto poMLS = poGeom->toMultiLineString(); + EXPECT_EQ(poMLS->getNumGeometries(), 2); + if (poMLS->getNumGeometries() == 2) + { + const auto poLS = poMLS->getGeometryRef(0); + EXPECT_EQ(poLS->getNumPoints(), 2); + if (poLS->getNumPoints() == 2) + { + EXPECT_NEAR(poLS->getX(0), 179, 1e-6); + EXPECT_NEAR(poLS->getY(0), 45, 1e-6); + EXPECT_NEAR(poLS->getX(1), 180, 1e-6); + EXPECT_NEAR(poLS->getY(1), 45.004384301691303, 1e-6); + } + } + } + + delete poGeom; + } +#endif +} + } // namespace diff --git a/ogr/ogrgeometryfactory.cpp b/ogr/ogrgeometryfactory.cpp index 90583391997e..532028a25507 100644 --- a/ogr/ogrgeometryfactory.cpp +++ b/ogr/ogrgeometryfactory.cpp @@ -3302,15 +3302,15 @@ static void AlterPole(OGRGeometry *poGeom, OGRPoint *poPole, } /************************************************************************/ -/* IsPolarToWGS84() */ +/* IsPolarToGeographic() */ /* */ /* Returns true if poCT transforms from a projection that includes one */ /* of the pole in a continuous way. */ /************************************************************************/ -static bool IsPolarToWGS84(OGRCoordinateTransformation *poCT, - OGRCoordinateTransformation *poRevCT, - bool &bIsNorthPolarOut) +static bool IsPolarToGeographic(OGRCoordinateTransformation *poCT, + OGRCoordinateTransformation *poRevCT, + bool &bIsNorthPolarOut) { bool bIsNorthPolar = false; bool bIsSouthPolar = false; @@ -3366,14 +3366,14 @@ static bool IsPolarToWGS84(OGRCoordinateTransformation *poCT, } /************************************************************************/ -/* TransformBeforePolarToWGS84() */ +/* TransformBeforePolarToGeographic() */ /* */ /* Transform the geometry (by intersection), so as to cut each geometry */ /* that crosses the pole, in 2 parts. Do also tricks for geometries */ /* that just touch the pole. */ /************************************************************************/ -static std::unique_ptr TransformBeforePolarToWGS84( +static std::unique_ptr TransformBeforePolarToGeographic( OGRCoordinateTransformation *poRevCT, bool bIsNorthPolar, std::unique_ptr poDstGeom, bool &bNeedPostCorrectionOut) { @@ -3449,15 +3449,15 @@ static std::unique_ptr TransformBeforePolarToWGS84( } /************************************************************************/ -/* IsAntimeridianProjToWGS84() */ +/* IsAntimeridianProjToGeographic() */ /* */ /* Returns true if poCT transforms from a projection that includes the */ /* antimeridian in a continuous way. */ /************************************************************************/ -static bool IsAntimeridianProjToWGS84(OGRCoordinateTransformation *poCT, - OGRCoordinateTransformation *poRevCT, - OGRGeometry *poDstGeometry) +static bool IsAntimeridianProjToGeographic(OGRCoordinateTransformation *poCT, + OGRCoordinateTransformation *poRevCT, + OGRGeometry *poDstGeometry) { const bool bBackupEmitErrors = poCT->GetEmitErrors(); poRevCT->SetEmitErrors(false); @@ -3633,13 +3633,13 @@ struct SortPointsByAscendingY }; /************************************************************************/ -/* TransformBeforeAntimeridianToWGS84() */ +/* TransformBeforeAntimeridianToGeographic() */ /* */ /* Transform the geometry (by intersection), so as to cut each geometry */ /* that crosses the antimeridian, in 2 parts. */ /************************************************************************/ -static std::unique_ptr TransformBeforeAntimeridianToWGS84( +static std::unique_ptr TransformBeforeAntimeridianToGeographic( OGRCoordinateTransformation *poCT, OGRCoordinateTransformation *poRevCT, std::unique_ptr poDstGeom, bool &bNeedPostCorrectionOut) { @@ -3820,9 +3820,22 @@ static void SnapCoordsCloseToLatLongBounds(OGRGeometry *poGeom) struct OGRGeometryFactory::TransformWithOptionsCache::Private { + const OGRSpatialReference *poSourceCRS = nullptr; + const OGRSpatialReference *poTargetCRS = nullptr; + const OGRCoordinateTransformation *poCT = nullptr; std::unique_ptr poRevCT{}; bool bIsPolar = false; bool bIsNorthPolar = false; + + void clear() + { + poSourceCRS = nullptr; + poTargetCRS = nullptr; + poCT = nullptr; + poRevCT.reset(); + bIsPolar = false; + bIsNorthPolar = false; + } }; /************************************************************************/ @@ -3858,52 +3871,76 @@ OGRGeometry *OGRGeometryFactory::transformWithOptions( char **papszOptions, CPL_UNUSED const TransformWithOptionsCache &cache) { auto poDstGeom = std::unique_ptr(poSrcGeom->clone()); - if (poCT != nullptr) + if (poCT) { #ifdef HAVE_GEOS bool bNeedPostCorrection = false; - - auto poSourceCRS = poCT->GetSourceCS(); - auto poTargetCRS = poCT->GetTargetCS(); - if (poSourceCRS != nullptr && poTargetCRS != nullptr && - poSourceCRS->IsProjected() && poTargetCRS->IsGeographic()) + const auto poSourceCRS = poCT->GetSourceCS(); + const auto poTargetCRS = poCT->GetTargetCS(); + const auto eSrcGeomType = wkbFlatten(poSrcGeom->getGeometryType()); + // Check if we are transforming from projected coordinates to + // geographic coordinates, with a chance that there might be polar or + // anti-meridian discontinuities. If so, create the inverse transform. + if (eSrcGeomType != wkbPoint && eSrcGeomType != wkbMultiPoint && + (poSourceCRS != cache.d->poSourceCRS || + poTargetCRS != cache.d->poTargetCRS || poCT != cache.d->poCT)) { - OGRSpatialReference oSRSWGS84; - oSRSWGS84.SetWellKnownGeogCS("WGS84"); - oSRSWGS84.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (poTargetCRS->IsSame(&oSRSWGS84)) + cache.d->clear(); + cache.d->poSourceCRS = poSourceCRS; + cache.d->poTargetCRS = poTargetCRS; + cache.d->poCT = poCT; + if (poSourceCRS && poTargetCRS && poSourceCRS->IsProjected() && + poTargetCRS->IsGeographic() && + poTargetCRS->GetAxisMappingStrategy() == + OAMS_TRADITIONAL_GIS_ORDER && + // check that angular units is degree + std::fabs(poTargetCRS->GetAngularUnits(nullptr) - + CPLAtof(SRS_UA_DEGREE_CONV)) <= + 1e-8 * CPLAtof(SRS_UA_DEGREE_CONV)) { - if (cache.d->poRevCT == nullptr || - !cache.d->poRevCT->GetTargetCS()->IsSame(poSourceCRS)) + double dfWestLong = 0.0; + double dfSouthLat = 0.0; + double dfEastLong = 0.0; + double dfNorthLat = 0.0; + if (poSourceCRS->GetAreaOfUse(&dfWestLong, &dfSouthLat, + &dfEastLong, &dfNorthLat, + nullptr) && + !(dfSouthLat == -90.0 || dfNorthLat == 90.0 || + dfWestLong == -180.0 || dfEastLong == 180.0 || + dfWestLong > dfEastLong)) + { + // Not a global geographic CRS + } + else { cache.d->poRevCT.reset(OGRCreateCoordinateTransformation( - &oSRSWGS84, poSourceCRS)); + poTargetCRS, poSourceCRS)); cache.d->bIsNorthPolar = false; cache.d->bIsPolar = false; + cache.d->poRevCT.reset(poCT->GetInverse()); if (cache.d->poRevCT && - IsPolarToWGS84(poCT, cache.d->poRevCT.get(), - cache.d->bIsNorthPolar)) + IsPolarToGeographic(poCT, cache.d->poRevCT.get(), + cache.d->bIsNorthPolar)) { cache.d->bIsPolar = true; } } - auto poRevCT = cache.d->poRevCT.get(); - if (poRevCT != nullptr) - { - if (cache.d->bIsPolar) - { - poDstGeom = TransformBeforePolarToWGS84( - poRevCT, cache.d->bIsNorthPolar, - std::move(poDstGeom), bNeedPostCorrection); - } - else if (IsAntimeridianProjToWGS84(poCT, poRevCT, - poDstGeom.get())) - { - poDstGeom = TransformBeforeAntimeridianToWGS84( - poCT, poRevCT, std::move(poDstGeom), - bNeedPostCorrection); - } - } + } + } + + if (auto poRevCT = cache.d->poRevCT.get()) + { + if (cache.d->bIsPolar) + { + poDstGeom = TransformBeforePolarToGeographic( + poRevCT, cache.d->bIsNorthPolar, std::move(poDstGeom), + bNeedPostCorrection); + } + else if (IsAntimeridianProjToGeographic(poCT, poRevCT, + poDstGeom.get())) + { + poDstGeom = TransformBeforeAntimeridianToGeographic( + poCT, poRevCT, std::move(poDstGeom), bNeedPostCorrection); } } #endif From 1e6c00c468f8f1134215ac041d535eeec6a0ee86 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 20:21:59 +0200 Subject: [PATCH 018/710] OGRCoordinateTransformation::TransformWithErrorCodes(): optimize when nCount == 1 --- ogr/ogrct.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ogr/ogrct.cpp b/ogr/ogrct.cpp index 49b83af8b096..4c62b1dc091b 100644 --- a/ogr/ogrct.cpp +++ b/ogr/ogrct.cpp @@ -2232,6 +2232,18 @@ int OGRCoordinateTransformation::TransformWithErrorCodes(size_t nCount, int *panErrorCodes) { + if (nCount == 1) + { + int nSuccess = 0; + const bool bOverallSuccess = + CPL_TO_BOOL(Transform(nCount, x, y, z, t, &nSuccess)); + if (panErrorCodes) + { + panErrorCodes[0] = nSuccess ? 0 : -1; + } + return bOverallSuccess; + } + std::vector abSuccess; try { From cfc5ff0c37fdfc0ce28527f4669b6d89d61f2120 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 20:28:40 +0200 Subject: [PATCH 019/710] OGRProjCT::TransformWithErrorCodes(): speed-up by avoiding OSRGetProjTLSContext() when possible --- ogr/ogrct.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ogr/ogrct.cpp b/ogr/ogrct.cpp index 4c62b1dc091b..afc3bbb489ea 100644 --- a/ogr/ogrct.cpp +++ b/ogr/ogrct.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include "cpl_conv.h" #include "cpl_error.h" @@ -765,6 +766,9 @@ class OGRProjCT : public OGRCoordinateTransformation double dfThreshold = 0.0; + PJ_CONTEXT *m_psLastContext = nullptr; + std::thread::id m_nLastContextThreadId{}; + PjPtr m_pj{}; bool m_bReversePj = false; @@ -1277,9 +1281,10 @@ OGRProjCT::OGRProjCT(const OGRProjCT &other) m_osTargetSRS(other.m_osTargetSRS), bWebMercatorToWGS84LongLat(other.bWebMercatorToWGS84LongLat), nErrorCount(other.nErrorCount), dfThreshold(other.dfThreshold), - m_pj(other.m_pj), m_bReversePj(other.m_bReversePj), - m_bEmitErrors(other.m_bEmitErrors), bNoTransform(other.bNoTransform), - m_eStrategy(other.m_eStrategy), + m_psLastContext(nullptr), + m_nLastContextThreadId(std::this_thread::get_id()), m_pj(other.m_pj), + m_bReversePj(other.m_bReversePj), m_bEmitErrors(other.m_bEmitErrors), + bNoTransform(other.bNoTransform), m_eStrategy(other.m_eStrategy), m_oTransformations(other.m_oTransformations), m_iCurTransformation(other.m_iCurTransformation), m_options(other.m_options) @@ -2544,7 +2549,15 @@ int OGRProjCT::TransformWithErrorCodes(size_t nCount, double *x, double *y, /* Select dynamically the best transformation for the data, if */ /* needed. */ /* -------------------------------------------------------------------- */ - auto ctx = OSRGetProjTLSContext(); + PJ_CONTEXT *ctx = m_psLastContext; + const auto nThisThreadId = std::this_thread::get_id(); + if (!ctx || nThisThreadId != m_nLastContextThreadId) + { + m_nLastContextThreadId = nThisThreadId; + m_psLastContext = OSRGetProjTLSContext(); + ctx = m_psLastContext; + } + PJ *pj = m_pj; if (!bTransformDone && !pj) { From 4c410fddd70d3388bf1ada7faa523c9d85959d15 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 20:43:21 +0200 Subject: [PATCH 020/710] ogr2ogr: implement CompositeCT::TransformWithErrorCodes() --- apps/ogr2ogr_lib.cpp | 59 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 42dea3bce9dc..9d28764d9d98 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -1224,20 +1224,24 @@ class GCPCoordTransformation : public OGRCoordinateTransformation class CompositeCT : public OGRCoordinateTransformation { + OGRCoordinateTransformation *const poCT1; + const bool bOwnCT1; + OGRCoordinateTransformation *const poCT2; + const bool bOwnCT2; + + // Working buffer + std::vector m_anErrorCode{}; + CompositeCT(const CompositeCT &other) : poCT1(other.poCT1 ? other.poCT1->Clone() : nullptr), bOwnCT1(true), - poCT2(other.poCT2 ? other.poCT2->Clone() : nullptr), bOwnCT2(true) + poCT2(other.poCT2 ? other.poCT2->Clone() : nullptr), bOwnCT2(true), + m_anErrorCode({}) { } CompositeCT &operator=(const CompositeCT &) = delete; public: - OGRCoordinateTransformation *poCT1; - bool bOwnCT1; - OGRCoordinateTransformation *poCT2; - bool bOwnCT2; - CompositeCT(OGRCoordinateTransformation *poCT1In, bool bOwnCT1In, OGRCoordinateTransformation *poCT2In, bool bOwnCT2In) : poCT1(poCT1In), bOwnCT1(bOwnCT1In), poCT2(poCT2In), bOwnCT2(bOwnCT2In) @@ -1299,6 +1303,35 @@ class CompositeCT : public OGRCoordinateTransformation return nResult; } + virtual int TransformWithErrorCodes(size_t nCount, double *x, double *y, + double *z, double *t, + int *panErrorCodes) override + { + if (poCT1 && poCT2 && panErrorCodes) + { + m_anErrorCode.resize(nCount); + int nResult = poCT1->TransformWithErrorCodes(nCount, x, y, z, t, + m_anErrorCode.data()); + if (nResult) + nResult = poCT2->TransformWithErrorCodes(nCount, x, y, z, t, + panErrorCodes); + for (size_t i = 0; i < nCount; ++i) + { + if (m_anErrorCode[i]) + panErrorCodes[i] = m_anErrorCode[i]; + } + return nResult; + } + int nResult = TRUE; + if (poCT1) + nResult = poCT1->TransformWithErrorCodes(nCount, x, y, z, t, + panErrorCodes); + if (nResult && poCT2) + nResult = poCT2->TransformWithErrorCodes(nCount, x, y, z, t, + panErrorCodes); + return nResult; + } + virtual OGRCoordinateTransformation *GetInverse() const override { if (!poCT1 && !poCT2) @@ -1386,6 +1419,20 @@ class AxisMappingCoordinateTransformation : public OGRCoordinateTransformation return true; } + virtual int TransformWithErrorCodes(size_t nCount, double *x, double *y, + double * /*z*/, double * /*t*/, + int *panErrorCodes) override + { + for (size_t i = 0; i < nCount; i++) + { + if (panErrorCodes) + panErrorCodes[i] = 0; + if (bSwapXY) + std::swap(x[i], y[i]); + } + return true; + } + virtual OGRCoordinateTransformation *GetInverse() const override { return new AxisMappingCoordinateTransformation(bSwapXY); From 6f88dcf12dc38211306e1e5a6982547cc740b6ad Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 22:02:25 +0200 Subject: [PATCH 021/710] Add OGRWKBTransform() for in-place coordinate transformation of WKB geometries --- autotest/cpp/test_ogr_wkb.cpp | 350 +++++++++++++++++++++++++++++++++ ogr/ogr_wkb.cpp | 351 ++++++++++++++++++++++++++++++++++ ogr/ogr_wkb.h | 42 ++++ 3 files changed, 743 insertions(+) diff --git a/autotest/cpp/test_ogr_wkb.cpp b/autotest/cpp/test_ogr_wkb.cpp index ed86e66fd6a6..dba3eedc46a0 100644 --- a/autotest/cpp/test_ogr_wkb.cpp +++ b/autotest/cpp/test_ogr_wkb.cpp @@ -524,4 +524,354 @@ INSTANTIATE_TEST_SUITE_P( OGRWKBIntersectsPessimisticFixture::ParamType> &l_info) { return std::get<6>(l_info.param); }); +class OGRWKBTransformFixture + : public test_ogr_wkb, + public ::testing::WithParamInterface< + std::tuple> +{ + public: + static std::vector< + std::tuple> + GetTupleValues() + { + return { + std::make_tuple("POINT EMPTY", wkbNDR, "POINT EMPTY", + "POINT_EMPTY_NDR"), + std::make_tuple("POINT EMPTY", wkbXDR, "POINT EMPTY", + "POINT_EMPTY_XDR"), + std::make_tuple("POINT (1 2)", wkbNDR, "POINT (2 4)", "POINT_NDR"), + std::make_tuple("POINT (1 2)", wkbXDR, "POINT (2 4)", "POINT_XDR"), + std::make_tuple("POINT Z EMPTY", wkbNDR, "POINT Z EMPTY", + "POINT_Z_EMPTY_NDR"), + std::make_tuple("POINT Z EMPTY", wkbXDR, "POINT Z EMPTY", + "POINT_Z_EMPTY_XDR"), + std::make_tuple("POINT Z (1 2 3)", wkbNDR, "POINT Z (2 4 6)", + "POINT_Z_NDR"), + std::make_tuple("POINT Z (1 2 3)", wkbXDR, "POINT Z (2 4 6)", + "POINT_Z_XDR"), + std::make_tuple("POINT M EMPTY", wkbNDR, "POINT M EMPTY", + "POINT_M_EMPTY_NDR"), + std::make_tuple("POINT M EMPTY", wkbXDR, "POINT M EMPTY", + "POINT_M_EMPTY_XDR"), + std::make_tuple("POINT M (1 2 -10)", wkbNDR, "POINT M (2 4 -10)", + "POINT_M_NDR"), + std::make_tuple("POINT M (1 2 -10)", wkbXDR, "POINT M (2 4 -10)", + "POINT_M_XDR"), + std::make_tuple("POINT ZM EMPTY", wkbNDR, "POINT ZM EMPTY", + "POINT_ZM_EMPTY_NDR"), + std::make_tuple("POINT ZM EMPTY", wkbXDR, "POINT ZM EMPTY", + "POINT_ZM_EMPTY_XDR"), + std::make_tuple("POINT ZM (1 2 3 10)", wkbNDR, + "POINT ZM (2 4 6 10)", "POINT_ZM_NDR"), + std::make_tuple("POINT ZM (1 2 3 10)", wkbXDR, + "POINT ZM (2 4 6 10)", "POINT_ZM_XDR"), + + std::make_tuple("LINESTRING EMPTY", wkbNDR, "LINESTRING EMPTY", + "LINESTRING_EMPTY"), + std::make_tuple("LINESTRING (1 2,11 12)", wkbNDR, + "LINESTRING (2 4,12 14)", "LINESTRING_NDR"), + std::make_tuple("LINESTRING (1 2,11 12)", wkbXDR, + "LINESTRING (2 4,12 14)", "LINESTRING_XDR"), + std::make_tuple("LINESTRING Z EMPTY", wkbNDR, "LINESTRING Z EMPTY", + "LINESTRING_Z_EMPTY"), + std::make_tuple("LINESTRING Z (1 2 3,11 12 13)", wkbNDR, + "LINESTRING Z (2 4 6,12 14 16)", + "LINESTRING_Z_NDR"), + std::make_tuple("LINESTRING Z (1 2 3,11 12 13)", wkbXDR, + "LINESTRING Z (2 4 6,12 14 16)", + "LINESTRING_Z_XDR"), + std::make_tuple("LINESTRING M EMPTY", wkbNDR, "LINESTRING M EMPTY", + "LINESTRING_M_EMPTY"), + std::make_tuple("LINESTRING M (1 2 -10,11 12 -20)", wkbNDR, + "LINESTRING M (2 4 -10,12 14 -20)", + "LINESTRING_M_NDR"), + std::make_tuple("LINESTRING M (1 2 -10,11 12 -20)", wkbXDR, + "LINESTRING M (2 4 -10,12 14 -20)", + "LINESTRING_M_XDR"), + std::make_tuple("LINESTRING ZM EMPTY", wkbNDR, + "LINESTRING ZM EMPTY", "LINESTRING_ZM_EMPTY"), + std::make_tuple("LINESTRING ZM (1 2 3 -10,11 12 13 -20)", wkbNDR, + "LINESTRING ZM (2 4 6 -10,12 14 16 -20)", + "LINESTRING_ZM_NDR"), + std::make_tuple("LINESTRING ZM (1 2 3 -10,11 12 13 -20)", wkbXDR, + "LINESTRING ZM (2 4 6 -10,12 14 16 -20)", + "LINESTRING_ZM_XDR"), + + // I know the polygon is invalid, but this is enough for our purposes + std::make_tuple("POLYGON EMPTY", wkbNDR, "POLYGON EMPTY", + "POLYGON_EMPTY"), + std::make_tuple("POLYGON ((1 2,11 12))", wkbNDR, + "POLYGON ((2 4,12 14))", "POLYGON_NDR"), + std::make_tuple("POLYGON ((1 2,11 12))", wkbXDR, + "POLYGON ((2 4,12 14))", "POLYGON_XDR"), + std::make_tuple("POLYGON ((1 2,11 12),(21 22,31 32))", wkbNDR, + "POLYGON ((2 4,12 14),(22 24,32 34))", + "POLYGON_TWO_RINGS"), + std::make_tuple("POLYGON Z EMPTY", wkbNDR, "POLYGON Z EMPTY", + "POLYGON_Z_EMPTY"), + std::make_tuple("POLYGON Z ((1 2 3,11 12 13))", wkbNDR, + "POLYGON Z ((2 4 6,12 14 16))", "POLYGON_Z_NDR"), + std::make_tuple("POLYGON Z ((1 2 3,11 12 13))", wkbXDR, + "POLYGON Z ((2 4 6,12 14 16))", "POLYGON_Z_XDR"), + std::make_tuple("POLYGON M EMPTY", wkbNDR, "POLYGON M EMPTY", + "POLYGON_M_EMPTY"), + std::make_tuple("POLYGON M ((1 2 -10,11 12 -20))", wkbNDR, + "POLYGON M ((2 4 -10,12 14 -20))", "POLYGON_M_NDR"), + std::make_tuple("POLYGON M ((1 2 -10,11 12 -20))", wkbXDR, + "POLYGON M ((2 4 -10,12 14 -20))", "POLYGON_M_XDR"), + std::make_tuple("POLYGON ZM EMPTY", wkbNDR, "POLYGON ZM EMPTY", + "POLYGON_ZM_EMPTY"), + std::make_tuple("POLYGON ZM ((1 2 3 -10,11 12 13 -20))", wkbNDR, + "POLYGON ZM ((2 4 6 -10,12 14 16 -20))", + "POLYGON_ZM_NDR"), + std::make_tuple("POLYGON ZM ((1 2 3 -10,11 12 13 -20))", wkbXDR, + "POLYGON ZM ((2 4 6 -10,12 14 16 -20))", + "POLYGON_ZM_XDR"), + + std::make_tuple("MULTIPOINT EMPTY", wkbNDR, "MULTIPOINT EMPTY", + "MULTIPOINT_EMPTY_NDR"), + std::make_tuple("MULTIPOINT ((1 2),(11 12))", wkbNDR, + "MULTIPOINT ((2 4),(12 14))", "MULTIPOINT_NDR"), + std::make_tuple("MULTIPOINT Z ((1 2 3),(11 12 13))", wkbXDR, + "MULTIPOINT Z ((2 4 6),(12 14 16))", + "MULTIPOINT_Z_XDR"), + + std::make_tuple("MULTILINESTRING ((1 2,11 12))", wkbNDR, + "MULTILINESTRING ((2 4,12 14))", + "MULTILINESTRING_NDR"), + + std::make_tuple("MULTIPOLYGON (((1 2,11 12)))", wkbNDR, + "MULTIPOLYGON (((2 4,12 14)))", "MULTIPOLYGON_NDR"), + + std::make_tuple("GEOMETRYCOLLECTION (POLYGON ((1 2,11 12)))", + wkbNDR, + "GEOMETRYCOLLECTION (POLYGON ((2 4,12 14)))", + "GEOMETRYCOLLECTION_NDR"), + + std::make_tuple("CIRCULARSTRING (1 2,11 12,21 22)", wkbNDR, + "CIRCULARSTRING (2 4,12 14,22 24)", + "CIRCULARSTRING_NDR"), + + std::make_tuple("COMPOUNDCURVE ((1 2,11 12))", wkbNDR, + "COMPOUNDCURVE ((2 4,12 14))", "COMPOUNDCURVE_NDR"), + + std::make_tuple("CURVEPOLYGON ((1 2,11 12,21 22,1 2))", wkbNDR, + "CURVEPOLYGON ((2 4,12 14,22 24,2 4))", + "CURVEPOLYGON_NDR"), + + std::make_tuple("MULTICURVE ((1 2,11 12))", wkbNDR, + "MULTICURVE ((2 4,12 14))", "MULTICURVE_NDR"), + + std::make_tuple("MULTISURFACE (((1 2,11 12)))", wkbNDR, + "MULTISURFACE (((2 4,12 14)))", "MULTISURFACE_NDR"), + + std::make_tuple("TRIANGLE ((1 2,11 12,21 22,1 2))", wkbNDR, + "TRIANGLE ((2 4,12 14,22 24,2 4))", "TRIANGLE_NDR"), + + std::make_tuple("POLYHEDRALSURFACE (((1 2,11 12,21 22,1 2)))", + wkbNDR, + "POLYHEDRALSURFACE (((2 4,12 14,22 24,2 4)))", + "POLYHEDRALSURFACE_NDR"), + + std::make_tuple("TIN (((1 2,11 12,21 22,1 2)))", wkbNDR, + "TIN (((2 4,12 14,22 24,2 4)))", "TIN_NDR"), + }; + } +}; + +struct MyCT final : public OGRCoordinateTransformation +{ + const bool m_bSuccess; + + explicit MyCT(bool bSuccess = true) : m_bSuccess(bSuccess) + { + } + + const OGRSpatialReference *GetSourceCS() const override + { + return nullptr; + } + + const OGRSpatialReference *GetTargetCS() const override + { + return nullptr; + } + + int Transform(size_t nCount, double *x, double *y, double *z, double *, + int *pabSuccess) override + { + for (size_t i = 0; i < nCount; ++i) + { + x[i] += 1; + y[i] += 2; + if (z) + z[i] += 3; + if (pabSuccess) + pabSuccess[i] = m_bSuccess; + } + return true; + } + + OGRCoordinateTransformation *Clone() const override + { + return new MyCT(); + } + + OGRCoordinateTransformation *GetInverse() const override + { + return nullptr; + } // unused +}; + +TEST_P(OGRWKBTransformFixture, test) +{ + const char *pszInput = std::get<0>(GetParam()); + OGRwkbByteOrder eByteOrder = std::get<1>(GetParam()); + const char *pszOutput = std::get<2>(GetParam()); + + MyCT oCT; + oCT.GetSourceCS(); // just for code coverage purpose + oCT.GetTargetCS(); // just for code coverage purpose + delete oCT.Clone(); // just for code coverage purpose + delete oCT.GetInverse(); // just for code coverage purpose + + OGRGeometry *poGeom = nullptr; + EXPECT_EQ(OGRGeometryFactory::createFromWkt(pszInput, nullptr, &poGeom), + OGRERR_NONE); + ASSERT_TRUE(poGeom != nullptr); + std::vector abyWkb(poGeom->WkbSize()); + poGeom->exportToWkb(eByteOrder, abyWkb.data(), wkbVariantIso); + delete poGeom; + + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv; + EXPECT_TRUE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + const auto abyWkbOri = abyWkb; + + poGeom = nullptr; + OGRGeometryFactory::createFromWkb(abyWkb.data(), nullptr, &poGeom, + abyWkb.size()); + ASSERT_TRUE(poGeom != nullptr); + char *pszWKT = nullptr; + poGeom->exportToWkt(&pszWKT, wkbVariantIso); + delete poGeom; + EXPECT_STREQ(pszWKT, pszOutput); + CPLFree(pszWKT); + + { + CPLErrorHandlerPusher oErrorHandler(CPLQuietErrorHandler); + + // Truncated geometry + for (size_t i = 0; i < abyWkb.size(); ++i) + { + abyWkb = abyWkbOri; + EXPECT_FALSE(OGRWKBTransform(abyWkb.data(), i, &oCT, oCache, sEnv)); + } + + // Check altering all bytes + for (size_t i = 0; i < abyWkb.size(); ++i) + { + abyWkb = abyWkbOri; + abyWkb[i] = 0xff; + CPL_IGNORE_RET_VAL(OGRWKBTransform(abyWkb.data(), abyWkb.size(), + &oCT, oCache, sEnv)); + } + + if (abyWkb.size() > 9) + { + abyWkb = abyWkbOri; + if (!STARTS_WITH(pszInput, "POINT")) + { + // Corrupt number of sub-geometries + abyWkb[5] = 0xff; + abyWkb[6] = 0xff; + abyWkb[7] = 0xff; + abyWkb[8] = 0xff; + EXPECT_FALSE(OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, + oCache, sEnv)); + } + } + } +} + +INSTANTIATE_TEST_SUITE_P( + test_ogr_wkb, OGRWKBTransformFixture, + ::testing::ValuesIn(OGRWKBTransformFixture::GetTupleValues()), + [](const ::testing::TestParamInfo + &l_info) { return std::get<3>(l_info.param); }); + +TEST_F(test_ogr_wkb, OGRWKBTransformFixture_rec_collection) +{ + std::vector abyWkb; + constexpr int BEYOND_ALLOWED_RECURSION_LEVEL = 128; + for (int i = 0; i < BEYOND_ALLOWED_RECURSION_LEVEL; ++i) + { + abyWkb.push_back(wkbNDR); + abyWkb.push_back(wkbGeometryCollection); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(1); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + } + { + abyWkb.push_back(wkbNDR); + abyWkb.push_back(wkbGeometryCollection); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + abyWkb.push_back(0); + } + + MyCT oCT; + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv; + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); +} + +TEST_F(test_ogr_wkb, OGRWKBTransformFixture_ct_failure) +{ + MyCT oCT(/* bSuccess = */ false); + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv; + { + OGRPoint p(1, 2); + std::vector abyWkb(p.WkbSize()); + static_cast(p).exportToWkb(wkbNDR, abyWkb.data(), + wkbVariantIso); + + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + } + { + OGRLineString ls; + ls.addPoint(1, 2); + std::vector abyWkb(ls.WkbSize()); + static_cast(ls).exportToWkb(wkbNDR, abyWkb.data(), + wkbVariantIso); + + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + } + { + OGRPolygon p; + auto poLR = std::make_unique(); + poLR->addPoint(1, 2); + p.addRing(std::move(poLR)); + std::vector abyWkb(p.WkbSize()); + static_cast(p).exportToWkb(wkbNDR, abyWkb.data(), + wkbVariantIso); + + EXPECT_FALSE( + OGRWKBTransform(abyWkb.data(), abyWkb.size(), &oCT, oCache, sEnv)); + } +} + } // namespace diff --git a/ogr/ogr_wkb.cpp b/ogr/ogr_wkb.cpp index 6b96f39e0144..c2b39d03b3ce 100644 --- a/ogr/ogr_wkb.cpp +++ b/ogr/ogr_wkb.cpp @@ -1072,6 +1072,357 @@ void OGRWKBFixupCounterClockWiseExternalRing(GByte *pabyWkb, size_t nWKBSize) pabyWkb, nWKBSize, iOffsetInOut, /* nRec = */ 0); } +/************************************************************************/ +/* OGRWKBPointUpdater() */ +/************************************************************************/ + +OGRWKBPointUpdater::OGRWKBPointUpdater() = default; + +/************************************************************************/ +/* OGRWKBIntersectsPointSequencePessimistic() */ +/************************************************************************/ + +static bool OGRWKBUpdatePointsSequence(uint8_t *data, const size_t size, + OGRWKBPointUpdater &oUpdater, + const OGRwkbByteOrder eByteOrder, + const int nDim, const bool bHasZ, + const bool bHasM, size_t &iOffsetInOut) +{ + const uint32_t nPoints = + OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); + if (nPoints > (size - iOffsetInOut) / (nDim * sizeof(double))) + { + return false; + } + const bool bNeedSwap = OGR_SWAP(eByteOrder); + for (uint32_t j = 0; j < nPoints; j++) + { + void *pdfX = data + iOffsetInOut; + void *pdfY = data + iOffsetInOut + sizeof(double); + void *pdfZ = bHasZ ? data + iOffsetInOut + 2 * sizeof(double) : nullptr; + void *pdfM = + bHasM ? data + iOffsetInOut + (bHasZ ? 3 : 2) * sizeof(double) + : nullptr; + if (!oUpdater.update(bNeedSwap, pdfX, pdfY, pdfZ, pdfM)) + return false; + + iOffsetInOut += nDim * sizeof(double); + } + + return true; +} + +/************************************************************************/ +/* OGRWKBVisitRingSequence() */ +/************************************************************************/ + +static bool OGRWKBVisitRingSequence(uint8_t *data, const size_t size, + OGRWKBPointUpdater &oUpdater, + const OGRwkbByteOrder eByteOrder, + const int nDim, const bool bHasZ, + const bool bHasM, size_t &iOffsetInOut) +{ + const uint32_t nRings = + OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); + if (nRings > (size - iOffsetInOut) / sizeof(uint32_t)) + { + return false; + } + + for (uint32_t i = 0; i < nRings; ++i) + { + if (iOffsetInOut + sizeof(uint32_t) > size) + { + return false; + } + if (!OGRWKBUpdatePointsSequence(data, size, oUpdater, eByteOrder, nDim, + bHasZ, bHasM, iOffsetInOut)) + { + return false; + } + } + return true; +} + +/************************************************************************/ +/* OGRWKBUpdatePoints() */ +/************************************************************************/ + +static bool OGRWKBUpdatePoints(uint8_t *data, const size_t size, + OGRWKBPointUpdater &oUpdater, + size_t &iOffsetInOut, const int nRec) +{ + if (size - iOffsetInOut < MIN_WKB_SIZE) + { + return false; + } + const int nByteOrder = DB2_V72_FIX_BYTE_ORDER(data[iOffsetInOut]); + if (!(nByteOrder == wkbXDR || nByteOrder == wkbNDR)) + { + return false; + } + const OGRwkbByteOrder eByteOrder = static_cast(nByteOrder); + + OGRwkbGeometryType eGeometryType = wkbUnknown; + OGRReadWKBGeometryType(data + iOffsetInOut, wkbVariantIso, &eGeometryType); + iOffsetInOut += 5; + const auto eFlatType = wkbFlatten(eGeometryType); + + if (eFlatType == wkbGeometryCollection || eFlatType == wkbCompoundCurve || + eFlatType == wkbCurvePolygon || eFlatType == wkbMultiPoint || + eFlatType == wkbMultiLineString || eFlatType == wkbMultiPolygon || + eFlatType == wkbMultiCurve || eFlatType == wkbMultiSurface || + eFlatType == wkbPolyhedralSurface || eFlatType == wkbTIN) + { + if (nRec == 128) + return false; + + const uint32_t nParts = + OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); + if (nParts > (size - iOffsetInOut) / MIN_WKB_SIZE) + { + return false; + } + for (uint32_t k = 0; k < nParts; k++) + { + if (!OGRWKBUpdatePoints(data, size, oUpdater, iOffsetInOut, + nRec + 1)) + return false; + } + return true; + } + + const bool bHasZ = OGR_GT_HasZ(eGeometryType); + const bool bHasM = OGR_GT_HasM(eGeometryType); + const int nDim = 2 + (bHasZ ? 1 : 0) + (bHasM ? 1 : 0); + + if (eFlatType == wkbPoint) + { + if (size - iOffsetInOut < nDim * sizeof(double)) + return false; + void *pdfX = data + iOffsetInOut; + void *pdfY = data + iOffsetInOut + sizeof(double); + void *pdfZ = bHasZ ? data + iOffsetInOut + 2 * sizeof(double) : nullptr; + void *pdfM = + bHasM ? data + iOffsetInOut + (bHasZ ? 3 : 2) * sizeof(double) + : nullptr; + const bool bNeedSwap = OGR_SWAP(eByteOrder); + if (!oUpdater.update(bNeedSwap, pdfX, pdfY, pdfZ, pdfM)) + return false; + iOffsetInOut += nDim * sizeof(double); + return true; + } + + if (eFlatType == wkbLineString || eFlatType == wkbCircularString) + { + return OGRWKBUpdatePointsSequence(data, size, oUpdater, eByteOrder, + nDim, bHasZ, bHasM, iOffsetInOut); + } + + if (eFlatType == wkbPolygon || eFlatType == wkbTriangle) + { + return OGRWKBVisitRingSequence(data, size, oUpdater, eByteOrder, nDim, + bHasZ, bHasM, iOffsetInOut); + } + + CPLDebug("OGR", "Unknown WKB geometry type"); + return false; +} + +/** Visit all points of a WKB geometry to update them. + */ +bool OGRWKBUpdatePoints(GByte *pabyWkb, size_t nWKBSize, + OGRWKBPointUpdater &oUpdater) +{ + size_t iOffsetInOut = 0; + return OGRWKBUpdatePoints(pabyWkb, nWKBSize, oUpdater, iOffsetInOut, + /* nRec = */ 0); +} + +/************************************************************************/ +/* OGRWKBTransformCache::clear() */ +/************************************************************************/ + +#ifdef OGR_WKB_TRANSFORM_ALL_AT_ONCE +void OGRWKBTransformCache::clear() +{ + abNeedSwap.clear(); + abIsEmpty.clear(); + apdfX.clear(); + apdfY.clear(); + apdfZ.clear(); + apdfM.clear(); + adfX.clear(); + adfY.clear(); + adfZ.clear(); + adfM.clear(); + anErrorCodes.clear(); +} +#endif + +/************************************************************************/ +/* OGRWKBTransform() */ +/************************************************************************/ + +/** Visit all points of a WKB geometry to transform them. + */ +bool OGRWKBTransform(GByte *pabyWkb, size_t nWKBSize, + OGRCoordinateTransformation *poCT, + [[maybe_unused]] OGRWKBTransformCache &oCache, + OGREnvelope3D &sEnvelope) +{ +#ifndef OGR_WKB_TRANSFORM_ALL_AT_ONCE + struct OGRWKBPointUpdaterReproj final : public OGRWKBPointUpdater + { + OGRCoordinateTransformation *m_poCT; + OGREnvelope3D &m_sEnvelope; + + explicit OGRWKBPointUpdaterReproj(OGRCoordinateTransformation *poCTIn, + OGREnvelope3D &sEnvelopeIn) + : m_poCT(poCTIn), m_sEnvelope(sEnvelopeIn) + { + } + + bool update(bool bNeedSwap, void *x, void *y, void *z, + void * /* m */) override + { + double dfX, dfY, dfZ; + memcpy(&dfX, x, sizeof(double)); + memcpy(&dfY, y, sizeof(double)); + if (bNeedSwap) + { + CPL_SWAP64PTR(&dfX); + CPL_SWAP64PTR(&dfY); + } + if (!(std::isnan(dfX) && std::isnan(dfY))) + { + if (z) + { + memcpy(&dfZ, z, sizeof(double)); + if (bNeedSwap) + { + CPL_SWAP64PTR(&dfZ); + } + } + else + dfZ = 0; + int nErrorCode = 0; + m_poCT->TransformWithErrorCodes(1, &dfX, &dfY, &dfZ, nullptr, + &nErrorCode); + if (nErrorCode) + return false; + m_sEnvelope.Merge(dfX, dfY, dfZ); + if (bNeedSwap) + { + CPL_SWAP64PTR(&dfX); + CPL_SWAP64PTR(&dfY); + CPL_SWAP64PTR(&dfZ); + } + memcpy(x, &dfX, sizeof(double)); + memcpy(y, &dfY, sizeof(double)); + if (z) + memcpy(z, &dfZ, sizeof(double)); + } + return true; + } + + private: + OGRWKBPointUpdaterReproj(const OGRWKBPointUpdaterReproj &) = delete; + OGRWKBPointUpdaterReproj & + operator=(const OGRWKBPointUpdaterReproj &) = delete; + }; + + sEnvelope = OGREnvelope3D(); + OGRWKBPointUpdaterReproj oUpdater(poCT, sEnvelope); + return OGRWKBUpdatePoints(pabyWkb, nWKBSize, oUpdater); + +#else + struct OGRWKBPointUpdaterReproj final : public OGRWKBPointUpdater + { + OGRWKBTransformCache &m_oCache; + + explicit OGRWKBPointUpdaterReproj(OGRWKBTransformCache &oCacheIn) + : m_oCache(oCacheIn) + { + } + + bool update(bool bNeedSwap, void *x, void *y, void *z, + void * /* m */) override + { + m_oCache.abNeedSwap.push_back(bNeedSwap); + m_oCache.apdfX.push_back(x); + m_oCache.apdfY.push_back(y); + m_oCache.apdfZ.push_back(z); + return true; + } + }; + + oCache.clear(); + OGRWKBPointUpdaterReproj oUpdater(oCache); + if (!OGRWKBUpdatePoints(pabyWkb, nWKBSize, oUpdater)) + return false; + + oCache.adfX.resize(oCache.apdfX.size()); + oCache.adfY.resize(oCache.apdfX.size()); + oCache.adfZ.resize(oCache.apdfX.size()); + + for (size_t i = 0; i < oCache.apdfX.size(); ++i) + { + memcpy(&oCache.adfX[i], oCache.apdfX[i], sizeof(double)); + memcpy(&oCache.adfY[i], oCache.apdfY[i], sizeof(double)); + if (oCache.apdfZ[i]) + memcpy(&oCache.adfZ[i], oCache.apdfZ[i], sizeof(double)); + if (oCache.abNeedSwap[i]) + { + CPL_SWAP64PTR(&oCache.adfX[i]); + CPL_SWAP64PTR(&oCache.adfY[i]); + CPL_SWAP64PTR(&oCache.adfZ[i]); + } + oCache.abIsEmpty.push_back(std::isnan(oCache.adfX[i]) && + std::isnan(oCache.adfY[i])); + } + + oCache.anErrorCodes.resize(oCache.apdfX.size()); + poCT->TransformWithErrorCodes(static_cast(oCache.apdfX.size()), + oCache.adfX.data(), oCache.adfY.data(), + oCache.adfZ.data(), nullptr, + oCache.anErrorCodes.data()); + + for (size_t i = 0; i < oCache.apdfX.size(); ++i) + { + if (!oCache.abIsEmpty[i] && oCache.anErrorCodes[i]) + return false; + } + + sEnvelope = OGREnvelope3D(); + for (size_t i = 0; i < oCache.apdfX.size(); ++i) + { + if (oCache.abIsEmpty[i]) + { + oCache.adfX[i] = std::numeric_limits::quiet_NaN(); + oCache.adfY[i] = std::numeric_limits::quiet_NaN(); + oCache.adfZ[i] = std::numeric_limits::quiet_NaN(); + } + else + { + sEnvelope.Merge(oCache.adfX[i], oCache.adfY[i], oCache.adfZ[i]); + } + if (oCache.abNeedSwap[i]) + { + CPL_SWAP64PTR(&oCache.adfX[i]); + CPL_SWAP64PTR(&oCache.adfY[i]); + CPL_SWAP64PTR(&oCache.adfZ[i]); + } + memcpy(oCache.apdfX[i], &oCache.adfX[i], sizeof(double)); + memcpy(oCache.apdfY[i], &oCache.adfY[i], sizeof(double)); + if (oCache.apdfZ[i]) + memcpy(oCache.apdfZ[i], &oCache.adfZ[i], sizeof(double)); + } + + return true; +#endif +} + /************************************************************************/ /* OGRAppendBuffer() */ /************************************************************************/ diff --git a/ogr/ogr_wkb.h b/ogr/ogr_wkb.h index 2c3a9770dc89..ef4bd409702f 100644 --- a/ogr/ogr_wkb.h +++ b/ogr/ogr_wkb.h @@ -32,6 +32,8 @@ #include "cpl_port.h" #include "ogr_core.h" +#include + bool CPL_DLL OGRWKBGetGeomType(const GByte *pabyWkb, size_t nWKBSize, bool &bNeedSwap, uint32_t &nType); bool OGRWKBPolygonGetArea(const GByte *&pabyWkb, size_t &nWKBSize, @@ -62,6 +64,46 @@ void CPL_DLL OGRWKBFixupCounterClockWiseExternalRing(GByte *pabyWkb, const GByte CPL_DLL *WKBFromEWKB(GByte *pabyEWKB, size_t nEWKBSize, size_t &nWKBSizeOut, int *pnSRIDOut); +/** Object to update point coordinates in a WKB geometry */ +class CPL_DLL OGRWKBPointUpdater +{ + public: + OGRWKBPointUpdater(); + virtual ~OGRWKBPointUpdater() = default; + + /** Update method */ + virtual bool update(bool bNeedSwap, void *x, void *y, void *z, void *m) = 0; +}; + +bool CPL_DLL OGRWKBUpdatePoints(GByte *pabyWkb, size_t nWKBSize, + OGRWKBPointUpdater &oUpdater); + +/** Transformation cache */ +struct CPL_DLL OGRWKBTransformCache +{ +#ifdef OGR_WKB_TRANSFORM_ALL_AT_ONCE + std::vector abNeedSwap{}; + std::vector abIsEmpty{}; + std::vector apdfX{}; + std::vector apdfY{}; + std::vector apdfZ{}; + std::vector apdfM{}; + std::vector adfX{}; + std::vector adfY{}; + std::vector adfZ{}; + std::vector adfM{}; + std::vector anErrorCodes{}; + + void clear(); +#endif +}; + +class OGRCoordinateTransformation; +bool CPL_DLL OGRWKBTransform(GByte *pabyWkb, size_t nWKBSize, + OGRCoordinateTransformation *poCT, + OGRWKBTransformCache &oCache, + OGREnvelope3D &sEnvelope); + /************************************************************************/ /* OGRAppendBuffer */ /************************************************************************/ From d4cafcd7d8e63ccfadad2a77c44d66c3e501a7d8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 22:02:46 +0200 Subject: [PATCH 022/710] GPKG: ST_Transform(): use OGRWKBTransform() --- ogr/ogrsf_frmts/gpkg/ogr_geopackage.h | 3 + .../gpkg/ogrgeopackagedatasource.cpp | 35 ++++++-- ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.cpp | 81 +++++++++++++++++++ ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.h | 4 + 4 files changed, 115 insertions(+), 8 deletions(-) diff --git a/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h b/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h index 46fc95e25509..720bdb56e9ed 100644 --- a/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h +++ b/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h @@ -37,6 +37,7 @@ #include "cpl_threadsafe_queue.hpp" #include "ograrrowarrayhelper.h" #include "ogr_p.h" +#include "ogr_wkb.h" #include #include @@ -185,6 +186,8 @@ class GDALGeoPackageDataset final : public OGRSQLiteBaseDataSource, int m_nLastCachedCTSrcSRId = -1; int m_nLastCachedCTDstSRId = -1; std::unique_ptr m_poLastCachedCT{}; + OGRWKBTransformCache m_oWKBTransformCache{}; + std::vector m_abyWKBTransformCache{}; int m_nOverviewCount = 0; GDALGeoPackageDataset **m_papoOverviewDS = nullptr; diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp index 61ee94970372..3bc3888fbb9f 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp @@ -8905,21 +8905,40 @@ void OGRGeoPackageTransform(sqlite3_context *pContext, int argc, poCT = poDS->m_poLastCachedCT.get(); } - auto poGeom = std::unique_ptr( - GPkgGeometryToOGR(pabyBLOB, nBLOBLen, nullptr)); - if (poGeom == nullptr) + if (sHeader.nHeaderLen >= 8) { - // Try also spatialite geometry blobs - OGRGeometry *poGeomSpatialite = nullptr; - if (OGRSQLiteImportSpatiaLiteGeometry(pabyBLOB, nBLOBLen, - &poGeomSpatialite) != OGRERR_NONE) + std::vector &abyNewBLOB = poDS->m_abyWKBTransformCache; + abyNewBLOB.resize(nBLOBLen); + memcpy(abyNewBLOB.data(), pabyBLOB, nBLOBLen); + + OGREnvelope3D oEnv3d; + if (!OGRWKBTransform(abyNewBLOB.data() + sHeader.nHeaderLen, + nBLOBLen - sHeader.nHeaderLen, poCT, + poDS->m_oWKBTransformCache, oEnv3d) || + !GPkgUpdateHeader(abyNewBLOB.data(), nBLOBLen, nDestSRID, + oEnv3d.MinX, oEnv3d.MaxX, oEnv3d.MinY, + oEnv3d.MaxY, oEnv3d.MinZ, oEnv3d.MaxZ)) { CPLError(CE_Failure, CPLE_AppDefined, "Invalid geometry"); sqlite3_result_blob(pContext, nullptr, 0, nullptr); return; } - poGeom.reset(poGeomSpatialite); + + sqlite3_result_blob(pContext, abyNewBLOB.data(), nBLOBLen, + SQLITE_TRANSIENT); + return; + } + + // Try also spatialite geometry blobs + OGRGeometry *poGeomSpatialite = nullptr; + if (OGRSQLiteImportSpatiaLiteGeometry(pabyBLOB, nBLOBLen, + &poGeomSpatialite) != OGRERR_NONE) + { + CPLError(CE_Failure, CPLE_AppDefined, "Invalid geometry"); + sqlite3_result_blob(pContext, nullptr, 0, nullptr); + return; } + auto poGeom = std::unique_ptr(poGeomSpatialite); if (poGeom->transform(poCT) != OGRERR_NONE) { diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.cpp index 70210463a029..4dd2b74f13a9 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.cpp @@ -487,6 +487,87 @@ OGRErr GPkgHeaderFromWKB(const GByte *pabyGpkg, size_t nGpkgLen, return OGRERR_NONE; } +bool GPkgUpdateHeader(GByte *pabyGpkg, size_t nGpkgLen, int nSrsId, double MinX, + double MaxX, double MinY, double MaxY, double MinZ, + double MaxZ) +{ + CPLAssert(nGpkgLen >= 8); + + /* Flags */ + const GByte byFlags = pabyGpkg[3]; + const auto eByteOrder = static_cast(byFlags & 0x01); + const OGRBoolean bSwap = OGR_SWAP(eByteOrder); + + /* SrsId */ + if (bSwap) + { + nSrsId = CPL_SWAP32(nSrsId); + } + memcpy(pabyGpkg + 4, &nSrsId, 4); + + /* Envelope */ + const int iEnvelope = (byFlags & (0x07 << 1)) >> 1; + int nEnvelopeDim = 0; + if (iEnvelope) + { + if (iEnvelope == 1) + { + nEnvelopeDim = 2; /* 2D envelope */ + } + else if (iEnvelope == 2) + { + nEnvelopeDim = 3; /* 2D+Z envelope */ + } + else if (iEnvelope == 3) + { + nEnvelopeDim = 3; /* 2D+M envelope */ + } + else if (iEnvelope == 4) + { + nEnvelopeDim = 4; /* 2D+ZM envelope */ + } + else + { + return false; + } + } + else + { + return true; + } + + if (nGpkgLen < static_cast(8 + 8 * 2 * nEnvelopeDim)) + { + // Not enough bytes + return false; + } + + /* Envelope */ + if (bSwap) + { + CPL_SWAPDOUBLE(&(MinX)); + CPL_SWAPDOUBLE(&(MaxX)); + CPL_SWAPDOUBLE(&(MinY)); + CPL_SWAPDOUBLE(&(MaxY)); + CPL_SWAPDOUBLE(&(MinZ)); + CPL_SWAPDOUBLE(&(MaxZ)); + } + + double *padPtr = reinterpret_cast(pabyGpkg + 8); + memcpy(&padPtr[0], &MinX, sizeof(double)); + memcpy(&padPtr[1], &MaxX, sizeof(double)); + memcpy(&padPtr[2], &MinY, sizeof(double)); + memcpy(&padPtr[3], &MaxY, sizeof(double)); + + if (iEnvelope == 2 || iEnvelope == 4) + { + memcpy(&padPtr[4], &MinZ, sizeof(double)); + memcpy(&padPtr[5], &MaxZ, sizeof(double)); + } + + return true; +} + OGRGeometry *GPkgGeometryToOGR(const GByte *pabyGpkg, size_t nGpkgLen, OGRSpatialReference *poSrs) { diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.h b/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.h index 19b07248994e..57c252c0a3b5 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.h +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackageutility.h @@ -72,4 +72,8 @@ bool OGRGeoPackageGetHeader(sqlite3_context *pContext, int /*argc*/, bool bNeedExtent, bool bNeedExtent3D, int iGeomIdx = 0); +bool GPkgUpdateHeader(GByte *pabyGpkg, size_t nGpkgLen, int nSrsId, double MinX, + double MaxX, double MinY, double MaxY, double MinZ, + double MaxZ); + #endif From 620cbba885e5225a0ab1811615f3d599b1e8150c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 22:03:12 +0200 Subject: [PATCH 023/710] Add OGRGeometryFactory::isTransformWithOptionsRegularTransform() --- ogr/ogr_geometry.h | 6 +++ ogr/ogrgeometryfactory.cpp | 89 ++++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/ogr/ogr_geometry.h b/ogr/ogr_geometry.h index e1f9de8c3575..0c680130791d 100644 --- a/ogr/ogr_geometry.h +++ b/ogr/ogr_geometry.h @@ -4285,6 +4285,12 @@ class CPL_DLL OGRGeometryFactory ~TransformWithOptionsCache(); }; + //! @cond Doxygen_Suppress + static bool isTransformWithOptionsRegularTransform( + const OGRSpatialReference *poSourceCRS, + const OGRSpatialReference *poTargetCRS, CSLConstList papszOptions); + //! @endcond + static OGRGeometry *transformWithOptions( const OGRGeometry *poSrcGeom, OGRCoordinateTransformation *poCT, char **papszOptions, diff --git a/ogr/ogrgeometryfactory.cpp b/ogr/ogrgeometryfactory.cpp index 532028a25507..ff9bad541cb8 100644 --- a/ogr/ogrgeometryfactory.cpp +++ b/ogr/ogrgeometryfactory.cpp @@ -3855,6 +3855,50 @@ OGRGeometryFactory::TransformWithOptionsCache::~TransformWithOptionsCache() { } +/************************************************************************/ +/* isTransformWithOptionsRegularTransform() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress +/*static */ +bool OGRGeometryFactory::isTransformWithOptionsRegularTransform( + [[maybe_unused]] const OGRSpatialReference *poSourceCRS, + [[maybe_unused]] const OGRSpatialReference *poTargetCRS, + CSLConstList papszOptions) +{ + if (papszOptions) + return false; + +#ifdef HAVE_GEOS + if (poSourceCRS->IsProjected() && poTargetCRS->IsGeographic() && + poTargetCRS->GetAxisMappingStrategy() == OAMS_TRADITIONAL_GIS_ORDER && + // check that angular units is degree + std::fabs(poTargetCRS->GetAngularUnits(nullptr) - + CPLAtof(SRS_UA_DEGREE_CONV)) <= + 1e-8 * CPLAtof(SRS_UA_DEGREE_CONV)) + { + double dfWestLong = 0.0; + double dfSouthLat = 0.0; + double dfEastLong = 0.0; + double dfNorthLat = 0.0; + if (poSourceCRS->GetAreaOfUse(&dfWestLong, &dfSouthLat, &dfEastLong, + &dfNorthLat, nullptr) && + !(dfSouthLat == -90.0 || dfNorthLat == 90.0 || + dfWestLong == -180.0 || dfEastLong == 180.0 || + dfWestLong > dfEastLong)) + { + // Not a global geographic CRS + return true; + } + return false; + } +#endif + + return true; +} + +//! @endcond + /************************************************************************/ /* transformWithOptions() */ /************************************************************************/ @@ -3889,41 +3933,20 @@ OGRGeometry *OGRGeometryFactory::transformWithOptions( cache.d->poSourceCRS = poSourceCRS; cache.d->poTargetCRS = poTargetCRS; cache.d->poCT = poCT; - if (poSourceCRS && poTargetCRS && poSourceCRS->IsProjected() && - poTargetCRS->IsGeographic() && - poTargetCRS->GetAxisMappingStrategy() == - OAMS_TRADITIONAL_GIS_ORDER && - // check that angular units is degree - std::fabs(poTargetCRS->GetAngularUnits(nullptr) - - CPLAtof(SRS_UA_DEGREE_CONV)) <= - 1e-8 * CPLAtof(SRS_UA_DEGREE_CONV)) + if (poSourceCRS && poTargetCRS && + !isTransformWithOptionsRegularTransform( + poSourceCRS, poTargetCRS, papszOptions)) { - double dfWestLong = 0.0; - double dfSouthLat = 0.0; - double dfEastLong = 0.0; - double dfNorthLat = 0.0; - if (poSourceCRS->GetAreaOfUse(&dfWestLong, &dfSouthLat, - &dfEastLong, &dfNorthLat, - nullptr) && - !(dfSouthLat == -90.0 || dfNorthLat == 90.0 || - dfWestLong == -180.0 || dfEastLong == 180.0 || - dfWestLong > dfEastLong)) + cache.d->poRevCT.reset(OGRCreateCoordinateTransformation( + poTargetCRS, poSourceCRS)); + cache.d->bIsNorthPolar = false; + cache.d->bIsPolar = false; + cache.d->poRevCT.reset(poCT->GetInverse()); + if (cache.d->poRevCT && + IsPolarToGeographic(poCT, cache.d->poRevCT.get(), + cache.d->bIsNorthPolar)) { - // Not a global geographic CRS - } - else - { - cache.d->poRevCT.reset(OGRCreateCoordinateTransformation( - poTargetCRS, poSourceCRS)); - cache.d->bIsNorthPolar = false; - cache.d->bIsPolar = false; - cache.d->poRevCT.reset(poCT->GetInverse()); - if (cache.d->poRevCT && - IsPolarToGeographic(poCT, cache.d->poRevCT.get(), - cache.d->bIsNorthPolar)) - { - cache.d->bIsPolar = true; - } + cache.d->bIsPolar = true; } } } From d4a125f92477686dad69b32fc872115cafad27c6 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 23:45:25 +0200 Subject: [PATCH 024/710] ogr2ogr: use OGRWKBTransform(), when possible, to speed-up -t_srs in Arrow code path Also use multi-threaded coordinate transformation by splitting the features within each batch in several sub-batches, each processed in its own thread. --- apps/ogr2ogr_lib.cpp | 329 +++++++++++++++++++++---- autotest/utilities/test_ogr2ogr_lib.py | 102 ++++++++ 2 files changed, 381 insertions(+), 50 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 9d28764d9d98..5aafc7b22e3b 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #include #include #include +#include #include #include @@ -69,8 +71,10 @@ #include "ogr_p.h" #include "ogr_recordbatch.h" #include "ogr_spatialref.h" +#include "ogrlayerarrow.h" #include "ogrlayerdecorator.h" #include "ogrsf_frmts.h" +#include "ogr_wkb.h" typedef enum { @@ -520,38 +524,39 @@ class SetupTargetLayer bool &bError); public: - GDALDataset *m_poSrcDS; - GDALDataset *m_poDstDS; - char **m_papszLCO; - OGRSpatialReference *m_poOutputSRS; + GDALDataset *m_poSrcDS = nullptr; + GDALDataset *m_poDstDS = nullptr; + CSLConstList m_papszLCO = nullptr; + const OGRSpatialReference *m_poUserSourceSRS = nullptr; + const OGRSpatialReference *m_poOutputSRS = nullptr; bool m_bTransform = false; - bool m_bNullifyOutputSRS; + bool m_bNullifyOutputSRS = false; bool m_bSelFieldsSet = false; - char **m_papszSelFields; - bool m_bAppend; - bool m_bAddMissingFields; - int m_eGType; - GeomTypeConversion m_eGeomTypeConversion; - int m_nCoordDim; - bool m_bOverwrite; - char **m_papszFieldTypesToString; - char **m_papszMapFieldType; - bool m_bUnsetFieldWidth; - bool m_bExplodeCollections; - const char *m_pszZField; - char **m_papszFieldMap; - const char *m_pszWHERE; - bool m_bExactFieldNameMatch; - bool m_bQuiet; - bool m_bForceNullable; - bool m_bResolveDomains; - bool m_bUnsetDefault; - bool m_bUnsetFid; - bool m_bPreserveFID; - bool m_bCopyMD; - bool m_bNativeData; - bool m_bNewDataSource; - const char *m_pszCTPipeline; + CSLConstList m_papszSelFields = nullptr; + bool m_bAppend = false; + bool m_bAddMissingFields = false; + int m_eGType = 0; + GeomTypeConversion m_eGeomTypeConversion = GTC_DEFAULT; + int m_nCoordDim = 0; + bool m_bOverwrite = false; + CSLConstList m_papszFieldTypesToString = nullptr; + CSLConstList m_papszMapFieldType = nullptr; + bool m_bUnsetFieldWidth = false; + bool m_bExplodeCollections = false; + const char *m_pszZField = nullptr; + CSLConstList m_papszFieldMap = nullptr; + const char *m_pszWHERE = nullptr; + bool m_bExactFieldNameMatch = false; + bool m_bQuiet = false; + bool m_bForceNullable = false; + bool m_bResolveDomains = false; + bool m_bUnsetDefault = false; + bool m_bUnsetFid = false; + bool m_bPreserveFID = false; + bool m_bCopyMD = false; + bool m_bNativeData = false; + bool m_bNewDataSource = false; + const char *m_pszCTPipeline = nullptr; std::unique_ptr Setup(OGRLayer *poSrcLayer, const char *pszNewLayerName, @@ -560,11 +565,10 @@ class SetupTargetLayer class LayerTranslator { - static bool TranslateArrow(const TargetLayerInfo *psInfo, - GIntBig nCountLayerFeatures, - GIntBig *pnReadFeatureCount, - GDALProgressFunc pfnProgress, void *pProgressArg, - const GDALVectorTranslateOptions *psOptions); + bool TranslateArrow(TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, + GIntBig *pnReadFeatureCount, + GDALProgressFunc pfnProgress, void *pProgressArg, + const GDALVectorTranslateOptions *psOptions); public: GDALDataset *m_poSrcDS = nullptr; @@ -572,9 +576,9 @@ class LayerTranslator bool m_bTransform = false; bool m_bWrapDateline = false; CPLString m_osDateLineOffset{}; - OGRSpatialReference *m_poOutputSRS = nullptr; + const OGRSpatialReference *m_poOutputSRS = nullptr; bool m_bNullifyOutputSRS = false; - OGRSpatialReference *m_poUserSourceSRS = nullptr; + const OGRSpatialReference *m_poUserSourceSRS = nullptr; OGRCoordinateTransformation *m_poGCPCoordTrans = nullptr; int m_eGType = -1; GeomTypeConversion m_eGeomTypeConversion = GTC_DEFAULT; @@ -586,20 +590,20 @@ class LayerTranslator OGRGeometry *m_poClipSrcOri = nullptr; bool m_bWarnedClipSrcSRS = false; - std::unique_ptr m_poClipSrcReprojectedToSrcSRS; + std::unique_ptr m_poClipSrcReprojectedToSrcSRS{}; const OGRSpatialReference *m_poClipSrcReprojectedToSrcSRS_SRS = nullptr; OGREnvelope m_oClipSrcEnv{}; OGRGeometry *m_poClipDstOri = nullptr; bool m_bWarnedClipDstSRS = false; - std::unique_ptr m_poClipDstReprojectedToDstSRS; + std::unique_ptr m_poClipDstReprojectedToDstSRS{}; const OGRSpatialReference *m_poClipDstReprojectedToDstSRS_SRS = nullptr; OGREnvelope m_oClipDstEnv{}; bool m_bExplodeCollections = false; bool m_bNativeData = false; GIntBig m_nLimit = -1; - OGRGeometryFactory::TransformWithOptionsCache m_transformWithOptionsCache; + OGRGeometryFactory::TransformWithOptionsCache m_transformWithOptionsCache{}; bool Translate(OGRFeature *poFeatureIn, TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, GIntBig *pnReadFeatureCount, @@ -2910,6 +2914,7 @@ GDALDatasetH GDALVectorTranslate(const char *pszDest, GDALDatasetH hDstDS, oSetup.m_poOutputSRS = oOutputSRSHolder.get(); oSetup.m_bTransform = psOptions->bTransform; oSetup.m_bNullifyOutputSRS = psOptions->bNullifyOutputSRS; + oSetup.m_poUserSourceSRS = poSourceSRS; oSetup.m_bSelFieldsSet = psOptions->bSelFieldsSet; oSetup.m_papszSelFields = psOptions->aosSelFields.List(); oSetup.m_bAppend = bAppend; @@ -3807,8 +3812,8 @@ static OGRwkbGeometryType ConvertType(GeomTypeConversion eGeomTypeConversion, static void DoFieldTypeConversion(GDALDataset *poDstDS, OGRFieldDefn &oFieldDefn, - char **papszFieldTypesToString, - char **papszMapFieldType, + CSLConstList papszFieldTypesToString, + CSLConstList papszMapFieldType, bool bUnsetFieldWidth, bool bQuiet, bool bForceNullable, bool bUnsetDefault) { @@ -3953,6 +3958,47 @@ static void DoFieldTypeConversion(GDALDataset *poDstDS, } } +/************************************************************************/ +/* GetArrowGeomFieldIndex() */ +/************************************************************************/ + +static int GetArrowGeomFieldIndex(const struct ArrowSchema *psLayerSchema, + const char *pszFieldName) +{ + if (strcmp(psLayerSchema->format, "+s") == 0) // struct + { + for (int i = 0; i < psLayerSchema->n_children; ++i) + { + const auto psSchema = psLayerSchema->children[i]; + if (strcmp(psSchema->format, "z") == 0) // binary + { + if (strcmp(psSchema->name, pszFieldName) == 0) + { + return i; + } + else + { + // Check if ARROW:extension:name = ogc.wkb or geoarrow.wkb + const char *pabyMetadata = psSchema->metadata; + if (pabyMetadata) + { + const auto oMetadata = + OGRParseArrowMetadata(pabyMetadata); + auto oIter = oMetadata.find(ARROW_EXTENSION_NAME_KEY); + if (oIter != oMetadata.end() && + (oIter->second == EXTENSION_NAME_OGC_WKB || + oIter->second == EXTENSION_NAME_GEOARROW_WKB)) + { + return i; + } + } + } + } + } + } + return -1; +} + /************************************************************************/ /* SetupTargetLayer::CanUseWriteArrowBatch() */ /************************************************************************/ @@ -3980,11 +4026,11 @@ bool SetupTargetLayer::CanUseWriteArrowBatch( !psOptions->aosLCO.FetchNameValue("BATCH_SIZE") && CPLTestBool(CPLGetConfigOption("OGR2OGR_USE_ARROW_API", "YES"))) || CPLTestBool(CPLGetConfigOption("OGR2OGR_USE_ARROW_API", "NO"))) && - !psOptions->bSkipFailures && !psOptions->bTransform && - !psOptions->poClipSrc && !psOptions->poClipDst && - psOptions->oGCPs.nGCPCount == 0 && !psOptions->bWrapDateline && - !m_papszSelFields && !m_bAddMissingFields && - m_eGType == GEOMTYPE_UNCHANGED && psOptions->eGeomOp == GEOMOP_NONE && + !psOptions->bSkipFailures && !psOptions->poClipSrc && + !psOptions->poClipDst && psOptions->oGCPs.nGCPCount == 0 && + !psOptions->bWrapDateline && !m_papszSelFields && + !m_bAddMissingFields && m_eGType == GEOMTYPE_UNCHANGED && + psOptions->eGeomOp == GEOMOP_NONE && m_eGeomTypeConversion == GTC_DEFAULT && m_nCoordDim < 0 && !m_papszFieldTypesToString && !m_papszMapFieldType && !m_bUnsetFieldWidth && !m_bExplodeCollections && !m_pszZField && @@ -3993,6 +4039,26 @@ bool SetupTargetLayer::CanUseWriteArrowBatch( psOptions->dfXYRes == OGRGeomCoordinatePrecision::UNKNOWN && !psOptions->bMakeValid && !psOptions->bSkipInvalidGeom) { + if (psOptions->bTransform) + { + // To simplify implementation for now + if (poSrcLayer->GetLayerDefn()->GetGeomFieldCount() != 1 || + poDstLayer->GetLayerDefn()->GetGeomFieldCount() != 1) + { + return false; + } + auto poSrcSRS = m_poUserSourceSRS ? m_poUserSourceSRS + : poSrcLayer->GetLayerDefn() + ->GetGeomFieldDefn(0) + ->GetSpatialRef(); + if (!poSrcSRS || + !OGRGeometryFactory::isTransformWithOptionsRegularTransform( + poSrcSRS, m_poOutputSRS, nullptr)) + { + return false; + } + } + struct ArrowArrayStream streamSrc; const char *const apszOptions[] = {"SILENCE_GET_SCHEMA_ERROR=YES", nullptr}; @@ -4001,6 +4067,15 @@ bool SetupTargetLayer::CanUseWriteArrowBatch( struct ArrowSchema schemaSrc; if (streamSrc.get_schema(&streamSrc, &schemaSrc) == 0) { + if (psOptions->bTransform && + GetArrowGeomFieldIndex(&schemaSrc, + poSrcLayer->GetGeometryColumn()) < 0) + { + schemaSrc.release(&schemaSrc); + streamSrc.release(&streamSrc); + return false; + } + std::string osErrorMsg; if (poDstLayer->IsArrowSchemaSupported(&schemaSrc, nullptr, osErrorMsg)) @@ -5699,7 +5774,7 @@ SetupCT(TargetLayerInfo *psInfo, OGRLayer *poSrcLayer, bool bTransform, /************************************************************************/ bool LayerTranslator::TranslateArrow( - const TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, + TargetLayerInfo *psInfo, GIntBig nCountLayerFeatures, GIntBig *pnReadFeatureCount, GDALProgressFunc pfnProgress, void *pProgressArg, const GDALVectorTranslateOptions *psOptions) { @@ -5749,10 +5824,52 @@ bool LayerTranslator::TranslateArrow( return false; } + int iArrowGeomFieldIndex = -1; + if (m_bTransform) + { + iArrowGeomFieldIndex = GetArrowGeomFieldIndex( + &schema, psInfo->m_poSrcLayer->GetGeometryColumn()); + if (!SetupCT(psInfo, psInfo->m_poSrcLayer, m_bTransform, + m_bWrapDateline, m_osDateLineOffset, m_poUserSourceSRS, + nullptr, m_poOutputSRS, m_poGCPCoordTrans, false)) + { + return false; + } + } + bool bRet = true; GIntBig nCount = 0; bool bGoOn = true; + std::vector abyModifiedWKB; + const int nNumReprojectionThreads = []() + { + const int nNumCPUs = CPLGetNumCPUs(); + if (nNumCPUs <= 1) + { + return 1; + } + else + { + const char *pszNumThreads = + CPLGetConfigOption("GDAL_NUM_THREADS", nullptr); + if (pszNumThreads) + { + if (EQUAL(pszNumThreads, "ALL_CPUS")) + return CPLGetNumCPUs(); + return std::min(atoi(pszNumThreads), 1024); + } + else + { + return std::max(2, nNumCPUs / 2); + } + } + }(); + + // Somewhat arbitrary threshold (config option only/mostly for autotest purposes) + const int MIN_FEATURES_FOR_THREADED_REPROJ = atoi(CPLGetConfigOption( + "OGR2OGR_MIN_FEATURES_FOR_THREADED_REPROJ", "10000")); + while (bGoOn) { struct ArrowArray array; @@ -5789,9 +5906,121 @@ bool LayerTranslator::TranslateArrow( nCount += array.length; } + const auto nArrayLength = array.length; + + // Coordinate reprojection + const void *backupGeomArrayBuffers2 = nullptr; + if (m_bTransform) + { + auto *psGeomArray = array.children[iArrowGeomFieldIndex]; + GByte *pabyWKB = static_cast( + const_cast(psGeomArray->buffers[2])); + const uint32_t *panOffsets = + static_cast(psGeomArray->buffers[1]); + auto poCT = psInfo->m_aoReprojectionInfo[0].m_poCT.get(); + + try + { + abyModifiedWKB.resize(panOffsets[nArrayLength]); + } + catch (const std::exception &) + { + CPLError(CE_Failure, CPLE_OutOfMemory, "Out of memory"); + bRet = false; + if (array.release) + array.release(&array); + break; + } + memcpy(abyModifiedWKB.data(), pabyWKB, panOffsets[nArrayLength]); + backupGeomArrayBuffers2 = psGeomArray->buffers[2]; + psGeomArray->buffers[2] = abyModifiedWKB.data(); + + std::atomic atomicRet{true}; + const auto oReprojectionLambda = + [psGeomArray, nArrayLength, panOffsets, &atomicRet, + &abyModifiedWKB, &poCT](int iThread, int nThreads) + { + OGRWKBTransformCache oCache; + OGREnvelope3D sEnv3D; + auto poThisCT = + std::unique_ptr(poCT->Clone()); + if (!poThisCT) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot clone OGRCoordinateTransformation"); + atomicRet = false; + return; + } + + const GByte *pabyValidity = + static_cast(psGeomArray->buffers[0]); + const size_t iStart = + static_cast(iThread * nArrayLength / nThreads); + const size_t iMax = static_cast( + (iThread + 1) * nArrayLength / nThreads); + for (size_t i = iStart; i < iMax; ++i) + { + const size_t iShifted = + static_cast(i + psGeomArray->offset); + if (!pabyValidity || (pabyValidity[iShifted >> 8] & + (1 << (iShifted % 8))) != 0) + { + const auto nWKBSize = + panOffsets[iShifted + 1] - panOffsets[iShifted]; + if (!OGRWKBTransform( + abyModifiedWKB.data() + panOffsets[iShifted], + nWKBSize, poThisCT.get(), oCache, sEnv3D)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Reprojection failed"); + atomicRet = false; + break; + } + } + } + }; + + if (nArrayLength >= MIN_FEATURES_FOR_THREADED_REPROJ && + nNumReprojectionThreads >= 2) + { + std::vector oThreads; + for (int iThread = 0; iThread < nNumReprojectionThreads; + ++iThread) + { + oThreads.emplace_back(oReprojectionLambda, iThread, + nNumReprojectionThreads); + } + for (auto &oThread : oThreads) + { + oThread.join(); + } + } + else + { + oReprojectionLambda(0, 1); + } + + bRet = atomicRet; + if (!bRet) + { + psGeomArray->buffers[2] = backupGeomArrayBuffers2; + if (array.release) + array.release(&array); + break; + } + } + // Write batch to target layer - if (!psInfo->m_poDstLayer->WriteArrowBatch( - &schema, &array, aosOptionsWriteArrowBatch.List())) + const bool bWriteOK = psInfo->m_poDstLayer->WriteArrowBatch( + &schema, &array, aosOptionsWriteArrowBatch.List()); + + if (backupGeomArrayBuffers2) + { + auto *psGeomArray = array.children[iArrowGeomFieldIndex]; + psGeomArray->buffers[2] = backupGeomArrayBuffers2; + } + + if (!bWriteOK) { CPLError(CE_Failure, CPLE_AppDefined, "WriteArrowBatch() failed"); if (array.release) diff --git a/autotest/utilities/test_ogr2ogr_lib.py b/autotest/utilities/test_ogr2ogr_lib.py index dbfec4cba68f..f51f7cd1ace3 100755 --- a/autotest/utilities/test_ogr2ogr_lib.py +++ b/autotest/utilities/test_ogr2ogr_lib.py @@ -2829,3 +2829,105 @@ def test_ogr2ogr_lib_skip_invalid(tmp_vsimem): lyr = ds.GetLayer(0) assert lyr.GetFeatureCount() == 1 ds = None + + +############################################################################### +# Test -t_srs in Arrow code path + + +@gdaltest.enable_exceptions() +@pytest.mark.parametrize("force_reproj_threading", [False, True]) +@pytest.mark.parametrize("source_driver", ["GPKG", "Parquet"]) +def test_ogr2ogr_lib_reproject_arrow(tmp_vsimem, source_driver, force_reproj_threading): + + src_driver = gdal.GetDriverByName(source_driver) + if src_driver is None: + pytest.skip(f"{source_driver} is not available") + src_filename = str(tmp_vsimem / ("in." + source_driver.lower())) + with src_driver.Create(src_filename, 0, 0, 0, gdal.GDT_Unknown) as srcDS: + srs = osr.SpatialReference() + srs.ImportFromEPSG(32631) + srcLayer = srcDS.CreateLayer("test", srs=srs) + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT(500000 4500000)")) + srcLayer.CreateFeature(f) + f = ogr.Feature(srcLayer.GetLayerDefn()) + srcLayer.CreateFeature(f) + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT(500000 4000000)")) + srcLayer.CreateFeature(f) + + config_options = {"CPL_DEBUG": "ON", "OGR2OGR_USE_ARROW_API": "YES"} + if force_reproj_threading: + config_options["OGR2OGR_MIN_FEATURES_FOR_THREADED_REPROJ"] = "0" + + with gdal.OpenEx(src_filename) as src_ds: + for i in range(2): + + got_msg = [] + + def my_handler(errorClass, errno, msg): + got_msg.append(msg) + return + + with gdaltest.error_handler(my_handler), gdaltest.config_options( + config_options + ): + ds = gdal.VectorTranslate( + "", src_ds, format="Memory", dstSRS="EPSG:4326" + ) + + assert "OGR2OGR: Using WriteArrowBatch()" in got_msg + + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 3 + f = lyr.GetNextFeature() + ogrtest.check_feature_geometry(f, "POINT(3 40.65085651557158)") + f = lyr.GetNextFeature() + assert f.GetGeometryRef() is None + f = lyr.GetNextFeature() + ogrtest.check_feature_geometry(f, "POINT(3 36.14471809881776)") + + +############################################################################### +# Test -t_srs in Arrow code path in a situation where it cannot be triggered +# currently (source CRS is crossing anti-meridian) + + +@gdaltest.enable_exceptions() +@pytest.mark.require_geos +@pytest.mark.require_driver("GPKG") +def test_ogr2ogr_lib_reproject_arrow_optim_cannot_trigger(tmp_vsimem): + + src_filename = str(tmp_vsimem / "in.gpkg") + with gdal.GetDriverByName("GPKG").Create( + src_filename, 0, 0, 0, gdal.GDT_Unknown + ) as srcDS: + srs = osr.SpatialReference() + srs.ImportFromEPSG(32660) + srcLayer = srcDS.CreateLayer("test", srs=srs) + f = ogr.Feature(srcLayer.GetLayerDefn()) + f.SetGeometry( + ogr.CreateGeometryFromWkt( + "LINESTRING(657630.64 4984896.17,815261.43 4990738.26)" + ) + ) + srcLayer.CreateFeature(f) + + got_msg = [] + + def my_handler(errorClass, errno, msg): + got_msg.append(msg) + return + + config_options = {"CPL_DEBUG": "ON", "OGR2OGR_USE_ARROW_API": "YES"} + with gdaltest.error_handler(my_handler), gdaltest.config_options(config_options): + ds = gdal.VectorTranslate("", src_filename, format="Memory", dstSRS="EPSG:4326") + + assert "OGR2OGR: Using WriteArrowBatch()" not in got_msg + + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + f = lyr.GetNextFeature() + assert f.GetGeometryRef().GetGeometryType() == ogr.wkbMultiLineString + assert f.GetGeometryRef().GetGeometryCount() == 2 From d0131e4684e55ba9e1e9abfc4f9755d2f335c970 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 2 Sep 2024 23:44:45 +0200 Subject: [PATCH 025/710] ogr2ogr: using std::async rather than std::thread --- apps/ogr2ogr_lib.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 5aafc7b22e3b..377d27ec4baa 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -41,6 +41,7 @@ #include #include +#include #include #include #include @@ -48,7 +49,6 @@ #include #include #include -#include #include #include @@ -5983,16 +5983,17 @@ bool LayerTranslator::TranslateArrow( if (nArrayLength >= MIN_FEATURES_FOR_THREADED_REPROJ && nNumReprojectionThreads >= 2) { - std::vector oThreads; + std::vector> oTasks; for (int iThread = 0; iThread < nNumReprojectionThreads; ++iThread) { - oThreads.emplace_back(oReprojectionLambda, iThread, - nNumReprojectionThreads); + oTasks.emplace_back(std::async(std::launch::async, + oReprojectionLambda, iThread, + nNumReprojectionThreads)); } - for (auto &oThread : oThreads) + for (auto &oTask : oTasks) { - oThread.join(); + oTask.get(); } } else From 58ec9e2004bfbdb3427a5d30f2ae92cb3c87b88c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 8 Sep 2024 01:21:57 +0200 Subject: [PATCH 026/710] Feather: add read support for StringView and BinaryView (but not in OGR generic Arrow code) --- autotest/generate_parquet_test_file.py | 89 +++++++++++++++ autotest/ogr/data/arrow/binaryview.feather | Bin 0 -> 618 bytes autotest/ogr/data/arrow/largelistview.feather | Bin 0 -> 762 bytes autotest/ogr/data/arrow/listview.feather | Bin 0 -> 746 bytes autotest/ogr/data/arrow/stringview.feather | Bin 0 -> 2618 bytes autotest/ogr/ogr_arrow.py | 55 +++++++++ autotest/ogr/ogr_parquet.py | 39 +++++++ .../arrow_common/ograrrowlayer.hpp | 108 ++++++++++++++++-- .../parquet/ogrparquetwriterlayer.cpp | 22 ++++ 9 files changed, 306 insertions(+), 7 deletions(-) create mode 100644 autotest/ogr/data/arrow/binaryview.feather create mode 100644 autotest/ogr/data/arrow/largelistview.feather create mode 100644 autotest/ogr/data/arrow/listview.feather create mode 100644 autotest/ogr/data/arrow/stringview.feather diff --git a/autotest/generate_parquet_test_file.py b/autotest/generate_parquet_test_file.py index 71e226d4ed9e..fd233ca21e55 100644 --- a/autotest/generate_parquet_test_file.py +++ b/autotest/generate_parquet_test_file.py @@ -1245,6 +1245,91 @@ def __arrow_ext_deserialize__(cls, storage_type, serialized): ) +def generate_arrow_stringview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + stringview = pa.array(["foo", "bar", "looooooooooong string"], pa.string_view()) + list_stringview = pa.array( + [None, [None], ["foo", "bar", "looooooooooong string"]], + pa.list_(pa.string_view()), + ) + list_of_list_stringview = pa.array( + [None, [None], [["foo", "bar", "looooooooooong string"]]], + pa.list_(pa.list_(pa.string_view())), + ) + map_stringview = pa.array( + [None, [], [("x", "x_val"), ("y", None)]], + type=pa.map_(pa.string_view(), pa.string_view()), + ) + + names = [ + "stringview", + "list_stringview", + "list_of_list_stringview", + "map_stringview", + ] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/stringview.feather") + + +def generate_arrow_binaryview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + binaryview = pa.array([b"foo", b"bar", b"looooooooooong binary"], pa.binary_view()) + + names = ["binaryview"] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/binaryview.feather") + + +def generate_arrow_listview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + listview = pa.array([[1]], pa.list_view(pa.int32())) + + names = ["listview"] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/listview.feather") + + +def generate_arrow_largelistview(): + import pathlib + + import pyarrow as pa + import pyarrow.feather as feather + + largelistview = pa.array([[1]], pa.large_list_view(pa.int32())) + + names = ["largelistview"] + + locals_ = locals() + table = pa.table([locals_[x] for x in names], names=names) + + HERE = pathlib.Path(__file__).parent + feather.write_feather(table, HERE / "ogr/data/arrow/largelistview.feather") + + if __name__ == "__main__": generate_test_parquet() generate_all_geoms_parquet() @@ -1252,3 +1337,7 @@ def __arrow_ext_deserialize__(cls, storage_type, serialized): generate_nested_types() generate_extension_custom() generate_extension_json() + generate_arrow_stringview() + generate_arrow_binaryview() + generate_arrow_listview() + generate_arrow_largelistview() diff --git a/autotest/ogr/data/arrow/binaryview.feather b/autotest/ogr/data/arrow/binaryview.feather new file mode 100644 index 0000000000000000000000000000000000000000..9f62bd82944c4fd7c8b8c7a4c5a2407336256330 GIT binary patch literal 618 zcmcIiJ!=9%5PcUnddQ($D2kO>n3M^mNEP%4h+t!@f{DW6fki&rkRoM%fNT->1|u_xBnhQd-(1tlq?oG3@&033jlo4|X+ ztQ3$XTE?20eQ$p2(axJ@v77)$${&D|3iJ_>BSXOxB}s`vRM4hwNgD!otR)&Sbv z8&g&B)_!b0+w~WZtk$PF6=OLbt#j#flb+_+mPg~h-K^hrE|^=eLOm%-O{077=`~vt zFz}lP>x_pK&*WDMx&$(%{l_`yo4j(=L-&pn@0vxT!#_D!yN`J0IJ$^qTzrrEf1bE& ztX)acm)F5+_DlFV Jw=uUjeggK;GpGOn literal 0 HcmV?d00001 diff --git a/autotest/ogr/data/arrow/stringview.feather b/autotest/ogr/data/arrow/stringview.feather new file mode 100644 index 0000000000000000000000000000000000000000..43ab1534e0f82dc7d119ceff6de9d322653a54b5 GIT binary patch literal 2618 zcmeHJyKWOv5S>jN$3jlBC?=Rku{;zEq6iX5RA|#tfUqJYD3FYi*0B}GMzIqTDTRgj z07?qV6mIA#T~MUp15)q-l=%Y8Id^BG^~M%7NE8|C?%a9adF@@VuB| zg>X*tl9d_BVF&OspaoP8&x}5~6Z!6myuvzk5|QV>hC>*fFMz(OJc#8g);!i`^=+lo z-DaiAHM?o-H=;c`f=$82;^is7t2&IIxm?6*Hagu(yH;^HkqGU| z30vk%|FpHU8r^6Uq=eAhk9AMwKpWGmL+2pjlQEc2`d`3G$MG5ZM}Oh_8+79OWK;F= zTugzU1>|NRkt$-~F9x_74$v&-NXW}&ot(U@x=d$WPY>D96PO3D03SK%_--zKpiu1h zV=VTSJuzTkSx3U3Fm)_G&c-Boq=HcKvf{d8v+XI(+7tRQa&yP+K~m!*U=Pf+_gY!{*I#=&Ys zxy7q`wCA-dAs+zc-^TU|l(#j9C7$pADfn{ijmzL)#-FVHnE0KT-xq?$53=WIJZuyk z{h#%TqrHSWmnE4bn1PG-B%ezlYs`I@9rR_er*cDM>f%^|Drj96L4M(VxLd)O&K;}@ zig+J$%X!M= 16.0.0") ogr.Open("vsi://data/arrow/test.feather") + + +############################################################################### + + +@gdaltest.enable_exceptions() +def test_ogr_arrow_string_view(): + + version = int( + ogr.GetDriverByName("ARROW").GetMetadataItem("ARROW_VERSION").split(".")[0] + ) + if version < 15: + pytest.skip("requires Arrow >= 15") + + with ogr.Open("data/arrow/stringview.feather") as ds: + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f["stringview"] == "foo" + assert f["list_stringview"] is None + assert f["list_of_list_stringview"] is None + assert f["map_stringview"] is None + + f = lyr.GetNextFeature() + assert f["stringview"] == "bar" + assert f["list_stringview"] == [""] + assert f["list_of_list_stringview"] == "[null]" + assert f["map_stringview"] == "{}" + + f = lyr.GetNextFeature() + assert f["stringview"] == "looooooooooong string" + assert f["list_stringview"] == ["foo", "bar", "looooooooooong string"] + assert f["list_of_list_stringview"] == '[["foo","bar","looooooooooong string"]]' + assert f["map_stringview"] == '{"x":"x_val","y":null}' + + +############################################################################### + + +@gdaltest.enable_exceptions() +def test_ogr_arrow_binary_view(): + + version = int( + ogr.GetDriverByName("ARROW").GetMetadataItem("ARROW_VERSION").split(".")[0] + ) + if version < 15: + pytest.skip("requires Arrow >= 15") + + with ogr.Open("data/arrow/binaryview.feather") as ds: + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + assert f.GetFieldAsBinary("binaryview") == b"foo" + f = lyr.GetNextFeature() + assert f.GetFieldAsBinary("binaryview") == b"bar" + f = lyr.GetNextFeature() + assert f.GetFieldAsBinary("binaryview") == b"looooooooooong binary" diff --git a/autotest/ogr/ogr_parquet.py b/autotest/ogr/ogr_parquet.py index 2563010535ac..d0baf5726ad7 100755 --- a/autotest/ogr/ogr_parquet.py +++ b/autotest/ogr/ogr_parquet.py @@ -4125,3 +4125,42 @@ def test_ogr_parquet_vsi_arrow_file_system(): ds = ogr.Open("PARQUET:vsi://data/parquet/test.parquet") lyr = ds.GetLayer(0) assert lyr.GetFeatureCount() > 0 + + +############################################################################### + + +@gdaltest.enable_exceptions() +@pytest.mark.require_driver("ARROW") +@pytest.mark.parametrize( + "src_filename,expected_error_msg", + [ + ("data/arrow/stringview.feather", "StringView not supported"), + ("data/arrow/binaryview.feather", "BinaryView not supported"), + ], +) +def test_ogr_parquet_IsArrowSchemaSupported_arrow_15_types( + src_filename, expected_error_msg, tmp_vsimem +): + + version = int( + ogr.GetDriverByName("ARROW").GetMetadataItem("ARROW_VERSION").split(".")[0] + ) + if version < 15: + pytest.skip("requires Arrow >= 15.0.0") + + src_ds = ogr.Open(src_filename) + src_lyr = src_ds.GetLayer(0) + + outfilename = str(tmp_vsimem / "test.parquet") + with ogr.GetDriverByName("Parquet").CreateDataSource(outfilename) as dst_ds: + dst_lyr = dst_ds.CreateLayer( + "test", srs=src_lyr.GetSpatialRef(), geom_type=ogr.wkbPoint, options=[] + ) + + stream = src_lyr.GetArrowStream() + schema = stream.GetSchema() + + success, error_msg = dst_lyr.IsArrowSchemaSupported(schema) + assert not success + assert error_msg == expected_error_msg diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp index 084a7273acea..341ca12803fa 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp @@ -248,6 +248,9 @@ inline bool OGRArrowLayer::IsHandledListOrMapType( itemTypeId == arrow::Type::DECIMAL256 || itemTypeId == arrow::Type::STRING || itemTypeId == arrow::Type::LARGE_STRING || +#if ARROW_VERSION_MAJOR >= 15 + itemTypeId == arrow::Type::STRING_VIEW || +#endif itemTypeId == arrow::Type::STRUCT || (itemTypeId == arrow::Type::MAP && IsHandledMapType( @@ -276,7 +279,12 @@ inline bool OGRArrowLayer::IsHandledListType( inline bool OGRArrowLayer::IsHandledMapType(const std::shared_ptr &mapType) { - return mapType->key_type()->id() == arrow::Type::STRING && + const auto typeId = mapType->key_type()->id(); + return (typeId == arrow::Type::STRING +#if ARROW_VERSION_MAJOR >= 15 + || typeId == arrow::Type::STRING_VIEW +#endif + ) && IsHandledListOrMapType(mapType->item_type()); } @@ -369,6 +377,9 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( break; case arrow::Type::STRING: case arrow::Type::LARGE_STRING: +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: +#endif bTypeOK = true; eType = OFTString; if (osExtensionName == EXTENSION_NAME_ARROW_JSON) @@ -376,6 +387,9 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( break; case arrow::Type::BINARY: case arrow::Type::LARGE_BINARY: +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::BINARY_VIEW: +#endif bTypeOK = true; eType = OFTBinary; break; @@ -476,6 +490,9 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( break; case arrow::Type::STRING: case arrow::Type::LARGE_STRING: +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: +#endif eType = OFTStringList; break; default: @@ -538,8 +555,6 @@ inline bool OGRArrowLayer::MapArrowTypeToOGR( case arrow::Type::RUN_END_ENCODED: #endif #if ARROW_VERSION_MAJOR >= 15 - case arrow::Type::STRING_VIEW: - case arrow::Type::BINARY_VIEW: case arrow::Type::LIST_VIEW: case arrow::Type::LARGE_LIST_VIEW: #endif @@ -1321,6 +1336,15 @@ static void AddToArray(CPLJSONArray &oArray, const arrow::Array *array, nIdx)); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + oArray.Add( + static_cast(array)->GetString( + nIdx)); + break; + } +#endif case arrow::Type::LIST: case arrow::Type::LARGE_LIST: case arrow::Type::FIXED_SIZE_LIST: @@ -1491,6 +1515,14 @@ static void AddToDict(CPLJSONObject &oDict, const std::string &osKey, ->GetString(nIdx)); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + oDict.Add(osKey, static_cast(array) + ->GetString(nIdx)); + break; + } +#endif case arrow::Type::LIST: case arrow::Type::LARGE_LIST: case arrow::Type::FIXED_SIZE_LIST: @@ -1514,12 +1546,12 @@ static void AddToDict(CPLJSONObject &oDict, const std::string &osKey, /* GetMapAsJSON() */ /************************************************************************/ +template static CPLJSONObject GetMapAsJSON(const arrow::Array *array, const size_t nIdxInArray) { const auto mapArray = static_cast(array); - const auto keys = - std::static_pointer_cast(mapArray->keys()); + const auto keys = std::static_pointer_cast(mapArray->keys()); const auto values = mapArray->items(); const auto nIdxStart = mapArray->value_offset(nIdxInArray); const int nCount = mapArray->value_length(nIdxInArray); @@ -1538,6 +1570,24 @@ static CPLJSONObject GetMapAsJSON(const arrow::Array *array, return oRoot; } +static CPLJSONObject GetMapAsJSON(const arrow::Array *array, + const size_t nIdxInArray) +{ + const auto mapArray = static_cast(array); + const auto eKeyType = mapArray->keys()->type()->id(); + if (eKeyType == arrow::Type::STRING) + return GetMapAsJSON(array, nIdxInArray); +#if ARROW_VERSION_MAJOR >= 15 + else if (eKeyType == arrow::Type::STRING_VIEW) + return GetMapAsJSON(array, nIdxInArray); +#endif + else + { + CPLAssert(false); + return CPLJSONObject(); + } +} + /************************************************************************/ /* GetStructureAsJSON() */ /************************************************************************/ @@ -1802,6 +1852,27 @@ static void ReadList(OGRFeature *poFeature, int i, int64_t nIdxInArray, poFeature->SetField(i, aosList.List()); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + const auto values = + std::static_pointer_cast( + array->values()); + const auto nIdxStart = array->value_offset(nIdxInArray); + const int nCount = array->value_length(nIdxInArray); + CPLStringList aosList; + for (int k = 0; k < nCount; k++) + { + if (values->IsNull(nIdxStart + k)) + aosList.AddString( + ""); // we cannot have null strings in a list + else + aosList.AddString(values->GetString(nIdxStart + k).c_str()); + } + poFeature->SetField(i, aosList.List()); + break; + } +#endif case arrow::Type::LIST: case arrow::Type::LARGE_LIST: case arrow::Type::FIXED_SIZE_LIST: @@ -2244,6 +2315,31 @@ inline OGRFeature *OGRArrowLayer::ReadFeature( poFeature->SetField(i, out_length, data); break; } +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::BINARY_VIEW: + { + const auto castArray = + static_cast(array); + const auto view = castArray->GetView(nIdxInBatch); + poFeature->SetField(i, static_cast(view.size()), + view.data()); + break; + } +#endif +#if ARROW_VERSION_MAJOR >= 15 + case arrow::Type::STRING_VIEW: + { + const auto castArray = + static_cast(array); + const auto strView = castArray->GetView(nIdxInBatch); + char *pszString = + static_cast(CPLMalloc(strView.length() + 1)); + memcpy(pszString, strView.data(), strView.length()); + pszString[strView.length()] = 0; + poFeature->SetFieldSameTypeUnsafe(i, pszString); + break; + } +#endif case arrow::Type::FIXED_SIZE_BINARY: { const auto castArray = @@ -2424,8 +2520,6 @@ inline OGRFeature *OGRArrowLayer::ReadFeature( case arrow::Type::RUN_END_ENCODED: #endif #if ARROW_VERSION_MAJOR >= 15 - case arrow::Type::STRING_VIEW: - case arrow::Type::BINARY_VIEW: case arrow::Type::LIST_VIEW: case arrow::Type::LARGE_LIST_VIEW: #endif diff --git a/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp b/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp index 8e369b13a595..f72610e96dc8 100644 --- a/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp +++ b/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp @@ -1281,6 +1281,28 @@ bool OGRParquetWriterLayer::IsArrowSchemaSupported( osErrorMsg = "float16 not supported"; return false; } + if (schema->format[0] == 'v' && schema->format[1] == 'u') + { + osErrorMsg = "StringView not supported"; + return false; + } + if (schema->format[0] == 'v' && schema->format[1] == 'z') + { + osErrorMsg = "BinaryView not supported"; + return false; + } + if (schema->format[0] == '+' && schema->format[1] == 'v' && + schema->format[1] == 'l') + { + osErrorMsg = "ListView not supported"; + return false; + } + if (schema->format[0] == '+' && schema->format[1] == 'v' && + schema->format[1] == 'L') + { + osErrorMsg = "LargeListView not supported"; + return false; + } for (int64_t i = 0; i < schema->n_children; ++i) { if (!IsArrowSchemaSupported(schema->children[i], papszOptions, From 6260dd94afb0721c3e30d05e90d6e93eacd5583f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 8 Sep 2024 17:47:46 +0200 Subject: [PATCH 027/710] GTiff: better error messages when trying to create too-large untiled JPEG/WEBP compressed files (or with huge tiles) --- autotest/gcore/tiff_write.py | 82 ++++++++++++++++++++++++++++++ frmts/gtiff/gtiffdataset_write.cpp | 49 ++++++++++++++++++ frmts/gtiff/libtiff/tif_jpeg.c | 7 ++- 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index 03aae9ca57e0..28da66db9805 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -11597,3 +11597,85 @@ def test_tiff_write_colormap_256_mult_factor(tmp_vsimem): and ct.GetColorEntry(1) == (0, 1, 2, 255) and ct.GetColorEntry(2) == (254, 254, 254, 255) ), "Wrong color table entry." + + +############################################################################### +@pytest.mark.require_creation_option("GTiff", "JPEG") +@pytest.mark.parametrize( + "xsize,ysize,options,expected_error_msg", + [ + ( + 65501, + 1, + ["COMPRESS=JPEG"], + "COMPRESS=JPEG is only compatible of un-tiled images whose width is lesser or equal to 65500 pixels", + ), + ( + 1, + 65501, + ["COMPRESS=JPEG", "BLOCKYSIZE=65501"], + "COMPRESS=JPEG is only compatible of images whose BLOCKYSIZE is lesser or equal to 65500 pixels", + ), + ( + 1, + 1, + ["COMPRESS=JPEG", "TILED=YES", "BLOCKXSIZE=65536"], + "COMPRESS=JPEG is only compatible of tiled images whose BLOCKXSIZE is lesser or equal to 65500 pixels", + ), + ( + 1, + 1, + ["COMPRESS=JPEG", "TILED=YES", "BLOCKYSIZE=65536"], + "COMPRESS=JPEG is only compatible of images whose BLOCKYSIZE is lesser or equal to 65500 pixels", + ), + ], +) +@gdaltest.enable_exceptions() +def test_tiff_write_too_large_jpeg( + tmp_vsimem, xsize, ysize, options, expected_error_msg +): + + filename = str(tmp_vsimem / "test.tif") + with pytest.raises(Exception, match=expected_error_msg): + gdal.GetDriverByName("GTiff").Create(filename, xsize, ysize, options=options) + + +############################################################################### +@pytest.mark.require_creation_option("GTiff", "WEBP") +@pytest.mark.parametrize( + "xsize,ysize,options,expected_error_msg", + [ + ( + 16384, + 1, + ["COMPRESS=WEBP"], + "COMPRESS=WEBP is only compatible of un-tiled images whose width is lesser or equal to 16383 pixels", + ), + ( + 1, + 16384, + ["COMPRESS=WEBP", "BLOCKYSIZE=16384"], + "COMPRESS=WEBP is only compatible of images whose BLOCKYSIZE is lesser or equal to 16383 pixels", + ), + ( + 1, + 1, + ["COMPRESS=WEBP", "TILED=YES", "BLOCKXSIZE=16384"], + "COMPRESS=WEBP is only compatible of tiled images whose BLOCKXSIZE is lesser or equal to 16383 pixels", + ), + ( + 1, + 1, + ["COMPRESS=WEBP", "TILED=YES", "BLOCKYSIZE=16384"], + "COMPRESS=WEBP is only compatible of images whose BLOCKYSIZE is lesser or equal to 16383 pixels", + ), + ], +) +@gdaltest.enable_exceptions() +def test_tiff_write_too_large_webp( + tmp_vsimem, xsize, ysize, options, expected_error_msg +): + + filename = str(tmp_vsimem / "test.tif") + with pytest.raises(Exception, match=expected_error_msg): + gdal.GetDriverByName("GTiff").Create(filename, xsize, ysize, options=options) diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 4146fcc84236..5fbe542bb937 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -5122,6 +5122,55 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, return nullptr; } + constexpr int JPEG_MAX_DIMENSION = 65500; // Defined in jpeglib.h + constexpr int WEBP_MAX_DIMENSION = 16383; + + const struct + { + int nCodecID; + const char *pszCodecName; + int nMaxDim; + } asLimitations[] = { + {COMPRESSION_JPEG, "JPEG", JPEG_MAX_DIMENSION}, + {COMPRESSION_WEBP, "WEBP", WEBP_MAX_DIMENSION}, + }; + + for (const auto &sLimitation : asLimitations) + { + if (l_nCompression == sLimitation.nCodecID && !bTiled && + nXSize > sLimitation.nMaxDim) + { + ReportError( + pszFilename, CE_Failure, CPLE_IllegalArg, + "COMPRESS=%s is only compatible of un-tiled images whose " + "width is lesser or equal to %d pixels. " + "To overcome this limitation, set the TILED=YES creation " + "option.", + sLimitation.pszCodecName, sLimitation.nMaxDim); + return nullptr; + } + else if (l_nCompression == sLimitation.nCodecID && bTiled && + l_nBlockXSize > sLimitation.nMaxDim) + { + ReportError(pszFilename, CE_Failure, CPLE_IllegalArg, + "COMPRESS=%s is only compatible of tiled images whose " + "BLOCKXSIZE is lesser or equal to %d pixels.", + sLimitation.pszCodecName, sLimitation.nMaxDim); + return nullptr; + } + else if (l_nCompression == sLimitation.nCodecID && + l_nBlockYSize > sLimitation.nMaxDim) + { + ReportError(pszFilename, CE_Failure, CPLE_IllegalArg, + "COMPRESS=%s is only compatible of images whose " + "BLOCKYSIZE is lesser or equal to %d pixels. " + "To overcome this limitation, set the TILED=YES " + "creation option", + sLimitation.pszCodecName, sLimitation.nMaxDim); + return nullptr; + } + } + /* -------------------------------------------------------------------- */ /* How many bits per sample? We have a special case if NBITS */ /* specified for GDT_Byte, GDT_UInt16, GDT_UInt32. */ diff --git a/frmts/gtiff/libtiff/tif_jpeg.c b/frmts/gtiff/libtiff/tif_jpeg.c index 10aed5463589..c14ca522fe94 100644 --- a/frmts/gtiff/libtiff/tif_jpeg.c +++ b/frmts/gtiff/libtiff/tif_jpeg.c @@ -2191,9 +2191,12 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); } - if (segment_width > 65535 || segment_height > 65535) + if (segment_width > (uint32_t)JPEG_MAX_DIMENSION || + segment_height > (uint32_t)JPEG_MAX_DIMENSION) { - TIFFErrorExtR(tif, module, "Strip/tile too large for JPEG"); + TIFFErrorExtR(tif, module, + "Strip/tile too large for JPEG. Maximum dimension is %d", + (int)JPEG_MAX_DIMENSION); return (0); } sp->cinfo.c.image_width = segment_width; From f04a6ce89e0d5732734f31470416adbba95c9d0a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 11:17:45 +0200 Subject: [PATCH 028/710] [RFC 101 text] More details about implementation --- .../rfc101_raster_dataset_threadsafety.rst | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst index 641afa164175..9545cc02964b 100644 --- a/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst +++ b/doc/source/development/rfc/rfc101_raster_dataset_threadsafety.rst @@ -273,14 +273,26 @@ which defines several classes (internal details): GDALThreadSafeDataset overloads its ReferenceUnderlyingDataset method, so that a thread-local dataset is opened the first-time a thread calls a method on the GDALThreadSafeDataset instance, cached for later use, and method call is - forwarded to it. + generally forwarded to it. There are exceptions for methods like + :cpp:func:`GDALDataset::GetSpatialRef`, :cpp:func:`GDALDataset::GetGCPSpatialRef`, + :cpp:func:`GDALDataset::GetGCPs`, :cpp:func:`GDALDataset::GetMetadata`, + :cpp:func:`GDALDataset::GetMetadataItem` that return non-primitive types + where the calls are forwarded to the dataset used to construct GDALThreadSafeDataset, + with a mutex being taken around them. If the call was otherwise forwarded to + a thread-local instance, there would be a risk of use-after-free situations + when the returned value is used by different threads. - ``GDALThreadSafeRasterBand`` extending :cpp:class:`GDALProxyRasterBand`. On instantiation, it creates child GDALThreadSafeRasterBand instances for band mask and overviews. Its ReferenceUnderlyingRasterBand method calls ReferenceUnderlyingDataset on the GDALThreadSafeDataset instance to get a thread-local dataset, fetches - the appropriate thread-local band and forwards its the method call. + the appropriate thread-local band and generally forwards its the method call. + There are exceptions for methods like + :cpp:func:`GDALRasterBand::GetUnitType`, :cpp:func:`GDALRasterBand::GetMetadata`, + :cpp:func:`GDALRasterBand::GetMetadataItem` that return non-primitive types where + the calls are forwarded to the band used to construct GDALThreadSafeRasterBand, + with a mutex being taken around them, and the returned value being . - ``GDALThreadLocalDatasetCache``. Instances of that class use thread-local storage. The main member of such instances is a LRU cache that maps @@ -334,6 +346,17 @@ wouldn't be thread-safe. - :cpp:func:`GDALRasterBand::EnablePixelTypeSignedByteWarning`: it should already have been made virtual for GDALProxyRasterBand needs. +Non-virtual methods :cpp:func:`GDALDataset::GetProjectionRef` and +:cpp:func:`GDALDataset::GetGCPProjection`, which cache the return value, have +been modify to apply a mutex when run on a dataset that IsThreadSafe() to be +effectively thread-safe. + +A SetThreadSafe() method has been added to :cpp:class:`OGRSpatialReference`. +When it is called, all methods of that class run under a per-instance (recursive) +mutex. This is used by GDALThreadSafeDataset for its implementation of the +:cpp:func:`GDALDataset::GetSpatialRef` and :cpp:func:`GDALDataset::GetGCPSpatialRef` +methods, such that the returned OGRSpatialReference instances are thread-safe. + Performance ----------- From b2e5b2387288abfa3fb145b91b262b2e659ece00 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 13:09:11 +0200 Subject: [PATCH 029/710] JSONFG: avoid Polyhedron/Prism geometry instantiation during initial scan During that phase, we only need the geometry type. --- autotest/ogr/ogr_jsonfg.py | 2 +- ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp | 64 ++++++++++++++++------ 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/autotest/ogr/ogr_jsonfg.py b/autotest/ogr/ogr_jsonfg.py index a94636f7af96..e95a7e5b92db 100755 --- a/autotest/ogr/ogr_jsonfg.py +++ b/autotest/ogr/ogr_jsonfg.py @@ -950,7 +950,7 @@ def test_jsonfg_write_several_layers(): [["GEOMETRYCOLLECTION (POINT (1 2))"], ogr.wkbGeometryCollection], [["GEOMETRYCOLLECTION Z (POINT Z (1 2 3))"], ogr.wkbGeometryCollection25D], [["POINT (1 2)", "LINESTRING (1 2,3 4)"], ogr.wkbUnknown], - [["POLYHEDRALSURFACE EMPTY"], ogr.wkbPolyhedralSurface], + [["POLYHEDRALSURFACE Z EMPTY"], ogr.wkbPolyhedralSurfaceZ], [ [ "POLYHEDRALSURFACE Z (((0 0 0,0 1 0,1 1 0,0 0 0)),((0 0 0,1 0 0,0 0 1,0 0 0)),((0 0 0,0 1 0,0 0 1,0 0 0)),((0 1 0,1 0 0,0 0 1,0 1 0)))" diff --git a/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp b/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp index a14ee55c9d6e..b6d95ae424cf 100644 --- a/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp +++ b/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp @@ -663,6 +663,50 @@ const char *OGRJSONFGReader::GetLayerNameForFeature(json_object *poObj) const return pszName; } +/************************************************************************/ +/* OGRJSONFGGetOGRGeometryType() */ +/************************************************************************/ + +static OGRwkbGeometryType OGRJSONFGGetOGRGeometryType(json_object *poObj) +{ + const auto eType = OGRGeoJSONGetOGRGeometryType(poObj); + if (eType != wkbUnknown) + return eType; + + json_object *poObjType = CPL_json_object_object_get(poObj, "type"); + const char *pszType = json_object_get_string(poObjType); + if (!pszType) + return wkbNone; + + if (strcmp(pszType, "Polyhedron") == 0) + { + return wkbPolyhedralSurfaceZ; + } + else if (strcmp(pszType, "Prism") == 0) + { + auto poBase = CPL_json_object_object_get(poObj, "base"); + if (!poBase || json_object_get_type(poBase) != json_type_object) + { + return wkbNone; + } + + const auto eBaseGeomType = OGRGeoJSONGetOGRGeometryType(poBase); + if (eBaseGeomType == wkbPoint) + { + return wkbLineString25D; + } + else if (eBaseGeomType == wkbLineString) + { + return wkbMultiPolygon25D; + } + else if (eBaseGeomType == wkbPolygon) + { + return wkbPolyhedralSurfaceZ; + } + } + return wkbNone; +} + /************************************************************************/ /* OGRJSONFGCreateNonGeoJSONGeometry() */ /************************************************************************/ @@ -708,6 +752,8 @@ OGRJSONFGCreateNonGeoJSONGeometry(json_object *poObj, bool bWarn) if (poGeom->addGeometryDirectly(poPoly) != OGRERR_NONE) return nullptr; } + if (nPolys == 0) + poGeom->set3D(true); return poGeom; } @@ -904,22 +950,8 @@ bool OGRJSONFGReader::GenerateLayerDefnFromFeature(json_object *poObj) (eGeometryElement_ != GeometryElement::PLACE); if (poPlace && json_object_get_type(poPlace) == json_type_object) { - const auto eType = OGRGeoJSONGetOGRGeometryType(poPlace); - if (eType == wkbUnknown) - { - auto poGeom = - OGRJSONFGCreateNonGeoJSONGeometry(poPlace, /*bWarn=*/true); - if (poGeom) - { - bFallbackToGeometry = false; - poContext->bDetectLayerGeomType = - OGRGeoJSONUpdateLayerGeomType( - poContext->bFirstGeometry, - poGeom->getGeometryType(), - poContext->eLayerGeomType); - } - } - else + const auto eType = OGRJSONFGGetOGRGeometryType(poPlace); + if (eType != wkbNone) { bFallbackToGeometry = false; poContext->bDetectLayerGeomType = OGRGeoJSONUpdateLayerGeomType( From 247a2017e7b093a4651fcf318dcbe64a1e9a772e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 18:12:19 +0200 Subject: [PATCH 030/710] Add OGRCloneArrowSchema() --- ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp | 129 ++++++++++++++++++++-- ogr/ogrsf_frmts/generic/ogrlayerarrow.h | 3 + 2 files changed, 123 insertions(+), 9 deletions(-) diff --git a/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp b/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp index b6f9a8ba1a61..5963df74a273 100644 --- a/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp +++ b/ogr/ogrsf_frmts/generic/ogrlayerarrow.cpp @@ -270,25 +270,29 @@ inline void UnsetBit(uint8_t *pabyData, size_t nIdx) /* DefaultReleaseSchema() */ /************************************************************************/ -static void OGRLayerDefaultReleaseSchema(struct ArrowSchema *schema) +static void OGRLayerReleaseSchema(struct ArrowSchema *schema, + bool bFullFreeFormat) { CPLAssert(schema->release != nullptr); - if (STARTS_WITH(schema->format, "w:") || + if (bFullFreeFormat || STARTS_WITH(schema->format, "w:") || STARTS_WITH(schema->format, "tsm:")) { CPLFree(const_cast(schema->format)); } CPLFree(const_cast(schema->name)); CPLFree(const_cast(schema->metadata)); - for (int i = 0; i < static_cast(schema->n_children); ++i) + if (schema->children) { - if (schema->children[i]->release) + for (int i = 0; i < static_cast(schema->n_children); ++i) { - schema->children[i]->release(schema->children[i]); - CPLFree(schema->children[i]); + if (schema->children[i] && schema->children[i]->release) + { + schema->children[i]->release(schema->children[i]); + CPLFree(schema->children[i]); + } } + CPLFree(schema->children); } - CPLFree(schema->children); if (schema->dictionary) { if (schema->dictionary->release) @@ -300,6 +304,16 @@ static void OGRLayerDefaultReleaseSchema(struct ArrowSchema *schema) schema->release = nullptr; } +static void OGRLayerPartialReleaseSchema(struct ArrowSchema *schema) +{ + OGRLayerReleaseSchema(schema, /* bFullFreeFormat = */ false); +} + +static void OGRLayerFullReleaseSchema(struct ArrowSchema *schema) +{ + OGRLayerReleaseSchema(schema, /* bFullFreeFormat = */ true); +} + /** Release a ArrowSchema. * * To be used by driver implementations that have a custom GetArrowStream() @@ -311,7 +325,7 @@ static void OGRLayerDefaultReleaseSchema(struct ArrowSchema *schema) void OGRLayer::ReleaseSchema(struct ArrowSchema *schema) { - OGRLayerDefaultReleaseSchema(schema); + OGRLayerPartialReleaseSchema(schema); } /************************************************************************/ @@ -355,7 +369,7 @@ static void AddDictToSchema(struct ArrowSchema *psChild, auto psChildDict = static_cast( CPLCalloc(1, sizeof(struct ArrowSchema))); psChild->dictionary = psChildDict; - psChildDict->release = OGRLayerDefaultReleaseSchema; + psChildDict->release = OGRLayerPartialReleaseSchema; psChildDict->name = CPLStrdup(poCodedDomain->GetName().c_str()); psChildDict->format = "u"; if (nCountNull) @@ -5464,6 +5478,103 @@ bool OGRCloneArrowArray(const struct ArrowSchema *schema, return OGRCloneArrowArray(schema, src_array, out_array, 0); } +/************************************************************************/ +/* OGRCloneArrowMetadata() */ +/************************************************************************/ + +static void *OGRCloneArrowMetadata(const void *pMetadata) +{ + if (!pMetadata) + return nullptr; + std::vector abyOut; + const GByte *pabyMetadata = static_cast(pMetadata); + int32_t nKVP; + abyOut.insert(abyOut.end(), pabyMetadata, pabyMetadata + sizeof(int32_t)); + memcpy(&nKVP, pabyMetadata, sizeof(int32_t)); + pabyMetadata += sizeof(int32_t); + for (int i = 0; i < nKVP; ++i) + { + int32_t nSizeKey; + abyOut.insert(abyOut.end(), pabyMetadata, + pabyMetadata + sizeof(int32_t)); + memcpy(&nSizeKey, pabyMetadata, sizeof(int32_t)); + pabyMetadata += sizeof(int32_t); + abyOut.insert(abyOut.end(), pabyMetadata, pabyMetadata + nSizeKey); + pabyMetadata += nSizeKey; + + int32_t nSizeValue; + abyOut.insert(abyOut.end(), pabyMetadata, + pabyMetadata + sizeof(int32_t)); + memcpy(&nSizeValue, pabyMetadata, sizeof(int32_t)); + pabyMetadata += sizeof(int32_t); + abyOut.insert(abyOut.end(), pabyMetadata, pabyMetadata + nSizeValue); + pabyMetadata += nSizeValue; + } + + GByte *pabyOut = static_cast(VSI_MALLOC_VERBOSE(abyOut.size())); + if (pabyOut) + memcpy(pabyOut, abyOut.data(), abyOut.size()); + return pabyOut; +} + +/************************************************************************/ +/* OGRCloneArrowSchema() */ +/************************************************************************/ + +/** Full/deep copy of a schema. + * + * In case of failure, out_schema will be let in a released state. + * + * @param schema Schema to clone. Must *NOT* be NULL. + * @param out_schema Output schema. Must *NOT* be NULL (but its content may be random) + * @return true if success. + */ +bool OGRCloneArrowSchema(const struct ArrowSchema *schema, + struct ArrowSchema *out_schema) +{ + memset(out_schema, 0, sizeof(*out_schema)); + out_schema->release = OGRLayerFullReleaseSchema; + out_schema->format = CPLStrdup(schema->format); + out_schema->name = CPLStrdup(schema->name); + out_schema->metadata = static_cast( + const_cast(OGRCloneArrowMetadata(schema->metadata))); + out_schema->flags = schema->flags; + if (schema->n_children) + { + out_schema->children = + static_cast(VSI_CALLOC_VERBOSE( + static_cast(schema->n_children), sizeof(ArrowSchema *))); + if (!out_schema->children) + { + out_schema->release(out_schema); + return false; + } + out_schema->n_children = schema->n_children; + for (int i = 0; i < static_cast(schema->n_children); ++i) + { + out_schema->children[i] = static_cast( + CPLMalloc(sizeof(ArrowSchema))); + if (!OGRCloneArrowSchema(schema->children[i], + out_schema->children[i])) + { + out_schema->release(out_schema); + return false; + } + } + } + if (schema->dictionary) + { + out_schema->dictionary = + static_cast(CPLMalloc(sizeof(ArrowSchema))); + if (!OGRCloneArrowSchema(schema->dictionary, out_schema->dictionary)) + { + out_schema->release(out_schema); + return false; + } + } + return true; +} + /************************************************************************/ /* OGRLayer::IsArrowSchemaSupported() */ /************************************************************************/ diff --git a/ogr/ogrsf_frmts/generic/ogrlayerarrow.h b/ogr/ogrsf_frmts/generic/ogrlayerarrow.h index dc9ed4e726ef..ae45d9135328 100644 --- a/ogr/ogrsf_frmts/generic/ogrlayerarrow.h +++ b/ogr/ogrsf_frmts/generic/ogrlayerarrow.h @@ -47,4 +47,7 @@ bool CPL_DLL OGRCloneArrowArray(const struct ArrowSchema *schema, const struct ArrowArray *array, struct ArrowArray *out_array); +bool CPL_DLL OGRCloneArrowSchema(const struct ArrowSchema *schema, + struct ArrowSchema *out_schema); + #endif // OGRLAYERARROW_H_DEFINED From fdbd6a8f78046ca5d87c55a5f480530d4c8a5e0e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 18:13:22 +0200 Subject: [PATCH 031/710] bench_ogr_batch: add a -sql switch --- perftests/bench_ogr_batch.cpp | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/perftests/bench_ogr_batch.cpp b/perftests/bench_ogr_batch.cpp index b53d5c2b504c..c9f6fc222457 100644 --- a/perftests/bench_ogr_batch.cpp +++ b/perftests/bench_ogr_batch.cpp @@ -39,7 +39,8 @@ static void Usage() { printf( "Usage: bench_ogr_batch [-where filter] [-spat xmin ymin xmax ymax]\n"); - printf(" [--stream-opt NAME=VALUE] [-v]*\n"); + printf(" [--stream-opt NAME=VALUE] [-v] [-sql " + "GetLayerCount() > 1) + if (pszSQL) + { + if (pszLayerName) + { + fprintf(stderr, "-sql is mutually exclusive with layer name.\n"); + CSLDestroy(argv); + exit(1); + } + } + else if (pszLayerName == nullptr && poDS->GetLayerCount() > 1) { fprintf(stderr, "A layer name must be specified because the dataset " "has several layers.\n"); CSLDestroy(argv); exit(1); } - OGRLayer *poLayer = - pszLayerName ? poDS->GetLayerByName(pszLayerName) : poDS->GetLayer(0); + OGRLayer *poLayer = pszSQL ? poDS->ExecuteSQL(pszSQL, nullptr, nullptr) + : pszLayerName ? poDS->GetLayerByName(pszLayerName) + : poDS->GetLayer(0); if (poLayer == nullptr) { fprintf(stderr, "Cannot find layer\n"); @@ -201,6 +218,9 @@ int main(int argc, char *argv[]) printf(CPL_FRMT_GUIB " features/rows selected\n", nFeatureCount); } + if (pszSQL) + poDS->ReleaseResultSet(poLayer); + poDS.reset(); CSLDestroy(argv); From e43fa19e54afd2de429e37a44d3c1a64f767765e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 18:13:07 +0200 Subject: [PATCH 032/710] GenSQL layer: implement OLCFastGetArrowStream when underlying layer does and for very simple SQL statements This can speed-up ``ogr2ogr -sql`` scenarios where the source is Arrow/Parquet/FlatGeoBuf and the SQL statement is something as simple as "SELECT column_a [AS renamed], ... FROM the_layer [WHERE some_cond] [LIMIT limit]" with all or a subset of columns, not reordered compared to the source layer, or repeated. --- autotest/ogr/ogr_flatgeobuf.py | 149 +++++++++++ ogr/ogrsf_frmts/generic/ogr_gensql.cpp | 356 ++++++++++++++++++++++++- ogr/ogrsf_frmts/generic/ogr_gensql.h | 18 ++ ogr/ogrsf_frmts/ogrsf_frmts.h | 3 + 4 files changed, 525 insertions(+), 1 deletion(-) diff --git a/autotest/ogr/ogr_flatgeobuf.py b/autotest/ogr/ogr_flatgeobuf.py index a37597cc044c..3b58eaf4832f 100644 --- a/autotest/ogr/ogr_flatgeobuf.py +++ b/autotest/ogr/ogr_flatgeobuf.py @@ -1419,3 +1419,152 @@ def test_ogr_flatgeobuf_write_mismatch_geom_type(tmp_vsimem): match="ICreateFeature: Mismatched geometry type. Feature geometry type is Line String, expected layer geometry type is Point", ): lyr.CreateFeature(f) + + +############################################################################### +# Test OGRGenSQLResultLayer::GetArrowStream() implementation. +# There isn't much specific of the FlatGeoBuf driver, except it is the +# only one in a default build that implements OLCFastGetArrowStream and doesn't +# have a specialized ExecuteSQL() implementation. + + +@gdaltest.enable_exceptions() +def test_ogr_flatgeobuf_sql_arrow(tmp_vsimem): + + filename = str(tmp_vsimem / "temp.fgb") + with ogr.GetDriverByName("FlatGeoBuf").CreateDataSource(filename) as ds: + lyr = ds.CreateLayer("test", geom_type=ogr.wkbPoint) + lyr.CreateField(ogr.FieldDefn("foo")) + lyr.CreateField(ogr.FieldDefn("bar")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "bar" + f["bar"] = "baz" + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (1 2)")) + lyr.CreateFeature(f) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "bar2" + f["bar"] = "baz2" + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (3 4)")) + lyr.CreateFeature(f) + + with ogr.Open(filename) as ds: + with ds.ExecuteSQL("SELECT 'a' FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + f = tmp_lyr.GetNextFeature() + assert f["FIELD_1"] == "a" + + with ds.ExecuteSQL("SELECT foo, foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT CONCAT(foo, 'x') FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT foo AS renamed, foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT bar, foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT CAST(foo AS float) FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT MIN(foo) FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT COUNT(*) FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test a JOIN test b ON a.foo = b.foo") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test OFFSET 1") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test ORDER BY foo") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT *, OGR_STYLE HIDDEN FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT DISTINCT foo FROM test") as lyr: + assert not lyr.TestCapability(ogr.OLCFastGetArrowStream) + + with ds.ExecuteSQL("SELECT * FROM test") as lyr: + try: + stream = lyr.GetArrowStreamAsNumPy() + except ImportError: + stream = None + if stream: + with pytest.raises( + Exception, + match=r"Calling get_next\(\) on a freed OGRLayer is not supported", + ): + [batch for batch in stream] + + sql = "SELECT foo, bar AS bar_renamed FROM test" + with ds.ExecuteSQL(sql) as lyr: + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetLayerDefn().GetFieldCount() == 2 + assert tmp_lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "foo" + assert tmp_lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "bar_renamed" + assert tmp_lyr.GetFeatureCount() == 2 + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar2" + assert f["bar_renamed"] == "baz2" + assert f.GetGeometryRef().ExportToWkt() == "POINT (3 4)" + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar" + assert f["bar_renamed"] == "baz" + assert f.GetGeometryRef().ExportToWkt() == "POINT (1 2)" + + sql = "SELECT bar FROM test LIMIT 1" + with ds.ExecuteSQL(sql) as lyr: + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetLayerDefn().GetFieldCount() == 1 + assert tmp_lyr.GetFeatureCount() == 1 + f = tmp_lyr.GetNextFeature() + assert f["bar"] == "baz2" + assert f.GetGeometryRef().ExportToWkt() == "POINT (3 4)" + + sql = "SELECT * EXCLUDE (\"_ogr_geometry_\") FROM test WHERE foo = 'bar'" + with ds.ExecuteSQL(sql) as lyr: + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetFeatureCount() == 1 + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar" + assert f["bar"] == "baz" + assert f.GetGeometryRef() is None + + sql = "SELECT * FROM test" + with ds.ExecuteSQL(sql) as lyr: + lyr.SetSpatialFilterRect(1, 2, 1, 2) + assert lyr.TestCapability(ogr.OLCFastGetArrowStream) + + tmp_ds = ogr.GetDriverByName("Memory").CreateDataSource("") + tmp_lyr = tmp_ds.CreateLayer("test") + tmp_lyr.WriteArrow(lyr) + assert tmp_lyr.GetLayerDefn().GetFieldCount() == 2 + assert tmp_lyr.GetLayerDefn().GetFieldDefn(0).GetName() == "foo" + assert tmp_lyr.GetLayerDefn().GetFieldDefn(1).GetName() == "bar" + assert tmp_lyr.GetFeatureCount() == 1 + f = tmp_lyr.GetNextFeature() + assert f["foo"] == "bar" + assert f["bar"] == "baz" + assert f.GetGeometryRef().ExportToWkt() == "POINT (1 2)" + f = tmp_lyr.GetNextFeature() diff --git a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp index ee0657331182..91307d3efb32 100644 --- a/ogr/ogrsf_frmts/generic/ogr_gensql.cpp +++ b/ogr/ogrsf_frmts/generic/ogr_gensql.cpp @@ -32,9 +32,13 @@ #include "ogr_gensql.h" #include "cpl_string.h" #include "ogr_api.h" +#include "ogr_recordbatch.h" +#include "ogrlayerarrow.h" #include "cpl_time.h" #include #include +#include +#include #include //! @cond Doxygen_Suppress @@ -745,7 +749,7 @@ GIntBig OGRGenSQLResultsLayer::GetFeatureCount(int bForce) int OGRGenSQLResultsLayer::TestCapability(const char *pszCap) { - swq_select *psSelectInfo = m_pSelectInfo.get(); + const swq_select *psSelectInfo = m_pSelectInfo.get(); if (EQUAL(pszCap, OLCFastSetNextByIndex)) { @@ -774,6 +778,58 @@ int OGRGenSQLResultsLayer::TestCapability(const char *pszCap) return m_poSrcLayer->TestCapability(pszCap); } + else if (EQUAL(pszCap, OLCFastGetArrowStream)) + { + // Make sure the SQL is something as simple as + // "SELECT field1 [AS renamed], ... FROM ... WHERE ....", without + // duplicated fields + if (m_bForwardWhereToSourceLayer && + psSelectInfo->query_mode == SWQM_RECORDSET && + psSelectInfo->offset == 0 && psSelectInfo->join_count == 0 && + psSelectInfo->order_specs == 0) + { + std::set oSetFieldIndex; + int nLastIdxRegularField = -1; + for (std::size_t iField = 0; + iField < psSelectInfo->column_defs.size(); iField++) + { + const swq_col_def *psColDef = + &psSelectInfo->column_defs[iField]; + if (psColDef->bHidden || psColDef->table_index < 0 || + psColDef->col_func != SWQCF_NONE || + cpl::contains(oSetFieldIndex, psColDef->field_index)) + { + return false; + } + + oSetFieldIndex.insert(psColDef->field_index); + + const auto poLayerDefn = + m_apoTableLayers[psColDef->table_index]->GetLayerDefn(); + + if (psColDef->field_index >= 0 && poLayerDefn != nullptr && + psColDef->field_index < poLayerDefn->GetFieldCount()) + { + // We do not support re-ordered fields + if (psColDef->field_index <= nLastIdxRegularField) + return false; + nLastIdxRegularField = psColDef->field_index; + } + else if (poLayerDefn != nullptr && + IS_GEOM_FIELD_INDEX(poLayerDefn, + psColDef->field_index)) + { + // ok + } + else + { + return false; + } + } + return m_poSrcLayer->TestCapability(pszCap); + } + } + return FALSE; } @@ -2738,4 +2794,302 @@ void OGRGenSQLResultsLayer::SetSpatialFilter(int iGeomField, OGRLayer::SetSpatialFilter(iGeomField, poGeom); } +/************************************************************************/ +/* OGRGenSQLResultsLayerArrowStreamPrivateData */ +/************************************************************************/ + +// Structure whose instances are set on the ArrowArrayStream::private_data +// member of the ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() +struct OGRGenSQLResultsLayerArrowStreamPrivateData +{ + // Member shared with OGRLayer::m_poSharedArrowArrayStreamPrivateData + // If the layer pointed by poShared->poLayer is destroyed, before its + // destruction, it nullifies poShared->poLayer, which we can detect. + std::shared_ptr poShared{}; + + // ArrowArrayStream to be used with poShared->poLayer + struct ArrowArrayStream *psSrcLayerStream = nullptr; + + // Original release() callback of the ArrowArrayStream passed to + // OGRGenSQLResultsLayer::GetArrowStream() + void (*release_backup)(struct ArrowArrayStream *) = nullptr; + + // Original private_data member of the ArrowArrayStream passed to + // OGRGenSQLResultsLayer::GetArrowStream() + void *private_data_backup = nullptr; + + // Set as the ArrowArrayStream::release callback member of the + // ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() + static void Release(struct ArrowArrayStream *self) + { + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + self->private_data); + + // Release source layer stream + if (psPrivateData->psSrcLayerStream->release) + psPrivateData->psSrcLayerStream->release( + psPrivateData->psSrcLayerStream); + CPLFree(psPrivateData->psSrcLayerStream); + + // Release ourselves using the base method + self->private_data = psPrivateData->private_data_backup; + self->release = psPrivateData->release_backup; + delete psPrivateData; + if (self->release) + self->release(self); + } + + // Set as the ArrowArrayStream::get_schema callback member of the + // ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() + static int GetSchema(struct ArrowArrayStream *self, struct ArrowSchema *out) + { + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + self->private_data); + auto poLayer = dynamic_cast( + psPrivateData->poShared->m_poLayer); + if (!poLayer) + { + CPLError( + CE_Failure, CPLE_NotSupported, + "Calling get_schema() on a freed OGRLayer is not supported"); + return EINVAL; + } + return poLayer->GetArrowSchemaForwarded(self, out); + } + + // Set as the ArrowArrayStream::get_next callback member of the + // ArrowArrayStream returned by OGRGenSQLResultsLayer::GetArrowStream() + static int GetNext(struct ArrowArrayStream *self, struct ArrowArray *out) + { + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + self->private_data); + auto poLayer = dynamic_cast( + psPrivateData->poShared->m_poLayer); + if (!poLayer) + { + CPLError(CE_Failure, CPLE_NotSupported, + "Calling get_next() on a freed OGRLayer is not supported"); + return EINVAL; + } + return poLayer->GetNextArrowArrayForwarded(self, out); + } +}; + +/************************************************************************/ +/* GetArrowStream() */ +/************************************************************************/ + +bool OGRGenSQLResultsLayer::GetArrowStream(struct ArrowArrayStream *out_stream, + CSLConstList papszOptions) +{ + if (!TestCapability(OLCFastGetArrowStream) || + CPLTestBool(CPLGetConfigOption("OGR_GENSQL_STREAM_BASE_IMPL", "NO"))) + { + CPLStringList aosOptions(papszOptions); + aosOptions.SetNameValue("OGR_GENSQL_STREAM_BASE_IMPL", "YES"); + return OGRLayer::GetArrowStream(out_stream, aosOptions.List()); + } + + const swq_select *psSelectInfo = m_pSelectInfo.get(); + if (m_nIteratedFeatures != -1) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GetArrowStream() not supported on non-rewinded layer"); + return false; + } + CPLStringList aosOptions(papszOptions); + if (psSelectInfo->limit > 0) + { + aosOptions.SetNameValue( + "MAX_FEATURES_IN_BATCH", + CPLSPrintf(CPL_FRMT_GIB, + std::min(psSelectInfo->limit, + CPLAtoGIntBig(aosOptions.FetchNameValueDef( + "MAX_FEATURES_IN_BATCH", "65536"))))); + } + bool bRet = OGRLayer::GetArrowStream(out_stream, aosOptions.List()); + if (bRet) + { + auto psSrcLayerStream = static_cast( + CPLMalloc(sizeof(ArrowArrayStream))); + if (m_poSrcLayer->GetArrowStream(psSrcLayerStream, aosOptions.List())) + { + auto psPrivateData = + new OGRGenSQLResultsLayerArrowStreamPrivateData; + CPLAssert(m_poSharedArrowArrayStreamPrivateData); + psPrivateData->poShared = m_poSharedArrowArrayStreamPrivateData; + psPrivateData->psSrcLayerStream = psSrcLayerStream; + psPrivateData->release_backup = out_stream->release; + psPrivateData->private_data_backup = out_stream->private_data; + out_stream->get_schema = + OGRGenSQLResultsLayerArrowStreamPrivateData::GetSchema; + out_stream->get_next = + OGRGenSQLResultsLayerArrowStreamPrivateData::GetNext; + out_stream->release = + OGRGenSQLResultsLayerArrowStreamPrivateData::Release; + out_stream->private_data = psPrivateData; + } + else + { + if (psSrcLayerStream->release) + psSrcLayerStream->release(psSrcLayerStream); + CPLFree(psSrcLayerStream); + if (out_stream->release) + out_stream->release(out_stream); + bRet = false; + } + } + return bRet; +} + +/************************************************************************/ +/* GetArrowSchema() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetArrowSchema(struct ArrowArrayStream *stream, + struct ArrowSchema *out_schema) +{ + if (m_aosArrowArrayStreamOptions.FetchNameValue( + "OGR_GENSQL_STREAM_BASE_IMPL") || + !TestCapability(OLCFastGetArrowStream)) + { + return OGRLayer::GetArrowSchema(stream, out_schema); + } + + return GetArrowSchemaForwarded(stream, out_schema); +} + +/************************************************************************/ +/* GetArrowSchemaForwarded() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetArrowSchemaForwarded( + struct ArrowArrayStream *stream, struct ArrowSchema *out_schema) const +{ + const swq_select *psSelectInfo = m_pSelectInfo.get(); + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + stream->private_data); + int ret = m_poSrcLayer->GetArrowSchema(psPrivateData->psSrcLayerStream, + out_schema); + if (ret == 0) + { + struct ArrowSchema newSchema; + ret = OGRCloneArrowSchema(out_schema, &newSchema) ? 0 : EIO; + if (out_schema->release) + out_schema->release(out_schema); + if (ret == 0) + { + std::map oMapSrcNameToRenamed; + for (std::size_t iField = 0; + iField < psSelectInfo->column_defs.size(); iField++) + { + const swq_col_def *psColDef = + &psSelectInfo->column_defs[iField]; + CPLAssert(!psColDef->bHidden); + CPLAssert(psColDef->table_index >= 0); + CPLAssert(psColDef->col_func == SWQCF_NONE); + + const auto poLayerDefn = + m_apoTableLayers[psColDef->table_index]->GetLayerDefn(); + CPLAssert(poLayerDefn); + + if (psColDef->field_index >= 0 && + psColDef->field_index < poLayerDefn->GetFieldCount()) + { + const auto poSrcFDefn = + poLayerDefn->GetFieldDefn(psColDef->field_index); + if (psColDef->field_alias) + oMapSrcNameToRenamed[poSrcFDefn->GetNameRef()] = + psColDef->field_alias; + } + else if (IS_GEOM_FIELD_INDEX(poLayerDefn, + psColDef->field_index)) + { + const int iSrcGeomField = + ALL_FIELD_INDEX_TO_GEOM_FIELD_INDEX( + poLayerDefn, psColDef->field_index); + const auto poSrcGFDefn = + poLayerDefn->GetGeomFieldDefn(iSrcGeomField); + if (psColDef->field_alias) + oMapSrcNameToRenamed[poSrcGFDefn->GetNameRef()] = + psColDef->field_alias; + } + } + + for (int i = 0; i < newSchema.n_children; ++i) + { + const auto oIter = + oMapSrcNameToRenamed.find(newSchema.children[i]->name); + if (oIter != oMapSrcNameToRenamed.end()) + { + CPLFree(const_cast(newSchema.children[i]->name)); + newSchema.children[i]->name = + CPLStrdup(oIter->second.c_str()); + } + } + + memcpy(out_schema, &newSchema, sizeof(newSchema)); + } + } + return ret; +} + +/************************************************************************/ +/* GetNextArrowArray() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetNextArrowArray(struct ArrowArrayStream *stream, + struct ArrowArray *out_array) +{ + if (m_aosArrowArrayStreamOptions.FetchNameValue( + "OGR_GENSQL_STREAM_BASE_IMPL") || + !TestCapability(OLCFastGetArrowStream)) + { + return OGRLayer::GetNextArrowArray(stream, out_array); + } + + return GetNextArrowArrayForwarded(stream, out_array); +} + +/************************************************************************/ +/* GetNextArrowArrayForwarded() */ +/************************************************************************/ + +int OGRGenSQLResultsLayer::GetNextArrowArrayForwarded( + struct ArrowArrayStream *stream, struct ArrowArray *out_array) +{ + const swq_select *psSelectInfo = m_pSelectInfo.get(); + if (psSelectInfo->limit >= 0 && m_nIteratedFeatures >= psSelectInfo->limit) + { + memset(out_array, 0, sizeof(*out_array)); + return 0; + } + + OGRGenSQLResultsLayerArrowStreamPrivateData *psPrivateData = + static_cast( + stream->private_data); + const int ret = m_poSrcLayer->GetNextArrowArray( + psPrivateData->psSrcLayerStream, out_array); + if (ret == 0 && psSelectInfo->limit >= 0) + { + if (m_nIteratedFeatures < 0) + m_nIteratedFeatures = 0; + m_nIteratedFeatures += out_array->length; + if (m_nIteratedFeatures > psSelectInfo->limit) + { + out_array->length -= m_nIteratedFeatures - psSelectInfo->limit; + for (int i = 0; i < out_array->n_children; ++i) + { + out_array->children[i]->length -= + m_nIteratedFeatures - psSelectInfo->limit; + } + } + } + return ret; +} + //! @endcond diff --git a/ogr/ogrsf_frmts/generic/ogr_gensql.h b/ogr/ogrsf_frmts/generic/ogr_gensql.h index 465d025805cd..30c3eacd9948 100644 --- a/ogr/ogrsf_frmts/generic/ogr_gensql.h +++ b/ogr/ogrsf_frmts/generic/ogr_gensql.h @@ -151,6 +151,24 @@ class OGRGenSQLResultsLayer final : public OGRLayer virtual void SetSpatialFilter(int iGeomField, OGRGeometry *) override; virtual OGRErr SetAttributeFilter(const char *) override; + + bool GetArrowStream(struct ArrowArrayStream *out_stream, + CSLConstList papszOptions = nullptr) override; + + int GetArrowSchema(struct ArrowArrayStream *stream, + struct ArrowSchema *out_schema) override; + + protected: + friend struct OGRGenSQLResultsLayerArrowStreamPrivateData; + + int GetArrowSchemaForwarded(struct ArrowArrayStream *stream, + struct ArrowSchema *out_schema) const; + + int GetNextArrowArray(struct ArrowArrayStream *stream, + struct ArrowArray *out_array) override; + + int GetNextArrowArrayForwarded(struct ArrowArrayStream *stream, + struct ArrowArray *out_array); }; /*! @endcond */ diff --git a/ogr/ogrsf_frmts/ogrsf_frmts.h b/ogr/ogrsf_frmts/ogrsf_frmts.h index 7878fcd81401..26749e3c59f8 100644 --- a/ogr/ogrsf_frmts/ogrsf_frmts.h +++ b/ogr/ogrsf_frmts/ogrsf_frmts.h @@ -133,6 +133,8 @@ class CPL_DLL OGRLayer : public GDALMajorObject //! @cond Doxygen_Suppress CPLStringList m_aosArrowArrayStreamOptions{}; + friend struct OGRGenSQLResultsLayerArrowStreamPrivateData; + struct ArrowArrayStreamPrivateData { bool m_bArrowArrayStreamInProgress = false; @@ -153,6 +155,7 @@ class CPL_DLL OGRLayer : public GDALMajorObject //! @endcond friend class OGRArrowArrayHelper; + friend class OGRGenSQLResultsLayer; static void ReleaseArray(struct ArrowArray *array); static void ReleaseSchema(struct ArrowSchema *schema); static void ReleaseStream(struct ArrowArrayStream *stream); From f9283394f58562eacaba1f705f717ddf000a2341 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 10 Sep 2024 17:12:05 +0200 Subject: [PATCH 033/710] Overview: fix nearest resampling to be exact with all data types Fixes #10758 Also make sure that for other resampling methods, the working data type is large enough (e.g using Float64 for Int32/UInt32/Int64/UInt64). While doing so, fix an underlying bug in the convolution based code, where a hard-coded pixel stride value of 4 was used instead of sizeof(TWork), which I believe wasn't visible before. --- autotest/gcore/rasterio.py | 136 ++++++++++++++++++++++++++++++----- gcore/gdalnodatamaskband.cpp | 3 +- gcore/overview.cpp | 96 ++++++++++++++++++------- 3 files changed, 190 insertions(+), 45 deletions(-) diff --git a/autotest/gcore/rasterio.py b/autotest/gcore/rasterio.py index cee967b15724..ed44b28ffdf2 100755 --- a/autotest/gcore/rasterio.py +++ b/autotest/gcore/rasterio.py @@ -884,29 +884,129 @@ def test_rasterio_12(): # Test cubic resampling with masking -def test_rasterio_13(): +@pytest.mark.parametrize( + "dt", + [ + "Byte", + "Int8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Float32", + "Float64", + ], +) +def test_rasterio_13(dt): numpy = pytest.importorskip("numpy") - for dt in [gdal.GDT_Byte, gdal.GDT_UInt16, gdal.GDT_UInt32]: + dt = gdal.GetDataTypeByName(dt) + mem_ds = gdal.GetDriverByName("MEM").Create("", 4, 3, 1, dt) + mem_ds.GetRasterBand(1).SetNoDataValue(0) + if dt == gdal.GDT_Int8: + x = (1 << 7) - 1 + elif dt == gdal.GDT_Byte: + x = (1 << 8) - 1 + elif dt == gdal.GDT_Int16: + x = (1 << 15) - 1 + elif dt == gdal.GDT_UInt16: + x = (1 << 16) - 1 + elif dt == gdal.GDT_Int32: + x = (1 << 31) - 1 + elif dt == gdal.GDT_UInt32: + x = (1 << 32) - 1 + elif dt == gdal.GDT_Int64: + x = (1 << 63) - 1 + elif dt == gdal.GDT_UInt64: + x = (1 << 64) - 2048 + elif dt == gdal.GDT_Float32: + x = 1.5 + else: + x = 1.23456 + mem_ds.GetRasterBand(1).WriteArray( + numpy.array([[0, 0, 0, 0], [0, x, 0, 0], [0, 0, 0, 0]]) + ) - mem_ds = gdal.GetDriverByName("MEM").Create("", 4, 3, 1, dt) - mem_ds.GetRasterBand(1).SetNoDataValue(0) - mem_ds.GetRasterBand(1).WriteArray( - numpy.array([[0, 0, 0, 0], [0, 255, 0, 0], [0, 0, 0, 0]]) - ) + ar_ds = mem_ds.ReadAsArray( + 0, 0, 4, 3, buf_xsize=8, buf_ysize=3, resample_alg=gdal.GRIORA_Cubic + ) - ar_ds = mem_ds.ReadAsArray( - 0, 0, 4, 3, buf_xsize=8, buf_ysize=3, resample_alg=gdal.GRIORA_Cubic - ) + expected_ar = numpy.array( + [ + [0, 0, 0, 0, 0, 0, 0, 0], + [0, x, x, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0], + ] + ) + assert numpy.array_equal(ar_ds, expected_ar) - expected_ar = numpy.array( - [ - [0, 0, 0, 0, 0, 0, 0, 0], - [0, 255, 255, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0], - ] - ) - assert numpy.array_equal(ar_ds, expected_ar), (ar_ds, dt) + +############################################################################### +# Test cubic resampling with masking + + +@pytest.mark.parametrize( + "dt", + [ + "Byte", + "Int8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "Float32", + "Float64", + "CInt16", + "CInt32", + "CFloat32", + "CFloat64", + ], +) +def test_rasterio_nearest(dt): + numpy = pytest.importorskip("numpy") + gdal_array = pytest.importorskip("osgeo.gdal_array") + + dt = gdal.GetDataTypeByName(dt) + mem_ds = gdal.GetDriverByName("MEM").Create("", 4, 4, 1, dt) + if dt == gdal.GDT_Int8: + x = (1 << 7) - 1 + elif dt == gdal.GDT_Byte: + x = (1 << 8) - 1 + elif dt == gdal.GDT_Int16 or dt == gdal.GDT_CInt16: + x = (1 << 15) - 1 + elif dt == gdal.GDT_UInt16: + x = (1 << 16) - 1 + elif dt == gdal.GDT_Int32 or dt == gdal.GDT_CInt32: + x = (1 << 31) - 1 + elif dt == gdal.GDT_UInt32: + x = (1 << 32) - 1 + elif dt == gdal.GDT_Int64: + x = (1 << 63) - 1 + elif dt == gdal.GDT_UInt64: + x = (1 << 64) - 1 + elif dt == gdal.GDT_Float32 or dt == gdal.GDT_CFloat32: + x = 1.5 + else: + x = 1.234567890123 + + if gdal.DataTypeIsComplex(dt): + x = complex(x, x) + + dtype = gdal_array.flip_code(dt) + mem_ds.GetRasterBand(1).WriteArray(numpy.full((4, 4), x, dtype=dtype)) + + ar_ds = mem_ds.ReadAsArray(0, 0, 4, 4, buf_xsize=1, buf_ysize=1) + + expected_ar = numpy.array([[x]]).astype(dtype) + assert numpy.array_equal(ar_ds, expected_ar) + + mem_ds.BuildOverviews("NEAR", [4]) + ar_ds = mem_ds.GetRasterBand(1).GetOverview(0).ReadAsArray() + assert numpy.array_equal(ar_ds, expected_ar) ############################################################################### diff --git a/gcore/gdalnodatamaskband.cpp b/gcore/gdalnodatamaskband.cpp index cd3b121acbbd..390957567d47 100644 --- a/gcore/gdalnodatamaskband.cpp +++ b/gcore/gdalnodatamaskband.cpp @@ -147,7 +147,8 @@ static GDALDataType GetWorkDataType(GDALDataType eDataType) eWrkDT = eDataType; break; - default: + case GDT_Unknown: + case GDT_TypeCount: CPLAssert(false); eWrkDT = GDT_Float64; break; diff --git a/gcore/overview.cpp b/gcore/overview.cpp index c962aa73bfc2..ccbf3f93028c 100644 --- a/gcore/overview.cpp +++ b/gcore/overview.cpp @@ -170,35 +170,58 @@ static CPLErr GDALResampleChunk_Near(const GDALOverviewResampleArgs &args, *peDstBufferDataType = args.eWrkDataType; switch (args.eWrkDataType) { + // For nearest resampling, as no computation is done, only the + // size of the data type matters. case GDT_Byte: + case GDT_Int8: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 1); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } + case GDT_Int16: case GDT_UInt16: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 2); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } + case GDT_CInt16: + case GDT_Int32: + case GDT_UInt32: case GDT_Float32: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 4); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } + case GDT_CInt32: + case GDT_CFloat32: + case GDT_Int64: + case GDT_UInt64: case GDT_Float64: { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 8); return GDALResampleChunk_NearT( - args, static_cast(pChunk), - reinterpret_cast(ppDstBuffer)); + args, static_cast(pChunk), + reinterpret_cast(ppDstBuffer)); } - default: + case GDT_CFloat64: + { + return GDALResampleChunk_NearT( + args, static_cast *>(pChunk), + reinterpret_cast **>(ppDstBuffer)); + } + + case GDT_Unknown: + case GDT_TypeCount: break; } CPLAssert(false); @@ -3032,6 +3055,7 @@ static CPLErr GDALResampleChunk_ConvolutionT( // cppcheck-suppress unreadVariable const int isIntegerDT = GDALDataTypeIsInteger(dstDataType); const auto nNodataValueInt64 = static_cast(dfNoDataValue); + constexpr int nWrkDataTypeSize = static_cast(sizeof(Twork)); // TODO: we should have some generic function to do this. Twork fDstMin = -std::numeric_limits::max(); @@ -3068,6 +3092,20 @@ static CPLErr GDALResampleChunk_ConvolutionT( // cppcheck-suppress unreadVariable fDstMax = static_cast(std::numeric_limits::max()); } + else if (dstDataType == GDT_UInt64) + { + // cppcheck-suppress unreadVariable + fDstMin = static_cast(std::numeric_limits::min()); + // cppcheck-suppress unreadVariable + fDstMax = static_cast(std::numeric_limits::max()); + } + else if (dstDataType == GDT_Int64) + { + // cppcheck-suppress unreadVariable + fDstMin = static_cast(std::numeric_limits::min()); + // cppcheck-suppress unreadVariable + fDstMax = static_cast(std::numeric_limits::max()); + } auto replaceValIfNodata = [bHasNoData, isIntegerDT, fDstMin, fDstMax, nNodataValueInt64, dfNoDataValue, @@ -3585,7 +3623,7 @@ static CPLErr GDALResampleChunk_ConvolutionT( if (pafWrkScanline) { - GDALCopyWords(pafWrkScanline, eWrkDataType, 4, + GDALCopyWords(pafWrkScanline, eWrkDataType, nWrkDataTypeSize, static_cast(pDstBuffer) + static_cast(iDstLine - nDstYOff) * nDstXSize * nDstDataTypeSize, @@ -4101,33 +4139,38 @@ GDALResampleFunction GDALGetResampleFunction(const char *pszResampling, GDALDataType GDALGetOvrWorkDataType(const char *pszResampling, GDALDataType eSrcDataType) { - if ((STARTS_WITH_CI(pszResampling, "NEAR") || - STARTS_WITH_CI(pszResampling, "AVER") || EQUAL(pszResampling, "RMS") || - EQUAL(pszResampling, "CUBIC") || EQUAL(pszResampling, "CUBICSPLINE") || - EQUAL(pszResampling, "LANCZOS") || EQUAL(pszResampling, "BILINEAR") || - EQUAL(pszResampling, "MODE")) && - eSrcDataType == GDT_Byte) + if (STARTS_WITH_CI(pszResampling, "NEAR")) + { + return eSrcDataType; + } + else if (eSrcDataType == GDT_Byte && + (STARTS_WITH_CI(pszResampling, "AVER") || + EQUAL(pszResampling, "RMS") || EQUAL(pszResampling, "CUBIC") || + EQUAL(pszResampling, "CUBICSPLINE") || + EQUAL(pszResampling, "LANCZOS") || + EQUAL(pszResampling, "BILINEAR") || EQUAL(pszResampling, "MODE"))) { return GDT_Byte; } - else if ((STARTS_WITH_CI(pszResampling, "NEAR") || - STARTS_WITH_CI(pszResampling, "AVER") || + else if (eSrcDataType == GDT_UInt16 && + (STARTS_WITH_CI(pszResampling, "AVER") || EQUAL(pszResampling, "RMS") || EQUAL(pszResampling, "CUBIC") || EQUAL(pszResampling, "CUBICSPLINE") || EQUAL(pszResampling, "LANCZOS") || - EQUAL(pszResampling, "BILINEAR") || - EQUAL(pszResampling, "MODE")) && - eSrcDataType == GDT_UInt16) + EQUAL(pszResampling, "BILINEAR") || EQUAL(pszResampling, "MODE"))) { return GDT_UInt16; } else if (EQUAL(pszResampling, "GAUSS")) return GDT_Float64; - if (eSrcDataType == GDT_Float64) - return GDT_Float64; - - return GDT_Float32; + if (eSrcDataType == GDT_Byte || eSrcDataType == GDT_Int8 || + eSrcDataType == GDT_UInt16 || eSrcDataType == GDT_Int16 || + eSrcDataType == GDT_Float32) + { + return GDT_Float32; + } + return GDT_Float64; } namespace @@ -4361,7 +4404,8 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, const GDALDataType eSrcDataType = poSrcBand->GetRasterDataType(); const GDALDataType eWrkDataType = - GDALDataTypeIsComplex(eSrcDataType) + (GDALDataTypeIsComplex(eSrcDataType) && + !STARTS_WITH_CI(pszResampling, "NEAR")) ? GDT_CFloat32 : GDALGetOvrWorkDataType(pszResampling, eSrcDataType); From 4a938e59d125448f00075e24bdfa8efa213cd893 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 10 Sep 2024 21:53:20 +0200 Subject: [PATCH 034/710] Overview: fix mode resampling to be exact with all data types Fixes #10758 --- autotest/gcore/rasterio.py | 46 +++++++--- gcore/overview.cpp | 181 ++++++++++++++++++++++++++++--------- gcore/rasterio.cpp | 7 +- 3 files changed, 178 insertions(+), 56 deletions(-) diff --git a/autotest/gcore/rasterio.py b/autotest/gcore/rasterio.py index ed44b28ffdf2..777f0841bfca 100755 --- a/autotest/gcore/rasterio.py +++ b/autotest/gcore/rasterio.py @@ -944,7 +944,7 @@ def test_rasterio_13(dt): ############################################################################### -# Test cubic resampling with masking +# Test nearest and mode resampling @pytest.mark.parametrize( @@ -966,7 +966,11 @@ def test_rasterio_13(dt): "CFloat64", ], ) -def test_rasterio_nearest(dt): +@pytest.mark.parametrize( + "resample_alg", [gdal.GRIORA_NearestNeighbour, gdal.GRIORA_Mode] +) +@pytest.mark.parametrize("use_nan", [True, False]) +def test_rasterio_nearest_or_mode(dt, resample_alg, use_nan): numpy = pytest.importorskip("numpy") gdal_array = pytest.importorskip("osgeo.gdal_array") @@ -989,24 +993,44 @@ def test_rasterio_nearest(dt): elif dt == gdal.GDT_UInt64: x = (1 << 64) - 1 elif dt == gdal.GDT_Float32 or dt == gdal.GDT_CFloat32: - x = 1.5 + x = float("nan") if use_nan else 1.5 else: - x = 1.234567890123 + x = float("nan") if use_nan else 1.234567890123 if gdal.DataTypeIsComplex(dt): - x = complex(x, x) + val = complex(x, x) + else: + val = x dtype = gdal_array.flip_code(dt) - mem_ds.GetRasterBand(1).WriteArray(numpy.full((4, 4), x, dtype=dtype)) + mem_ds.GetRasterBand(1).WriteArray(numpy.full((4, 4), val, dtype=dtype)) - ar_ds = mem_ds.ReadAsArray(0, 0, 4, 4, buf_xsize=1, buf_ysize=1) + ar_ds = mem_ds.ReadAsArray( + 0, 0, 4, 4, buf_xsize=1, buf_ysize=1, resample_alg=resample_alg + ) - expected_ar = numpy.array([[x]]).astype(dtype) - assert numpy.array_equal(ar_ds, expected_ar) + expected_ar = numpy.array([[val]]).astype(dtype) + if math.isnan(x): + if gdal.DataTypeIsComplex(dt): + assert math.isnan(ar_ds[0][0].real) and math.isnan(ar_ds[0][0].imag) + else: + assert math.isnan(ar_ds[0][0]) + else: + assert numpy.array_equal(ar_ds, expected_ar) - mem_ds.BuildOverviews("NEAR", [4]) + resample_alg_mapping = { + gdal.GRIORA_NearestNeighbour: "NEAR", + gdal.GRIORA_Mode: "MODE", + } + mem_ds.BuildOverviews(resample_alg_mapping[resample_alg], [4]) ar_ds = mem_ds.GetRasterBand(1).GetOverview(0).ReadAsArray() - assert numpy.array_equal(ar_ds, expected_ar) + if math.isnan(x): + if gdal.DataTypeIsComplex(dt): + assert math.isnan(ar_ds[0][0].real) and math.isnan(ar_ds[0][0].imag) + else: + assert math.isnan(ar_ds[0][0]) + else: + assert numpy.array_equal(ar_ds, expected_ar) ############################################################################### diff --git a/gcore/overview.cpp b/gcore/overview.cpp index ccbf3f93028c..74d99b32bb8a 100644 --- a/gcore/overview.cpp +++ b/gcore/overview.cpp @@ -2099,9 +2099,39 @@ static CPLErr GDALResampleChunk_Gauss(const GDALOverviewResampleArgs &args, /* GDALResampleChunk_Mode() */ /************************************************************************/ +template static inline bool IsSame(T a, T b) +{ + return a == b; +} + +template <> bool IsSame(float a, float b) +{ + return a == b || (std::isnan(a) && std::isnan(b)); +} + +template <> bool IsSame(double a, double b) +{ + return a == b || (std::isnan(a) && std::isnan(b)); +} + +template <> +bool IsSame>(std::complex a, std::complex b) +{ + return a == b || (std::isnan(a.real()) && std::isnan(a.imag()) && + std::isnan(b.real()) && std::isnan(b.imag())); +} + +template <> +bool IsSame>(std::complex a, + std::complex b) +{ + return a == b || (std::isnan(a.real()) && std::isnan(a.imag()) && + std::isnan(b.real()) && std::isnan(b.imag())); +} + template -static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, - const T *pChunk, T *const pDstBuffer) +static CPLErr GDALResampleChunk_ModeT(const GDALOverviewResampleArgs &args, + const T *pChunk, T *const pDstBuffer) { const double dfXRatioDstToSrc = args.dfXRatioDstToSrc; @@ -2118,19 +2148,25 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, const int nDstYOff = args.nDstYOff; const int nDstYOff2 = args.nDstYOff2; const bool bHasNoData = args.bHasNoData; - const double dfNoDataValue = args.dfNoDataValue; const GDALColorTable *poColorTable = args.poColorTable; - const GDALDataType eSrcDataType = args.eSrcDataType; const int nDstXSize = nDstXOff2 - nDstXOff; T tNoDataValue; - if (!bHasNoData || !GDALIsValueInRange(dfNoDataValue)) + if constexpr (std::is_same>::value || + std::is_same>::value) + { + using BaseT = typename T::value_type; + tNoDataValue = + std::complex(std::numeric_limits::quiet_NaN(), + std::numeric_limits::quiet_NaN()); + } + else if (!bHasNoData || !GDALIsValueInRange(args.dfNoDataValue)) tNoDataValue = 0; else - tNoDataValue = static_cast(dfNoDataValue); + tNoDataValue = static_cast(args.dfNoDataValue); size_t nMaxNumPx = 0; - T *padfVals = nullptr; + T *paVals = nullptr; int *panSums = nullptr; const int nChunkRightXOff = nChunkXOff + nChunkXSize; @@ -2213,8 +2249,13 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, if (nSrcXOff2 > nChunkRightXOff) nSrcXOff2 = nChunkRightXOff; - if (eSrcDataType != GDT_Byte || - (poColorTable && poColorTable->GetColorEntryCount() > 256)) + bool bRegularProcessing = false; + if constexpr (!std::is_same::value) + bRegularProcessing = true; + else if (poColorTable && poColorTable->GetColorEntryCount() > 256) + bRegularProcessing = true; + + if (bRegularProcessing) { // Not sure how much sense it makes to run a majority // filter on floating point data, but here it is for the sake @@ -2229,7 +2270,7 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, { CPLError(CE_Failure, CPLE_NotSupported, "Too big downsampling factor"); - CPLFree(padfVals); + CPLFree(paVals); CPLFree(panSums); return CE_Failure; } @@ -2240,19 +2281,19 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, size_t iMaxVal = 0; bool biMaxValdValid = false; - if (padfVals == nullptr || nNumPx > nMaxNumPx) + if (paVals == nullptr || nNumPx > nMaxNumPx) { - T *padfValsNew = static_cast( - VSI_REALLOC_VERBOSE(padfVals, nNumPx * sizeof(T))); + T *paValsNew = static_cast( + VSI_REALLOC_VERBOSE(paVals, nNumPx * sizeof(T))); int *panSumsNew = static_cast( VSI_REALLOC_VERBOSE(panSums, nNumPx * sizeof(int))); - if (padfValsNew != nullptr) - padfVals = padfValsNew; + if (paValsNew != nullptr) + paVals = paValsNew; if (panSumsNew != nullptr) panSums = panSumsNew; - if (padfValsNew == nullptr || panSumsNew == nullptr) + if (paValsNew == nullptr || panSumsNew == nullptr) { - CPLFree(padfVals); + CPLFree(paVals); CPLFree(panSums); return CE_Failure; } @@ -2269,12 +2310,12 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, if (pabySrcScanlineNodataMask == nullptr || pabySrcScanlineNodataMask[iX + iTotYOff]) { - const T dfVal = paSrcScanline[iX + iTotYOff]; + const T val = paSrcScanline[iX + iTotYOff]; size_t i = 0; // Used after for. // Check array for existing entry. for (; i < iMaxInd; ++i) - if (padfVals[i] == dfVal && + if (IsSame(paVals[i], val) && ++panSums[i] > panSums[iMaxVal]) { iMaxVal = i; @@ -2285,7 +2326,7 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, // Add to arr if entry not already there. if (i == iMaxInd) { - padfVals[iMaxInd] = dfVal; + paVals[iMaxInd] = val; panSums[iMaxInd] = 1; if (!biMaxValdValid) @@ -2303,9 +2344,10 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, if (!biMaxValdValid) paDstScanline[iDstPixel - nDstXOff] = tNoDataValue; else - paDstScanline[iDstPixel - nDstXOff] = padfVals[iMaxVal]; + paDstScanline[iDstPixel - nDstXOff] = paVals[iMaxVal]; } - else // if( eSrcDataType == GDT_Byte && nEntryCount < 256 ) + else if constexpr (std::is_same::value) + // ( eSrcDataType == GDT_Byte && nEntryCount < 256 ) { // So we go here for a paletted or non-paletted byte band. // The input values are then between 0 and 255. @@ -2348,7 +2390,7 @@ static CPLErr GDALResampleChunk_Mode_T(const GDALOverviewResampleArgs &args, } } - CPLFree(padfVals); + CPLFree(paVals); CPLFree(panSums); return CE_None; @@ -2366,38 +2408,89 @@ static CPLErr GDALResampleChunk_Mode(const GDALOverviewResampleArgs &args, return CE_Failure; } + CPLAssert(args.eSrcDataType == args.eWrkDataType); + *peDstBufferDataType = args.eWrkDataType; switch (args.eWrkDataType) { + // For mode resampling, as no computation is done, only the + // size of the data type matters... except for Byte where we have + // special processing. And for floating point values case GDT_Byte: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); } + case GDT_Int8: + { + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); + } + + case GDT_Int16: case GDT_UInt16: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 2); + return GDALResampleChunk_ModeT( + args, static_cast(pChunk), + static_cast(*ppDstBuffer)); + } + + case GDT_CInt16: + case GDT_Int32: + case GDT_UInt32: + { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 4); + return GDALResampleChunk_ModeT( + args, static_cast(pChunk), + static_cast(*ppDstBuffer)); } case GDT_Float32: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 4); + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); + } + + case GDT_CInt32: + case GDT_Int64: + case GDT_UInt64: + { + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 8); + return GDALResampleChunk_ModeT( + args, static_cast(pChunk), + static_cast(*ppDstBuffer)); } case GDT_Float64: { - return GDALResampleChunk_Mode_T( - args, static_cast(pChunk), - static_cast(*ppDstBuffer)); + CPLAssert(GDALGetDataTypeSizeBytes(args.eWrkDataType) == 8); + return GDALResampleChunk_ModeT(args, + static_cast(pChunk), + static_cast(*ppDstBuffer)); } - default: + case GDT_CFloat32: + { + return GDALResampleChunk_ModeT( + args, static_cast *>(pChunk), + static_cast *>(*ppDstBuffer)); + } + + case GDT_CFloat64: + { + return GDALResampleChunk_ModeT( + args, static_cast *>(pChunk), + static_cast *>(*ppDstBuffer)); + } + + case GDT_Unknown: + case GDT_TypeCount: break; } @@ -4139,7 +4232,7 @@ GDALResampleFunction GDALGetResampleFunction(const char *pszResampling, GDALDataType GDALGetOvrWorkDataType(const char *pszResampling, GDALDataType eSrcDataType) { - if (STARTS_WITH_CI(pszResampling, "NEAR")) + if (STARTS_WITH_CI(pszResampling, "NEAR") || EQUAL(pszResampling, "MODE")) { return eSrcDataType; } @@ -4403,11 +4496,13 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, poSrcBand->GetBlockSize(&nFRXBlockSize, &nFRYBlockSize); const GDALDataType eSrcDataType = poSrcBand->GetRasterDataType(); + const bool bUseGenericResampleFn = STARTS_WITH_CI(pszResampling, "NEAR") || + EQUAL(pszResampling, "MODE") || + !GDALDataTypeIsComplex(eSrcDataType); const GDALDataType eWrkDataType = - (GDALDataTypeIsComplex(eSrcDataType) && - !STARTS_WITH_CI(pszResampling, "NEAR")) - ? GDT_CFloat32 - : GDALGetOvrWorkDataType(pszResampling, eSrcDataType); + bUseGenericResampleFn + ? GDALGetOvrWorkDataType(pszResampling, eSrcDataType) + : GDT_CFloat32; const int nWidth = poSrcBand->GetXSize(); const int nHeight = poSrcBand->GetYSize(); @@ -4509,6 +4604,7 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, int nDstWidth = 0; GDALOverviewResampleArgs args{}; const void *pChunk = nullptr; + bool bUseGenericResampleFn = false; // Output values of resampling function CPLErr eErr = CE_Failure; @@ -4538,7 +4634,7 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, { OvrJob *poJob = static_cast(pData); - if (poJob->args.eWrkDataType != GDT_CFloat32) + if (poJob->bUseGenericResampleFn) { poJob->eErr = poJob->pfnResampleFn(poJob->args, poJob->pChunk, &(poJob->pDstBuffer), @@ -4862,6 +4958,7 @@ CPLErr GDALRegenerateOverviewsEx(GDALRasterBandH hSrcBand, int nOverviewCount, auto poJob = std::make_unique(); poJob->pfnResampleFn = pfnResampleFn; + poJob->bUseGenericResampleFn = bUseGenericResampleFn; poJob->args.eOvrDataType = poDstBand->GetRasterDataType(); poJob->args.nOvrXSize = poDstBand->GetXSize(); poJob->args.nOvrYSize = poDstBand->GetYSize(); diff --git a/gcore/rasterio.cpp b/gcore/rasterio.cpp index c2eae1398301..04be0bc5968b 100644 --- a/gcore/rasterio.cpp +++ b/gcore/rasterio.cpp @@ -1005,9 +1005,10 @@ CPLErr GDALRasterBand::RasterIOResampled( GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg) { // Determine if we use warping resampling or overview resampling - bool bUseWarp = false; - if (GDALDataTypeIsComplex(eDataType)) - bUseWarp = true; + const bool bUseWarp = + (GDALDataTypeIsComplex(eDataType) && + psExtraArg->eResampleAlg != GRIORA_NearestNeighbour && + psExtraArg->eResampleAlg != GRIORA_Mode); double dfXOff = nXOff; double dfYOff = nYOff; From 21dba706f70a41478a805504ec27e94a329f2fbc Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 13 Sep 2024 18:48:00 +0200 Subject: [PATCH 035/710] MEM layer: fix UpdateFeature() that didn't mark the layer as updated, which caused GeoJSON files to not be updated This amends https://github.com/OSGeo/gdal/pull/10197 which didn't fix the issue of https://github.com/qgis/QGIS/pull/57736#discussion_r1636226384 --- autotest/ogr/ogr_geojson.py | 18 +++++++++++++++--- ogr/ogrsf_frmts/mem/ogrmemlayer.cpp | 3 +++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/autotest/ogr/ogr_geojson.py b/autotest/ogr/ogr_geojson.py index 59da4f8c4e10..4bc636f9e01a 100755 --- a/autotest/ogr/ogr_geojson.py +++ b/autotest/ogr/ogr_geojson.py @@ -1806,7 +1806,11 @@ def test_ogr_geojson_48(tmp_vsimem): # Test UpdateFeature() support -def test_ogr_geojson_update_feature(tmp_vsimem): +@pytest.mark.parametrize("check_after_update_before_reopen", [True, False]) +@pytest.mark.parametrize("sync_to_disk_after_update", [True, False]) +def test_ogr_geojson_update_feature( + tmp_vsimem, check_after_update_before_reopen, sync_to_disk_after_update +): filename = str(tmp_vsimem / "test.json") @@ -1821,13 +1825,21 @@ def test_ogr_geojson_update_feature(tmp_vsimem): lyr = ds.GetLayer(0) f = ogr.Feature(lyr.GetLayerDefn()) f.SetFID(0) - f["int64list"] = [123456790123, -123456790123] + f["int64list"] = [-123456790123, 123456790123] lyr.UpdateFeature(f, [0], [], False) + if sync_to_disk_after_update: + lyr.SyncToDisk() + + if check_after_update_before_reopen: + lyr.ResetReading() + f = lyr.GetNextFeature() + assert f["int64list"] == [-123456790123, 123456790123] + with ogr.Open(filename) as ds: lyr = ds.GetLayer(0) f = lyr.GetNextFeature() - assert f["int64list"] == [123456790123, -123456790123] + assert f["int64list"] == [-123456790123, 123456790123] ############################################################################### diff --git a/ogr/ogrsf_frmts/mem/ogrmemlayer.cpp b/ogr/ogrsf_frmts/mem/ogrmemlayer.cpp index 0bc25abc0145..43a302fd036e 100644 --- a/ogr/ogrsf_frmts/mem/ogrmemlayer.cpp +++ b/ogr/ogrsf_frmts/mem/ogrmemlayer.cpp @@ -498,6 +498,9 @@ OGRErr OGRMemLayer::IUpdateFeature(OGRFeature *poFeature, { poFeatureRef->SetStyleString(poFeature->GetStyleString()); } + + m_bUpdated = true; + return OGRERR_NONE; } From d742a7c3055b64b648106bf6b74484ca1916f673 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 14 Sep 2024 00:39:10 +0200 Subject: [PATCH 036/710] TileDB: make Identify() method return false if passed object is not a directory --- frmts/tiledb/tiledbcommon.cpp | 50 +++++++------------------------ frmts/tiledb/tiledbdrivercore.cpp | 19 +++++------- frmts/tiledb/tiledbdrivercore.h | 2 ++ 3 files changed, 19 insertions(+), 52 deletions(-) diff --git a/frmts/tiledb/tiledbcommon.cpp b/frmts/tiledb/tiledbcommon.cpp index 120276dcc08d..2c297f139942 100644 --- a/frmts/tiledb/tiledbcommon.cpp +++ b/frmts/tiledb/tiledbcommon.cpp @@ -118,54 +118,24 @@ CPLErr TileDBDataset::AddFilter(tiledb::Context &ctx, int TileDBDataset::Identify(GDALOpenInfo *poOpenInfo) { - if (STARTS_WITH_CI(poOpenInfo->pszFilename, "TILEDB:")) + int nRet = TileDBDriverIdentifySimplified(poOpenInfo); + if (nRet == GDAL_IDENTIFY_UNKNOWN) { - return TRUE; - } - - if (poOpenInfo->IsSingleAllowedDriver("TileDB")) - { - return TRUE; - } - - try - { - const char *pszConfig = - CSLFetchNameValue(poOpenInfo->papszOpenOptions, "TILEDB_CONFIG"); - - if (pszConfig != nullptr) - { - return TRUE; - } - - const bool bIsS3OrGS = - STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || - STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/"); - // If this is a /vsi virtual file systems, bail out, except if it is S3 or GS. - if (!bIsS3OrGS && STARTS_WITH(poOpenInfo->pszFilename, "/vsi")) - { - return false; - } - - if (poOpenInfo->bIsDirectory || - (bIsS3OrGS && - !EQUAL(CPLGetExtension(poOpenInfo->pszFilename), "tif"))) + try { tiledb::Context ctx; CPLString osArrayPath = TileDBDataset::VSI_to_tiledb_uri(poOpenInfo->pszFilename); const auto eType = tiledb::Object::object(ctx, osArrayPath).type(); - if (eType == tiledb::Object::Type::Array || - eType == tiledb::Object::Type::Group) - return true; + nRet = (eType == tiledb::Object::Type::Array || + eType == tiledb::Object::Type::Group); + } + catch (...) + { + nRet = FALSE; } - - return FALSE; - } - catch (...) - { - return FALSE; } + return nRet; } /************************************************************************/ diff --git a/frmts/tiledb/tiledbdrivercore.cpp b/frmts/tiledb/tiledbdrivercore.cpp index 4737945ff783..f3f5917a866f 100644 --- a/frmts/tiledb/tiledbdrivercore.cpp +++ b/frmts/tiledb/tiledbdrivercore.cpp @@ -35,7 +35,7 @@ /* TileDBDriverIdentifySimplified() */ /************************************************************************/ -static int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) +int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) { if (STARTS_WITH_CI(poOpenInfo->pszFilename, "TILEDB:")) @@ -56,6 +56,11 @@ static int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) return TRUE; } + if (!poOpenInfo->bIsDirectory) + { + return false; + } + const bool bIsS3OrGS = STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIS3/") || STARTS_WITH_CI(poOpenInfo->pszFilename, "/VSIGS/"); // If this is a /vsi virtual file systems, bail out, except if it is S3 or GS. @@ -64,17 +69,7 @@ static int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo) return false; } - if (poOpenInfo->bIsDirectory) - { - return GDAL_IDENTIFY_UNKNOWN; - } - - if (bIsS3OrGS && !EQUAL(CPLGetExtension(poOpenInfo->pszFilename), "tif")) - { - return GDAL_IDENTIFY_UNKNOWN; - } - - return FALSE; + return GDAL_IDENTIFY_UNKNOWN; } /************************************************************************/ diff --git a/frmts/tiledb/tiledbdrivercore.h b/frmts/tiledb/tiledbdrivercore.h index 26e0e16aea83..632d945bee80 100644 --- a/frmts/tiledb/tiledbdrivercore.h +++ b/frmts/tiledb/tiledbdrivercore.h @@ -36,6 +36,8 @@ constexpr const char *DRIVER_NAME = "TileDB"; #define TileDBDriverSetCommonMetadata \ PLUGIN_SYMBOL_NAME(TileDBDriverSetCommonMetadata) +int TileDBDriverIdentifySimplified(GDALOpenInfo *poOpenInfo); + void TileDBDriverSetCommonMetadata(GDALDriver *poDriver); #endif From ce9bbb5ba9bdc857a0072e8954f5c26aadeb3d71 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 14 Sep 2024 17:46:37 +0200 Subject: [PATCH 037/710] CPLRecode(): make ISO-8859-2 and -15 and CP437/CP1250/CP1251/CP1252 to UTF-8 always available (but not other direction yet) This will help for common use cases when using a minimal GDAL build lacking iconv support. --- autotest/cpp/test_cpl.cpp | 55 +- port/CMakeLists.txt | 9 + port/character_set_conv_table_generator.c | 124 ++++ port/cpl_character_sets.c | 818 ++++++++++++++++++++++ port/cpl_character_sets.h | 8 + port/cpl_recode.cpp | 25 +- port/cpl_recode_stub.cpp | 140 ++-- 7 files changed, 1112 insertions(+), 67 deletions(-) create mode 100644 port/character_set_conv_table_generator.c create mode 100644 port/cpl_character_sets.c create mode 100644 port/cpl_character_sets.h diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp index be5292265689..710aa876f490 100644 --- a/autotest/cpp/test_cpl.cpp +++ b/autotest/cpp/test_cpl.cpp @@ -2846,22 +2846,23 @@ TEST_F(test_cpl, CPLJSONDocument) } // Test CPLRecodeIconv() with re-allocation +// (this test also passed on Windows using its native recoding API) TEST_F(test_cpl, CPLRecodeIconv) { -#ifdef CPL_RECODE_ICONV +#if defined(CPL_RECODE_ICONV) || defined(_WIN32) int N = 32800; char *pszIn = static_cast(CPLMalloc(N + 1)); for (int i = 0; i < N; i++) - pszIn[i] = '\xE9'; + pszIn[i] = '\xA1'; pszIn[N] = 0; char *pszExpected = static_cast(CPLMalloc(N * 2 + 1)); for (int i = 0; i < N; i++) { - pszExpected[2 * i] = '\xC3'; - pszExpected[2 * i + 1] = '\xA9'; + pszExpected[2 * i] = '\xD0'; + pszExpected[2 * i + 1] = '\x81'; } pszExpected[N * 2] = 0; - char *pszRet = CPLRecode(pszIn, "ISO-8859-2", CPL_ENC_UTF8); + char *pszRet = CPLRecode(pszIn, "ISO-8859-5", CPL_ENC_UTF8); EXPECT_EQ(memcmp(pszExpected, pszRet, N * 2 + 1), 0); CPLFree(pszIn); CPLFree(pszRet); @@ -2871,6 +2872,50 @@ TEST_F(test_cpl, CPLRecodeIconv) #endif } +// Test CP1252 to UTF-8 +TEST_F(test_cpl, CPLRecodeStubCP1252_to_UTF8_strict_alloc) +{ + CPLClearRecodeWarningFlags(); + CPLErrorReset(); + CPLPushErrorHandler(CPLQuietErrorHandler); + // Euro character expands to 3-bytes + char *pszRet = CPLRecode("\x80", "CP1252", CPL_ENC_UTF8); + CPLPopErrorHandler(); + EXPECT_STREQ(CPLGetLastErrorMsg(), ""); + EXPECT_EQ(memcmp(pszRet, "\xE2\x82\xAC\x00", 4), 0); + CPLFree(pszRet); +} + +// Test CP1252 to UTF-8 +TEST_F(test_cpl, CPLRecodeStubCP1252_to_UTF8_with_ascii) +{ + CPLClearRecodeWarningFlags(); + CPLErrorReset(); + CPLPushErrorHandler(CPLQuietErrorHandler); + char *pszRet = CPLRecode("x\x80y", "CP1252", CPL_ENC_UTF8); + CPLPopErrorHandler(); + EXPECT_STREQ(CPLGetLastErrorMsg(), ""); + EXPECT_EQ(memcmp(pszRet, "x\xE2\x82\xACy\x00", 6), 0); + CPLFree(pszRet); +} + +// Test CP1252 to UTF-8 +TEST_F(test_cpl, CPLRecodeStubCP1252_to_UTF8_with_warning) +{ + CPLClearRecodeWarningFlags(); + CPLErrorReset(); + CPLPushErrorHandler(CPLQuietErrorHandler); + // \x90 is an invalid CP1252 character. Will be skipped + char *pszRet = CPLRecode("\x90\x80", "CP1252", CPL_ENC_UTF8); + CPLPopErrorHandler(); + EXPECT_STREQ( + CPLGetLastErrorMsg(), + "One or several characters couldn't be converted correctly from CP1252 " + "to UTF-8. This warning will not be emitted anymore"); + EXPECT_EQ(memcmp(pszRet, "\xE2\x82\xAC\x00", 4), 0); + CPLFree(pszRet); +} + // Test CPLHTTPParseMultipartMime() TEST_F(test_cpl, CPLHTTPParseMultipartMime) { diff --git a/port/CMakeLists.txt b/port/CMakeLists.txt index bb87bc8c3e37..26469f1b47a0 100644 --- a/port/CMakeLists.txt +++ b/port/CMakeLists.txt @@ -374,3 +374,12 @@ if (NOT CMAKE_CROSSCOMPILING AND BUILD_VSIPRELOAD AND "${CMAKE_SYSTEM}" MATCHES endforeach() endif () endif () + +# Utility to generate cpl_character_sets.h and .c +add_executable(character_set_conv_table_generator EXCLUDE_FROM_ALL character_set_conv_table_generator.c) + +# Custom target that must be manually invoked if character_set_conv_table_generator.c is modified +add_custom_target(generate_cpl_character_sets + COMMAND $ + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + DEPENDS character_set_conv_table_generator) diff --git a/port/character_set_conv_table_generator.c b/port/character_set_conv_table_generator.c new file mode 100644 index 000000000000..9b8928f6bac6 --- /dev/null +++ b/port/character_set_conv_table_generator.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: MIT +// Copyright 2024 Even Rouault + +#include +#include +#include +#include + +#define ENCODING_MAX_LEN 256 + +static void launder_name(const char *srcEncoding, + char srcEncodingLaundered[ENCODING_MAX_LEN]) +{ + snprintf(srcEncodingLaundered, ENCODING_MAX_LEN, "%s", srcEncoding); + for (int i = 0; srcEncodingLaundered[i]; ++i) + { + if (srcEncodingLaundered[i] == '-') + srcEncodingLaundered[i] = '_'; + } +} + +static void generate(FILE *c_file, FILE *h_file, const char *srcEncoding, + const char *comment) +{ + iconv_t sConv = iconv_open("UTF-8", srcEncoding); + if (sConv == (iconv_t)(-1)) + { + fprintf(stderr, "iconv_open(%s) failed\n", srcEncoding); + exit(1); + } + char srcEncodingLaundered[ENCODING_MAX_LEN]; + launder_name(srcEncoding, srcEncodingLaundered); + fprintf(c_file, "/* %s */\n", comment); + fprintf(c_file, "static const CPLCodePageConvTable CPL_%s_to_UTF8 = {\n", + srcEncodingLaundered); + for (int i = 0; i <= 255; ++i) + { + unsigned char c = (unsigned char)i; + size_t size_in = 1; + unsigned char out[4] = {0, 0, 0, 0}; + size_t size_out = sizeof(out); + char *p_in = (char *)&c; + char *p_out = (char *)out; + size_t nConverted = iconv(sConv, &p_in, &size_in, &p_out, &size_out); + if (i <= 127) + { + assert(out[0] == i); + continue; + } + if (nConverted != (size_t)-1) + { + const size_t needed = sizeof(out) - size_out; + assert(needed <= 3); + fprintf(c_file, " {0x%02X, 0x%02X, 0x%02X},\n", out[0], out[1], + out[2]); + } + else + { + fprintf(c_file, " {0, 0, 0}, /* invalid */\n"); + } + } + fprintf(c_file, "};\n\n"); + iconv_close(sConv); +} + +int main() +{ + FILE *c_file = fopen("cpl_character_sets.c", "wb"); + FILE *h_file = fopen("cpl_character_sets.h", "wb"); + fprintf(c_file, "/* This file has been generated by " + "generate_character_set_conv_tables.c */\n"); + fprintf(c_file, "/* DO NOT EDIT !*/\n\n"); + fprintf(c_file, "/* clang-format off */\n"); + fprintf(c_file, "#include \"cpl_port.h\"\n"); + fprintf(c_file, "#include \"cpl_character_sets.h\"\n\n"); + + fprintf(h_file, "/* This file has been generated by " + "generate_character_set_conv_tables.c */\n"); + fprintf(h_file, "/* DO NOT EDIT !*/\n\n"); + fprintf(h_file, "/* clang-format off */\n"); + fprintf(h_file, "typedef unsigned char CPLCodePageConvTable[128][3];\n"); + + const struct + { + const char *name; + const char *comment; + } encodings[] = { + {"CP437", "Character set of original IBM PC"}, + {"CP1250", "Central and eastern Europe languages"}, + {"CP1251", "Cyrillic script"}, + {"CP1252", + "Legacy Windows single-byte character set used in a lot of countries"}, + {"ISO-8859-2", "Central Europe languages"}, + {"ISO-8859-15", "New Western Europe"}, + {NULL, NULL}}; + + for (int i = 0; encodings[i].name; ++i) + { + generate(c_file, h_file, encodings[i].name, encodings[i].comment); + } + fprintf(h_file, "\n"); + fprintf(h_file, "const CPLCodePageConvTable* " + "CPLGetConversionTableToUTF8(const char* pszEncoding);\n"); + + fprintf(c_file, "\nconst CPLCodePageConvTable* " + "CPLGetConversionTableToUTF8(const char* pszEncoding)\n"); + fprintf(c_file, "{\n"); + for (int i = 0; encodings[i].name; ++i) + { + char srcEncodingLaundered[ENCODING_MAX_LEN]; + launder_name(encodings[i].name, srcEncodingLaundered); + fprintf(c_file, " if (EQUAL(pszEncoding, \"%s\"))\n", + encodings[i].name); + fprintf(c_file, " return &CPL_%s_to_UTF8;\n", + srcEncodingLaundered); + } + fprintf(c_file, " return CPL_NULLPTR;\n"); + fprintf(c_file, "}\n"); + fprintf(c_file, "/* clang-format on */\n"); + fprintf(h_file, "/* clang-format on */\n"); + fclose(c_file); + fclose(h_file); + return 0; +} diff --git a/port/cpl_character_sets.c b/port/cpl_character_sets.c new file mode 100644 index 000000000000..65a326059785 --- /dev/null +++ b/port/cpl_character_sets.c @@ -0,0 +1,818 @@ +/* This file has been generated by generate_character_set_conv_tables.c */ +/* DO NOT EDIT !*/ + +/* clang-format off */ +#include "cpl_port.h" +#include "cpl_character_sets.h" + +/* Character set of original IBM PC */ +static const CPLCodePageConvTable CPL_CP437_to_UTF8 = { + {0xC3, 0x87, 0x00}, + {0xC3, 0xBC, 0x00}, + {0xC3, 0xA9, 0x00}, + {0xC3, 0xA2, 0x00}, + {0xC3, 0xA4, 0x00}, + {0xC3, 0xA0, 0x00}, + {0xC3, 0xA5, 0x00}, + {0xC3, 0xA7, 0x00}, + {0xC3, 0xAA, 0x00}, + {0xC3, 0xAB, 0x00}, + {0xC3, 0xA8, 0x00}, + {0xC3, 0xAF, 0x00}, + {0xC3, 0xAE, 0x00}, + {0xC3, 0xAC, 0x00}, + {0xC3, 0x84, 0x00}, + {0xC3, 0x85, 0x00}, + {0xC3, 0x89, 0x00}, + {0xC3, 0xA6, 0x00}, + {0xC3, 0x86, 0x00}, + {0xC3, 0xB4, 0x00}, + {0xC3, 0xB6, 0x00}, + {0xC3, 0xB2, 0x00}, + {0xC3, 0xBB, 0x00}, + {0xC3, 0xB9, 0x00}, + {0xC3, 0xBF, 0x00}, + {0xC3, 0x96, 0x00}, + {0xC3, 0x9C, 0x00}, + {0xC2, 0xA2, 0x00}, + {0xC2, 0xA3, 0x00}, + {0xC2, 0xA5, 0x00}, + {0xE2, 0x82, 0xA7}, + {0xC6, 0x92, 0x00}, + {0xC3, 0xA1, 0x00}, + {0xC3, 0xAD, 0x00}, + {0xC3, 0xB3, 0x00}, + {0xC3, 0xBA, 0x00}, + {0xC3, 0xB1, 0x00}, + {0xC3, 0x91, 0x00}, + {0xC2, 0xAA, 0x00}, + {0xC2, 0xBA, 0x00}, + {0xC2, 0xBF, 0x00}, + {0xE2, 0x8C, 0x90}, + {0xC2, 0xAC, 0x00}, + {0xC2, 0xBD, 0x00}, + {0xC2, 0xBC, 0x00}, + {0xC2, 0xA1, 0x00}, + {0xC2, 0xAB, 0x00}, + {0xC2, 0xBB, 0x00}, + {0xE2, 0x96, 0x91}, + {0xE2, 0x96, 0x92}, + {0xE2, 0x96, 0x93}, + {0xE2, 0x94, 0x82}, + {0xE2, 0x94, 0xA4}, + {0xE2, 0x95, 0xA1}, + {0xE2, 0x95, 0xA2}, + {0xE2, 0x95, 0x96}, + {0xE2, 0x95, 0x95}, + {0xE2, 0x95, 0xA3}, + {0xE2, 0x95, 0x91}, + {0xE2, 0x95, 0x97}, + {0xE2, 0x95, 0x9D}, + {0xE2, 0x95, 0x9C}, + {0xE2, 0x95, 0x9B}, + {0xE2, 0x94, 0x90}, + {0xE2, 0x94, 0x94}, + {0xE2, 0x94, 0xB4}, + {0xE2, 0x94, 0xAC}, + {0xE2, 0x94, 0x9C}, + {0xE2, 0x94, 0x80}, + {0xE2, 0x94, 0xBC}, + {0xE2, 0x95, 0x9E}, + {0xE2, 0x95, 0x9F}, + {0xE2, 0x95, 0x9A}, + {0xE2, 0x95, 0x94}, + {0xE2, 0x95, 0xA9}, + {0xE2, 0x95, 0xA6}, + {0xE2, 0x95, 0xA0}, + {0xE2, 0x95, 0x90}, + {0xE2, 0x95, 0xAC}, + {0xE2, 0x95, 0xA7}, + {0xE2, 0x95, 0xA8}, + {0xE2, 0x95, 0xA4}, + {0xE2, 0x95, 0xA5}, + {0xE2, 0x95, 0x99}, + {0xE2, 0x95, 0x98}, + {0xE2, 0x95, 0x92}, + {0xE2, 0x95, 0x93}, + {0xE2, 0x95, 0xAB}, + {0xE2, 0x95, 0xAA}, + {0xE2, 0x94, 0x98}, + {0xE2, 0x94, 0x8C}, + {0xE2, 0x96, 0x88}, + {0xE2, 0x96, 0x84}, + {0xE2, 0x96, 0x8C}, + {0xE2, 0x96, 0x90}, + {0xE2, 0x96, 0x80}, + {0xCE, 0xB1, 0x00}, + {0xC3, 0x9F, 0x00}, + {0xCE, 0x93, 0x00}, + {0xCF, 0x80, 0x00}, + {0xCE, 0xA3, 0x00}, + {0xCF, 0x83, 0x00}, + {0xC2, 0xB5, 0x00}, + {0xCF, 0x84, 0x00}, + {0xCE, 0xA6, 0x00}, + {0xCE, 0x98, 0x00}, + {0xCE, 0xA9, 0x00}, + {0xCE, 0xB4, 0x00}, + {0xE2, 0x88, 0x9E}, + {0xCF, 0x86, 0x00}, + {0xCE, 0xB5, 0x00}, + {0xE2, 0x88, 0xA9}, + {0xE2, 0x89, 0xA1}, + {0xC2, 0xB1, 0x00}, + {0xE2, 0x89, 0xA5}, + {0xE2, 0x89, 0xA4}, + {0xE2, 0x8C, 0xA0}, + {0xE2, 0x8C, 0xA1}, + {0xC3, 0xB7, 0x00}, + {0xE2, 0x89, 0x88}, + {0xC2, 0xB0, 0x00}, + {0xE2, 0x88, 0x99}, + {0xC2, 0xB7, 0x00}, + {0xE2, 0x88, 0x9A}, + {0xE2, 0x81, 0xBF}, + {0xC2, 0xB2, 0x00}, + {0xE2, 0x96, 0xA0}, + {0xC2, 0xA0, 0x00}, +}; + +/* Central and eastern Europe languages */ +static const CPLCodePageConvTable CPL_CP1250_to_UTF8 = { + {0xE2, 0x82, 0xAC}, + {0, 0, 0}, /* invalid */ + {0xE2, 0x80, 0x9A}, + {0, 0, 0}, /* invalid */ + {0xE2, 0x80, 0x9E}, + {0xE2, 0x80, 0xA6}, + {0xE2, 0x80, 0xA0}, + {0xE2, 0x80, 0xA1}, + {0, 0, 0}, /* invalid */ + {0xE2, 0x80, 0xB0}, + {0xC5, 0xA0, 0x00}, + {0xE2, 0x80, 0xB9}, + {0xC5, 0x9A, 0x00}, + {0xC5, 0xA4, 0x00}, + {0xC5, 0xBD, 0x00}, + {0xC5, 0xB9, 0x00}, + {0, 0, 0}, /* invalid */ + {0xE2, 0x80, 0x98}, + {0xE2, 0x80, 0x99}, + {0xE2, 0x80, 0x9C}, + {0xE2, 0x80, 0x9D}, + {0xE2, 0x80, 0xA2}, + {0xE2, 0x80, 0x93}, + {0xE2, 0x80, 0x94}, + {0, 0, 0}, /* invalid */ + {0xE2, 0x84, 0xA2}, + {0xC5, 0xA1, 0x00}, + {0xE2, 0x80, 0xBA}, + {0xC5, 0x9B, 0x00}, + {0xC5, 0xA5, 0x00}, + {0xC5, 0xBE, 0x00}, + {0xC5, 0xBA, 0x00}, + {0xC2, 0xA0, 0x00}, + {0xCB, 0x87, 0x00}, + {0xCB, 0x98, 0x00}, + {0xC5, 0x81, 0x00}, + {0xC2, 0xA4, 0x00}, + {0xC4, 0x84, 0x00}, + {0xC2, 0xA6, 0x00}, + {0xC2, 0xA7, 0x00}, + {0xC2, 0xA8, 0x00}, + {0xC2, 0xA9, 0x00}, + {0xC5, 0x9E, 0x00}, + {0xC2, 0xAB, 0x00}, + {0xC2, 0xAC, 0x00}, + {0xC2, 0xAD, 0x00}, + {0xC2, 0xAE, 0x00}, + {0xC5, 0xBB, 0x00}, + {0xC2, 0xB0, 0x00}, + {0xC2, 0xB1, 0x00}, + {0xCB, 0x9B, 0x00}, + {0xC5, 0x82, 0x00}, + {0xC2, 0xB4, 0x00}, + {0xC2, 0xB5, 0x00}, + {0xC2, 0xB6, 0x00}, + {0xC2, 0xB7, 0x00}, + {0xC2, 0xB8, 0x00}, + {0xC4, 0x85, 0x00}, + {0xC5, 0x9F, 0x00}, + {0xC2, 0xBB, 0x00}, + {0xC4, 0xBD, 0x00}, + {0xCB, 0x9D, 0x00}, + {0xC4, 0xBE, 0x00}, + {0xC5, 0xBC, 0x00}, + {0xC5, 0x94, 0x00}, + {0xC3, 0x81, 0x00}, + {0xC3, 0x82, 0x00}, + {0xC4, 0x82, 0x00}, + {0xC3, 0x84, 0x00}, + {0xC4, 0xB9, 0x00}, + {0xC4, 0x86, 0x00}, + {0xC3, 0x87, 0x00}, + {0xC4, 0x8C, 0x00}, + {0xC3, 0x89, 0x00}, + {0xC4, 0x98, 0x00}, + {0xC3, 0x8B, 0x00}, + {0xC4, 0x9A, 0x00}, + {0xC3, 0x8D, 0x00}, + {0xC3, 0x8E, 0x00}, + {0xC4, 0x8E, 0x00}, + {0xC4, 0x90, 0x00}, + {0xC5, 0x83, 0x00}, + {0xC5, 0x87, 0x00}, + {0xC3, 0x93, 0x00}, + {0xC3, 0x94, 0x00}, + {0xC5, 0x90, 0x00}, + {0xC3, 0x96, 0x00}, + {0xC3, 0x97, 0x00}, + {0xC5, 0x98, 0x00}, + {0xC5, 0xAE, 0x00}, + {0xC3, 0x9A, 0x00}, + {0xC5, 0xB0, 0x00}, + {0xC3, 0x9C, 0x00}, + {0xC3, 0x9D, 0x00}, + {0xC5, 0xA2, 0x00}, + {0xC3, 0x9F, 0x00}, + {0xC5, 0x95, 0x00}, + {0xC3, 0xA1, 0x00}, + {0xC3, 0xA2, 0x00}, + {0xC4, 0x83, 0x00}, + {0xC3, 0xA4, 0x00}, + {0xC4, 0xBA, 0x00}, + {0xC4, 0x87, 0x00}, + {0xC3, 0xA7, 0x00}, + {0xC4, 0x8D, 0x00}, + {0xC3, 0xA9, 0x00}, + {0xC4, 0x99, 0x00}, + {0xC3, 0xAB, 0x00}, + {0xC4, 0x9B, 0x00}, + {0xC3, 0xAD, 0x00}, + {0xC3, 0xAE, 0x00}, + {0xC4, 0x8F, 0x00}, + {0xC4, 0x91, 0x00}, + {0xC5, 0x84, 0x00}, + {0xC5, 0x88, 0x00}, + {0xC3, 0xB3, 0x00}, + {0xC3, 0xB4, 0x00}, + {0xC5, 0x91, 0x00}, + {0xC3, 0xB6, 0x00}, + {0xC3, 0xB7, 0x00}, + {0xC5, 0x99, 0x00}, + {0xC5, 0xAF, 0x00}, + {0xC3, 0xBA, 0x00}, + {0xC5, 0xB1, 0x00}, + {0xC3, 0xBC, 0x00}, + {0xC3, 0xBD, 0x00}, + {0xC5, 0xA3, 0x00}, + {0xCB, 0x99, 0x00}, +}; + +/* Cyrillic script */ +static const CPLCodePageConvTable CPL_CP1251_to_UTF8 = { + {0xD0, 0x82, 0x00}, + {0xD0, 0x83, 0x00}, + {0xE2, 0x80, 0x9A}, + {0xD1, 0x93, 0x00}, + {0xE2, 0x80, 0x9E}, + {0xE2, 0x80, 0xA6}, + {0xE2, 0x80, 0xA0}, + {0xE2, 0x80, 0xA1}, + {0xE2, 0x82, 0xAC}, + {0xE2, 0x80, 0xB0}, + {0xD0, 0x89, 0x00}, + {0xE2, 0x80, 0xB9}, + {0xD0, 0x8A, 0x00}, + {0xD0, 0x8C, 0x00}, + {0xD0, 0x8B, 0x00}, + {0xD0, 0x8F, 0x00}, + {0xD1, 0x92, 0x00}, + {0xE2, 0x80, 0x98}, + {0xE2, 0x80, 0x99}, + {0xE2, 0x80, 0x9C}, + {0xE2, 0x80, 0x9D}, + {0xE2, 0x80, 0xA2}, + {0xE2, 0x80, 0x93}, + {0xE2, 0x80, 0x94}, + {0, 0, 0}, /* invalid */ + {0xE2, 0x84, 0xA2}, + {0xD1, 0x99, 0x00}, + {0xE2, 0x80, 0xBA}, + {0xD1, 0x9A, 0x00}, + {0xD1, 0x9C, 0x00}, + {0xD1, 0x9B, 0x00}, + {0xD1, 0x9F, 0x00}, + {0xC2, 0xA0, 0x00}, + {0xD0, 0x8E, 0x00}, + {0xD1, 0x9E, 0x00}, + {0xD0, 0x88, 0x00}, + {0xC2, 0xA4, 0x00}, + {0xD2, 0x90, 0x00}, + {0xC2, 0xA6, 0x00}, + {0xC2, 0xA7, 0x00}, + {0xD0, 0x81, 0x00}, + {0xC2, 0xA9, 0x00}, + {0xD0, 0x84, 0x00}, + {0xC2, 0xAB, 0x00}, + {0xC2, 0xAC, 0x00}, + {0xC2, 0xAD, 0x00}, + {0xC2, 0xAE, 0x00}, + {0xD0, 0x87, 0x00}, + {0xC2, 0xB0, 0x00}, + {0xC2, 0xB1, 0x00}, + {0xD0, 0x86, 0x00}, + {0xD1, 0x96, 0x00}, + {0xD2, 0x91, 0x00}, + {0xC2, 0xB5, 0x00}, + {0xC2, 0xB6, 0x00}, + {0xC2, 0xB7, 0x00}, + {0xD1, 0x91, 0x00}, + {0xE2, 0x84, 0x96}, + {0xD1, 0x94, 0x00}, + {0xC2, 0xBB, 0x00}, + {0xD1, 0x98, 0x00}, + {0xD0, 0x85, 0x00}, + {0xD1, 0x95, 0x00}, + {0xD1, 0x97, 0x00}, + {0xD0, 0x90, 0x00}, + {0xD0, 0x91, 0x00}, + {0xD0, 0x92, 0x00}, + {0xD0, 0x93, 0x00}, + {0xD0, 0x94, 0x00}, + {0xD0, 0x95, 0x00}, + {0xD0, 0x96, 0x00}, + {0xD0, 0x97, 0x00}, + {0xD0, 0x98, 0x00}, + {0xD0, 0x99, 0x00}, + {0xD0, 0x9A, 0x00}, + {0xD0, 0x9B, 0x00}, + {0xD0, 0x9C, 0x00}, + {0xD0, 0x9D, 0x00}, + {0xD0, 0x9E, 0x00}, + {0xD0, 0x9F, 0x00}, + {0xD0, 0xA0, 0x00}, + {0xD0, 0xA1, 0x00}, + {0xD0, 0xA2, 0x00}, + {0xD0, 0xA3, 0x00}, + {0xD0, 0xA4, 0x00}, + {0xD0, 0xA5, 0x00}, + {0xD0, 0xA6, 0x00}, + {0xD0, 0xA7, 0x00}, + {0xD0, 0xA8, 0x00}, + {0xD0, 0xA9, 0x00}, + {0xD0, 0xAA, 0x00}, + {0xD0, 0xAB, 0x00}, + {0xD0, 0xAC, 0x00}, + {0xD0, 0xAD, 0x00}, + {0xD0, 0xAE, 0x00}, + {0xD0, 0xAF, 0x00}, + {0xD0, 0xB0, 0x00}, + {0xD0, 0xB1, 0x00}, + {0xD0, 0xB2, 0x00}, + {0xD0, 0xB3, 0x00}, + {0xD0, 0xB4, 0x00}, + {0xD0, 0xB5, 0x00}, + {0xD0, 0xB6, 0x00}, + {0xD0, 0xB7, 0x00}, + {0xD0, 0xB8, 0x00}, + {0xD0, 0xB9, 0x00}, + {0xD0, 0xBA, 0x00}, + {0xD0, 0xBB, 0x00}, + {0xD0, 0xBC, 0x00}, + {0xD0, 0xBD, 0x00}, + {0xD0, 0xBE, 0x00}, + {0xD0, 0xBF, 0x00}, + {0xD1, 0x80, 0x00}, + {0xD1, 0x81, 0x00}, + {0xD1, 0x82, 0x00}, + {0xD1, 0x83, 0x00}, + {0xD1, 0x84, 0x00}, + {0xD1, 0x85, 0x00}, + {0xD1, 0x86, 0x00}, + {0xD1, 0x87, 0x00}, + {0xD1, 0x88, 0x00}, + {0xD1, 0x89, 0x00}, + {0xD1, 0x8A, 0x00}, + {0xD1, 0x8B, 0x00}, + {0xD1, 0x8C, 0x00}, + {0xD1, 0x8D, 0x00}, + {0xD1, 0x8E, 0x00}, + {0xD1, 0x8F, 0x00}, +}; + +/* Legacy Windows single-byte character set used in a lot of countries */ +static const CPLCodePageConvTable CPL_CP1252_to_UTF8 = { + {0xE2, 0x82, 0xAC}, + {0, 0, 0}, /* invalid */ + {0xE2, 0x80, 0x9A}, + {0xC6, 0x92, 0x00}, + {0xE2, 0x80, 0x9E}, + {0xE2, 0x80, 0xA6}, + {0xE2, 0x80, 0xA0}, + {0xE2, 0x80, 0xA1}, + {0xCB, 0x86, 0x00}, + {0xE2, 0x80, 0xB0}, + {0xC5, 0xA0, 0x00}, + {0xE2, 0x80, 0xB9}, + {0xC5, 0x92, 0x00}, + {0, 0, 0}, /* invalid */ + {0xC5, 0xBD, 0x00}, + {0, 0, 0}, /* invalid */ + {0, 0, 0}, /* invalid */ + {0xE2, 0x80, 0x98}, + {0xE2, 0x80, 0x99}, + {0xE2, 0x80, 0x9C}, + {0xE2, 0x80, 0x9D}, + {0xE2, 0x80, 0xA2}, + {0xE2, 0x80, 0x93}, + {0xE2, 0x80, 0x94}, + {0xCB, 0x9C, 0x00}, + {0xE2, 0x84, 0xA2}, + {0xC5, 0xA1, 0x00}, + {0xE2, 0x80, 0xBA}, + {0xC5, 0x93, 0x00}, + {0, 0, 0}, /* invalid */ + {0xC5, 0xBE, 0x00}, + {0xC5, 0xB8, 0x00}, + {0xC2, 0xA0, 0x00}, + {0xC2, 0xA1, 0x00}, + {0xC2, 0xA2, 0x00}, + {0xC2, 0xA3, 0x00}, + {0xC2, 0xA4, 0x00}, + {0xC2, 0xA5, 0x00}, + {0xC2, 0xA6, 0x00}, + {0xC2, 0xA7, 0x00}, + {0xC2, 0xA8, 0x00}, + {0xC2, 0xA9, 0x00}, + {0xC2, 0xAA, 0x00}, + {0xC2, 0xAB, 0x00}, + {0xC2, 0xAC, 0x00}, + {0xC2, 0xAD, 0x00}, + {0xC2, 0xAE, 0x00}, + {0xC2, 0xAF, 0x00}, + {0xC2, 0xB0, 0x00}, + {0xC2, 0xB1, 0x00}, + {0xC2, 0xB2, 0x00}, + {0xC2, 0xB3, 0x00}, + {0xC2, 0xB4, 0x00}, + {0xC2, 0xB5, 0x00}, + {0xC2, 0xB6, 0x00}, + {0xC2, 0xB7, 0x00}, + {0xC2, 0xB8, 0x00}, + {0xC2, 0xB9, 0x00}, + {0xC2, 0xBA, 0x00}, + {0xC2, 0xBB, 0x00}, + {0xC2, 0xBC, 0x00}, + {0xC2, 0xBD, 0x00}, + {0xC2, 0xBE, 0x00}, + {0xC2, 0xBF, 0x00}, + {0xC3, 0x80, 0x00}, + {0xC3, 0x81, 0x00}, + {0xC3, 0x82, 0x00}, + {0xC3, 0x83, 0x00}, + {0xC3, 0x84, 0x00}, + {0xC3, 0x85, 0x00}, + {0xC3, 0x86, 0x00}, + {0xC3, 0x87, 0x00}, + {0xC3, 0x88, 0x00}, + {0xC3, 0x89, 0x00}, + {0xC3, 0x8A, 0x00}, + {0xC3, 0x8B, 0x00}, + {0xC3, 0x8C, 0x00}, + {0xC3, 0x8D, 0x00}, + {0xC3, 0x8E, 0x00}, + {0xC3, 0x8F, 0x00}, + {0xC3, 0x90, 0x00}, + {0xC3, 0x91, 0x00}, + {0xC3, 0x92, 0x00}, + {0xC3, 0x93, 0x00}, + {0xC3, 0x94, 0x00}, + {0xC3, 0x95, 0x00}, + {0xC3, 0x96, 0x00}, + {0xC3, 0x97, 0x00}, + {0xC3, 0x98, 0x00}, + {0xC3, 0x99, 0x00}, + {0xC3, 0x9A, 0x00}, + {0xC3, 0x9B, 0x00}, + {0xC3, 0x9C, 0x00}, + {0xC3, 0x9D, 0x00}, + {0xC3, 0x9E, 0x00}, + {0xC3, 0x9F, 0x00}, + {0xC3, 0xA0, 0x00}, + {0xC3, 0xA1, 0x00}, + {0xC3, 0xA2, 0x00}, + {0xC3, 0xA3, 0x00}, + {0xC3, 0xA4, 0x00}, + {0xC3, 0xA5, 0x00}, + {0xC3, 0xA6, 0x00}, + {0xC3, 0xA7, 0x00}, + {0xC3, 0xA8, 0x00}, + {0xC3, 0xA9, 0x00}, + {0xC3, 0xAA, 0x00}, + {0xC3, 0xAB, 0x00}, + {0xC3, 0xAC, 0x00}, + {0xC3, 0xAD, 0x00}, + {0xC3, 0xAE, 0x00}, + {0xC3, 0xAF, 0x00}, + {0xC3, 0xB0, 0x00}, + {0xC3, 0xB1, 0x00}, + {0xC3, 0xB2, 0x00}, + {0xC3, 0xB3, 0x00}, + {0xC3, 0xB4, 0x00}, + {0xC3, 0xB5, 0x00}, + {0xC3, 0xB6, 0x00}, + {0xC3, 0xB7, 0x00}, + {0xC3, 0xB8, 0x00}, + {0xC3, 0xB9, 0x00}, + {0xC3, 0xBA, 0x00}, + {0xC3, 0xBB, 0x00}, + {0xC3, 0xBC, 0x00}, + {0xC3, 0xBD, 0x00}, + {0xC3, 0xBE, 0x00}, + {0xC3, 0xBF, 0x00}, +}; + +/* Central Europe languages */ +static const CPLCodePageConvTable CPL_ISO_8859_2_to_UTF8 = { + {0xC2, 0x80, 0x00}, + {0xC2, 0x81, 0x00}, + {0xC2, 0x82, 0x00}, + {0xC2, 0x83, 0x00}, + {0xC2, 0x84, 0x00}, + {0xC2, 0x85, 0x00}, + {0xC2, 0x86, 0x00}, + {0xC2, 0x87, 0x00}, + {0xC2, 0x88, 0x00}, + {0xC2, 0x89, 0x00}, + {0xC2, 0x8A, 0x00}, + {0xC2, 0x8B, 0x00}, + {0xC2, 0x8C, 0x00}, + {0xC2, 0x8D, 0x00}, + {0xC2, 0x8E, 0x00}, + {0xC2, 0x8F, 0x00}, + {0xC2, 0x90, 0x00}, + {0xC2, 0x91, 0x00}, + {0xC2, 0x92, 0x00}, + {0xC2, 0x93, 0x00}, + {0xC2, 0x94, 0x00}, + {0xC2, 0x95, 0x00}, + {0xC2, 0x96, 0x00}, + {0xC2, 0x97, 0x00}, + {0xC2, 0x98, 0x00}, + {0xC2, 0x99, 0x00}, + {0xC2, 0x9A, 0x00}, + {0xC2, 0x9B, 0x00}, + {0xC2, 0x9C, 0x00}, + {0xC2, 0x9D, 0x00}, + {0xC2, 0x9E, 0x00}, + {0xC2, 0x9F, 0x00}, + {0xC2, 0xA0, 0x00}, + {0xC4, 0x84, 0x00}, + {0xCB, 0x98, 0x00}, + {0xC5, 0x81, 0x00}, + {0xC2, 0xA4, 0x00}, + {0xC4, 0xBD, 0x00}, + {0xC5, 0x9A, 0x00}, + {0xC2, 0xA7, 0x00}, + {0xC2, 0xA8, 0x00}, + {0xC5, 0xA0, 0x00}, + {0xC5, 0x9E, 0x00}, + {0xC5, 0xA4, 0x00}, + {0xC5, 0xB9, 0x00}, + {0xC2, 0xAD, 0x00}, + {0xC5, 0xBD, 0x00}, + {0xC5, 0xBB, 0x00}, + {0xC2, 0xB0, 0x00}, + {0xC4, 0x85, 0x00}, + {0xCB, 0x9B, 0x00}, + {0xC5, 0x82, 0x00}, + {0xC2, 0xB4, 0x00}, + {0xC4, 0xBE, 0x00}, + {0xC5, 0x9B, 0x00}, + {0xCB, 0x87, 0x00}, + {0xC2, 0xB8, 0x00}, + {0xC5, 0xA1, 0x00}, + {0xC5, 0x9F, 0x00}, + {0xC5, 0xA5, 0x00}, + {0xC5, 0xBA, 0x00}, + {0xCB, 0x9D, 0x00}, + {0xC5, 0xBE, 0x00}, + {0xC5, 0xBC, 0x00}, + {0xC5, 0x94, 0x00}, + {0xC3, 0x81, 0x00}, + {0xC3, 0x82, 0x00}, + {0xC4, 0x82, 0x00}, + {0xC3, 0x84, 0x00}, + {0xC4, 0xB9, 0x00}, + {0xC4, 0x86, 0x00}, + {0xC3, 0x87, 0x00}, + {0xC4, 0x8C, 0x00}, + {0xC3, 0x89, 0x00}, + {0xC4, 0x98, 0x00}, + {0xC3, 0x8B, 0x00}, + {0xC4, 0x9A, 0x00}, + {0xC3, 0x8D, 0x00}, + {0xC3, 0x8E, 0x00}, + {0xC4, 0x8E, 0x00}, + {0xC4, 0x90, 0x00}, + {0xC5, 0x83, 0x00}, + {0xC5, 0x87, 0x00}, + {0xC3, 0x93, 0x00}, + {0xC3, 0x94, 0x00}, + {0xC5, 0x90, 0x00}, + {0xC3, 0x96, 0x00}, + {0xC3, 0x97, 0x00}, + {0xC5, 0x98, 0x00}, + {0xC5, 0xAE, 0x00}, + {0xC3, 0x9A, 0x00}, + {0xC5, 0xB0, 0x00}, + {0xC3, 0x9C, 0x00}, + {0xC3, 0x9D, 0x00}, + {0xC5, 0xA2, 0x00}, + {0xC3, 0x9F, 0x00}, + {0xC5, 0x95, 0x00}, + {0xC3, 0xA1, 0x00}, + {0xC3, 0xA2, 0x00}, + {0xC4, 0x83, 0x00}, + {0xC3, 0xA4, 0x00}, + {0xC4, 0xBA, 0x00}, + {0xC4, 0x87, 0x00}, + {0xC3, 0xA7, 0x00}, + {0xC4, 0x8D, 0x00}, + {0xC3, 0xA9, 0x00}, + {0xC4, 0x99, 0x00}, + {0xC3, 0xAB, 0x00}, + {0xC4, 0x9B, 0x00}, + {0xC3, 0xAD, 0x00}, + {0xC3, 0xAE, 0x00}, + {0xC4, 0x8F, 0x00}, + {0xC4, 0x91, 0x00}, + {0xC5, 0x84, 0x00}, + {0xC5, 0x88, 0x00}, + {0xC3, 0xB3, 0x00}, + {0xC3, 0xB4, 0x00}, + {0xC5, 0x91, 0x00}, + {0xC3, 0xB6, 0x00}, + {0xC3, 0xB7, 0x00}, + {0xC5, 0x99, 0x00}, + {0xC5, 0xAF, 0x00}, + {0xC3, 0xBA, 0x00}, + {0xC5, 0xB1, 0x00}, + {0xC3, 0xBC, 0x00}, + {0xC3, 0xBD, 0x00}, + {0xC5, 0xA3, 0x00}, + {0xCB, 0x99, 0x00}, +}; + +/* New Western Europe */ +static const CPLCodePageConvTable CPL_ISO_8859_15_to_UTF8 = { + {0xC2, 0x80, 0x00}, + {0xC2, 0x81, 0x00}, + {0xC2, 0x82, 0x00}, + {0xC2, 0x83, 0x00}, + {0xC2, 0x84, 0x00}, + {0xC2, 0x85, 0x00}, + {0xC2, 0x86, 0x00}, + {0xC2, 0x87, 0x00}, + {0xC2, 0x88, 0x00}, + {0xC2, 0x89, 0x00}, + {0xC2, 0x8A, 0x00}, + {0xC2, 0x8B, 0x00}, + {0xC2, 0x8C, 0x00}, + {0xC2, 0x8D, 0x00}, + {0xC2, 0x8E, 0x00}, + {0xC2, 0x8F, 0x00}, + {0xC2, 0x90, 0x00}, + {0xC2, 0x91, 0x00}, + {0xC2, 0x92, 0x00}, + {0xC2, 0x93, 0x00}, + {0xC2, 0x94, 0x00}, + {0xC2, 0x95, 0x00}, + {0xC2, 0x96, 0x00}, + {0xC2, 0x97, 0x00}, + {0xC2, 0x98, 0x00}, + {0xC2, 0x99, 0x00}, + {0xC2, 0x9A, 0x00}, + {0xC2, 0x9B, 0x00}, + {0xC2, 0x9C, 0x00}, + {0xC2, 0x9D, 0x00}, + {0xC2, 0x9E, 0x00}, + {0xC2, 0x9F, 0x00}, + {0xC2, 0xA0, 0x00}, + {0xC2, 0xA1, 0x00}, + {0xC2, 0xA2, 0x00}, + {0xC2, 0xA3, 0x00}, + {0xE2, 0x82, 0xAC}, + {0xC2, 0xA5, 0x00}, + {0xC5, 0xA0, 0x00}, + {0xC2, 0xA7, 0x00}, + {0xC5, 0xA1, 0x00}, + {0xC2, 0xA9, 0x00}, + {0xC2, 0xAA, 0x00}, + {0xC2, 0xAB, 0x00}, + {0xC2, 0xAC, 0x00}, + {0xC2, 0xAD, 0x00}, + {0xC2, 0xAE, 0x00}, + {0xC2, 0xAF, 0x00}, + {0xC2, 0xB0, 0x00}, + {0xC2, 0xB1, 0x00}, + {0xC2, 0xB2, 0x00}, + {0xC2, 0xB3, 0x00}, + {0xC5, 0xBD, 0x00}, + {0xC2, 0xB5, 0x00}, + {0xC2, 0xB6, 0x00}, + {0xC2, 0xB7, 0x00}, + {0xC5, 0xBE, 0x00}, + {0xC2, 0xB9, 0x00}, + {0xC2, 0xBA, 0x00}, + {0xC2, 0xBB, 0x00}, + {0xC5, 0x92, 0x00}, + {0xC5, 0x93, 0x00}, + {0xC5, 0xB8, 0x00}, + {0xC2, 0xBF, 0x00}, + {0xC3, 0x80, 0x00}, + {0xC3, 0x81, 0x00}, + {0xC3, 0x82, 0x00}, + {0xC3, 0x83, 0x00}, + {0xC3, 0x84, 0x00}, + {0xC3, 0x85, 0x00}, + {0xC3, 0x86, 0x00}, + {0xC3, 0x87, 0x00}, + {0xC3, 0x88, 0x00}, + {0xC3, 0x89, 0x00}, + {0xC3, 0x8A, 0x00}, + {0xC3, 0x8B, 0x00}, + {0xC3, 0x8C, 0x00}, + {0xC3, 0x8D, 0x00}, + {0xC3, 0x8E, 0x00}, + {0xC3, 0x8F, 0x00}, + {0xC3, 0x90, 0x00}, + {0xC3, 0x91, 0x00}, + {0xC3, 0x92, 0x00}, + {0xC3, 0x93, 0x00}, + {0xC3, 0x94, 0x00}, + {0xC3, 0x95, 0x00}, + {0xC3, 0x96, 0x00}, + {0xC3, 0x97, 0x00}, + {0xC3, 0x98, 0x00}, + {0xC3, 0x99, 0x00}, + {0xC3, 0x9A, 0x00}, + {0xC3, 0x9B, 0x00}, + {0xC3, 0x9C, 0x00}, + {0xC3, 0x9D, 0x00}, + {0xC3, 0x9E, 0x00}, + {0xC3, 0x9F, 0x00}, + {0xC3, 0xA0, 0x00}, + {0xC3, 0xA1, 0x00}, + {0xC3, 0xA2, 0x00}, + {0xC3, 0xA3, 0x00}, + {0xC3, 0xA4, 0x00}, + {0xC3, 0xA5, 0x00}, + {0xC3, 0xA6, 0x00}, + {0xC3, 0xA7, 0x00}, + {0xC3, 0xA8, 0x00}, + {0xC3, 0xA9, 0x00}, + {0xC3, 0xAA, 0x00}, + {0xC3, 0xAB, 0x00}, + {0xC3, 0xAC, 0x00}, + {0xC3, 0xAD, 0x00}, + {0xC3, 0xAE, 0x00}, + {0xC3, 0xAF, 0x00}, + {0xC3, 0xB0, 0x00}, + {0xC3, 0xB1, 0x00}, + {0xC3, 0xB2, 0x00}, + {0xC3, 0xB3, 0x00}, + {0xC3, 0xB4, 0x00}, + {0xC3, 0xB5, 0x00}, + {0xC3, 0xB6, 0x00}, + {0xC3, 0xB7, 0x00}, + {0xC3, 0xB8, 0x00}, + {0xC3, 0xB9, 0x00}, + {0xC3, 0xBA, 0x00}, + {0xC3, 0xBB, 0x00}, + {0xC3, 0xBC, 0x00}, + {0xC3, 0xBD, 0x00}, + {0xC3, 0xBE, 0x00}, + {0xC3, 0xBF, 0x00}, +}; + + +const CPLCodePageConvTable* CPLGetConversionTableToUTF8(const char* pszEncoding) +{ + if (EQUAL(pszEncoding, "CP437")) + return &CPL_CP437_to_UTF8; + if (EQUAL(pszEncoding, "CP1250")) + return &CPL_CP1250_to_UTF8; + if (EQUAL(pszEncoding, "CP1251")) + return &CPL_CP1251_to_UTF8; + if (EQUAL(pszEncoding, "CP1252")) + return &CPL_CP1252_to_UTF8; + if (EQUAL(pszEncoding, "ISO-8859-2")) + return &CPL_ISO_8859_2_to_UTF8; + if (EQUAL(pszEncoding, "ISO-8859-15")) + return &CPL_ISO_8859_15_to_UTF8; + return CPL_NULLPTR; +} + +/* clang-format on */ diff --git a/port/cpl_character_sets.h b/port/cpl_character_sets.h new file mode 100644 index 000000000000..358887e8166e --- /dev/null +++ b/port/cpl_character_sets.h @@ -0,0 +1,8 @@ +/* This file has been generated by generate_character_set_conv_tables.c */ +/* DO NOT EDIT !*/ + +/* clang-format off */ +typedef unsigned char CPLCodePageConvTable[128][3]; + +const CPLCodePageConvTable* CPLGetConversionTableToUTF8(const char* pszEncoding); +/* clang-format on */ diff --git a/port/cpl_recode.cpp b/port/cpl_recode.cpp index e22f5d6e095e..f4bb24f63a79 100644 --- a/port/cpl_recode.cpp +++ b/port/cpl_recode.cpp @@ -29,6 +29,7 @@ #include #include "cpl_conv.h" +#include "cpl_character_sets.h" #include "utf8.h" @@ -91,27 +92,11 @@ char CPL_DLL *CPLRecode(const char *pszSource, const char *pszSrcEncoding, EQUAL(pszDstEncoding, CPL_ENC_ISO8859_1))) return CPLStrdup(pszSource); - /* -------------------------------------------------------------------- */ - /* For ZIP file handling */ - /* (CP437 might be missing even on some iconv, like on Mac) */ - /* -------------------------------------------------------------------- */ - if (EQUAL(pszSrcEncoding, "CP437") && - EQUAL(pszDstEncoding, CPL_ENC_UTF8)) // + // A few hard coded CPxxx/ISO-8859-x to UTF-8 tables + if (EQUAL(pszDstEncoding, CPL_ENC_UTF8) && + CPLGetConversionTableToUTF8(pszSrcEncoding)) { - bool bIsAllPrintableASCII = true; - const size_t nCharCount = strlen(pszSource); - for (size_t i = 0; i < nCharCount; i++) - { - if (pszSource[i] < 32 || pszSource[i] > 126) - { - bIsAllPrintableASCII = false; - break; - } - } - if (bIsAllPrintableASCII) - { - return CPLStrdup(pszSource); - } + return CPLRecodeStub(pszSource, pszSrcEncoding, pszDstEncoding); } #ifdef CPL_RECODE_ICONV diff --git a/port/cpl_recode_stub.cpp b/port/cpl_recode_stub.cpp index 62e7d5cbf037..e44da896c0cb 100644 --- a/port/cpl_recode_stub.cpp +++ b/port/cpl_recode_stub.cpp @@ -36,6 +36,7 @@ #include "cpl_conv.h" #include "cpl_error.h" +#include "cpl_character_sets.c" static unsigned utf8decode(const char *p, const char *end, int *len); static unsigned utf8towc(const char *src, unsigned srclen, wchar_t *dst, @@ -161,39 +162,118 @@ char *CPLRecodeStub(const char *pszSource, const char *pszSrcEncoding, return pszResult; } + // A few hard coded CPxxx/ISO-8859-x to UTF-8 tables + if (EQUAL(pszDstEncoding, CPL_ENC_UTF8)) + { + const auto pConvTable = CPLGetConversionTableToUTF8(pszSrcEncoding); + if (pConvTable) + { + const auto convTable = *pConvTable; + const size_t nCharCount = strlen(pszSource); + char *pszResult = + static_cast(CPLCalloc(1, nCharCount * 3 + 1)); + size_t iDst = 0; + unsigned char *pabyResult = + reinterpret_cast(pszResult); + for (size_t i = 0; i < nCharCount; ++i) + { + const unsigned char nChar = + static_cast(pszSource[i]); + if (nChar <= 127) + { + pszResult[iDst] = pszSource[i]; + ++iDst; + } + else + { + const unsigned char nShiftedChar = nChar - 128; + if (convTable[nShiftedChar][0]) + { + pabyResult[iDst] = convTable[nShiftedChar][0]; + ++iDst; + CPLAssert(convTable[nShiftedChar][1]); + pabyResult[iDst] = convTable[nShiftedChar][1]; + ++iDst; + if (convTable[nShiftedChar][2]) + { + pabyResult[iDst] = convTable[nShiftedChar][2]; + ++iDst; + } + } + else + { + // Skip the invalid sequence in the input string. + if (!bHaveWarned2) + { + bHaveWarned2 = true; + CPLError(CE_Warning, CPLE_AppDefined, + "One or several characters couldn't be " + "converted correctly from %s to %s. " + "This warning will not be emitted anymore", + pszSrcEncoding, pszDstEncoding); + } + } + } + } + + pszResult[iDst] = 0; + return pszResult; + } + } + #ifdef _WIN32 + const auto MapEncodingToWindowsCodePage = [](const char *pszEncoding) + { + // Cf https://learn.microsoft.com/fr-fr/windows/win32/intl/code-page-identifiers + if (STARTS_WITH(pszEncoding, "CP")) + { + const int nCode = atoi(pszEncoding + strlen("CP")); + if (nCode > 0) + return nCode; + else if (EQUAL(pszEncoding, "CP_OEMCP")) + return CP_OEMCP; + else if (EQUAL(pszEncoding, "CP_ACP")) + return CP_ACP; + } + else if (STARTS_WITH(pszEncoding, "WINDOWS-")) + { + const int nCode = atoi(pszEncoding + strlen("WINDOWS-")); + if (nCode > 0) + return nCode; + } + else if (STARTS_WITH(pszEncoding, "ISO-8859-")) + { + const int nCode = atoi(pszEncoding + strlen("ISO-8859-")); + if ((nCode >= 1 && nCode <= 9) || nCode == 13 || nCode == 15) + return 28590 + nCode; + } + + // Return a negative value, since CP_ACP = 0 + return -1; + }; + /* ---------------------------------------------------------------------*/ - /* CPXXX to UTF8 */ + /* XXX to UTF8 */ /* ---------------------------------------------------------------------*/ - if (STARTS_WITH(pszSrcEncoding, "CP") && - strcmp(pszDstEncoding, CPL_ENC_UTF8) == 0) + if (strcmp(pszDstEncoding, CPL_ENC_UTF8) == 0) { - int nCode = atoi(pszSrcEncoding + 2); - if (nCode > 0) + const int nCode = MapEncodingToWindowsCodePage(pszSrcEncoding); + if (nCode >= 0) { return CPLWin32Recode(pszSource, nCode, CP_UTF8); } - else if (EQUAL(pszSrcEncoding, "CP_OEMCP")) - return CPLWin32Recode(pszSource, CP_OEMCP, CP_UTF8); - else if (EQUAL(pszSrcEncoding, "CP_ACP")) - return CPLWin32Recode(pszSource, CP_ACP, CP_UTF8); } /* ---------------------------------------------------------------------*/ - /* UTF8 to CPXXX */ + /* UTF8 to XXX */ /* ---------------------------------------------------------------------*/ - if (strcmp(pszSrcEncoding, CPL_ENC_UTF8) == 0 && - STARTS_WITH(pszDstEncoding, "CP")) + if (strcmp(pszSrcEncoding, CPL_ENC_UTF8) == 0) { - int nCode = atoi(pszDstEncoding + 2); - if (nCode > 0) + const int nCode = MapEncodingToWindowsCodePage(pszDstEncoding); + if (nCode >= 0) { return CPLWin32Recode(pszSource, CP_UTF8, nCode); } - else if (EQUAL(pszDstEncoding, "CP_OEMCP")) - return CPLWin32Recode(pszSource, CP_UTF8, CP_OEMCP); - else if (EQUAL(pszDstEncoding, "CP_ACP")) - return CPLWin32Recode(pszSource, CP_UTF8, CP_ACP); } #endif @@ -220,30 +300,6 @@ char *CPLRecodeStub(const char *pszSource, const char *pszSrcEncoding, return pszResult; } - /* -------------------------------------------------------------------- */ - /* UTF-8 to anything else is treated as UTF-8 to ISO-8859-1 */ - /* with a warning. */ - /* -------------------------------------------------------------------- */ - if (strcmp(pszSrcEncoding, CPL_ENC_UTF8) == 0 && - strcmp(pszDstEncoding, CPL_ENC_ISO8859_1) == 0) - { - int nCharCount = static_cast(strlen(pszSource)); - char *pszResult = static_cast(CPLCalloc(1, nCharCount + 1)); - - if (!bHaveWarned2) - { - bHaveWarned2 = true; - CPLError(CE_Warning, CPLE_AppDefined, - "Recode from UTF-8 to %s not supported, " - "treated as UTF-8 to ISO-8859-1.", - pszDstEncoding); - } - - utf8toa(pszSource, nCharCount, pszResult, nCharCount + 1); - - return pszResult; - } - /* -------------------------------------------------------------------- */ /* Everything else is treated as a no-op with a warning. */ /* -------------------------------------------------------------------- */ From 5c37c41bafd0082abef161b4efb23532181b1b79 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 14 Sep 2024 19:36:07 +0200 Subject: [PATCH 038/710] autotest: fixes for SWIG master Commit https://github.com/swig/swig/commit/9bf4842002b34e11a310a9691bd07c2e1ec6df94 changed the Python exception emitted on NULL detection from ValueError to TypeError --- autotest/gcore/relationship.py | 2 +- autotest/gcore/vsifile.py | 22 +++++++++------------- autotest/gdrivers/memmultidim.py | 14 +++++++------- autotest/utilities/test_gdalwarp_lib.py | 2 +- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/autotest/gcore/relationship.py b/autotest/gcore/relationship.py index 60d32f24085e..57569ec3598d 100755 --- a/autotest/gcore/relationship.py +++ b/autotest/gcore/relationship.py @@ -35,7 +35,7 @@ def test_gdal_relationship(): - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.Relationship(None, None, None, gdal.GRC_ONE_TO_ONE) relationship = gdal.Relationship( diff --git a/autotest/gcore/vsifile.py b/autotest/gcore/vsifile.py index fe96a8b808c4..57b359a35661 100755 --- a/autotest/gcore/vsifile.py +++ b/autotest/gcore/vsifile.py @@ -776,12 +776,8 @@ def test_vsifile_19(): def test_vsifile_20(): - try: + with pytest.raises(Exception): gdal.VSIFReadL(1, 1, None) - except ValueError: - return - - pytest.fail() ############################################################################### @@ -1710,7 +1706,7 @@ def test_vsifile_MultipartUpload(): with gdal.quiet_errors(): assert gdal.MultipartUploadGetCapabilities("foo") is None with gdal.ExceptionMgr(useExceptions=True): - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadGetCapabilities(None) with pytest.raises( @@ -1719,7 +1715,7 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadGetCapabilities("foo") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadStart(None) with pytest.raises( @@ -1728,9 +1724,9 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadStart("foo") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAddPart(None, "", 1, 0, b"") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAddPart("", None, 1, 0, b"") with pytest.raises( @@ -1739,9 +1735,9 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadAddPart("", "", 1, 0, b"") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadEnd(None, "", [], 0) - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadEnd("", None, [], 0) with pytest.raises( @@ -1750,9 +1746,9 @@ def test_vsifile_MultipartUpload(): ): gdal.MultipartUploadEnd("", "", [], 0) - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAbort(None, "") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.MultipartUploadAbort("", None) with pytest.raises( diff --git a/autotest/gdrivers/memmultidim.py b/autotest/gdrivers/memmultidim.py index 0c59a4367536..96d15806d8d0 100755 --- a/autotest/gdrivers/memmultidim.py +++ b/autotest/gdrivers/memmultidim.py @@ -90,7 +90,7 @@ def test_mem_md_subgroup(): with gdal.quiet_errors(): assert not rg.CreateGroup("") # unnamed group not supported - with pytest.raises(ValueError): + with pytest.raises(Exception): assert not rg.CreateGroup(None) subg = rg.CreateGroup("subgroup") @@ -339,12 +339,12 @@ def test_mem_md_datatypes(): assert dt_byte.GetNumericDataType() == gdal.GDT_Byte assert dt_byte.GetSize() == 1 assert dt_byte.CanConvertTo(dt_byte) - with pytest.raises(ValueError): + with pytest.raises(Exception): assert dt_byte.CanConvertTo(None) assert dt_byte == gdal.ExtendedDataType.Create(gdal.GDT_Byte) assert not dt_byte != gdal.ExtendedDataType.Create(gdal.GDT_Byte) assert dt_byte.Equals(dt_byte) - with pytest.raises(ValueError): + with pytest.raises(Exception): assert dt_byte.Equals(None) assert not dt_byte.GetComponents() @@ -762,9 +762,9 @@ def test_mem_md_array_invalid_args(): rg.CreateMDArray("myarray", [None], edt) with pytest.raises((TypeError, SystemError)): rg.CreateMDArray("myarray", [1], edt) - with pytest.raises(ValueError): + with pytest.raises(Exception): rg.CreateMDArray("myarray", [dim], None) - with pytest.raises(ValueError): + with pytest.raises(Exception): rg.CreateMDArray(None, [dim], edt) @@ -837,7 +837,7 @@ def test_mem_md_group_attribute_single_numeric(): float64dt = gdal.ExtendedDataType.Create(gdal.GDT_Float64) with gdal.quiet_errors(): assert not rg.CreateAttribute("", [1], float64dt) # unnamed attr not supported - with pytest.raises(ValueError): + with pytest.raises(Exception): rg.CreateAttribute(None, [1], float64dt) attr = rg.CreateAttribute("attr", [1], float64dt) @@ -955,7 +955,7 @@ def test_mem_md_array_attribute(): assert not myarray.CreateAttribute( "", [1], float64dt ) # unnamed attr not supported - with pytest.raises(ValueError): + with pytest.raises(Exception): myarray.CreateAttribute(None, [1], float64dt) attr = myarray.CreateAttribute("attr", [1], float64dt) diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index 3134d0a8d9dd..d7a4869d646d 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -4102,7 +4102,7 @@ def DISABLED_test_gdalwarp_lib_to_projection_without_inverse_method(): def test_gdalwarp_lib_no_crash_on_none_dst(): ds1 = gdal.Open("../gcore/data/byte.tif") - with pytest.raises(ValueError): + with pytest.raises(Exception): gdal.Warp(None, ds1) From 06e7a7b34b974660adfc4b85d06f840d4a6ec675 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 15 Sep 2024 19:23:56 +0200 Subject: [PATCH 039/710] Use std::sort() instead of qsort() --- alg/gdalwarpoperation.cpp | 32 +++++++++++------------------- ogr/ogrfeaturequery.cpp | 20 ++----------------- ogr/ogrsf_frmts/gml/gmlhandler.cpp | 23 ++++----------------- port/cplstringlist.cpp | 22 +++++++------------- 4 files changed, 25 insertions(+), 72 deletions(-) diff --git a/alg/gdalwarpoperation.cpp b/alg/gdalwarpoperation.cpp index 1448de37f95b..d84d2c08b943 100644 --- a/alg/gdalwarpoperation.cpp +++ b/alg/gdalwarpoperation.cpp @@ -832,22 +832,6 @@ void GDALDestroyWarpOperation(GDALWarpOperationH hOperation) /* CollectChunkList() */ /************************************************************************/ -static int OrderWarpChunk(const void *_a, const void *_b) -{ - const GDALWarpChunk *a = static_cast(_a); - const GDALWarpChunk *b = static_cast(_b); - if (a->dy < b->dy) - return -1; - else if (a->dy > b->dy) - return 1; - else if (a->dx < b->dx) - return -1; - else if (a->dx > b->dx) - return 1; - else - return 0; -} - void GDALWarpOperation::CollectChunkList(int nDstXOff, int nDstYOff, int nDstXSize, int nDstYSize) @@ -859,10 +843,18 @@ void GDALWarpOperation::CollectChunkList(int nDstXOff, int nDstYOff, CollectChunkListInternal(nDstXOff, nDstYOff, nDstXSize, nDstYSize); // Sort chunks from top to bottom, and for equal y, from left to right. - // TODO(schwehr): Use std::sort. - if (pasChunkList) - qsort(pasChunkList, nChunkListCount, sizeof(GDALWarpChunk), - OrderWarpChunk); + if (nChunkListCount > 1) + { + std::sort(pasChunkList, pasChunkList + nChunkListCount, + [](const GDALWarpChunk &a, const GDALWarpChunk &b) + { + if (a.dy < b.dy) + return true; + if (a.dy > b.dy) + return false; + return a.dx < b.dx; + }); + } /* -------------------------------------------------------------------- */ /* Find the global source window. */ diff --git a/ogr/ogrfeaturequery.cpp b/ogr/ogrfeaturequery.cpp index 4cc77749d7f8..371672d89551 100644 --- a/ogr/ogrfeaturequery.cpp +++ b/ogr/ogrfeaturequery.cpp @@ -33,7 +33,6 @@ #include "ogr_swq.h" #include -#include #include #include "cpl_conv.h" @@ -407,18 +406,6 @@ int OGRFeatureQuery::CanUseIndex(const swq_expr_node *psExpr, OGRLayer *poLayer) /* multi-part queries with ranges. */ /************************************************************************/ -static int CompareGIntBig(const void *pa, const void *pb) -{ - const GIntBig a = *(reinterpret_cast(pa)); - const GIntBig b = *(reinterpret_cast(pb)); - if (a < b) - return -1; - else if (a > b) - return 1; - else - return 0; -} - GIntBig *OGRFeatureQuery::EvaluateAgainstIndices(OGRLayer *poLayer, OGRErr *peErr) @@ -668,8 +655,7 @@ GIntBig *OGRFeatureQuery::EvaluateAgainstIndices(const swq_expr_node *psExpr, if (nFIDCount > 1) { // The returned FIDs are expected to be in sorted order. - qsort(panFIDs, static_cast(nFIDCount), sizeof(GIntBig), - CompareGIntBig); + std::sort(panFIDs, panFIDs + nFIDCount); } return panFIDs; } @@ -712,9 +698,7 @@ GIntBig *OGRFeatureQuery::EvaluateAgainstIndices(const swq_expr_node *psExpr, if (nFIDCount > 1) { // The returned FIDs are expected to be sorted. - // TODO(schwehr): Use std::sort. - qsort(panFIDs, static_cast(nFIDCount), sizeof(GIntBig), - CompareGIntBig); + std::sort(panFIDs, panFIDs + nFIDCount); } return panFIDs; } diff --git a/ogr/ogrsf_frmts/gml/gmlhandler.cpp b/ogr/ogrsf_frmts/gml/gmlhandler.cpp index 82b16111576a..6cdd85f99fc4 100644 --- a/ogr/ogrsf_frmts/gml/gmlhandler.cpp +++ b/ogr/ogrsf_frmts/gml/gmlhandler.cpp @@ -31,6 +31,7 @@ #include "gmlreader.h" #include "gmlreaderp.h" +#include #include #include #include @@ -486,23 +487,6 @@ struct _GeometryNamesStruct const char *pszName; }; -/************************************************************************/ -/* GMLHandlerSortGeometryElements() */ -/************************************************************************/ - -static int GMLHandlerSortGeometryElements(const void *pAIn, const void *pBIn) -{ - const GeometryNamesStruct *pA = - static_cast(pAIn); - const GeometryNamesStruct *pB = - static_cast(pBIn); - CPLAssert(pA->nHash != pB->nHash); - if (pA->nHash < pB->nHash) - return -1; - else - return 1; -} - /************************************************************************/ /* GMLHandler() */ /************************************************************************/ @@ -520,8 +504,9 @@ GMLHandler::GMLHandler(GMLReader *poReader) pasGeometryNames[i].nHash = CPLHashSetHashStr(pasGeometryNames[i].pszName); } - qsort(pasGeometryNames, GML_GEOMETRY_TYPE_COUNT, - sizeof(GeometryNamesStruct), GMLHandlerSortGeometryElements); + std::sort(pasGeometryNames, pasGeometryNames + GML_GEOMETRY_TYPE_COUNT, + [](const GeometryNamesStruct &a, const GeometryNamesStruct &b) + { return a.nHash < b.nHash; }); stateStack[0] = STATE_TOP; } diff --git a/port/cplstringlist.cpp b/port/cplstringlist.cpp index 8e92d3af7f75..b3957c24f0af 100644 --- a/port/cplstringlist.cpp +++ b/port/cplstringlist.cpp @@ -653,6 +653,7 @@ char **CPLStringList::StealList() return papszRetList; } +/* Case insensitive comparison function */ static int CPLCompareKeyValueString(const char *pszKVa, const char *pszKVb) { const char *pszItera = pszKVa; @@ -685,19 +686,6 @@ static int CPLCompareKeyValueString(const char *pszKVa, const char *pszKVb) } } -/************************************************************************/ -/* llCompareStr() */ -/* */ -/* Note this is case insensitive! This is because we normally */ -/* treat key value keywords as case insensitive. */ -/************************************************************************/ -static int llCompareStr(const void *a, const void *b) -{ - return CPLCompareKeyValueString( - *static_cast(const_cast(a)), - *static_cast(const_cast(b))); -} - /************************************************************************/ /* Sort() */ /************************************************************************/ @@ -721,8 +709,12 @@ CPLStringList &CPLStringList::Sort() if (!MakeOurOwnCopy()) return *this; - if (nCount) - qsort(papszList, nCount, sizeof(char *), llCompareStr); + if (nCount > 1) + { + std::sort(papszList, papszList + nCount, + [](const char *a, const char *b) + { return CPLCompareKeyValueString(a, b) < 0; }); + } bIsSorted = true; return *this; From 640dafca4f90fa315819d6cfc5595ab7c99914b8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 15 Sep 2024 22:47:35 +0200 Subject: [PATCH 040/710] Test that GML driver is robust to XML billion laugh attack --- autotest/ogr/data/gml/billionlaugh.gml | 21 ++++++++ autotest/ogr/data/gml/billionlaugh.xsd | 69 ++++++++++++++++++++++++++ autotest/ogr/ogr_gml.py | 19 +++++++ 3 files changed, 109 insertions(+) create mode 100644 autotest/ogr/data/gml/billionlaugh.gml create mode 100644 autotest/ogr/data/gml/billionlaugh.xsd diff --git a/autotest/ogr/data/gml/billionlaugh.gml b/autotest/ogr/data/gml/billionlaugh.gml new file mode 100644 index 000000000000..6fc911bf771f --- /dev/null +++ b/autotest/ogr/data/gml/billionlaugh.gml @@ -0,0 +1,21 @@ + + + + + + + + + + + +]> + + &lol9; + diff --git a/autotest/ogr/data/gml/billionlaugh.xsd b/autotest/ogr/data/gml/billionlaugh.xsd new file mode 100644 index 000000000000..1e8c85b3fa03 --- /dev/null +++ b/autotest/ogr/data/gml/billionlaugh.xsd @@ -0,0 +1,69 @@ + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/autotest/ogr/ogr_gml.py b/autotest/ogr/ogr_gml.py index d4624b100a7b..c5483b60da55 100755 --- a/autotest/ogr/ogr_gml.py +++ b/autotest/ogr/ogr_gml.py @@ -2558,6 +2558,25 @@ def test_ogr_gml_64(parser): assert feat is not None, parser +############################################################################### +# Test we don't spend too much time parsing documents featuring the billion +# laugh attack + + +@gdaltest.enable_exceptions() +@pytest.mark.parametrize("parser", ("XERCES", "EXPAT")) +def test_ogr_gml_billion_laugh(parser): + + with gdal.config_option("GML_PARSER", parser), pytest.raises( + Exception, match="File probably corrupted" + ): + with gdal.OpenEx("data/gml/billionlaugh.gml") as ds: + assert ds.GetDriver().GetDescription() == "GML" + for lyr in ds: + for f in lyr: + pass + + ############################################################################### # Test SRSDIMENSION_LOC=GEOMETRY option (#5606) From e8182bb1170b832405b5fb05189b34ad3873aa93 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 15 Sep 2024 22:48:37 +0200 Subject: [PATCH 041/710] NAS: make it robust to XML billion laugh attack --- autotest/ogr/data/nas/billionlaugh.xml | 45 ++++++++++++++++++++++++++ autotest/ogr/ogr_nas.py | 14 ++++++++ ogr/ogrsf_frmts/nas/nashandler.cpp | 18 +++++++++++ ogr/ogrsf_frmts/nas/nasreader.cpp | 7 ++++ ogr/ogrsf_frmts/nas/nasreaderp.h | 4 +++ 5 files changed, 88 insertions(+) create mode 100644 autotest/ogr/data/nas/billionlaugh.xml diff --git a/autotest/ogr/data/nas/billionlaugh.xml b/autotest/ogr/data/nas/billionlaugh.xml new file mode 100644 index 000000000000..24f73f6e6614 --- /dev/null +++ b/autotest/ogr/data/nas/billionlaugh.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + +]> + + + &lol9; + <?xml version="1.0" encoding="utf-8"?><Protocol xmlns="http://www.aed-sicad.de/namespaces/va" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema"><auftragsid>AMGR000000161077</auftragsid><profilkennung>NBABenutzerLA</profilkennung><auftragsnummer>Admin_100413082150_001_0006</auftragsnummer><antragsnummer>Admin_100413082150_0006</antragsnummer><nasoperation>NutzerbezogeneBestandsdatenaktualisierung_NBA</nasoperation><status>beendet</status><startzeitreal>2010-10-13T22:29:43.66646+02:00</startzeitreal><endzeitreal>2010-10-13T22:29:43.66646+02:00</endzeitreal><Message><Category>NOTHING</Category><MessageLevel>Info</MessageLevel><MessageObject><FeatureClass /><Id /></MessageObject><MessageText>Das Ergebnis wurde mit der 3A-Version 6.0.9.3 erstellt.</MessageText><ProcessingTime>2010-10-13T22:29:43.66646+02:00</ProcessingTime></Message></Protocol> + true + Admin_100413082150_0006 + + + + + + + + true + + + + + + + + Admin_100413082150_001_0006 + + + NBABenutzerLA + 2010-10-13T20:10:14Z + 16 + 130 + 484560.000 5753705.000 + + + diff --git a/autotest/ogr/ogr_nas.py b/autotest/ogr/ogr_nas.py index c548635f9706..2190fe6d47f0 100755 --- a/autotest/ogr/ogr_nas.py +++ b/autotest/ogr/ogr_nas.py @@ -304,3 +304,17 @@ def test_ogr_nas_force_opening(tmp_vsimem): with gdal.quiet_errors(): ds = gdal.OpenEx(filename, allowed_drivers=["NAS"]) assert ds.GetDriver().GetDescription() == "NAS" + + +############################################################################### +# Test we don't spend too much time parsing documents featuring the billion +# laugh attack + + +def test_ogr_nas_billion_laugh(): + + with gdal.config_option("NAS_GFS_TEMPLATE", ""): + with gdal.quiet_errors(), pytest.raises( + Exception, match="File probably corrupted" + ): + ogr.Open("data/nas/billionlaugh.xml") diff --git a/ogr/ogrsf_frmts/nas/nashandler.cpp b/ogr/ogrsf_frmts/nas/nashandler.cpp index e542e7f979c9..63d47eecf910 100644 --- a/ogr/ogrsf_frmts/nas/nashandler.cpp +++ b/ogr/ogrsf_frmts/nas/nashandler.cpp @@ -181,6 +181,8 @@ void NASHandler::startElement(const XMLCh *const /* uri */, const Attributes &attrs) { + m_nEntityCounter = 0; + GMLReadState *poState = m_poReader->GetState(); transcode(localname, m_osElementName); @@ -438,6 +440,8 @@ void NASHandler::endElement(const XMLCh *const /* uri */, const XMLCh *const /* qname */) { + m_nEntityCounter = 0; + GMLReadState *poState = m_poReader->GetState(); transcode(localname, m_osElementName); @@ -662,6 +666,20 @@ void NASHandler::endElement(const XMLCh *const /* uri */, } } +/************************************************************************/ +/* startEntity() */ +/************************************************************************/ + +void NASHandler::startEntity(const XMLCh *const /* name */) +{ + m_nEntityCounter++; + if (m_nEntityCounter > 1000 && !m_poReader->HasStoppedParsing()) + { + throw SAXNotSupportedException( + "File probably corrupted (million laugh pattern)"); + } +} + /************************************************************************/ /* characters() */ /************************************************************************/ diff --git a/ogr/ogrsf_frmts/nas/nasreader.cpp b/ogr/ogrsf_frmts/nas/nasreader.cpp index 40bc5bafac7a..f38dbab76e19 100644 --- a/ogr/ogrsf_frmts/nas/nasreader.cpp +++ b/ogr/ogrsf_frmts/nas/nasreader.cpp @@ -266,6 +266,13 @@ GMLFeature *NASReader::NextFeature() CPLDebug("NAS", "Error during NextFeature()! Message:\n%s", transcode(toCatch.getMessage()).c_str()); } + catch (const SAXException &toCatch) + { + CPLString osErrMsg; + transcode(toCatch.getMessage(), osErrMsg); + CPLError(CE_Failure, CPLE_AppDefined, "%s", osErrMsg.c_str()); + m_bStopParsing = true; + } return poReturn; } diff --git a/ogr/ogrsf_frmts/nas/nasreaderp.h b/ogr/ogrsf_frmts/nas/nasreaderp.h index 9e935a75402a..962849a17a21 100644 --- a/ogr/ogrsf_frmts/nas/nasreaderp.h +++ b/ogr/ogrsf_frmts/nas/nasreaderp.h @@ -84,6 +84,8 @@ class NASHandler final : public DefaultHandler const Locator *m_Locator; + int m_nEntityCounter = 0; + public: explicit NASHandler(NASReader *poReader); virtual ~NASHandler(); @@ -101,6 +103,8 @@ class NASHandler final : public DefaultHandler void fatalError(const SAXParseException &) override; + void startEntity(const XMLCh *const name) override; + void setDocumentLocator(const Locator *locator) override; CPLString GetAttributes(const Attributes *attr); From 29d44e663ddcb59594b37932c12fe911eb2054c4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 15 Sep 2024 22:49:07 +0200 Subject: [PATCH 042/710] GMLAS: make it robust to XML billion laugh attack --- autotest/ogr/ogr_gmlas.py | 16 ++++++++++++++++ ogr/ogrsf_frmts/gmlas/ogr_gmlas.h | 5 +++++ ogr/ogrsf_frmts/gmlas/ogrgmlasreader.cpp | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/autotest/ogr/ogr_gmlas.py b/autotest/ogr/ogr_gmlas.py index d3a82e6467c9..5cafaeafde7f 100755 --- a/autotest/ogr/ogr_gmlas.py +++ b/autotest/ogr/ogr_gmlas.py @@ -3485,3 +3485,19 @@ def test_ogr_gmlas_ossfuzz_70511(): Exception, match="Cannot get type definition for attribute y" ): gdal.OpenEx("GMLAS:", open_options=["XSD=data/gmlas/test_ossfuzz_70511.xsd"]) + + +############################################################################### +# Test we don't spend too much time parsing documents featuring the billion +# laugh attack + + +@gdaltest.enable_exceptions() +def test_ogr_gmlas_billion_laugh(): + + with gdal.quiet_errors(), pytest.raises(Exception, match="File probably corrupted"): + with gdal.OpenEx("GMLAS:data/gml/billionlaugh.gml") as ds: + assert ds.GetDriver().GetDescription() == "GMLAS" + for lyr in ds: + for f in lyr: + pass diff --git a/ogr/ogrsf_frmts/gmlas/ogr_gmlas.h b/ogr/ogrsf_frmts/gmlas/ogr_gmlas.h index bbd8ff20a040..4a9b1ee4f36e 100644 --- a/ogr/ogrsf_frmts/gmlas/ogr_gmlas.h +++ b/ogr/ogrsf_frmts/gmlas/ogr_gmlas.h @@ -1792,6 +1792,9 @@ class GMLASReader final : public DefaultHandler /** Stack of contexts to build XML tree of GML Geometry */ std::vector m_apsXMLNodeStack{}; + /** Counter used to prevent XML billion laugh attacks */ + int m_nEntityCounter = 0; + /** Maximum allowed number of XML nesting level */ int m_nMaxLevel = 100; @@ -2048,6 +2051,8 @@ class GMLASReader final : public DefaultHandler virtual void characters(const XMLCh *const chars, const XMLSize_t length) override; + void startEntity(const XMLCh *const name) override; + bool RunFirstPass(GDALProgressFunc pfnProgress, void *pProgressData, bool bRemoveUnusedLayers, bool bRemoveUnusedFields, bool bProcessSWEDataArray, diff --git a/ogr/ogrsf_frmts/gmlas/ogrgmlasreader.cpp b/ogr/ogrsf_frmts/gmlas/ogrgmlasreader.cpp index 9b4a91f61cad..ca99999e5982 100644 --- a/ogr/ogrsf_frmts/gmlas/ogrgmlasreader.cpp +++ b/ogr/ogrsf_frmts/gmlas/ogrgmlasreader.cpp @@ -1131,6 +1131,8 @@ void GMLASReader::startElement(const XMLCh *const uri, , const Attributes &attrs) { + m_nEntityCounter = 0; + const CPLString &osLocalname(transcode(localname, m_osLocalname)); const CPLString &osNSURI(transcode(uri, m_osNSUri)); const CPLString &osNSPrefix(m_osNSPrefix = m_oMapURIToPrefix[osNSURI]); @@ -2387,6 +2389,8 @@ void GMLASReader::endElement(const XMLCh *const uri, #endif ) { + m_nEntityCounter = 0; + m_nLevel--; #ifdef DEBUG_VERBOSE @@ -2675,6 +2679,20 @@ void GMLASReader::endElement(const XMLCh *const uri, } } +/************************************************************************/ +/* startEntity() */ +/************************************************************************/ + +void GMLASReader::startEntity(const XMLCh *const /* name */) +{ + m_nEntityCounter++; + if (m_nEntityCounter > 1000 && !m_bParsingError) + { + throw SAXNotSupportedException( + "File probably corrupted (million laugh pattern)"); + } +} + /************************************************************************/ /* SetSWEValue() */ /************************************************************************/ From a763a24e13a35fc2f1def08d93a60eebcf0ba290 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 15 Sep 2024 22:57:33 +0200 Subject: [PATCH 043/710] GML: honour IsSingleAllowedDriver() --- autotest/ogr/ogr_gml.py | 15 +++++++++++++++ ogr/ogrsf_frmts/gml/ogrgmldriver.cpp | 3 +++ 2 files changed, 18 insertions(+) diff --git a/autotest/ogr/ogr_gml.py b/autotest/ogr/ogr_gml.py index d4624b100a7b..ed21fe892650 100755 --- a/autotest/ogr/ogr_gml.py +++ b/autotest/ogr/ogr_gml.py @@ -4626,3 +4626,18 @@ def test_ogr_gml_get_layers_by_name_from_imported_schema_more_tests(tmp_path): assert isinstance( dat_erst, list ), f"Expected 'dat_erst' to be of type 'list', but got {type(dat_erst)}" + + +############################################################################### +# Test force opening a GML file + + +def test_ogr_gml_force_opening(tmp_vsimem): + + filename = str(tmp_vsimem / "test.gml") + gdal.FileFromMemBuffer(filename, open("data/nas/empty_nas.xml", "rb").read()) + + # Would be opened by NAS driver if not forced + with gdal.config_option("NAS_GFS_TEMPLATE", ""): + ds = gdal.OpenEx(filename, allowed_drivers=["GML"]) + assert ds.GetDriver().GetDescription() == "GML" diff --git a/ogr/ogrsf_frmts/gml/ogrgmldriver.cpp b/ogr/ogrsf_frmts/gml/ogrgmldriver.cpp index 9f633360c769..c180c298388b 100644 --- a/ogr/ogrsf_frmts/gml/ogrgmldriver.cpp +++ b/ogr/ogrsf_frmts/gml/ogrgmldriver.cpp @@ -75,6 +75,9 @@ static int OGRGMLDriverIdentify(GDALOpenInfo *poOpenInfo) if (!poOpenInfo->TryToIngest(4096)) return FALSE; + if (poOpenInfo->IsSingleAllowedDriver("GML")) + return TRUE; + return OGRGMLDataSource::CheckHeader( reinterpret_cast(poOpenInfo->pabyHeader)); } From 421627106e9eb11f2cbeceadfc4b883840a384ec Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 15 Sep 2024 22:57:45 +0200 Subject: [PATCH 044/710] LVBAG: honour IsSingleAllowedDriver() --- autotest/ogr/ogr_lvbag.py | 11 +++++++++++ ogr/ogrsf_frmts/lvbag/ogrlvbagdriver.cpp | 7 +++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/autotest/ogr/ogr_lvbag.py b/autotest/ogr/ogr_lvbag.py index 24154c580b6d..250aeeafffbb 100644 --- a/autotest/ogr/ogr_lvbag.py +++ b/autotest/ogr/ogr_lvbag.py @@ -603,3 +603,14 @@ def test_ogr_lvbag_test_ogrsf_num(): ) assert "INFO" in ret and "ERROR" not in ret + + +############################################################################### +# Test force opening + + +def test_ogr_lvbag_force_opening(): + + # Would be opened by GML driver if not forced + ds = gdal.OpenEx("data/gml/empty.gml", allowed_drivers=["LVBAG"]) + assert ds.GetDriver().GetDescription() == "LVBAG" diff --git a/ogr/ogrsf_frmts/lvbag/ogrlvbagdriver.cpp b/ogr/ogrsf_frmts/lvbag/ogrlvbagdriver.cpp index b6e3b5a094de..e86dface9039 100644 --- a/ogr/ogrsf_frmts/lvbag/ogrlvbagdriver.cpp +++ b/ogr/ogrsf_frmts/lvbag/ogrlvbagdriver.cpp @@ -39,13 +39,16 @@ static int OGRLVBAGDriverIdentify(GDALOpenInfo *poOpenInfo) return FALSE; if (poOpenInfo->bIsDirectory) return -1; // Check later - if (poOpenInfo->fpL == nullptr) + if (poOpenInfo->fpL == nullptr || poOpenInfo->nHeaderBytes == 0) return FALSE; auto pszPtr = reinterpret_cast(poOpenInfo->pabyHeader); - if (poOpenInfo->nHeaderBytes == 0 || pszPtr[0] != '<') + if (pszPtr[0] != '<') return FALSE; + if (poOpenInfo->IsSingleAllowedDriver("LVBAG")) + return TRUE; + // Can't handle mutations just yet if (strstr(pszPtr, "http://www.kadaster.nl/schemas/mutatielevering-generiek/1.0") != From 4606665ece618aea3f50866244b223e8f6fbd5da Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 15 Sep 2024 23:25:54 +0200 Subject: [PATCH 045/710] Python bindings: clarify destNameOrDestDS semantics regarding updates Fixes #10805 --- swig/include/python/gdal_python.i | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index 0f6cb1bfc347..da9cc8e441d3 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -2992,7 +2992,12 @@ def Warp(destNameOrDestDS, srcDSOrSrcDSTab, **kwargs): Parameters ---------- destNameOrDestDS: - Output dataset name or object + Output dataset name or object. + + If passed as a dataset name, a potentially existing output dataset of + the same name will be overwritten. To update an existing output dataset, + it must be passed as a dataset object. + srcDSOrSrcDSTab: an array of Dataset objects or filenames, or a Dataset object or a filename kwargs: @@ -3408,6 +3413,13 @@ def VectorTranslate(destNameOrDestDS, srcDS, **kwargs): ---------- destNameOrDestDS: Output dataset name or object + + If passed as a dataset name, a potentially existing output dataset of + the same name will be overwritten. To update an existing output dataset, + it must be passed as a dataset object. Note that the accessMode parameter + also controls, at the layer level, if existing layers must be overwritten + or updated. + srcDS: a Dataset object or a filename kwargs: @@ -3675,6 +3687,11 @@ def Nearblack(destNameOrDestDS, srcDS, **kwargs): ---------- destNameOrDestDS: Output dataset name or object + + If passed as a dataset name, a potentially existing output dataset of + the same name will be overwritten. To update an existing output dataset, + it must be passed as a dataset object. + srcDS: a Dataset object or a filename kwargs: @@ -4030,7 +4047,12 @@ def Rasterize(destNameOrDestDS, srcDS, **kwargs): Parameters ---------- destNameOrDestDS: - Output dataset name or object + Output dataset name or object. + + If passed as a dataset name, a potentially existing output dataset of + the same name will be overwritten. To update an existing output dataset, + it must be passed as a dataset object. + srcDS: a Dataset object or a filename kwargs: @@ -4199,6 +4221,11 @@ def Footprint(destNameOrDestDS, srcDS, **kwargs): ---------- destNameOrDestDS: Output dataset name or object + + If passed as a dataset name, a potentially existing output dataset of + the same name will be overwritten. To update an existing output dataset, + it must be passed as a dataset object. + srcDS: a Dataset object or a filename kwargs: From f12f3f29fac9464f7751a7ac6e013cda810152b9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 01:54:36 +0200 Subject: [PATCH 046/710] Fix compiler warning with -Wdocumentation of clang 19 --- frmts/pcraster/pcrasterutil.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frmts/pcraster/pcrasterutil.cpp b/frmts/pcraster/pcrasterutil.cpp index a424278308b2..0228e81ed24f 100644 --- a/frmts/pcraster/pcrasterutil.cpp +++ b/frmts/pcraster/pcrasterutil.cpp @@ -393,8 +393,6 @@ CSF_CR GDALType2CellRepresentation(GDALDataType type, bool exact) /*! \param cellRepresentation Cell representation of the data. \return Missing value. - \exception . - \sa . */ double missingValue(CSF_CR cellRepresentation) { @@ -468,9 +466,6 @@ double missingValue(CSF_CR cellRepresentation) /*! \param filename Filename of raster to open. \return Pointer to CSF MAP structure. - \exception . - \warning . - \sa . */ MAP *mapOpen(std::string const &filename, MOPEN_PERM mode) { From 00db7a6f9a231493dd44ab7cd29ce461e02bb229 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 10:39:24 +0200 Subject: [PATCH 047/710] GTiff: GTiffWriteJPEGTables()/CreateLL(): avoid code relying on vsimem filename --- frmts/gtiff/geotiff.cpp | 4 +++- frmts/gtiff/gtiffdataset.h | 3 --- frmts/gtiff/gtiffdataset_write.cpp | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/frmts/gtiff/geotiff.cpp b/frmts/gtiff/geotiff.cpp index 55b1d9ebf6f9..a0d763f23dff 100644 --- a/frmts/gtiff/geotiff.cpp +++ b/frmts/gtiff/geotiff.cpp @@ -702,7 +702,7 @@ void GTiffWriteJPEGTables(TIFF *hTIFF, const char *pszPhotometric, l_nBitsPerSample = 1; CPLString osTmpFilenameIn; - osTmpFilenameIn.Printf("%s%p", szJPEGGTiffDatasetTmpPrefix, hTIFF); + osTmpFilenameIn.Printf("/vsimem/gtiffdataset_jpg_tmp_%p", hTIFF); VSILFILE *fpTmp = nullptr; CPLString osTmp; char **papszLocalParameters = nullptr; @@ -723,6 +723,8 @@ void GTiffWriteJPEGTables(TIFF *hTIFF, const char *pszPhotometric, CPLSPrintf("%u", l_nBitsPerSample)); papszLocalParameters = CSLSetNameValue(papszLocalParameters, "JPEGTABLESMODE", pszJPEGTablesMode); + papszLocalParameters = + CSLSetNameValue(papszLocalParameters, "WRITE_JPEGTABLE_TAG", "NO"); TIFF *hTIFFTmp = GTiffDataset::CreateLL(osTmpFilenameIn, nInMemImageWidth, diff --git a/frmts/gtiff/gtiffdataset.h b/frmts/gtiff/gtiffdataset.h index 2c54e910a31e..ab8af5725f8d 100644 --- a/frmts/gtiff/gtiffdataset.h +++ b/frmts/gtiff/gtiffdataset.h @@ -54,9 +54,6 @@ enum class GTiffProfile : GByte // This must be a #define, since it is used in a XSTRINGIFY() macro #define DEFAULT_WEBP_LEVEL 75 -constexpr const char *const szJPEGGTiffDatasetTmpPrefix = - "/vsimem/gtiffdataset_jpg_tmp_"; - class GTiffBitmapBand; class GTiffDataset; class GTiffJPEGOverviewBand; diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 4146fcc84236..576f78f19373 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -5875,7 +5875,6 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, // strip/tile writing, which is too late, since we have already crystalized // the directory. This way we avoid a directory rewriting. if (l_nCompression == COMPRESSION_JPEG && - !STARTS_WITH(pszFilename, szJPEGGTiffDatasetTmpPrefix) && CPLTestBool( CSLFetchNameValueDef(papszParamList, "WRITE_JPEGTABLE_TAG", "YES"))) { From 6bdfda3a083d111ff69d0d4538e9beeefdcfba35 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 15:53:42 +0200 Subject: [PATCH 048/710] autotest/gcore/vsifile.py: more tests --- autotest/gcore/vsifile.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/autotest/gcore/vsifile.py b/autotest/gcore/vsifile.py index 57b359a35661..4b648807a813 100755 --- a/autotest/gcore/vsifile.py +++ b/autotest/gcore/vsifile.py @@ -150,6 +150,26 @@ def vsifile_generic(filename, options=[]): gdal.Unlink(filename) + if not filename.startswith("/vsicrypt/"): + assert gdal.RmdirRecursive(filename + "/i_dont_exist") == -1 + + subdir = filename + "/subdir" + assert gdal.MkdirRecursive(subdir + "/subsubdir", 0o755) == 0 + + assert gdal.VSIStatL(subdir) is not None + assert gdal.VSIStatL(subdir + "/subsubdir") is not None + + if not filename.startswith("/vsimem/"): + assert gdal.Rmdir(subdir) == -1 + assert gdal.VSIStatL(subdir) is not None + + # Safety belt... + assert filename.startswith("tmp/") or filename.startswith("/vsimem/") + assert gdal.RmdirRecursive(filename) == 0 + + assert gdal.VSIStatL(subdir) is None + assert gdal.VSIStatL(subdir + "/subsubdir") is None + ############################################################################### # Test /vsimem From 07fa2670d4194581313383e099ca85dadfc33290 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:32:49 +0200 Subject: [PATCH 049/710] Add VSIMemGenerateHiddenFilename() --- autotest/cpp/test_cpl.cpp | 190 +++++++++++++++++++++++++++++ port/cpl_vsi.h | 2 + port/cpl_vsi_mem.cpp | 245 ++++++++++++++++++++++++++++++++++---- 3 files changed, 414 insertions(+), 23 deletions(-) diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp index be5292265689..77f5636b2072 100644 --- a/autotest/cpp/test_cpl.cpp +++ b/autotest/cpp/test_cpl.cpp @@ -5286,4 +5286,194 @@ TEST_F(test_cpl, CPLSpawn) } #endif +static bool ENDS_WITH(const char *pszStr, const char *pszEnd) +{ + return strlen(pszStr) >= strlen(pszEnd) && + strcmp(pszStr + strlen(pszStr) - strlen(pszEnd), pszEnd) == 0; +} + +TEST_F(test_cpl, VSIMemGenerateHiddenFilename) +{ + { + // Initial cleanup + VSIRmdirRecursive("/vsimem/"); + VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."); + + // Generate unlisted filename + const std::string osFilename1 = VSIMemGenerateHiddenFilename(nullptr); + const char *pszFilename1 = osFilename1.c_str(); + EXPECT_TRUE(STARTS_WITH(pszFilename1, "/vsimem/.#!HIDDEN!#./")); + EXPECT_TRUE(ENDS_WITH(pszFilename1, "/unnamed")); + + { + // Check the file doesn't exist yet + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(pszFilename1, &sStat), -1); + } + + // Create the file with some content + GByte abyDummyData[1] = {0}; + VSIFCloseL(VSIFileFromMemBuffer(pszFilename1, abyDummyData, + sizeof(abyDummyData), false)); + + { + // Check the file exists now + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(pszFilename1, &sStat), 0); + } + + // Get's back content + EXPECT_EQ(VSIGetMemFileBuffer(pszFilename1, nullptr, false), + abyDummyData); + + { + // Check the hidden file doesn't popup + const CPLStringList aosFiles(VSIReadDir("/vsimem/")); + EXPECT_EQ(aosFiles.size(), 0); + } + + { + // Check that we can list the below directory if we know it exists + // and there's just one subdir + const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_EQ(aosFiles.size(), 1); + } + + { + // but that it is not an explicit directory + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL("/vsimem/.#!HIDDEN!#.", &sStat), -1); + } + + // Creates second file + const std::string osFilename2 = VSIMemGenerateHiddenFilename(nullptr); + const char *pszFilename2 = osFilename2.c_str(); + EXPECT_TRUE(strcmp(pszFilename1, pszFilename2) != 0); + + // Create it + VSIFCloseL(VSIFileFromMemBuffer(pszFilename2, abyDummyData, + sizeof(abyDummyData), false)); + + { + // Check that we can list the root hidden dir if we know it exists + const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_EQ(aosFiles.size(), 2); + } + + { + // Create an explicit subdirectory in a hidden directory + const std::string osBaseName = + VSIMemGenerateHiddenFilename(nullptr); + const std::string osSubDir = + CPLFormFilename(osBaseName.c_str(), "mysubdir", nullptr); + EXPECT_EQ(VSIMkdir(osSubDir.c_str(), 0), 0); + + // Check the subdirectory exists + { + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osSubDir.c_str(), &sStat), 0); + } + + // but not its hidden parent + { + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osBaseName.c_str(), &sStat), -1); + } + + // Create file within the subdirectory + VSIFCloseL(VSIFileFromMemBuffer( + CPLFormFilename(osSubDir.c_str(), "my.bin", nullptr), + abyDummyData, sizeof(abyDummyData), false)); + + { + // Check that we can list the subdirectory + const CPLStringList aosFiles(VSIReadDir(osSubDir.c_str())); + EXPECT_EQ(aosFiles.size(), 1); + } + + { + // Check that we can list the root hidden dir if we know it exists + const CPLStringList aosFiles( + VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_EQ(aosFiles.size(), 3); + } + } + + // Directly create a directory with the return of VSIMemGenerateHiddenFilename() + { + const std::string osDirname = VSIMemGenerateHiddenFilename(nullptr); + EXPECT_EQ(VSIMkdir(osDirname.c_str(), 0), 0); + + // Check the subdirectory exists + { + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), 0); + } + + // Create file within the subdirectory + VSIFCloseL(VSIFileFromMemBuffer( + CPLFormFilename(osDirname.c_str(), "my.bin", nullptr), + abyDummyData, sizeof(abyDummyData), false)); + + { + // Check there's a file in this subdirectory + const CPLStringList aosFiles(VSIReadDir(osDirname.c_str())); + EXPECT_EQ(aosFiles.size(), 1); + } + + EXPECT_EQ(VSIRmdirRecursive(osDirname.c_str()), 0); + + { + // Check there's no longer any file in this subdirectory + const CPLStringList aosFiles(VSIReadDir(osDirname.c_str())); + EXPECT_EQ(aosFiles.size(), 0); + } + + { + // Check that it no longer exists + VSIStatBufL sStat; + EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), -1); + } + } + + // Check that operations on "/vsimem/" do not interfere with hidden files + { + // Create regular file + VSIFCloseL(VSIFileFromMemBuffer("/vsimem/regular_file", + abyDummyData, sizeof(abyDummyData), + false)); + + // Check it is visible + EXPECT_EQ(CPLStringList(VSIReadDir("/vsimem/")).size(), 1); + + // Clean root /vsimem/ + VSIRmdirRecursive("/vsimem/"); + + // No more user files + EXPECT_TRUE(CPLStringList(VSIReadDir("/vsimem/")).empty()); + + // But still hidden files + EXPECT_TRUE( + !CPLStringList(VSIReadDir("/vsimem/.#!HIDDEN!#.")).empty()); + } + + // Clean-up hidden files + EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0); + + { + // Check the root hidden dir is empty + const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#.")); + EXPECT_TRUE(aosFiles.empty()); + } + + EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0); + } + + { + const std::string osFilename = VSIMemGenerateHiddenFilename("foo.bar"); + const char *pszFilename = osFilename.c_str(); + EXPECT_TRUE(STARTS_WITH(pszFilename, "/vsimem/.#!HIDDEN!#./")); + EXPECT_TRUE(ENDS_WITH(pszFilename, "/foo.bar")); + } +} } // namespace diff --git a/port/cpl_vsi.h b/port/cpl_vsi.h index 634f599451e8..5133f17b1296 100644 --- a/port/cpl_vsi.h +++ b/port/cpl_vsi.h @@ -547,6 +547,8 @@ GByte CPL_DLL *VSIGetMemFileBuffer(const char *pszFilename, vsi_l_offset *pnDataLength, int bUnlinkAndSeize); +const char CPL_DLL *VSIMemGenerateHiddenFilename(const char *pszFilename); + /** Callback used by VSIStdoutSetRedirection() */ typedef size_t (*VSIWriteFunction)(const void *ptr, size_t size, size_t nmemb, FILE *stream); diff --git a/port/cpl_vsi_mem.cpp b/port/cpl_vsi_mem.cpp index e5099e9c20ba..9f90a033046b 100644 --- a/port/cpl_vsi_mem.cpp +++ b/port/cpl_vsi_mem.cpp @@ -43,10 +43,12 @@ #endif #include +#include #include #include #include #include +#include #include // c++17 or VS2017 @@ -70,6 +72,39 @@ //! @cond Doxygen_Suppress +// szHIDDEN_DIRNAME is for files created by VSIMemGenerateHiddenFilename(pszFilename). +// Such files are of the form "/vsimem/.#!HIDDEN!#./{counter}/{pszFilename}" +// +// The high-level design constraint is that "/vsimem/.#!HIDDEN!#." acts as a +// "side" hiearchy, but still under the "/vsimem/" namespace, so that code +// having special processing of filenames starting with /vsimem/ can still work. +// The structure of the returned filename is also such that those files form +// independent hierarchies, i.e. the tree generated by a +// VSIMemGenerateHiddenFilename() is "invisible" from the one returned by +// another call to it. +// +// As a consequence: +// - we don't want ".#!HIDDEN!#." to be listed in VSIReadDir("/vsimem/") +// - we don't want content under ""/vsimem/.#!HIDDEN!#" to be deleted by +// VSIRmdirRecursive("/vsimem/") +// - we don't want the creation of a file (or directory) called +// "/vsimem/.#!HIDDEN!#./{counter}/{pszFilename}" +// to cause the implicit creation of "/vsimem/.#!HIDDEN!#./{counter}" and +// "/vsimem/.#!HIDDEN!#". This is done so that users don't have to care about +// cleaning such implicit directories that are upper in the hiearchy w.r.t. +// to what we return to them. +// - But we want the creation of file or directory +// "/vsimem/.#!HIDDEN!#./{counter}/{pszFilename}/something_added_by_user" +// to cause "/vsimem/.#!HIDDEN!#./{counter}/{pszFilename}" to be implicitly +// created as a directory, so they can list it, or recursively delete it. +// - we want VSIReadDirRecursive("/vsimem/.#!HIDDEN!#.") to list everything +// under it (for debugging purposes) +// - we want VSIRmdirRecursive("/vsimem/.#!HIDDEN!#.") to remove everything +// under it (for debugging purposes) +// + +constexpr const char *szHIDDEN_DIRNAME = "/vsimem/.#!HIDDEN!#."; + /* ** Notes on Multithreading: ** @@ -195,6 +230,7 @@ class VSIMemFilesystemHandler final : public VSIFilesystemHandler int Unlink(const char *pszFilename) override; int Mkdir(const char *pszDirname, long nMode) override; int Rmdir(const char *pszDirname) override; + int RmdirRecursive(const char *pszDirname) override; char **ReadDirEx(const char *pszDirname, int nMaxFiles) override; int Rename(const char *oldpath, const char *newpath) override; GIntBig GetDiskFreeSpace(const char *pszDirname) override; @@ -807,6 +843,18 @@ int VSIMemFilesystemHandler::Mkdir(const char *pszPathname, long /* nMode */) CPLMutexHolder oHolder(&hMutex); const CPLString osPathname = NormalizePath(pszPathname); + if (STARTS_WITH(osPathname.c_str(), szHIDDEN_DIRNAME)) + { + if (osPathname.size() == strlen(szHIDDEN_DIRNAME)) + return 0; + // "/vsimem/.#!HIDDEN!#./{unique_counter}" + else if (osPathname.find('/', strlen(szHIDDEN_DIRNAME) + 1) == + std::string::npos) + return 0; + + // If "/vsimem/.#!HIDDEN!#./{unique_counter}/user_directory", then + // accept creating an explicit directory + } if (oFileList.find(osPathname) != oFileList.end()) { @@ -836,6 +884,69 @@ int VSIMemFilesystemHandler::Rmdir(const char *pszPathname) return Unlink(pszPathname); } +/************************************************************************/ +/* RmdirRecursive() */ +/************************************************************************/ + +int VSIMemFilesystemHandler::RmdirRecursive(const char *pszDirname) +{ + CPLMutexHolder oHolder(&hMutex); + + const CPLString osPath = NormalizePath(pszDirname); + const size_t nPathLen = osPath.size(); + int ret = 0; + if (osPath == "/vsimem") + { + // Clean-up all files under pszDirname, except hidden directories + // if called from "/vsimem" + for (auto iter = oFileList.begin(); iter != oFileList.end(); + /* no automatic increment */) + { + const char *pszFilePath = iter->second->osFilename.c_str(); + const size_t nFileLen = iter->second->osFilename.size(); + if (nFileLen > nPathLen && + memcmp(pszFilePath, osPath.c_str(), nPathLen) == 0 && + pszFilePath[nPathLen] == '/' && + !STARTS_WITH(pszFilePath, szHIDDEN_DIRNAME)) + { + iter = oFileList.erase(iter); + } + else + { + ++iter; + } + } + } + else + { + ret = -1; + for (auto iter = oFileList.begin(); iter != oFileList.end(); + /* no automatic increment */) + { + const char *pszFilePath = iter->second->osFilename.c_str(); + const size_t nFileLen = iter->second->osFilename.size(); + if (nFileLen >= nPathLen && + memcmp(pszFilePath, osPath.c_str(), nPathLen) == 0 && + (nFileLen == nPathLen || pszFilePath[nPathLen] == '/')) + { + // If VSIRmdirRecursive() is used correctly, it should at + // least delete the directory on which it has been called + ret = 0; + iter = oFileList.erase(iter); + } + else + { + ++iter; + } + } + + // Make sure that it always succeed on the root hidden directory + if (osPath == szHIDDEN_DIRNAME) + ret = 0; + } + return ret; +} + /************************************************************************/ /* ReadDirEx() */ /************************************************************************/ @@ -848,41 +959,89 @@ char **VSIMemFilesystemHandler::ReadDirEx(const char *pszPath, int nMaxFiles) const CPLString osPath = NormalizePath(pszPath); char **papszDir = nullptr; - size_t nPathLen = osPath.size(); - - if (nPathLen > 0 && osPath.back() == '/') - nPathLen--; + const size_t nPathLen = osPath.size(); // In case of really big number of files in the directory, CSLAddString // can be slow (see #2158). We then directly build the list. int nItems = 0; int nAllocatedItems = 0; - for (const auto &iter : oFileList) + if (osPath == szHIDDEN_DIRNAME) { - const char *pszFilePath = iter.second->osFilename.c_str(); - if (EQUALN(osPath, pszFilePath, nPathLen) && - pszFilePath[nPathLen] == '/' && - strstr(pszFilePath + nPathLen + 1, "/") == nullptr) + // Special mode for hidden filenames. + // "/vsimem/.#!HIDDEN!#./{counter}" subdirectories are not explicitly + // created so they do not appear in oFileList, but their subcontent + // (e.g "/vsimem/.#!HIDDEN!#./{counter}/foo") does + std::set oSetSubDirs; + for (const auto &iter : oFileList) { - if (nItems == 0) + const char *pszFilePath = iter.second->osFilename.c_str(); + if (iter.second->osFilename.size() > nPathLen && + memcmp(pszFilePath, osPath.c_str(), nPathLen) == 0) { - papszDir = static_cast(CPLCalloc(2, sizeof(char *))); - nAllocatedItems = 1; + char *pszItem = CPLStrdup(pszFilePath + nPathLen + 1); + char *pszSlash = strchr(pszItem, '/'); + if (pszSlash) + *pszSlash = 0; + if (cpl::contains(oSetSubDirs, pszItem)) + { + CPLFree(pszItem); + continue; + } + oSetSubDirs.insert(pszItem); + + if (nItems == 0) + { + papszDir = + static_cast(CPLCalloc(2, sizeof(char *))); + nAllocatedItems = 1; + } + else if (nItems >= nAllocatedItems) + { + nAllocatedItems = nAllocatedItems * 2; + papszDir = static_cast(CPLRealloc( + papszDir, (nAllocatedItems + 2) * sizeof(char *))); + } + + papszDir[nItems] = pszItem; + papszDir[nItems + 1] = nullptr; + + nItems++; + if (nMaxFiles > 0 && nItems > nMaxFiles) + break; } - else if (nItems >= nAllocatedItems) + } + } + else + { + for (const auto &iter : oFileList) + { + const char *pszFilePath = iter.second->osFilename.c_str(); + if (iter.second->osFilename.size() > nPathLen && + memcmp(pszFilePath, osPath.c_str(), nPathLen) == 0 && + pszFilePath[nPathLen] == '/' && + strstr(pszFilePath + nPathLen + 1, "/") == nullptr) { - nAllocatedItems = nAllocatedItems * 2; - papszDir = static_cast(CPLRealloc( - papszDir, (nAllocatedItems + 2) * sizeof(char *))); + if (nItems == 0) + { + papszDir = + static_cast(CPLCalloc(2, sizeof(char *))); + nAllocatedItems = 1; + } + else if (nItems >= nAllocatedItems) + { + nAllocatedItems = nAllocatedItems * 2; + papszDir = static_cast(CPLRealloc( + papszDir, (nAllocatedItems + 2) * sizeof(char *))); + } + + papszDir[nItems] = CPLStrdup(pszFilePath + nPathLen + 1); + papszDir[nItems + 1] = nullptr; + + nItems++; + if (nMaxFiles > 0 && nItems > nMaxFiles) + break; } - - papszDir[nItems] = CPLStrdup(pszFilePath + nPathLen + 1); - papszDir[nItems + 1] = nullptr; - - nItems++; - if (nMaxFiles > 0 && nItems > nMaxFiles) - break; } } @@ -1184,3 +1343,43 @@ GByte *VSIGetMemFileBuffer(const char *pszFilename, vsi_l_offset *pnDataLength, return pabyData; } + +/************************************************************************/ +/* VSIMemGenerateHiddenFilename() */ +/************************************************************************/ + +/** + * \brief Generates a unique filename that can be used with the /vsimem/ + * virtual file system. + * + * This function returns a (short-lived) string containing a unique filename, + * (using an atomic counter), designed for temporary files that must remain + * invisible for other users working at the "/vsimem/" top-level, i.e. + * such files are not returned by VSIReadDir("/vsimem/") or + * VSIReadDirRecursive("/vsimem/)". + * + * The function does not create the file per se. Such filename can be used to + * create a regular file with VSIFOpenL() or VSIFileFromMemBuffer(), or create + * a directory with VSIMkdir() + * + * Once created, deletion of those files using VSIUnlink(), VSIRmdirRecursive(), + * etc. is of the responsibility of the user. The user should not attempt to + * work with the "parent" directory returned by CPLGetPath() / CPLGetDirname() + * on the returned filename, and work only with files at the same level or + * in subdirectories of what is returned by this function. + * + * @param pszFilename the filename to be appended at the end of the returned + * filename. If not specified, defaults to "unnamed". + * + * @return pointer to a short-lived string (rotating buffer of strings in + * thread-local storage). It is recommended to use CPLStrdup() or std::string() + * immediately on it. + * + * @since GDAL 3.10 + */ +const char *VSIMemGenerateHiddenFilename(const char *pszFilename) +{ + static std::atomic nCounter{0}; + return CPLSPrintf("%s/%u/%s", szHIDDEN_DIRNAME, ++nCounter, + pszFilename ? pszFilename : "unnamed"); +} From af528db8f1275e1138fe74b89eab692cbf4207bf Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:33:05 +0200 Subject: [PATCH 050/710] VSIMemGenerateHiddenFilename(): use in gdalwarp --- apps/gdalwarp_lib.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 7e43acdce3ad..bf4ef59c1d08 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -1175,8 +1175,8 @@ static bool DealWithCOGOptions(CPLStringList &aosCreateOptions, int nSrcCount, GDALWarpAppOptions oClonedOptions(*psOptions); oClonedOptions.bQuiet = true; - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/gdalwarp/%p.tif", &oClonedOptions); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("gdalwarp_tmp.tif")); CPLStringList aosTmpGTiffCreateOptions; aosTmpGTiffCreateOptions.SetNameValue("SPARSE_OK", "YES"); aosTmpGTiffCreateOptions.SetNameValue("TILED", "YES"); From fb2f9ce202f261fdc61406d8fcf4db7ab312f523 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:33:46 +0200 Subject: [PATCH 051/710] VSIMemGenerateHiddenFilename(): use in gcore/ --- autotest/gdrivers/jp2openjpeg.py | 4 ++-- gcore/gdaljp2abstractdataset.cpp | 15 +++++++++------ gcore/gdaljp2metadata.cpp | 10 ++++++---- gcore/gdaljp2structure.cpp | 7 +++---- gcore/gdalmultidim.cpp | 8 ++++---- gcore/gdalmultidim_gridded.cpp | 5 ++--- gcore/gdalorienteddataset.cpp | 3 +-- gcore/gdalpython.cpp | 2 +- 8 files changed, 28 insertions(+), 26 deletions(-) diff --git a/autotest/gdrivers/jp2openjpeg.py b/autotest/gdrivers/jp2openjpeg.py index 46820ee810e7..cc64df8dabf1 100755 --- a/autotest/gdrivers/jp2openjpeg.py +++ b/autotest/gdrivers/jp2openjpeg.py @@ -2744,10 +2744,10 @@ def test_jp2openjpeg_45(): ) del out_ds - dircontent = gdal.ReadDir("/vsimem/") + dircontent = gdal.ReadDir("/vsimem/.#!HIDDEN!#.") if dircontent: for filename in dircontent: - assert not filename.startswith("gmljp2") + assert "gmljp2" not in filename ds = ogr.Open("/vsimem/jp2openjpeg_45.jp2") assert ds.GetLayerCount() == 1 diff --git a/gcore/gdaljp2abstractdataset.cpp b/gcore/gdaljp2abstractdataset.cpp index fc63d9e28578..98b37bec1270 100644 --- a/gcore/gdaljp2abstractdataset.cpp +++ b/gcore/gdaljp2abstractdataset.cpp @@ -368,6 +368,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) return; } + const std::string osTmpDir = VSIMemGenerateHiddenFilename("gmljp2"); + // Find feature collections. int nLayersAtCC = 0; int nLayersAtGC = 0; @@ -451,7 +453,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) if (psFC != nullptr) { - osGMLTmpFile = CPLSPrintf("/vsimem/gmljp2_%p/my.gml", this); + osGMLTmpFile = + CPLFormFilename(osTmpDir.c_str(), "my.gml", nullptr); // Create temporary .gml file. CPLSerializeXMLTreeToFile(psFC, osGMLTmpFile); } @@ -486,8 +489,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) CPLSPrintf("xml:%s", pszBoxName)); if (papszBoxData != nullptr) { - osXSDTmpFile = CPLSPrintf( - "/vsimem/gmljp2_%p/my.xsd", this); + osXSDTmpFile = CPLFormFilename( + osTmpDir.c_str(), "my.xsd", nullptr); CPL_IGNORE_RET_VAL( VSIFCloseL(VSIFileFromMemBuffer( osXSDTmpFile, @@ -551,7 +554,7 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) "No GML driver found to read feature collection"); } - VSIRmdirRecursive(CPLSPrintf("/vsimem/gmljp2_%p", this)); + VSIRmdirRecursive(osTmpDir.c_str()); } } @@ -589,8 +592,8 @@ void GDALJP2AbstractDataset::LoadVectorLayers(int bOpenRemoteResources) // Create temporary .kml file. CPLXMLNode *const psKML = psGCorGMLJP2FeaturesChildIter->psChild; - CPLString osKMLTmpFile( - CPLSPrintf("/vsimem/gmljp2_%p_my.kml", this)); + const CPLString osKMLTmpFile( + VSIMemGenerateHiddenFilename("my.kml")); CPLSerializeXMLTreeToFile(psKML, osKMLTmpFile); GDALDatasetUniquePtr poTmpDS(GDALDataset::Open( diff --git a/gcore/gdaljp2metadata.cpp b/gcore/gdaljp2metadata.cpp index 198a80ef9d9f..3852ecd93fd1 100644 --- a/gcore/gdaljp2metadata.cpp +++ b/gcore/gdaljp2metadata.cpp @@ -2593,6 +2593,8 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, "\n", osRootGMLId.c_str(), osGridCoverage.c_str()); + const std::string osTmpDir = VSIMemGenerateHiddenFilename("gmljp2"); + /* -------------------------------------------------------------------- */ /* Process metadata, annotations and features collections. */ /* -------------------------------------------------------------------- */ @@ -2756,7 +2758,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, if (hSrcDS) { CPLString osTmpFile = - CPLSPrintf("/vsimem/gmljp2_%p/%d/%s.gml", this, i, + CPLSPrintf("%s/%d/%s.gml", osTmpDir.c_str(), i, CPLGetBasename(aoGMLFiles[i].osFile)); char **papszOptions = nullptr; papszOptions = @@ -2864,7 +2866,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, !aoGMLFiles[i].osRemoteResource.empty()) { osTmpFile = - CPLSPrintf("/vsimem/gmljp2_%p/%d/%s.gml", this, i, + CPLSPrintf("%s/%d/%s.gml", osTmpDir.c_str(), i, CPLGetBasename(aoGMLFiles[i].osFile)); GMLJP2V2BoxDesc oDesc; @@ -3044,7 +3046,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, if (hSrcDS) { CPLString osTmpFile = - CPLSPrintf("/vsimem/gmljp2_%p/%d/%s.kml", this, i, + CPLSPrintf("%s/%d/%s.kml", osTmpDir.c_str(), i, CPLGetBasename(aoAnnotations[i].osFile)); char **papszOptions = nullptr; if (aoAnnotations.size() > 1) @@ -3260,7 +3262,7 @@ GDALJP2Box *GDALJP2Metadata::CreateGMLJP2V2(int nXSize, int nYSize, for (auto &poGMLBox : apoGMLBoxes) delete poGMLBox; - VSIRmdirRecursive(CPLSPrintf("/vsimem/gmljp2_%p", this)); + VSIRmdirRecursive(osTmpDir.c_str()); return poGMLData; } diff --git a/gcore/gdaljp2structure.cpp b/gcore/gdaljp2structure.cpp index 6520f901ab02..d115e9cc782d 100644 --- a/gcore/gdaljp2structure.cpp +++ b/gcore/gdaljp2structure.cpp @@ -251,8 +251,7 @@ static void DumpGeoTIFFBox(CPLXMLNode *psBox, GDALJP2Box &oBox, static_cast(GDALGetDriverByName("VRT")); if (pabyBoxData && poVRTDriver) { - CPLString osTmpFilename( - CPLSPrintf("/vsimem/tmp_%p.tif", oBox.GetFILE())); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("tmp.tif")); CPL_IGNORE_RET_VAL(VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename, pabyBoxData, nBoxDataLength, FALSE))); CPLPushErrorHandler(CPLQuietErrorHandler); @@ -267,8 +266,8 @@ static void DumpGeoTIFFBox(CPLXMLNode *psBox, GDALJP2Box &oBox, } if (poDS) { - CPLString osTmpVRTFilename( - CPLSPrintf("/vsimem/tmp_%p.vrt", oBox.GetFILE())); + const CPLString osTmpVRTFilename( + CPLResetExtension(osTmpFilename.c_str(), "vrt")); GDALDataset *poVRTDS = poVRTDriver->CreateCopy( osTmpVRTFilename, poDS, FALSE, nullptr, nullptr, nullptr); GDALClose(poVRTDS); diff --git a/gcore/gdalmultidim.cpp b/gcore/gdalmultidim.cpp index b46bf17780fe..77576028b34b 100644 --- a/gcore/gdalmultidim.cpp +++ b/gcore/gdalmultidim.cpp @@ -8187,10 +8187,10 @@ std::shared_ptr GDALMDArrayResampled::Create( "Setting geolocation array from variables %s and %s", poLongVar->GetName().c_str(), poLatVar->GetName().c_str()); - std::string osFilenameLong = - CPLSPrintf("/vsimem/%p/longitude.tif", poParent.get()); - std::string osFilenameLat = - CPLSPrintf("/vsimem/%p/latitude.tif", poParent.get()); + const std::string osFilenameLong = + VSIMemGenerateHiddenFilename("longitude.tif"); + const std::string osFilenameLat = + VSIMemGenerateHiddenFilename("latitude.tif"); std::unique_ptr poTmpLongDS( longDimCount == 1 ? poLongVar->AsClassicDataset(0, 0) diff --git a/gcore/gdalmultidim_gridded.cpp b/gcore/gdalmultidim_gridded.cpp index 4a0741a903c7..56142a31f4fe 100644 --- a/gcore/gdalmultidim_gridded.cpp +++ b/gcore/gdalmultidim_gridded.cpp @@ -665,9 +665,8 @@ GDALMDArray::GetGridded(const std::string &osGridOptions, } // Create a in-memory vector layer with (X,Y) points - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/GDALMDArray::GetGridded_%p_%p.%s", this, - pOptions, pszExt); + const std::string osTmpFilename(VSIMemGenerateHiddenFilename( + std::string("tmp.").append(pszExt).c_str())); auto poDS = std::unique_ptr( poDrv->Create(osTmpFilename.c_str(), 0, 0, 0, GDT_Unknown, nullptr)); if (!poDS) diff --git a/gcore/gdalorienteddataset.cpp b/gcore/gdalorienteddataset.cpp index 11e49b4ebcd7..15e00d6941b0 100644 --- a/gcore/gdalorienteddataset.cpp +++ b/gcore/gdalorienteddataset.cpp @@ -149,8 +149,7 @@ CPLErr GDALOrientedRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, } else { - osTmpName = - CPLSPrintf("/vsimem/_gdalorienteddataset/%p.tif", this); + osTmpName = VSIMemGenerateHiddenFilename(nullptr); } } GDALTranslateOptions *psOptions = diff --git a/gcore/gdalpython.cpp b/gcore/gdalpython.cpp index af964c91400d..4f944c52e3ae 100644 --- a/gcore/gdalpython.cpp +++ b/gcore/gdalpython.cpp @@ -378,7 +378,7 @@ static bool LoadPythonAPI() "-c", pszPrintVersion, nullptr}; const CPLString osTmpFilename( - "/vsimem/LoadPythonAPI/out.txt"); + VSIMemGenerateHiddenFilename("out.txt")); VSILFILE *fout = VSIFOpenL(osTmpFilename, "wb+"); if (CPLSpawn(apszArgv, nullptr, fout, FALSE) == 0) { From 5e0b6f9fe41f598cd31aedf11f574dec1a6a2a66 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:34:01 +0200 Subject: [PATCH 052/710] VSIMemGenerateHiddenFilename(): use in WCS driver --- frmts/wcs/wcsdataset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frmts/wcs/wcsdataset.cpp b/frmts/wcs/wcsdataset.cpp index 009b035fcfd5..c37e62374a51 100644 --- a/frmts/wcs/wcsdataset.cpp +++ b/frmts/wcs/wcsdataset.cpp @@ -720,7 +720,7 @@ GDALDataset *WCSDataset::GDALOpenResult(CPLHTTPResult *psResult) #endif // Eventually we should be looking at mime info and stuff to figure // out an optimal filename, but for now we just use a fixed one. - osResultFilename = CPLString().Printf("/vsimem/wcs/%p/wcsresult.dat", this); + osResultFilename = VSIMemGenerateHiddenFilename("wcsresult.dat"); VSILFILE *fp = VSIFileFromMemBuffer(osResultFilename.c_str(), pabyData, nDataLen, FALSE); From 045bc82aea64c2236c0a973ccaff95471103db43 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:34:15 +0200 Subject: [PATCH 053/710] VSIMemGenerateHiddenFilename(): use in MRF driver --- frmts/mrf/PNG_band.cpp | 2 +- frmts/mrf/Tif_band.cpp | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/frmts/mrf/PNG_band.cpp b/frmts/mrf/PNG_band.cpp index b868e6324232..b0582f346a95 100644 --- a/frmts/mrf/PNG_band.cpp +++ b/frmts/mrf/PNG_band.cpp @@ -153,7 +153,7 @@ CPLErr PNG_Codec::DecompressPNG(buf_mgr &dst, buf_mgr &src) { // Use the PNG driver for decompression of 8-bit images, as it // has optimizations for whole image decompression. - CPLString osTmpFilename(CPLSPrintf("/vsimem/mrf/%p.png", &dst)); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("mrf.png")); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), reinterpret_cast(src_ori.buffer), src_ori.size, false)); diff --git a/frmts/mrf/Tif_band.cpp b/frmts/mrf/Tif_band.cpp index 1d69ecabad9b..04a22bc15920 100644 --- a/frmts/mrf/Tif_band.cpp +++ b/frmts/mrf/Tif_band.cpp @@ -51,9 +51,7 @@ #include "marfa.h" NAMESPACE_MRF_START -// Returns a string in /vsimem/ + prefix + count that doesn't exist when this -// function gets called It is not thread safe, open the result as soon as -// possible +// Returns a unique filename static CPLString uniq_memfname(const char *prefix) { // Define MRF_LOCAL_TMP to use local files instead of RAM @@ -61,14 +59,7 @@ static CPLString uniq_memfname(const char *prefix) #if defined(MRF_LOCAL_TMP) return CPLGenerateTempFilename(prefix); #else - CPLString fname; - VSIStatBufL statb; - static unsigned int cnt = 0; - do - { - fname.Printf("/vsimem/%s_%08x", prefix, cnt++); - } while (!VSIStatL(fname, &statb)); - return fname; + return VSIMemGenerateHiddenFilename(prefix); #endif } From f6e473f72ad94148478b87a32915bd525ab2d245 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:34:31 +0200 Subject: [PATCH 054/710] VSIMemGenerateHiddenFilename(): use in JPIPKAK driver --- frmts/jpipkak/jpipkakdataset.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frmts/jpipkak/jpipkakdataset.cpp b/frmts/jpipkak/jpipkakdataset.cpp index 9057a3f7e198..f8195d9d19c0 100644 --- a/frmts/jpipkak/jpipkakdataset.cpp +++ b/frmts/jpipkak/jpipkakdataset.cpp @@ -749,8 +749,7 @@ int JPIPKAKDataset::Initialize(const char *pszDatasetName, int bReinitializing) } // create in memory file using vsimem - CPLString osFileBoxName; - osFileBoxName.Printf("/vsimem/jpip/%s.dat", pszCid); + const CPLString osFileBoxName(VSIMemGenerateHiddenFilename("jpip")); VSILFILE *fpLL = VSIFOpenL(osFileBoxName.c_str(), "w+"); poCache->set_read_scope(KDU_META_DATABIN, nCodestream, 0); kdu_byte *pabyBuffer = (kdu_byte *)CPLMalloc(nLen); From 3296b1a8d935592d20f254550026a70d3be33530 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:35:07 +0200 Subject: [PATCH 055/710] VSIMemGenerateHiddenFilename(): use in GTiff driver --- frmts/gtiff/geotiff.cpp | 4 ++-- frmts/gtiff/gt_jpeg_copy.cpp | 25 ++++++++++++------------- frmts/gtiff/gt_wkt_srs.cpp | 29 +++++++++++++---------------- frmts/gtiff/gtiffdataset_read.cpp | 9 ++++----- frmts/gtiff/gtiffdataset_write.cpp | 14 ++++++-------- frmts/gtiff/gtiffjpegoverviewds.cpp | 4 ++-- 6 files changed, 39 insertions(+), 46 deletions(-) diff --git a/frmts/gtiff/geotiff.cpp b/frmts/gtiff/geotiff.cpp index a0d763f23dff..77b6c0ee6b88 100644 --- a/frmts/gtiff/geotiff.cpp +++ b/frmts/gtiff/geotiff.cpp @@ -701,8 +701,8 @@ void GTiffWriteJPEGTables(TIFF *hTIFF, const char *pszPhotometric, if (!TIFFGetField(hTIFF, TIFFTAG_BITSPERSAMPLE, &(l_nBitsPerSample))) l_nBitsPerSample = 1; - CPLString osTmpFilenameIn; - osTmpFilenameIn.Printf("/vsimem/gtiffdataset_jpg_tmp_%p", hTIFF); + const CPLString osTmpFilenameIn( + VSIMemGenerateHiddenFilename("gtiffdataset_jpg_tmp")); VSILFILE *fpTmp = nullptr; CPLString osTmp; char **papszLocalParameters = nullptr; diff --git a/frmts/gtiff/gt_jpeg_copy.cpp b/frmts/gtiff/gt_jpeg_copy.cpp index 73262092f210..2a647ff190fb 100644 --- a/frmts/gtiff/gt_jpeg_copy.cpp +++ b/frmts/gtiff/gt_jpeg_copy.cpp @@ -377,13 +377,10 @@ static void GTIFF_ErrorExitJPEG(j_common_ptr cinfo) /************************************************************************/ static void GTIFF_Set_TIFFTAG_JPEGTABLES(TIFF *hTIFF, - jpeg_decompress_struct &sDInfo, jpeg_compress_struct &sCInfo) { - char szTmpFilename[128] = {'\0'}; - snprintf(szTmpFilename, sizeof(szTmpFilename), "/vsimem/tables_%p", - &sDInfo); - VSILFILE *fpTABLES = VSIFOpenL(szTmpFilename, "wb+"); + const std::string osTmpFilename(VSIMemGenerateHiddenFilename("tables")); + VSILFILE *fpTABLES = VSIFOpenL(osTmpFilename.c_str(), "wb+"); uint16_t nPhotometric = 0; TIFFGetField(hTIFF, TIFFTAG_PHOTOMETRIC, &nPhotometric); @@ -409,11 +406,11 @@ static void GTIFF_Set_TIFFTAG_JPEGTABLES(TIFF *hTIFF, vsi_l_offset nSizeTables = 0; GByte *pabyJPEGTablesData = - VSIGetMemFileBuffer(szTmpFilename, &nSizeTables, FALSE); + VSIGetMemFileBuffer(osTmpFilename.c_str(), &nSizeTables, FALSE); TIFFSetField(hTIFF, TIFFTAG_JPEGTABLES, static_cast(nSizeTables), pabyJPEGTablesData); - VSIUnlink(szTmpFilename); + VSIUnlink(osTmpFilename.c_str()); } /************************************************************************/ @@ -476,7 +473,7 @@ CPLErr GTIFF_CopyFromJPEG_WriteAdditionalTags(TIFF *hTIFF, GDALDataset *poSrcDS) jpeg_CreateCompress(&sCInfo, JPEG_LIB_VERSION, sizeof(sCInfo)); bCallDestroyCompress = true; jpeg_copy_critical_parameters(&sDInfo, &sCInfo); - GTIFF_Set_TIFFTAG_JPEGTABLES(hTIFF, sDInfo, sCInfo); + GTIFF_Set_TIFFTAG_JPEGTABLES(hTIFF, sCInfo); bCallDestroyCompress = false; jpeg_abort_compress(&sCInfo); jpeg_destroy_compress(&sCInfo); @@ -578,8 +575,9 @@ typedef struct static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) { - CPLString osTmpFilename(CPLSPrintf("/vsimem/%p", psArgs->psDInfo)); - VSILFILE *fpMEM = VSIFOpenL(osTmpFilename, "wb+"); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("GTIFF_CopyBlockFromJPEG.tif")); + VSILFILE *fpMEM = VSIFOpenL(osTmpFilename.c_str(), "wb+"); /* -------------------------------------------------------------------- */ /* Initialization of the compressor */ @@ -588,7 +586,7 @@ static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) if (setjmp(setjmp_buffer)) { CPL_IGNORE_RET_VAL(VSIFCloseL(fpMEM)); - VSIUnlink(osTmpFilename); + VSIUnlink(osTmpFilename.c_str()); return CE_Failure; } @@ -775,7 +773,8 @@ static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) /* Write the JPEG content with libtiff raw API */ /* -------------------------------------------------------------------- */ vsi_l_offset nSize = 0; - GByte *pabyJPEGData = VSIGetMemFileBuffer(osTmpFilename, &nSize, FALSE); + GByte *pabyJPEGData = + VSIGetMemFileBuffer(osTmpFilename.c_str(), &nSize, FALSE); CPLErr eErr = CE_None; @@ -794,7 +793,7 @@ static CPLErr GTIFF_CopyBlockFromJPEG(GTIFF_CopyBlockFromJPEGArgs *psArgs) eErr = CE_Failure; } - VSIUnlink(osTmpFilename); + VSIUnlink(osTmpFilename.c_str()); return eErr; } diff --git a/frmts/gtiff/gt_wkt_srs.cpp b/frmts/gtiff/gt_wkt_srs.cpp index 919829f2d511..1bffb47d8f1d 100644 --- a/frmts/gtiff/gt_wkt_srs.cpp +++ b/frmts/gtiff/gt_wkt_srs.cpp @@ -42,7 +42,6 @@ #include "cpl_conv.h" #include "cpl_error.h" -#include "cpl_multiproc.h" #include "cpl_string.h" #include "cpl_vsi.h" #include "gt_citation.h" @@ -3513,10 +3512,8 @@ CPLErr GTIFWktFromMemBufEx(int nSize, unsigned char *pabyBuffer, char ***ppapszRPCMD) { - char szFilename[100] = {}; - - snprintf(szFilename, sizeof(szFilename), "/vsimem/wkt_from_mem_buf_%ld.tif", - static_cast(CPLGetPID())); + const std::string osFilename( + VSIMemGenerateHiddenFilename("wkt_from_mem_buf.tif")); /* -------------------------------------------------------------------- */ /* Initialization of libtiff and libgeotiff. */ @@ -3527,20 +3524,21 @@ CPLErr GTIFWktFromMemBufEx(int nSize, unsigned char *pabyBuffer, /* -------------------------------------------------------------------- */ /* Create a memory file from the buffer. */ /* -------------------------------------------------------------------- */ - VSILFILE *fp = VSIFileFromMemBuffer(szFilename, pabyBuffer, nSize, FALSE); + VSILFILE *fp = + VSIFileFromMemBuffer(osFilename.c_str(), pabyBuffer, nSize, FALSE); if (fp == nullptr) return CE_Failure; /* -------------------------------------------------------------------- */ /* Initialize access to the memory geotiff structure. */ /* -------------------------------------------------------------------- */ - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "rc", fp); + TIFF *hTIFF = VSI_TIFFOpen(osFilename.c_str(), "rc", fp); if (hTIFF == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, "TIFF/GeoTIFF structure is corrupt."); - VSIUnlink(szFilename); + VSIUnlink(osFilename.c_str()); CPL_IGNORE_RET_VAL(VSIFCloseL(fp)); return CE_Failure; } @@ -3678,7 +3676,7 @@ CPLErr GTIFWktFromMemBufEx(int nSize, unsigned char *pabyBuffer, XTIFFClose(hTIFF); CPL_IGNORE_RET_VAL(VSIFCloseL(fp)); - VSIUnlink(szFilename); + VSIUnlink(osFilename.c_str()); if (phSRS && *phSRS == nullptr) return CE_Failure; @@ -3709,10 +3707,8 @@ CPLErr GTIFMemBufFromSRS(OGRSpatialReferenceH hSRS, char **papszRPCMD) { - char szFilename[100] = {}; - - snprintf(szFilename, sizeof(szFilename), "/vsimem/wkt_from_mem_buf_%ld.tif", - static_cast(CPLGetPID())); + const std::string osFilename( + VSIMemGenerateHiddenFilename("wkt_from_mem_buf.tif")); /* -------------------------------------------------------------------- */ /* Initialization of libtiff and libgeotiff. */ @@ -3723,17 +3719,18 @@ CPLErr GTIFMemBufFromSRS(OGRSpatialReferenceH hSRS, /* -------------------------------------------------------------------- */ /* Initialize access to the memory geotiff structure. */ /* -------------------------------------------------------------------- */ - VSILFILE *fpL = VSIFOpenL(szFilename, "w"); + VSILFILE *fpL = VSIFOpenL(osFilename.c_str(), "w"); if (fpL == nullptr) return CE_Failure; - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "w", fpL); + TIFF *hTIFF = VSI_TIFFOpen(osFilename.c_str(), "w", fpL); if (hTIFF == nullptr) { CPLError(CE_Failure, CPLE_AppDefined, "TIFF/GeoTIFF structure is corrupt."); CPL_IGNORE_RET_VAL(VSIFCloseL(fpL)); + VSIUnlink(osFilename.c_str()); return CE_Failure; } @@ -3885,7 +3882,7 @@ CPLErr GTIFMemBufFromSRS(OGRSpatialReferenceH hSRS, /* -------------------------------------------------------------------- */ GUIntBig nBigLength = 0; - *ppabyBuffer = VSIGetMemFileBuffer(szFilename, &nBigLength, TRUE); + *ppabyBuffer = VSIGetMemFileBuffer(osFilename.c_str(), &nBigLength, TRUE); *pnSize = static_cast(nBigLength); return CE_None; diff --git a/frmts/gtiff/gtiffdataset_read.cpp b/frmts/gtiff/gtiffdataset_read.cpp index 2cdcdbeada92..7b8d3e24b374 100644 --- a/frmts/gtiff/gtiffdataset_read.cpp +++ b/frmts/gtiff/gtiffdataset_read.cpp @@ -704,8 +704,8 @@ static void CPL_STDCALL ThreadDecompressionFuncErrorHandler( { // Generate a dummy in-memory TIFF file that has all the needed tags // from the original file - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/decompress_%p.tif", psJob); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("decompress.tif")); VSILFILE *fpTmp = VSIFOpenL(osTmpFilename.c_str(), "wb+"); TIFF *hTIFFTmp = VSI_TIFFOpen(osTmpFilename.c_str(), @@ -3524,9 +3524,8 @@ static bool GTIFFExtendMemoryFile(const CPLString &osTmpFilename, static bool GTIFFMakeBufferedStream(GDALOpenInfo *poOpenInfo) { - CPLString osTmpFilename; - static int nCounter = 0; - osTmpFilename.Printf("/vsimem/stream_%d.tif", ++nCounter); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("GTIFFMakeBufferedStream.tif")); VSILFILE *fpTemp = VSIFOpenL(osTmpFilename, "wb+"); if (fpTemp == nullptr) return false; diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp index 576f78f19373..1ce0ea1c68d0 100644 --- a/frmts/gtiff/gtiffdataset_write.cpp +++ b/frmts/gtiff/gtiffdataset_write.cpp @@ -947,8 +947,8 @@ void GTiffDataset::InitCompressionThreads(bool bUpdateMode, i < static_cast(m_asCompressionJobs.size()); ++i) { m_asCompressionJobs[i].pszTmpFilename = - CPLStrdup(CPLSPrintf("/vsimem/gtiff/thread/job/%p", - &m_asCompressionJobs[i])); + CPLStrdup(VSIMemGenerateHiddenFilename( + CPLSPrintf("thread_job_%d.tif", i))); m_asCompressionJobs[i].nStripOrTile = -1; } @@ -1383,7 +1383,7 @@ bool GTiffDataset::SubmitCompressionJob(int nStripOrTile, GByte *pabyData, memset(&sJob, 0, sizeof(sJob)); SetupJob(sJob); sJob.pszTmpFilename = - CPLStrdup(CPLSPrintf("/vsimem/gtiff/%p", this)); + CPLStrdup(VSIMemGenerateHiddenFilename("temp.tif")); ThreadCompressionFunc(&sJob); @@ -5344,8 +5344,7 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize, } if (bStreaming) { - static int nCounter = 0; - l_osTmpFilename = CPLSPrintf("/vsimem/vsistdout_%d.tif", ++nCounter); + l_osTmpFilename = VSIMemGenerateHiddenFilename("vsistdout.tif"); pszFilename = l_osTmpFilename.c_str(); } @@ -6067,9 +6066,8 @@ int GTiffDataset::GuessJPEGQuality(bool &bOutHasQuantizationTable, papszLocalParameters = CSLSetNameValue(papszLocalParameters, "NBITS", "12"); - CPLString osTmpFilenameIn; - osTmpFilenameIn.Printf("/vsimem/gtiffdataset_guess_jpeg_quality_tmp_%p", - this); + const CPLString osTmpFilenameIn( + VSIMemGenerateHiddenFilename("gtiffdataset_guess_jpeg_quality_tmp")); int nRet = -1; for (int nQuality = 0; nQuality <= 100 && nRet < 0; ++nQuality) diff --git a/frmts/gtiff/gtiffjpegoverviewds.cpp b/frmts/gtiff/gtiffjpegoverviewds.cpp index cc3ca61e3db6..86e4205107e0 100644 --- a/frmts/gtiff/gtiffjpegoverviewds.cpp +++ b/frmts/gtiff/gtiffjpegoverviewds.cpp @@ -71,7 +71,7 @@ GTiffJPEGOverviewDS::GTiffJPEGOverviewDS(GTiffDataset *poParentDSIn, { ShareLockWithParentDataset(poParentDSIn); - m_osTmpFilenameJPEGTable.Printf("/vsimem/jpegtable_%p", this); + m_osTmpFilenameJPEGTable = VSIMemGenerateHiddenFilename("jpegtable"); const GByte abyAdobeAPP14RGB[] = {0xFF, 0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, 0x65, 0x00, 0x64, 0x00, @@ -221,7 +221,7 @@ CPLErr GTiffJPEGOverviewBand::IReadBlock(int nBlockXOff, int nBlockYOff, nByteCount -= 2; CPLString osFileToOpen; - m_poGDS->m_osTmpFilename.Printf("/vsimem/sparse_%p", m_poGDS); + m_poGDS->m_osTmpFilename = VSIMemGenerateHiddenFilename("sparse"); VSILFILE *fp = VSIFOpenL(m_poGDS->m_osTmpFilename, "wb+"); // If the size of the JPEG strip/tile is small enough, we will From d1711228a3a785225d70514a9f370c3d63f232ce Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:35:51 +0200 Subject: [PATCH 056/710] VSIMemGenerateHiddenFilename(): use in DAAS driver --- frmts/daas/daasdataset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frmts/daas/daasdataset.cpp b/frmts/daas/daasdataset.cpp index 0c68c9e31439..8672c941a699 100644 --- a/frmts/daas/daasdataset.cpp +++ b/frmts/daas/daasdataset.cpp @@ -2420,7 +2420,7 @@ CPLErr GDALDAASRasterBand::GetBlocks(int nBlockXOff, int nBlockYOff, } else { - CPLString osTmpMemFile = CPLSPrintf("/vsimem/daas_%p", this); + const CPLString osTmpMemFile = VSIMemGenerateHiddenFilename("daas"); VSIFCloseL(VSIFileFromMemBuffer( osTmpMemFile, psResult->pasMimePart[iDataPart].pabyData, psResult->pasMimePart[iDataPart].nDataLen, false)); From dc3536eab7ba4ff5e1cce7c02465b72cd8aebf39 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:36:14 +0200 Subject: [PATCH 057/710] VSIMemGenerateHiddenFilename(): use in HEIF driver --- frmts/heif/heifdataset.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frmts/heif/heifdataset.cpp b/frmts/heif/heifdataset.cpp index 8fff6737fe9a..86e00c177b9b 100644 --- a/frmts/heif/heifdataset.cpp +++ b/frmts/heif/heifdataset.cpp @@ -392,8 +392,8 @@ void GDALHEIFDataset::ReadMetadata() } } - CPLString osTempFile; - osTempFile.Printf("/vsimem/heif_exif_%p.tif", this); + const CPLString osTempFile( + VSIMemGenerateHiddenFilename("heif_exif.tif")); VSILFILE *fpTemp = VSIFileFromMemBuffer(osTempFile, &data[nTIFFFileOffset], nCount - nTIFFFileOffset, FALSE); From 4de646803e74485d233ca96b4f3f6bfc22eaffc0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:38:41 +0200 Subject: [PATCH 058/710] VSIMemGenerateHiddenFilename(): use in GeoRaster driver --- frmts/georaster/georaster_wrapper.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frmts/georaster/georaster_wrapper.cpp b/frmts/georaster/georaster_wrapper.cpp index cdb9dbd8c9cb..7114592c1131 100644 --- a/frmts/georaster/georaster_wrapper.cpp +++ b/frmts/georaster/georaster_wrapper.cpp @@ -4236,13 +4236,13 @@ void GeoRasterWrapper::UncompressJpeg(unsigned long nInSize) // Load JPEG in a virtual file // -------------------------------------------------------------------- - const char *pszMemFile = CPLSPrintf("/vsimem/geor_%p.jpg", pabyBlockBuf); + const CPLString osMemFile = VSIMemGenerateHiddenFilename("geor.jpg"); - VSILFILE *fpImage = VSIFOpenL(pszMemFile, "wb"); + VSILFILE *fpImage = VSIFOpenL(osMemFile, "wb"); VSIFWriteL(pabyBlockBuf, nInSize, 1, fpImage); VSIFCloseL(fpImage); - fpImage = VSIFOpenL(pszMemFile, "rb"); + fpImage = VSIFOpenL(osMemFile, "rb"); // -------------------------------------------------------------------- // Initialize decompressor @@ -4299,7 +4299,7 @@ void GeoRasterWrapper::UncompressJpeg(unsigned long nInSize) VSIFCloseL(fpImage); - VSIUnlink(pszMemFile); + VSIUnlink(osMemFile); } // --------------------------------------------------------------------------- @@ -4312,9 +4312,9 @@ unsigned long GeoRasterWrapper::CompressJpeg(void) // Load JPEG in a virtual file // -------------------------------------------------------------------- - const char *pszMemFile = CPLSPrintf("/vsimem/geor_%p.jpg", pabyBlockBuf); + const CPLString osMemFile = VSIMemGenerateHiddenFilename("geor.jpg"); - VSILFILE *fpImage = VSIFOpenL(pszMemFile, "wb"); + VSILFILE *fpImage = VSIFOpenL(osMemFile, "wb"); bool write_all_tables = TRUE; @@ -4389,11 +4389,11 @@ unsigned long GeoRasterWrapper::CompressJpeg(void) VSIFCloseL(fpImage); - fpImage = VSIFOpenL(pszMemFile, "rb"); + fpImage = VSIFOpenL(osMemFile, "rb"); size_t nSize = VSIFReadL(pabyCompressBuf, 1, nBlockBytes, fpImage); VSIFCloseL(fpImage); - VSIUnlink(pszMemFile); + VSIUnlink(osMemFile); return (unsigned long)nSize; } From e294fb3a87f25525fc38e71400bf5291c03288c6 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:39:37 +0200 Subject: [PATCH 059/710] VSIMemGenerateHiddenFilename(): use in CALS driver --- frmts/cals/calsdataset.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frmts/cals/calsdataset.cpp b/frmts/cals/calsdataset.cpp index f8267075379f..389308524764 100644 --- a/frmts/cals/calsdataset.cpp +++ b/frmts/cals/calsdataset.cpp @@ -331,7 +331,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo) // Create a TIFF header for a single-strip CCITTFAX4 file. poDS->osTIFFHeaderFilename = - CPLSPrintf("/vsimem/cals/header_%p.tiff", poDS); + VSIMemGenerateHiddenFilename("cals_header.tiff"); VSILFILE *fp = VSIFOpenL(poDS->osTIFFHeaderFilename, "wb"); const int nTagCount = 10; const int nHeaderSize = 4 + 4 + 2 + nTagCount * 12 + 4; @@ -359,7 +359,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo) // Create a /vsisparse/ description file assembling the TIFF header // with the FAX4 codestream that starts at offset 2048 of the CALS file. - poDS->osSparseFilename = CPLSPrintf("/vsimem/cals/sparse_%p.xml", poDS); + poDS->osSparseFilename = VSIMemGenerateHiddenFilename("cals_sparse.xml"); fp = VSIFOpenL(poDS->osSparseFilename, "wb"); CPLAssert(fp); VSIFPrintfL(fp, @@ -473,7 +473,8 @@ GDALDataset *CALSDataset::CreateCopy(const char *pszFilename, // Write a in-memory TIFF with just the TIFF header to figure out // how large it will be. - CPLString osTmpFilename(CPLSPrintf("/vsimem/cals/tmp_%p", poSrcDS)); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("tmp_tif_header")); char **papszOptions = nullptr; papszOptions = CSLSetNameValue(papszOptions, "COMPRESS", "CCITTFAX4"); papszOptions = CSLSetNameValue(papszOptions, "NBITS", "1"); From b230384049755dc2bf611f1b9c166be04a5c8b4a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 16:42:34 +0200 Subject: [PATCH 060/710] VSIMemGenerateHiddenFilename(): use in KMLSuperOverlay driver --- frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp b/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp index 62ba490713b8..4b06cb443f31 100644 --- a/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp +++ b/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp @@ -31,7 +31,6 @@ #include "kmlsuperoverlaydataset.h" #include -#include #include #include #include @@ -1799,9 +1798,7 @@ KmlSuperOverlayLoadIcon(const char *pszBaseFilename, const char *pszIcon) return nullptr; } - static std::atomic nInc = 0; - osSubFilename = - CPLSPrintf("/vsimem/kmlsuperoverlay_%d_%p", nInc++, pszBaseFilename); + osSubFilename = VSIMemGenerateHiddenFilename("kmlsuperoverlay"); VSIFCloseL(VSIFileFromMemBuffer(osSubFilename, pabyBuffer, nRead, TRUE)); auto poDSIcon = std::unique_ptr(GDALDataset::Open( From b8884e4a6b16aa996dc536f323409699bec782fd Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:04:25 +0200 Subject: [PATCH 061/710] VSIMemGenerateHiddenFilename(): use in PDF driver --- frmts/pdf/pdfcreatecopy.cpp | 13 +++++++------ frmts/pdf/pdfdataset.cpp | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/frmts/pdf/pdfcreatecopy.cpp b/frmts/pdf/pdfcreatecopy.cpp index 462c2b158bc6..8c72aebf4501 100644 --- a/frmts/pdf/pdfcreatecopy.cpp +++ b/frmts/pdf/pdfcreatecopy.cpp @@ -4311,7 +4311,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( eCompressMethod == COMPRESS_JPEG2000) { GDALDriver *poJPEGDriver = nullptr; - char szTmp[64]; + std::string osTmpfilename; char **papszOptions = nullptr; bool bEcwEncodeKeyRequiredButNotFound = false; @@ -4321,7 +4321,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( if (poJPEGDriver != nullptr && nJPEGQuality > 0) papszOptions = CSLAddString( papszOptions, CPLSPrintf("QUALITY=%d", nJPEGQuality)); - snprintf(szTmp, sizeof(szTmp), "/vsimem/pdftemp/%p.jpg", this); + osTmpfilename = VSIMemGenerateHiddenFilename("pdf_temp.jpg"); } else { @@ -4374,7 +4374,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( papszOptions = CSLAddString(papszOptions, "GMLJP2=OFF"); } } - snprintf(szTmp, sizeof(szTmp), "/vsimem/pdftemp/%p.jp2", this); + osTmpfilename = VSIMemGenerateHiddenFilename("pdf_temp.jp2"); } if (poJPEGDriver == nullptr) @@ -4396,8 +4396,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( } GDALDataset *poJPEGDS = - poJPEGDriver->CreateCopy(szTmp, poBlockSrcDS, FALSE, papszOptions, - pfnProgress, pProgressData); + poJPEGDriver->CreateCopy(osTmpfilename.c_str(), poBlockSrcDS, FALSE, + papszOptions, pfnProgress, pProgressData); CSLDestroy(papszOptions); if (poJPEGDS == nullptr) @@ -4409,7 +4409,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( GDALClose(poJPEGDS); vsi_l_offset nJPEGDataSize = 0; - GByte *pabyJPEGData = VSIGetMemFileBuffer(szTmp, &nJPEGDataSize, TRUE); + GByte *pabyJPEGData = + VSIGetMemFileBuffer(osTmpfilename.c_str(), &nJPEGDataSize, TRUE); VSIFWriteL(pabyJPEGData, static_cast(nJPEGDataSize), 1, m_fp); CPLFree(pabyJPEGData); } diff --git a/frmts/pdf/pdfdataset.cpp b/frmts/pdf/pdfdataset.cpp index cab477ab4da9..845057fd68a9 100644 --- a/frmts/pdf/pdfdataset.cpp +++ b/frmts/pdf/pdfdataset.cpp @@ -2120,7 +2120,7 @@ CPLErr PDFDataset::ReadPixels(int nReqXOff, int nReqYOff, int nReqXSize, } papszArgs = CSLAddString(papszArgs, m_osFilename.c_str()); - osTmpFilename = CPLSPrintf("/vsimem/pdf/temp_%p.ppm", this); + osTmpFilename = VSIMemGenerateHiddenFilename("pdf_temp.ppm"); VSILFILE *fpOut = VSIFOpenL(osTmpFilename, "wb"); if (fpOut != nullptr) { From 259418a1f15de89959e8ce4958f4f4c15837dadd Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:06:42 +0200 Subject: [PATCH 062/710] JPEGX: use anonymous /vsimem/ file --- frmts/jpegxl/jpegxl.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frmts/jpegxl/jpegxl.cpp b/frmts/jpegxl/jpegxl.cpp index 17b6ba51dc6a..826315d228ba 100644 --- a/frmts/jpegxl/jpegxl.cpp +++ b/frmts/jpegxl/jpegxl.cpp @@ -358,11 +358,9 @@ bool JPEGXLDataset::Open(GDALOpenInfo *poOpenInfo) { CPL_LSBPTR32(&nTiffDirStart); } - const std::string osTmpFilename = - CPLSPrintf("/vsimem/jxl/%p", this); - VSILFILE *fpEXIF = VSIFileFromMemBuffer( - osTmpFilename.c_str(), abyBoxBuffer.data() + 4, - abyBoxBuffer.size() - 4, false); + VSILFILE *fpEXIF = + VSIFileFromMemBuffer(nullptr, abyBoxBuffer.data() + 4, + abyBoxBuffer.size() - 4, false); int nExifOffset = 0; int nInterOffset = 0; int nGPSOffset = 0; From 2d1588effe70591180cc710b1961e5ced5ca4a02 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:07:23 +0200 Subject: [PATCH 063/710] VSIMemGenerateHiddenFilename(): use in ESRIC driver --- frmts/esric/esric_dataset.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frmts/esric/esric_dataset.cpp b/frmts/esric/esric_dataset.cpp index 7ff5fdd585f4..2d5768762b59 100644 --- a/frmts/esric/esric_dataset.cpp +++ b/frmts/esric/esric_dataset.cpp @@ -896,9 +896,7 @@ CPLErr ECBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pData) GUInt64(size), GUInt64(offset)); return CE_Failure; } - CPLString magic; - // Should use some sort of unique - magic.Printf("/vsimem/esric_%p.tmp", this); + const CPLString magic(VSIMemGenerateHiddenFilename("esric.tmp")); auto mfh = VSIFileFromMemBuffer(magic.c_str(), fbuffer.data(), size, false); VSIFCloseL(mfh); // Can't open a raster by handle? From 75de26a4205e7000740a97535c02844d86b599b9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:12:27 +0200 Subject: [PATCH 064/710] VSIMemGenerateHiddenFilename(): use in JPEG driver --- frmts/jpeg/jpgdataset.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frmts/jpeg/jpgdataset.cpp b/frmts/jpeg/jpgdataset.cpp index 2d5f593efaea..fae0b2b47617 100644 --- a/frmts/jpeg/jpgdataset.cpp +++ b/frmts/jpeg/jpgdataset.cpp @@ -2859,7 +2859,8 @@ GDALDataset *JPGDatasetCommon::OpenFLIRRawThermalImage() GByte *pabyData = static_cast(CPLMalloc(m_abyRawThermalImage.size())); - const std::string osTmpFilename(CPLSPrintf("/vsimem/jpeg/%p", pabyData)); + const std::string osTmpFilename( + VSIMemGenerateHiddenFilename("jpeg_flir_raw")); memcpy(pabyData, m_abyRawThermalImage.data(), m_abyRawThermalImage.size()); VSILFILE *fpRaw = VSIFileFromMemBuffer(osTmpFilename.c_str(), pabyData, m_abyRawThermalImage.size(), true); @@ -4097,7 +4098,7 @@ void JPGAddEXIF(GDALDataType eWorkDT, GDALDataset *poSrcDS, char **papszOptions, return; } - CPLString osTmpFile(CPLSPrintf("/vsimem/ovrjpg%p", poMemDS)); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("ovrjpg")); GDALDataset *poOutDS = pCreateCopy(osTmpFile, poMemDS, 0, nullptr, GDALDummyProgress, nullptr); const bool bExifOverviewSuccess = poOutDS != nullptr; From 5fd2bb43dc8bcc9a8a421810872bde5f64cd57ed Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:15:58 +0200 Subject: [PATCH 065/710] VSIMemGenerateHiddenFilename(): use in STACTA driver --- frmts/stacta/stactadataset.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frmts/stacta/stactadataset.cpp b/frmts/stacta/stactadataset.cpp index 55c4cdfc47e9..0603efb561a3 100644 --- a/frmts/stacta/stactadataset.cpp +++ b/frmts/stacta/stactadataset.cpp @@ -489,8 +489,13 @@ CPLErr STACTARawDataset::IRasterIO( return CE_Failure; } VSIFCloseL(fp); - const CPLString osMEMFilename("/vsimem/stacta/" + - osURL); + const CPLString osMEMFilename( + VSIMemGenerateHiddenFilename( + std::string("stacta_") + .append(CPLString(osURL) + .replaceAll("/", "_") + .replaceAll("\\", "_")) + .c_str())); VSIFCloseL(VSIFileFromMemBuffer(osMEMFilename, pabyBuf, nSize, TRUE)); poTileDS = std::unique_ptr( From b4d43a8dc5edc6aa684101fe47fddd10ba6d49c9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:16:48 +0200 Subject: [PATCH 066/710] VSIMemGenerateHiddenFilename(): use in EEDA driver --- frmts/eeda/eedaidataset.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frmts/eeda/eedaidataset.cpp b/frmts/eeda/eedaidataset.cpp index 9a6c41be78b8..e6df9d0d2a8a 100644 --- a/frmts/eeda/eedaidataset.cpp +++ b/frmts/eeda/eedaidataset.cpp @@ -433,7 +433,7 @@ bool GDALEEDAIRasterBand::DecodeGDALDataset(const GByte *pabyData, int nDataLen, { GDALEEDAIDataset *poGDS = reinterpret_cast(poDS); - CPLString osTmpFilename(CPLSPrintf("/vsimem/eeai/%p", this)); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("eedai")); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename, const_cast(pabyData), nDataLen, false)); const char *const apszDrivers[] = {"PNG", "JPEG", "GTIFF", nullptr}; From 76ca1d2675ab63dbecdb204c8f38a47fc7238aee Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:18:44 +0200 Subject: [PATCH 067/710] VSIMemGenerateHiddenFilename(): use in VICAR driver --- frmts/pds/vicardataset.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/frmts/pds/vicardataset.cpp b/frmts/pds/vicardataset.cpp index 26a51bdf7790..8f022e60f24b 100644 --- a/frmts/pds/vicardataset.cpp +++ b/frmts/pds/vicardataset.cpp @@ -1913,8 +1913,8 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) // Create a in-memory GeoTIFF file - char szFilename[100] = {}; - snprintf(szFilename, sizeof(szFilename), "/vsimem/vicar_tmp_%p.tif", this); + const std::string osTmpFilename( + VSIMemGenerateHiddenFilename("vicar_tmp.tif")); GDALDriver *poGTiffDriver = GDALDriver::FromHandle(GDALGetDriverByName("GTiff")); if (poGTiffDriver == nullptr) @@ -1923,8 +1923,8 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) return; } const char *const apszOptions[] = {"GEOTIFF_VERSION=1.0", nullptr}; - auto poDS = std::unique_ptr( - poGTiffDriver->Create(szFilename, 1, 1, 1, GDT_Byte, apszOptions)); + auto poDS = std::unique_ptr(poGTiffDriver->Create( + osTmpFilename.c_str(), 1, 1, 1, GDT_Byte, apszOptions)); if (!poDS) return; poDS->SetSpatialRef(&m_oSRS); @@ -1935,14 +1935,14 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) poDS.reset(); // Open it with libtiff/libgeotiff - VSILFILE *fpL = VSIFOpenL(szFilename, "r"); + VSILFILE *fpL = VSIFOpenL(osTmpFilename.c_str(), "r"); if (fpL == nullptr) { - VSIUnlink(szFilename); + VSIUnlink(osTmpFilename.c_str()); return; } - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "r", fpL); + TIFF *hTIFF = VSI_TIFFOpen(osTmpFilename.c_str(), "r", fpL); CPLAssert(hTIFF); GTIF *hGTIF = GTIFNew(hTIFF); @@ -2008,7 +2008,7 @@ void VICARDataset::BuildLabelPropertyGeoTIFF(CPLJSONObject &oLabel) XTIFFClose(hTIFF); CPL_IGNORE_RET_VAL(VSIFCloseL(fpL)); - VSIUnlink(szFilename); + VSIUnlink(osTmpFilename.c_str()); } /************************************************************************/ @@ -2320,8 +2320,8 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() // We will build a in-memory temporary GeoTIFF file from the VICAR GEOTIFF // metadata items. - char szFilename[100] = {}; - snprintf(szFilename, sizeof(szFilename), "/vsimem/vicar_tmp_%p.tif", this); + const std::string osTmpFilename( + VSIMemGenerateHiddenFilename("vicar_tmp.tif")); /* -------------------------------------------------------------------- */ /* Initialization of libtiff and libgeotiff. */ @@ -2332,11 +2332,11 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() /* -------------------------------------------------------------------- */ /* Initialize access to the memory geotiff structure. */ /* -------------------------------------------------------------------- */ - VSILFILE *fpL = VSIFOpenL(szFilename, "w"); + VSILFILE *fpL = VSIFOpenL(osTmpFilename.c_str(), "w"); if (fpL == nullptr) return; - TIFF *hTIFF = VSI_TIFFOpen(szFilename, "w", fpL); + TIFF *hTIFF = VSI_TIFFOpen(osTmpFilename.c_str(), "w", fpL); if (hTIFF == nullptr) { @@ -2451,7 +2451,7 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() /* Get georeferencing from file. */ /* -------------------------------------------------------------------- */ auto poGTiffDS = - std::unique_ptr(GDALDataset::Open(szFilename)); + std::unique_ptr(GDALDataset::Open(osTmpFilename.c_str())); if (poGTiffDS) { auto poSRS = poGTiffDS->GetSpatialRef(); @@ -2469,7 +2469,7 @@ void VICARDataset::ReadProjectionFromGeoTIFFGroup() GDALDataset::SetMetadataItem(GDALMD_AREA_OR_POINT, pszAreaOrPoint); } - VSIUnlink(szFilename); + VSIUnlink(osTmpFilename.c_str()); } /************************************************************************/ From ada9b7f1c27c871bb3c77bb9c6200dc07da02a79 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:21:18 +0200 Subject: [PATCH 068/710] VSIMemGenerateHiddenFilename(): use in ISIS3 driver --- frmts/pds/isis3dataset.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frmts/pds/isis3dataset.cpp b/frmts/pds/isis3dataset.cpp index bacaab84372f..41aa9c2c6808 100644 --- a/frmts/pds/isis3dataset.cpp +++ b/frmts/pds/isis3dataset.cpp @@ -3618,8 +3618,7 @@ void ISIS3Dataset::WriteLabel() CPLString ISIS3Dataset::SerializeAsPDL(const CPLJSONObject &oObj) { - CPLString osTmpFile( - CPLSPrintf("/vsimem/isis3_%p", oObj.GetInternalHandle())); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("isis3_pdl")); VSILFILE *fpTmp = VSIFOpenL(osTmpFile, "wb+"); SerializeAsPDL(fpTmp, oObj); VSIFCloseL(fpTmp); From 137fb59ad5612ba45693d9a89f4a50296bfad559 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:23:44 +0200 Subject: [PATCH 069/710] VSIMemGenerateHiddenFilename(): use in RasterLite driver --- frmts/rasterlite/rasterlitecreatecopy.cpp | 4 ++-- frmts/rasterlite/rasterlitedataset.cpp | 8 ++++---- frmts/rasterlite/rasterliteoverviews.cpp | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frmts/rasterlite/rasterlitecreatecopy.cpp b/frmts/rasterlite/rasterlitecreatecopy.cpp index fba77104823c..9de3a39f5e0d 100644 --- a/frmts/rasterlite/rasterlitecreatecopy.cpp +++ b/frmts/rasterlite/rasterlitecreatecopy.cpp @@ -581,8 +581,8 @@ GDALDataset *RasterliteCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, return nullptr; } - CPLString osTempFileName; - osTempFileName.Printf("/vsimem/%p", hDS); + const CPLString osTempFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); int nTileId = 0; int nBlocks = 0; diff --git a/frmts/rasterlite/rasterlitedataset.cpp b/frmts/rasterlite/rasterlitedataset.cpp index a85cf6fafa4a..8f12d77d64fd 100644 --- a/frmts/rasterlite/rasterlitedataset.cpp +++ b/frmts/rasterlite/rasterlitedataset.cpp @@ -156,8 +156,8 @@ CPLErr RasterliteBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) return CE_None; } - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); #ifdef RASTERLITE_DEBUG if (nBand == 1) @@ -912,8 +912,8 @@ int RasterliteDataset::GetBlockParams(OGRLayerH hRasterLyr, int nLevelIn, return FALSE; } - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); VSILFILE *fp = VSIFileFromMemBuffer(osMemFileName.c_str(), pabyData, nDataSize, FALSE); VSIFCloseL(fp); diff --git a/frmts/rasterlite/rasterliteoverviews.cpp b/frmts/rasterlite/rasterliteoverviews.cpp index bc6599d9fb0c..2a0c9cc3c8f1 100644 --- a/frmts/rasterlite/rasterliteoverviews.cpp +++ b/frmts/rasterlite/rasterliteoverviews.cpp @@ -350,8 +350,8 @@ CPLErr RasterliteDataset::CreateOverviewLevel(const char *pszResampling, return CE_Failure; } - CPLString osTempFileName; - osTempFileName.Printf("/vsimem/%p", hDS); + const CPLString osTempFileName( + VSIMemGenerateHiddenFilename("rasterlite_tile")); int nTileId = 0; int nBlocks = 0; From f54ede49d0e0adfa13f6893ac7866ae415ba7ba0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:26:41 +0200 Subject: [PATCH 070/710] VSIMemGenerateHiddenFilename(): use in ECW driver --- frmts/ecw/ecwdataset.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frmts/ecw/ecwdataset.cpp b/frmts/ecw/ecwdataset.cpp index dd7bde5a9b1a..6c94add614c1 100644 --- a/frmts/ecw/ecwdataset.cpp +++ b/frmts/ecw/ecwdataset.cpp @@ -2850,10 +2850,11 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000) /* There are issues at least in the 5.x series. */ /* -------------------------------------------------------------------- */ #if ECWSDK_VERSION >= 40 + constexpr const char *szDETECT_BUG_FILENAME = + "__detect_ecw_uint32_bug__.j2k"; if (bIsJPEG2000 && poDS->eNCSRequestDataType == NCSCT_UINT32 && CPLTestBool(CPLGetConfigOption("ECW_CHECK_CORRECT_DECODING", "TRUE")) && - !STARTS_WITH_CI(poOpenInfo->pszFilename, - "/vsimem/detect_ecw_uint32_bug")) + strstr(poOpenInfo->pszFilename, szDETECT_BUG_FILENAME) == nullptr) { static bool bUINT32_Ok = false; { @@ -2878,7 +2879,7 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000) 0xDF, 0xFF, 0x7F, 0x5F, 0xFF, 0xD9}; const std::string osTmpFilename = - CPLSPrintf("/vsimem/detect_ecw_uint32_bug_%p.j2k", poDS); + VSIMemGenerateHiddenFilename(szDETECT_BUG_FILENAME); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), const_cast(abyTestUInt32ImageData), From aa0ba75ecd8d0e4e3eff982607e60117c0a9c354 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:30:21 +0200 Subject: [PATCH 071/710] VSIMemGenerateHiddenFilename(): use in GRIB driver --- frmts/grib/degrib/g2clib/dec_jpeg2000.cpp | 4 ++-- frmts/grib/gribcreatecopy.cpp | 4 ++-- frmts/grib/gribdataset.cpp | 7 +------ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp b/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp index a22109e823f8..cf0961c637e8 100644 --- a/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp +++ b/frmts/grib/degrib/g2clib/dec_jpeg2000.cpp @@ -52,8 +52,8 @@ int dec_jpeg2000(const void *injpc,g2int bufsize,g2int **outfld,g2int outpixels) { // create "memory file" from buffer - CPLString osFileName; - osFileName.Printf( "/vsimem/work_grib_%p.jpc", injpc ); + const CPLString osFileName( + VSIMemGenerateHiddenFilename("temp_grib.jpc")); VSIFCloseL( VSIFileFromMemBuffer( osFileName, (unsigned char*)injpc, bufsize, diff --git a/frmts/grib/gribcreatecopy.cpp b/frmts/grib/gribcreatecopy.cpp index 6a886d410656..6a9637bca749 100644 --- a/frmts/grib/gribcreatecopy.cpp +++ b/frmts/grib/gribcreatecopy.cpp @@ -1664,7 +1664,7 @@ bool GRIB2Section567Writer::WritePNG() GDALDataset *poMEMDS = WrapArrayAsMemDataset(m_nXSize, m_nYSize, eReducedDT, panData); - CPLString osTmpFile(CPLSPrintf("/vsimem/grib_driver_%p.png", m_poSrcDS)); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("grib_driver.png")); GDALDataset *poPNGDS = poPNGDriver->CreateCopy( osTmpFile, poMEMDS, FALSE, aosPNGOptions.List(), nullptr, nullptr); if (poPNGDS == nullptr) @@ -1850,7 +1850,7 @@ bool GRIB2Section567Writer::WriteJPEG2000(char **papszOptions) GDALDataset *poMEMDS = WrapArrayAsMemDataset(m_nXSize, m_nYSize, eReducedDT, panData); - CPLString osTmpFile(CPLSPrintf("/vsimem/grib_driver_%p.j2k", m_poSrcDS)); + const CPLString osTmpFile(VSIMemGenerateHiddenFilename("grib_driver.j2k")); GDALDataset *poJ2KDS = poJ2KDriver->CreateCopy( osTmpFile, poMEMDS, FALSE, aosJ2KOptions.List(), nullptr, nullptr); if (poJ2KDS == nullptr) diff --git a/frmts/grib/gribdataset.cpp b/frmts/grib/gribdataset.cpp index f098463b23c6..c7a42e496819 100644 --- a/frmts/grib/gribdataset.cpp +++ b/frmts/grib/gribdataset.cpp @@ -1489,10 +1489,7 @@ GDALDataset *GRIBDataset::Open(GDALOpenInfo *poOpenInfo) // for other thread safe formats CPLMutexHolderD(&hGRIBMutex); - CPLString tmpFilename; - tmpFilename.Printf("/vsimem/gribdataset-%p", poOpenInfo); - - VSILFILE *memfp = VSIFileFromMemBuffer(tmpFilename, poOpenInfo->pabyHeader, + VSILFILE *memfp = VSIFileFromMemBuffer(nullptr, poOpenInfo->pabyHeader, poOpenInfo->nHeaderBytes, FALSE); if (memfp == nullptr || ReadSECT0(memfp, &buff, &buffLen, -1, sect0, &gribLen, &version) < 0) @@ -1500,7 +1497,6 @@ GDALDataset *GRIBDataset::Open(GDALOpenInfo *poOpenInfo) if (memfp != nullptr) { VSIFCloseL(memfp); - VSIUnlink(tmpFilename); } free(buff); char *errMsg = errSprintf(nullptr); @@ -1510,7 +1506,6 @@ GDALDataset *GRIBDataset::Open(GDALOpenInfo *poOpenInfo) return nullptr; } VSIFCloseL(memfp); - VSIUnlink(tmpFilename); free(buff); // Confirm the requested access is supported. From e8d1b9ee79dc80572114bd899d46f66e03fd8871 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:31:19 +0200 Subject: [PATCH 072/710] VSIMemGenerateHiddenFilename(): use in OGCAPI driver --- frmts/ogcapi/gdalogcapidataset.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frmts/ogcapi/gdalogcapidataset.cpp b/frmts/ogcapi/gdalogcapidataset.cpp index 7157d1979104..69750ab7526c 100644 --- a/frmts/ogcapi/gdalogcapidataset.cpp +++ b/frmts/ogcapi/gdalogcapidataset.cpp @@ -638,8 +638,7 @@ OGCAPIDataset::OpenTile(const CPLString &osURLPattern, int nMatrix, int nColumn, if (bEmptyContent) return nullptr; - CPLString osTempFile; - osTempFile.Printf("/vsimem/ogcapi/%p", this); + const CPLString osTempFile(VSIMemGenerateHiddenFilename("ogcapi")); VSIFCloseL(VSIFileFromMemBuffer(osTempFile.c_str(), reinterpret_cast(&m_osTileData[0]), m_osTileData.size(), false)); From f0206699babc0737ca36ccbd9494b00941e06102 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:34:17 +0200 Subject: [PATCH 073/710] VSIMemGenerateHiddenFilename(): use in NITF driver --- frmts/nitf/nitfbilevel.cpp | 7 ++----- frmts/nitf/nitfdump.c | 5 +++-- frmts/nitf/nitfimage.c | 5 +++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/frmts/nitf/nitfbilevel.cpp b/frmts/nitf/nitfbilevel.cpp index 879dbad39582..3370c5eee608 100644 --- a/frmts/nitf/nitfbilevel.cpp +++ b/frmts/nitf/nitfbilevel.cpp @@ -33,7 +33,6 @@ #include #include "cpl_conv.h" -#include "cpl_multiproc.h" #include "cpl_string.h" #include "cpl_vsi.h" #include "gdal.h" @@ -57,10 +56,8 @@ int NITFUncompressBILEVEL(NITFImage *psImage, GByte *pabyInputData, const int nOutputBytes = (psImage->nBlockWidth * psImage->nBlockHeight + 7) / 8; - CPLString osFilename; - - osFilename.Printf("/vsimem/nitf-wrk-%ld.tif", (long)CPLGetPID()); - + const CPLString osFilename( + VSIMemGenerateHiddenFilename("nitf_bilevel.tif")); VSILFILE *fpL = VSIFOpenL(osFilename, "w+"); if (fpL == nullptr) return FALSE; diff --git a/frmts/nitf/nitfdump.c b/frmts/nitf/nitfdump.c index 6d8934490893..4dc24987d42e 100644 --- a/frmts/nitf/nitfdump.c +++ b/frmts/nitf/nitfdump.c @@ -629,10 +629,11 @@ int main(int nArgc, char **papszArgv) "CSSHPA DES")) { char szFilename[40]; - char szRadix[32]; + char szRadix[256]; if (bExtractSHPInMem) snprintf(szRadix, sizeof(szRadix), - "/vsimem/nitf_segment_%d", iSegment + 1); + VSIMemGenerateHiddenFilename( + CPLSPrintf("nitf_segment_%d", iSegment + 1))); else snprintf(szRadix, sizeof(szRadix), "nitf_segment_%d", iSegment + 1); diff --git a/frmts/nitf/nitfimage.c b/frmts/nitf/nitfimage.c index 185127fc48fd..090edc96b166 100644 --- a/frmts/nitf/nitfimage.c +++ b/frmts/nitf/nitfimage.c @@ -3847,7 +3847,7 @@ static void NITFLoadLocationTable(NITFImage *psImage) GUInt32 nHeaderOffset = 0; int i; int nTRESize; - char szTempFileName[32]; + char szTempFileName[256]; VSILFILE *fpTemp; pszTRE = @@ -3855,7 +3855,8 @@ static void NITFLoadLocationTable(NITFImage *psImage) if (pszTRE == NULL) return; - snprintf(szTempFileName, sizeof(szTempFileName), "/vsimem/%p", pszTRE); + snprintf(szTempFileName, sizeof(szTempFileName), "%s", + VSIMemGenerateHiddenFilename("nitf_tre")); fpTemp = VSIFileFromMemBuffer(szTempFileName, (GByte *)pszTRE, nTRESize, FALSE); psImage->pasLocations = From 05d3dc07c05791d204a0e28495073af12ec0fbe7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:36:30 +0200 Subject: [PATCH 074/710] VSIMemGenerateHiddenFilename(): use in PLMosaic driver --- frmts/plmosaic/plmosaicdataset.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frmts/plmosaic/plmosaicdataset.cpp b/frmts/plmosaic/plmosaicdataset.cpp index a8632e275d1a..fd759dd1a60c 100644 --- a/frmts/plmosaic/plmosaicdataset.cpp +++ b/frmts/plmosaic/plmosaicdataset.cpp @@ -1329,6 +1329,7 @@ GDALDataset *PLMosaicDataset::GetMetaTile(int tile_x, int tile_y) CreateMosaicCachePathIfNecessary(); + bool bUnlink = false; VSILFILE *fp = osCachePathRoot.size() ? VSIFOpenL(osTmpFilename, "wb") : nullptr; if (fp) @@ -1349,9 +1350,10 @@ GDALDataset *PLMosaicDataset::GetMetaTile(int tile_x, int tile_y) FlushDatasetsCache(); nCacheMaxSize = 1; } - osTmpFilename = - CPLSPrintf("/vsimem/single_tile_plmosaic_cache/%s/%d_%d.tif", - osMosaic.c_str(), tile_x, tile_y); + bUnlink = true; + osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("single_tile_plmosaic_cache_%s_%d_%d.tif", + osMosaic.c_str(), tile_x, tile_y)); fp = VSIFOpenL(osTmpFilename, "wb"); if (fp) { @@ -1362,7 +1364,7 @@ GDALDataset *PLMosaicDataset::GetMetaTile(int tile_x, int tile_y) CPLHTTPDestroyResult(psResult); GDALDataset *poDS = OpenAndInsertNewDataset(osTmpFilename, osTilename); - if (STARTS_WITH(osTmpFilename, "/vsimem/single_tile_plmosaic_cache/")) + if (bUnlink) VSIUnlink(osTilename); return poDS; From e9288d3708f4fcf8f4ab4a30dc2cc9398bc22ca2 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:37:15 +0200 Subject: [PATCH 075/710] VSIMemGenerateHiddenFilename(): use in RMF driver --- frmts/rmf/rmfjpeg.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/frmts/rmf/rmfjpeg.cpp b/frmts/rmf/rmfjpeg.cpp index 8c4b7fd78f73..b7dc56abbbcc 100644 --- a/frmts/rmf/rmfjpeg.cpp +++ b/frmts/rmf/rmfjpeg.cpp @@ -31,6 +31,7 @@ #include #include "cpl_conv.h" +#include "cpl_vsi.h" #include "rmfdataset.h" #include "../mem/memdataset.h" @@ -46,13 +47,10 @@ size_t RMFDataset::JPEGDecompress(const GByte *pabyIn, GUInt32 nSizeIn, nSizeIn < 2) return 0; - CPLString osTmpFilename; - VSILFILE *fp; + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("rmfjpeg.jpg")); - osTmpFilename.Printf("/vsimem/rmfjpeg/%p.jpg", pabyIn); - - fp = VSIFileFromMemBuffer(osTmpFilename, const_cast(pabyIn), - nSizeIn, FALSE); + VSILFILE *fp = VSIFileFromMemBuffer( + osTmpFilename, const_cast(pabyIn), nSizeIn, FALSE); if (fp == nullptr) { @@ -165,8 +163,7 @@ size_t RMFDataset::JPEGCompress(const GByte *pabyIn, GUInt32 nSizeIn, poMemDS->AddMEMBand(hBand); } - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/rmfjpeg/%p.jpg", pabyIn); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("rmfjpeg.jpg")); char szQuality[32] = {}; if (poDS != nullptr && poDS->sHeader.iJpegQuality > 0) From 5c0983313ccca48b66ff24f3cc4ee5ab2d15be2d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:40:51 +0200 Subject: [PATCH 076/710] VSIMemGenerateHiddenFilename(): use in MBTiles driver --- frmts/mbtiles/mbtilesdataset.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/frmts/mbtiles/mbtilesdataset.cpp b/frmts/mbtiles/mbtilesdataset.cpp index 1d3549ca0d53..6ef43a125b20 100644 --- a/frmts/mbtiles/mbtilesdataset.cpp +++ b/frmts/mbtiles/mbtilesdataset.cpp @@ -1710,8 +1710,8 @@ GIntBig MBTilesVectorLayer::GetFeatureCount(int bForce) { VSIUnlink(m_osTmpFilename); } - m_osTmpFilename = - CPLSPrintf("/vsimem/mvt_%p_%d_%d.pbf", this, m_nX, m_nY); + m_osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("mvt_%d_%d.pbf", m_nX, m_nY)); VSIFCloseL(VSIFileFromMemBuffer(m_osTmpFilename, pabyDataDup, nDataSize, true)); @@ -1794,8 +1794,8 @@ OGRFeature *MBTilesVectorLayer::GetNextSrcFeature() { VSIUnlink(m_osTmpFilename); } - m_osTmpFilename = - CPLSPrintf("/vsimem/mvt_%p_%d_%d.pbf", this, m_nX, m_nY); + m_osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("mvt_%d_%d.pbf", m_nX, m_nY)); VSIFCloseL(VSIFileFromMemBuffer(m_osTmpFilename, pabyDataDup, nDataSize, true)); @@ -1903,8 +1903,8 @@ OGRFeature *MBTilesVectorLayer::GetFeature(GIntBig nFID) OGR_F_Destroy(hFeat); OGR_DS_ReleaseResultSet(m_poDS->hDS, hSQLLyr); - CPLString osTmpFilename = - CPLSPrintf("/vsimem/mvt_getfeature_%p_%d_%d.pbf", this, nX, nY); + const CPLString osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("mvt_get_feature_%d_%d.pbf", m_nX, m_nY)); VSIFCloseL( VSIFileFromMemBuffer(osTmpFilename, pabyDataDup, nDataSize, true)); @@ -1979,7 +1979,8 @@ void MBTilesDataset::InitVector(double dfMinX, double dfMinY, double dfMaxX, OGR_DS_ReleaseResultSet(hDS, hSQLLyr); } - m_osMetadataMemFilename = CPLSPrintf("/vsimem/%p_metadata.json", this); + m_osMetadataMemFilename = + VSIMemGenerateHiddenFilename("mbtiles_metadata.json"); oDoc.Save(m_osMetadataMemFilename); CPLJSONArray oVectorLayers; @@ -2556,8 +2557,7 @@ static int MBTilesGetBandCountAndTileSize(bool bIsVSICURL, OGRDataSourceH &hDS, break; } - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/%p", hSQLLyr); + const CPLString osMemFileName(VSIMemGenerateHiddenFilename("mvt_temp.db")); int nDataSize = 0; GByte *pabyData = OGR_F_GetFieldAsBinary(hFeat, 0, &nDataSize); From 2a822ceaaf9cd60bfd9e615f5fc6537bc21cf6d4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:41:55 +0200 Subject: [PATCH 077/710] VSIMemGenerateHiddenFilename(): use in WMS driver --- frmts/wms/wmsutils.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frmts/wms/wmsutils.cpp b/frmts/wms/wmsutils.cpp index 75ddddc6598a..b9f044c929d0 100644 --- a/frmts/wms/wmsutils.cpp +++ b/frmts/wms/wmsutils.cpp @@ -70,9 +70,7 @@ void URLPrepare(CPLString &url) CPLString BufferToVSIFile(GByte *buffer, size_t size) { - CPLString file_name; - - file_name.Printf("/vsimem/wms/%p/wmsresult.dat", buffer); + const CPLString file_name(VSIMemGenerateHiddenFilename("wmsresult.dat")); VSILFILE *f = VSIFileFromMemBuffer(file_name.c_str(), buffer, size, false); if (f == nullptr) return CPLString(); From 7a19cb27966bf5c95ebb402c1cb135731671f5df Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:44:44 +0200 Subject: [PATCH 078/710] VSIMemGenerateHiddenFilename(): use in HTTP driver --- frmts/http/httpdriver.cpp | 12 +++--------- frmts/jpeg/jpgdataset.cpp | 3 ++- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/frmts/http/httpdriver.cpp b/frmts/http/httpdriver.cpp index 1c17d82eb5d1..153a4863b6c5 100644 --- a/frmts/http/httpdriver.cpp +++ b/frmts/http/httpdriver.cpp @@ -29,7 +29,6 @@ #include "cpl_string.h" #include "cpl_http.h" -#include "cpl_atomic_ops.h" #include "gdal_frmts.h" #include "gdal_pam.h" @@ -87,8 +86,6 @@ static std::string HTTPFetchContentDispositionFilename(char **papszHeaders) static GDALDataset *HTTPOpen(GDALOpenInfo *poOpenInfo) { - static volatile int nCounter = 0; - if (poOpenInfo->nHeaderBytes != 0) return nullptr; @@ -116,10 +113,6 @@ static GDALDataset *HTTPOpen(GDALOpenInfo *poOpenInfo) /* -------------------------------------------------------------------- */ /* Create a memory file from the result. */ /* -------------------------------------------------------------------- */ - CPLString osResultFilename; - - int nNewCounter = CPLAtomicInc(&nCounter); - std::string osFilename = HTTPFetchContentDispositionFilename(psResult->papszHeaders); if (osFilename.empty()) @@ -130,8 +123,9 @@ static GDALDataset *HTTPOpen(GDALOpenInfo *poOpenInfo) osFilename = "file.dat"; } - osResultFilename.Printf("/vsimem/http_%d_%s", nNewCounter, - osFilename.c_str()); + // If changing the _gdal_http_ marker, change jpgdataset.cpp that tests for it + const CPLString osResultFilename = VSIMemGenerateHiddenFilename( + std::string("_gdal_http_").append(osFilename).c_str()); VSILFILE *fp = VSIFileFromMemBuffer(osResultFilename, psResult->pabyData, psResult->nDataLen, TRUE); diff --git a/frmts/jpeg/jpgdataset.cpp b/frmts/jpeg/jpgdataset.cpp index fae0b2b47617..a9a8029c2dc9 100644 --- a/frmts/jpeg/jpgdataset.cpp +++ b/frmts/jpeg/jpgdataset.cpp @@ -3236,7 +3236,8 @@ JPGDatasetCommon *JPGDataset::OpenStage2(JPGDatasetOpenArgs *psArgs, // will unlink the temporary /vsimem file just after GDALOpen(), so // later VSIFOpenL() when reading internal overviews would fail. // Initialize them now. - if (STARTS_WITH(real_filename, "/vsimem/http_")) + if (STARTS_WITH(real_filename, "/vsimem/") && + strstr(real_filename, "_gdal_http_")) { poDS->InitInternalOverviews(); } From 1654b361e24467d6af39cd952757820b7a5e234f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 17:47:32 +0200 Subject: [PATCH 079/710] VSIMemGenerateHiddenFilename(): use in OAPIF driver --- ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp b/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp index 652d4a0d816e..d0b038aa03f9 100644 --- a/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp +++ b/ogr/ogrsf_frmts/wfs/ogroapifdriver.cpp @@ -1890,7 +1890,7 @@ void OGROAPIFLayer::EstablishFeatureDefn() if (!m_poDS->DownloadJSon(osURL, oDoc)) return; - CPLString osTmpFilename(CPLSPrintf("/vsimem/oapif_%p.json", this)); + const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("oapif.json")); oDoc.Save(osTmpFilename); std::unique_ptr poDS(GDALDataset::FromHandle( GDALOpenEx(osTmpFilename, GDAL_OF_VECTOR | GDAL_OF_INTERNAL, nullptr, @@ -2134,7 +2134,8 @@ OGRFeature *OGROAPIFLayer::GetNextRawFeature() } } - CPLString osTmpFilename(CPLSPrintf("/vsimem/oapif_%p.json", this)); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("oapif.json")); m_oCurDoc.Save(osTmpFilename); m_poUnderlyingDS = std::unique_ptr(GDALDataset::FromHandle( From 7155f5e616f3ed762aa8970164032985e894a991 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:24:04 +0200 Subject: [PATCH 080/710] VSIMemGenerateHiddenFilename(): use in WFS driver --- autotest/ogr/ogr_wfs.py | 4 ++ ogr/ogrsf_frmts/csw/ogrcswdataset.cpp | 2 +- ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp | 3 +- ogr/ogrsf_frmts/wfs/ogr_wfs.h | 11 ++- ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp | 6 +- ogr/ogrsf_frmts/wfs/ogrwfsjoinlayer.cpp | 21 +++--- ogr/ogrsf_frmts/wfs/ogrwfslayer.cpp | 86 +++++++----------------- 7 files changed, 53 insertions(+), 80 deletions(-) diff --git a/autotest/ogr/ogr_wfs.py b/autotest/ogr/ogr_wfs.py index 7873234f9d2a..3fe3b91695ed 100755 --- a/autotest/ogr/ogr_wfs.py +++ b/autotest/ogr/ogr_wfs.py @@ -63,9 +63,13 @@ def ogr_wfs_init(): if gml_ds is None: pytest.skip("cannot read GML files") + vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") + with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"): yield + assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before + @pytest.fixture( params=["NO", None], scope="module", ids=["without-streaming", "with-streaming"] diff --git a/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp b/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp index 82ae661544f0..d5292839c335 100644 --- a/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp +++ b/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp @@ -269,7 +269,7 @@ OGRCSWLayer::~OGRCSWLayer() poFeatureDefn->Release(); GDALClose(poBaseDS); CPLString osTmpDirName = CPLSPrintf("/vsimem/tempcsw_%p", this); - OGRWFSRecursiveUnlink(osTmpDirName); + VSIRmdirRecursive(osTmpDirName); } /************************************************************************/ diff --git a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp index 9848c665a36a..3c57a20cb8bb 100644 --- a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp +++ b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp @@ -532,7 +532,8 @@ bool OGRGMLDataSource::Open(GDALOpenInfo *poOpenInfo) } } } - else if (STARTS_WITH(pszFilename, "/vsimem/tempwfs_")) + else if (STARTS_WITH(pszFilename, "/vsimem/") && + strstr(pszFilename, "_ogr_wfs_")) { // http://regis.intergraph.com/wfs/dcmetro/request.asp? returns a // Who knows what servers can return? When diff --git a/ogr/ogrsf_frmts/wfs/ogr_wfs.h b/ogr/ogrsf_frmts/wfs/ogr_wfs.h index 91ca09f34f3d..6178c496e16a 100644 --- a/ogr/ogrsf_frmts/wfs/ogr_wfs.h +++ b/ogr/ogrsf_frmts/wfs/ogr_wfs.h @@ -41,7 +41,7 @@ #include "ogr_swq.h" const CPLXMLNode *WFSFindNode(const CPLXMLNode *psXML, const char *pszRootName); -void OGRWFSRecursiveUnlink(const char *pszName); + CPLString WFS_TurnSQLFilterToOGCFilter(const swq_expr_node *poExpr, OGRDataSource *poDS, OGRFeatureDefn *poFDefn, int nVersion, @@ -145,6 +145,8 @@ class OGRWFSLayer final : public OGRLayer std::vector m_aosSupportedCRSList{}; OGRLayer::GetSupportedSRSListRetType m_apoSupportedCRSList{}; + std::string m_osTmpDir{}; + public: OGRWFSLayer(OGRWFSDataSource *poDS, OGRSpatialReference *poSRS, int bAxisOrderAlreadyInverted, const char *pszBaseURL, @@ -257,6 +259,11 @@ class OGRWFSLayer final : public OGRLayer OGRErr SetActiveSRS(int iGeomField, const OGRSpatialReference *poSRS) override; + + const std::string &GetTmpDir() const + { + return m_osTmpDir; + } }; /************************************************************************/ @@ -290,6 +297,8 @@ class OGRWFSJoinLayer final : public OGRLayer CPLString osFeatureTypes; + std::string m_osTmpDir{}; + OGRWFSJoinLayer(OGRWFSDataSource *poDS, const swq_select *psSelectInfo, const CPLString &osGlobalFilter); CPLString MakeGetFeatureURL(int bRequestHits = FALSE); diff --git a/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp b/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp index c87ae09dd349..b591f084908b 100644 --- a/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp +++ b/ogr/ogrsf_frmts/wfs/ogrwfsdatasource.cpp @@ -267,7 +267,7 @@ OGRLayer *OGRWFSDataSource::GetLayerByName(const char *pszNameIn) return poLayerMetadataLayer; osLayerMetadataTmpFileName = - CPLSPrintf("/vsimem/tempwfs_%p/WFSLayerMetadata.csv", this); + VSIMemGenerateHiddenFilename("WFSLayerMetadata.csv"); osLayerMetadataCSV = "layer_name,title,abstract\n" + osLayerMetadataCSV; VSIFCloseL(VSIFileFromMemBuffer(osLayerMetadataTmpFileName, @@ -1911,9 +1911,7 @@ void OGRWFSDataSource::LoadMultipleLayerDefn(const char *pszLayerName, return; } - CPLString osTmpFileName; - - osTmpFileName = CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", this); + const CPLString osTmpFileName = VSIMemGenerateHiddenFilename("file.xsd"); CPLSerializeXMLTreeToFile(psSchema, osTmpFileName); std::vector aosClasses; diff --git a/ogr/ogrsf_frmts/wfs/ogrwfsjoinlayer.cpp b/ogr/ogrsf_frmts/wfs/ogrwfsjoinlayer.cpp index f024b1d02b76..0e8c82e84749 100644 --- a/ogr/ogrsf_frmts/wfs/ogrwfsjoinlayer.cpp +++ b/ogr/ogrsf_frmts/wfs/ogrwfsjoinlayer.cpp @@ -40,7 +40,9 @@ OGRWFSJoinLayer::OGRWFSJoinLayer(OGRWFSDataSource *poDSIn, bDistinct(psSelectInfo->query_mode == SWQM_DISTINCT_LIST), poBaseDS(nullptr), poBaseLayer(nullptr), bReloadNeeded(false), bHasFetched(false), bPagingActive(false), nPagingStartIndex(0), - nFeatureRead(0), nFeatureCountRequested(0) + nFeatureRead(0), nFeatureCountRequested(0), + // If changing that, change in the GML driver too + m_osTmpDir(VSIMemGenerateHiddenFilename("_ogr_wfs_")) { CPLString osName("join_"); CPLString osLayerName = psSelectInfo->table_defs[0].table_name; @@ -178,7 +180,7 @@ OGRWFSJoinLayer::OGRWFSJoinLayer(OGRWFSDataSource *poDSIn, for (int i = 0; i < (int)apoLayers.size(); i++) { CPLString osTmpFileName = - CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", apoLayers[i]); + CPLSPrintf("%s/file.xsd", apoLayers[i]->GetTmpDir().c_str()); CPLPushErrorHandler(CPLQuietErrorHandler); CPLXMLNode *psSchema = CPLParseXMLFile(osTmpFileName); CPLPopErrorHandler(); @@ -199,8 +201,7 @@ OGRWFSJoinLayer::OGRWFSJoinLayer(OGRWFSDataSource *poDSIn, } if (psGlobalSchema) { - CPLString osTmpFileName = - CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", this); + CPLString osTmpFileName = CPLSPrintf("%s/file.xsd", m_osTmpDir.c_str()); CPLSerializeXMLTreeToFile(psGlobalSchema, osTmpFileName); CPLDestroyXMLNode(psGlobalSchema); } @@ -217,8 +218,7 @@ OGRWFSJoinLayer::~OGRWFSJoinLayer() if (poBaseDS != nullptr) GDALClose(poBaseDS); - CPLString osTmpDirName = CPLSPrintf("/vsimem/tempwfs_%p", this); - OGRWFSRecursiveUnlink(osTmpDirName); + VSIRmdirRecursive(m_osTmpDir.c_str()); } /************************************************************************/ @@ -430,7 +430,7 @@ GDALDataset *OGRWFSJoinLayer::FetchGetFeature() /* Try streaming when the output format is GML and that we have a .xsd */ /* that we are able to understand */ - CPLString osXSDFileName = CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", this); + CPLString osXSDFileName = CPLSPrintf("%s/file.xsd", m_osTmpDir.c_str()); VSIStatBufL sBuf; if (CPLTestBool(CPLGetConfigOption("OGR_WFS_USE_STREAMING", "YES")) && VSIStatL(osXSDFileName, &sBuf) == 0 && @@ -487,8 +487,7 @@ GDALDataset *OGRWFSJoinLayer::FetchGetFeature() return nullptr; } - CPLString osTmpDirName = CPLSPrintf("/vsimem/tempwfs_%p", this); - VSIMkdir(osTmpDirName, 0); + VSIMkdir(m_osTmpDir.c_str(), 0); GByte *pabyData = psResult->pabyData; int nDataLen = psResult->nDataLen; @@ -504,10 +503,10 @@ GDALDataset *OGRWFSJoinLayer::FetchGetFeature() CPLString osTmpFileName; - osTmpFileName = osTmpDirName + "/file.gfs"; + osTmpFileName = m_osTmpDir + "/file.gfs"; VSIUnlink(osTmpFileName); - osTmpFileName = osTmpDirName + "/file.gml"; + osTmpFileName = m_osTmpDir + "/file.gml"; VSILFILE *fp = VSIFileFromMemBuffer(osTmpFileName, pabyData, nDataLen, TRUE); diff --git a/ogr/ogrsf_frmts/wfs/ogrwfslayer.cpp b/ogr/ogrsf_frmts/wfs/ogrwfslayer.cpp index e1d384f3a04a..5773e38c75e3 100644 --- a/ogr/ogrsf_frmts/wfs/ogrwfslayer.cpp +++ b/ogr/ogrsf_frmts/wfs/ogrwfslayer.cpp @@ -33,44 +33,6 @@ #include "cpl_http.h" #include "parsexsd.h" -/************************************************************************/ -/* OGRWFSRecursiveUnlink() */ -/************************************************************************/ - -void OGRWFSRecursiveUnlink(const char *pszName) - -{ - char **papszFileList = VSIReadDir(pszName); - - for (int i = 0; papszFileList != nullptr && papszFileList[i] != nullptr; - i++) - { - VSIStatBufL sStatBuf; - - if (EQUAL(papszFileList[i], ".") || EQUAL(papszFileList[i], "..")) - continue; - - CPLString osFullFilename = - CPLFormFilename(pszName, papszFileList[i], nullptr); - - if (VSIStatL(osFullFilename, &sStatBuf) == 0) - { - if (VSI_ISREG(sStatBuf.st_mode)) - { - VSIUnlink(osFullFilename); - } - else if (VSI_ISDIR(sStatBuf.st_mode)) - { - OGRWFSRecursiveUnlink(osFullFilename); - } - } - } - - CSLDestroy(papszFileList); - - VSIRmdir(pszName); -} - /************************************************************************/ /* OGRWFSLayer() */ /************************************************************************/ @@ -93,6 +55,9 @@ OGRWFSLayer::OGRWFSLayer(OGRWFSDataSource *poDSIn, OGRSpatialReference *poSRSIn, nPagingStartIndex(0), nFeatureRead(0), pszRequiredOutputFormat(nullptr) { SetDescription(pszName); + + // If changing that, change in the GML driver too + m_osTmpDir = VSIMemGenerateHiddenFilename("_ogr_wfs_"); } /************************************************************************/ @@ -114,9 +79,9 @@ OGRWFSLayer *OGRWFSLayer::Clone() pszRequiredOutputFormat ? CPLStrdup(pszRequiredOutputFormat) : nullptr; /* Copy existing schema file if already found */ - CPLString osSrcFileName = CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", this); + CPLString osSrcFileName = CPLSPrintf("%s/file.xsd", m_osTmpDir.c_str()); CPLString osTargetFileName = - CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", poDupLayer); + CPLSPrintf("%s/file.xsd", poDupLayer->m_osTmpDir.c_str()); CPL_IGNORE_RET_VAL(CPLCopyFile(osTargetFileName, osSrcFileName)); return poDupLayer; @@ -148,8 +113,7 @@ OGRWFSLayer::~OGRWFSLayer() delete poFetchedFilterGeom; - CPLString osTmpDirName = CPLSPrintf("/vsimem/tempwfs_%p", this); - OGRWFSRecursiveUnlink(osTmpDirName); + VSIRmdirRecursive(m_osTmpDir.c_str()); CPLFree(pszRequiredOutputFormat); } @@ -312,7 +276,7 @@ OGRFeatureDefn *OGRWFSLayer::ParseSchema(const CPLXMLNode *psSchema) CPLString osTmpFileName; - osTmpFileName = CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", this); + osTmpFileName = CPLSPrintf("%s/file.xsd", m_osTmpDir.c_str()); CPLSerializeXMLTreeToFile(psSchema, osTmpFileName); std::vector aosClasses; @@ -803,8 +767,7 @@ GDALDataset *OGRWFSLayer::FetchGetFeature(int nRequestMaxFeatures) /* Try streaming when the output format is GML and that we have a .xsd */ /* that we are able to understand */ - CPLString osXSDFileName = - CPLSPrintf("/vsimem/tempwfs_%p/file.xsd", this); + CPLString osXSDFileName = CPLSPrintf("%s/file.xsd", m_osTmpDir.c_str()); VSIStatBufL sBuf; GDALDriver *poDriver = nullptr; if ((osOutputFormat.empty() || @@ -922,8 +885,7 @@ GDALDataset *OGRWFSLayer::FetchGetFeature(int nRequestMaxFeatures) if (psResult->pszContentType) pszContentType = psResult->pszContentType; - const std::string osTmpDirName = CPLSPrintf("/vsimem/tempwfs_%p", this); - VSIMkdir(osTmpDirName.c_str(), 0); + VSIMkdir(m_osTmpDir.c_str(), 0); GByte *pabyData = psResult->pabyData; int nDataLen = psResult->nDataLen; @@ -934,11 +896,11 @@ GDALDataset *OGRWFSLayer::FetchGetFeature(int nRequestMaxFeatures) CPLHTTPParseMultipartMime(psResult)) { bIsMultiPart = true; - OGRWFSRecursiveUnlink(osTmpDirName.c_str()); - VSIMkdir(osTmpDirName.c_str(), 0); + VSIRmdirRecursive(m_osTmpDir.c_str()); + VSIMkdir(m_osTmpDir.c_str(), 0); for (int i = 0; i < psResult->nMimePartCount; i++) { - CPLString osTmpFileName = osTmpDirName + "/"; + CPLString osTmpFileName = m_osTmpDir + "/"; pszAttachmentFilename = OGRWFSFetchContentDispositionFilename( psResult->pasMimePart[i].papszHeaders); @@ -1038,31 +1000,31 @@ GDALDataset *OGRWFSLayer::FetchGetFeature(int nRequestMaxFeatures) if (!bIsMultiPart) { if (bJSON) - osTmpFileName = osTmpDirName + "/file.geojson"; + osTmpFileName = m_osTmpDir + "/file.geojson"; else if (bZIP) - osTmpFileName = osTmpDirName + "/file.zip"; + osTmpFileName = m_osTmpDir + "/file.zip"; else if (bCSV) - osTmpFileName = osTmpDirName + "/file.csv"; + osTmpFileName = m_osTmpDir + "/file.csv"; else if (bKML) - osTmpFileName = osTmpDirName + "/file.kml"; + osTmpFileName = m_osTmpDir + "/file.kml"; else if (bKMZ) - osTmpFileName = osTmpDirName + "/file.kmz"; + osTmpFileName = m_osTmpDir + "/file.kmz"; else if (bFlatGeobuf) - osTmpFileName = osTmpDirName + "/file.fgb"; + osTmpFileName = m_osTmpDir + "/file.fgb"; /* GML is a special case. It needs the .xsd file that has been saved */ /* as file.xsd, so we cannot used the attachment filename */ else if (pszAttachmentFilename && !EQUAL(CPLGetExtension(pszAttachmentFilename), "GML")) { - osTmpFileName = osTmpDirName + "/"; + osTmpFileName = m_osTmpDir + "/"; osTmpFileName += pszAttachmentFilename; } else { - osTmpFileName = osTmpDirName + "/file.gfs"; + osTmpFileName = m_osTmpDir + "/file.gfs"; VSIUnlink(osTmpFileName); - osTmpFileName = osTmpDirName + "/file.gml"; + osTmpFileName = m_osTmpDir + "/file.gml"; } VSILFILE *fp = @@ -1083,7 +1045,7 @@ GDALDataset *OGRWFSLayer::FetchGetFeature(int nRequestMaxFeatures) { pabyData = nullptr; nDataLen = 0; - osTmpFileName = osTmpDirName; + osTmpFileName = m_osTmpDir; } CPLHTTPDestroyResult(psResult); @@ -1620,8 +1582,8 @@ GIntBig OGRWFSLayer::ExecuteGetFeatureResultTypeHits() if (psResult->pszContentType != nullptr && strstr(psResult->pszContentType, "application/zip") != nullptr) { - CPLString osTmpFileName; - osTmpFileName.Printf("/vsimem/wfstemphits_%p.zip", this); + const CPLString osTmpFileName( + VSIMemGenerateHiddenFilename("wfstemphits.zip")); VSILFILE *fp = VSIFileFromMemBuffer(osTmpFileName, psResult->pabyData, psResult->nDataLen, FALSE); VSIFCloseL(fp); From d175f4de756509f788d0198b86854a29e225d3f9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:29:47 +0200 Subject: [PATCH 081/710] VSIMemGenerateHiddenFilename(): use in CSW driver --- autotest/ogr/ogr_csw.py | 5 +++++ ogr/ogrsf_frmts/csw/ogrcswdataset.cpp | 14 ++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/autotest/ogr/ogr_csw.py b/autotest/ogr/ogr_csw.py index 653de9c73981..b6fb800d4688 100755 --- a/autotest/ogr/ogr_csw.py +++ b/autotest/ogr/ogr_csw.py @@ -46,9 +46,14 @@ def module_disable_exceptions(): ############################################################################### @pytest.fixture(autouse=True, scope="module") def setup_and_cleanup(): + + vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") + with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"): yield + assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before + ############################################################################### # Test underlying OGR drivers diff --git a/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp b/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp index d5292839c335..b3c5d419254e 100644 --- a/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp +++ b/ogr/ogrsf_frmts/csw/ogrcswdataset.cpp @@ -56,6 +56,8 @@ class OGRCSWLayer final : public OGRLayer CPLString osQuery; CPLString osCSWWhere; + std::string m_osTmpDir{}; + GDALDataset *FetchGetRecords(); GIntBig GetFeatureCountWithHits(); void BuildQuery(); @@ -258,6 +260,8 @@ OGRCSWLayer::OGRCSWLayer(OGRCSWDataSource *poDSIn) } poSRS->Release(); + + m_osTmpDir = VSIMemGenerateHiddenFilename("csw"); } /************************************************************************/ @@ -268,8 +272,7 @@ OGRCSWLayer::~OGRCSWLayer() { poFeatureDefn->Release(); GDALClose(poBaseDS); - CPLString osTmpDirName = CPLSPrintf("/vsimem/tempcsw_%p", this); - VSIRmdirRecursive(osTmpDirName); + VSIRmdirRecursive(m_osTmpDir.c_str()); } /************************************************************************/ @@ -531,8 +534,7 @@ GDALDataset *OGRCSWLayer::FetchGetRecords() return nullptr; } - CPLString osTmpDirName = CPLSPrintf("/vsimem/tempcsw_%p", this); - VSIMkdir(osTmpDirName, 0); + VSIMkdir(m_osTmpDir.c_str(), 0); GByte *pabyData = psResult->pabyData; int nDataLen = psResult->nDataLen; @@ -549,10 +551,10 @@ GDALDataset *OGRCSWLayer::FetchGetRecords() CPLString osTmpFileName; - osTmpFileName = osTmpDirName + "/file.gfs"; + osTmpFileName = m_osTmpDir + "/file.gfs"; VSIUnlink(osTmpFileName); - osTmpFileName = osTmpDirName + "/file.gml"; + osTmpFileName = m_osTmpDir + "/file.gml"; VSILFILE *fp = VSIFileFromMemBuffer(osTmpFileName, pabyData, nDataLen, TRUE); From 2c743fee8f44a37f47453fd88f6a9ca935d060be Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:30:31 +0200 Subject: [PATCH 082/710] VSIMemGenerateHiddenFilename(): use in CSV driver --- ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp index cf8a7cd6b81d..98e387324589 100644 --- a/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp +++ b/ogr/ogrsf_frmts/csv/ogrcsvlayer.cpp @@ -960,7 +960,7 @@ char **OGRCSVLayer::AutodetectFieldTypes(CSLConstList papszOpenOptions, nRead = VSIFReadL(pszData, 1, nRequested, fpCSV); pszData[nRead] = 0; - osTmpMemFile = CPLSPrintf("/vsimem/tmp%p", this); + osTmpMemFile = VSIMemGenerateHiddenFilename("temp.csv"); fp = VSIFileFromMemBuffer(osTmpMemFile.c_str(), reinterpret_cast(pszData), nRead, FALSE); From bb6bca3727a8dc98a25685679821273d1f76f877 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:31:46 +0200 Subject: [PATCH 083/710] VSIMemGenerateHiddenFilename(): use in Shapefile driver --- ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp b/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp index eb102e8c3b8a..9e747cd2d06d 100644 --- a/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp +++ b/ogr/ogrsf_frmts/shape/ogrshapedatasource.cpp @@ -1556,7 +1556,7 @@ bool OGRShapeDataSource::UncompressIfNeeded() nTotalUncompressedSize < static_cast(CPLGetUsablePhysicalRAM() / 10))) { - osTemporaryDir = CPLSPrintf("/vsimem/_shapedriver/%p", this); + osTemporaryDir = VSIMemGenerateHiddenFilename("shapedriver"); } CPLDebug("Shape", "Uncompressing to %s", osTemporaryDir.c_str()); From c5e912bc8db4d04fe1e7315f1e4f0303f570c4fb Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:33:58 +0200 Subject: [PATCH 084/710] VSIMemGenerateHiddenFilename(): use in (raster-side) OpenFileGDB driver --- .../openfilegdb/gdalopenfilegdbrasterband.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp b/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp index 45920d639f2a..c39eb52bcf13 100644 --- a/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp +++ b/ogr/ogrsf_frmts/openfilegdb/gdalopenfilegdbrasterband.cpp @@ -906,8 +906,8 @@ void OGROpenFileGDBDataSource::GuessJPEGQuality(int nOverviewCount) } if (nJPEGSize) { - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/_openfilegdb/%p.jpg", this); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("openfilegdb.jpg")); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), const_cast(pabyData + nJPEGOffset), nJPEGSize, @@ -1551,8 +1551,8 @@ CPLErr GDALOpenFileGDBRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, return CE_Failure; } - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/_openfilegdb/%p.jpg", this); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("openfilegdb.jpg")); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), const_cast(pabyData + nJPEGOffset), nJPEGSize, false)); @@ -1674,8 +1674,8 @@ CPLErr GDALOpenFileGDBRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, return CE_Failure; } - CPLString osTmpFilename; - osTmpFilename.Printf("/vsimem/_openfilegdb/%p.j2k", this); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("openfilegdb.j2k")); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), const_cast(pabyData + nJPEGOffset), nJPEGSize, false)); From 9245f47cb870c0cbb017106f14ab0d697172e493 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:34:28 +0200 Subject: [PATCH 085/710] VSIMemGenerateHiddenFilename(): use in Arrow driver --- ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp b/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp index 8e5dad1c0053..27639faf4421 100644 --- a/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp +++ b/ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp @@ -90,7 +90,7 @@ static bool IsArrowIPCStream(GDALOpenInfo *poOpenInfo) } const std::string osTmpFilename( - CPLSPrintf("/vsimem/_arrow/%p", poOpenInfo)); + VSIMemGenerateHiddenFilename("arrow")); auto fp = VSIVirtualHandleUniquePtr(VSIFileFromMemBuffer( osTmpFilename.c_str(), poOpenInfo->pabyHeader, nSizeToRead, false)); From bbd825e48981216d873f2222ecbe87ebd4b2a872 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:35:26 +0200 Subject: [PATCH 086/710] VSIMemGenerateHiddenFilename(): use in GMLAS driver --- ogr/ogrsf_frmts/gmlas/ogrgmlasxsdcache.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ogr/ogrsf_frmts/gmlas/ogrgmlasxsdcache.cpp b/ogr/ogrsf_frmts/gmlas/ogrgmlasxsdcache.cpp index b855d71020ee..acbe77fe5fee 100644 --- a/ogr/ogrsf_frmts/gmlas/ogrgmlasxsdcache.cpp +++ b/ogr/ogrsf_frmts/gmlas/ogrgmlasxsdcache.cpp @@ -152,7 +152,8 @@ bool GMLASXSDCache::CacheAllGML321() CPLHTTPResult *psResult = CPLHTTPFetch(pszHTTPZIP, nullptr); if (psResult && psResult->nDataLen) { - const std::string osZIPFilename(CPLSPrintf("/vsimem/%p.zip", this)); + const std::string osZIPFilename( + VSIMemGenerateHiddenFilename("temp.zip")); auto fpZIP = VSIFileFromMemBuffer(osZIPFilename.c_str(), psResult->pabyData, psResult->nDataLen, FALSE); @@ -220,7 +221,8 @@ bool GMLASXSDCache::CacheAllISO20070417() CPLHTTPResult *psResult = CPLHTTPFetch(pszHTTPZIP, nullptr); if (psResult && psResult->nDataLen) { - const std::string osZIPFilename(CPLSPrintf("/vsimem/%p.zip", this)); + const std::string osZIPFilename( + VSIMemGenerateHiddenFilename("temp.zip")); auto fpZIP = VSIFileFromMemBuffer(osZIPFilename.c_str(), psResult->pabyData, psResult->nDataLen, FALSE); From 25cf6a1f5d3fa16417ce930be1a66878988e4ac3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:36:43 +0200 Subject: [PATCH 087/710] VSIMemGenerateHiddenFilename(): use in GeoJSON driver --- ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp index 4d7b4510a193..426f07ef90ef 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp @@ -195,8 +195,9 @@ int OGRGeoJSONDataSource::Open(GDALOpenInfo *poOpenInfo, bool bEmitError = true; if (eGeoJSONSourceService == nSrcType) { - const CPLString osTmpFilename = CPLSPrintf( - "/vsimem/%p/%s", this, CPLGetFilename(poOpenInfo->pszFilename)); + const CPLString osTmpFilename = + VSIMemGenerateHiddenFilename(CPLSPrintf( + "geojson_%s", CPLGetFilename(poOpenInfo->pszFilename))); VSIFCloseL(VSIFileFromMemBuffer(osTmpFilename, (GByte *)pszGeoData_, nGeoDataLen_, TRUE)); pszGeoData_ = nullptr; From 9902f63470d2b1afc87406ef743c96ff6277fa5b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:37:20 +0200 Subject: [PATCH 088/710] VSIMemGenerateHiddenFilename(): use in GeoJSONSeq driver --- ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp index 0ac1eb612e30..b1fa548ec798 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp @@ -818,7 +818,7 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, if (poOpenInfo->eAccess == GA_Update) return false; - m_osTmpFile = CPLSPrintf("/vsimem/geojsonseq/%p", this); + m_osTmpFile = VSIMemGenerateHiddenFilename("geojsonseq"); m_fp = VSIFileFromMemBuffer( m_osTmpFile.c_str(), reinterpret_cast(CPLStrdup(poOpenInfo->pszFilename)), @@ -842,7 +842,7 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, } else { - m_osTmpFile = CPLSPrintf("/vsimem/geojsonseq/%p", this); + m_osTmpFile = VSIMemGenerateHiddenFilename("geojsonseq"); m_fp = VSIFileFromMemBuffer( m_osTmpFile.c_str(), reinterpret_cast(pszStoredContent), @@ -873,7 +873,7 @@ bool OGRGeoJSONSeqDataSource::Open(GDALOpenInfo *poOpenInfo, return false; } - m_osTmpFile = CPLSPrintf("/vsimem/geojsonseq/%p", this); + m_osTmpFile = VSIMemGenerateHiddenFilename("geojsonseq"); m_fp = VSIFileFromMemBuffer(m_osTmpFile.c_str(), pResult->pabyData, pResult->nDataLen, true); pResult->pabyData = nullptr; From c3e0f3ac060329066126e1794a325ce9e119c998 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:39:23 +0200 Subject: [PATCH 089/710] VSIMemGenerateHiddenFilename(): use in GPKG driver --- .../gpkg/gdalgeopackagerasterband.cpp | 16 ++++++++-------- ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ogr/ogrsf_frmts/gpkg/gdalgeopackagerasterband.cpp b/ogr/ogrsf_frmts/gpkg/gdalgeopackagerasterband.cpp index 23d6f530e237..abc99854405b 100644 --- a/ogr/ogrsf_frmts/gpkg/gdalgeopackagerasterband.cpp +++ b/ogr/ogrsf_frmts/gpkg/gdalgeopackagerasterband.cpp @@ -243,8 +243,8 @@ GDALColorTable *GDALGPKGMBTilesLikeRasterBand::GetColorTable() const int nBytes = sqlite3_column_bytes(hStmt, 0); GByte *pabyRawData = reinterpret_cast( const_cast(sqlite3_column_blob(hStmt, 0))); - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/gpkg_read_tile_%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("gpkg_read_tile")); VSILFILE *fp = VSIFileFromMemBuffer( osMemFileName.c_str(), pabyRawData, nBytes, FALSE); VSIFCloseL(fp); @@ -915,8 +915,8 @@ GByte *GDALGPKGMBTilesLikePseudoDataset::ReadTile(int nRow, int nCol, (m_eDT == GDT_Byte) ? 0 : sqlite3_column_int64(hStmt, 1); GByte *pabyRawData = static_cast( const_cast(sqlite3_column_blob(hStmt, 0))); - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/gpkg_read_tile_%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("gpkg_read_tile")); VSILFILE *fp = VSIFileFromMemBuffer(osMemFileName.c_str(), pabyRawData, nBytes, FALSE); VSIFCloseL(fp); @@ -1847,8 +1847,8 @@ CPLErr GDALGPKGMBTilesLikePseudoDataset::WriteTileInternal() nRow, nCol, m_nZoomLevel); } - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/gpkg_write_tile_%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("gpkg_write_tile")); const char *pszDriverName = "PNG"; bool bTileDriverSupports1Band = false; bool bTileDriverSupports2Bands = false; @@ -2727,8 +2727,8 @@ GDALGPKGMBTilesLikePseudoDataset::FlushRemainingShiftedTiles(bool bPartialFlush) GByte *pabyRawData = const_cast(static_cast( sqlite3_column_blob(hNewStmt, 0))); - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/gpkg_read_tile_%p", this); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("gpkg_read_tile")); VSILFILE *fp = VSIFileFromMemBuffer( osMemFileName.c_str(), pabyRawData, nBytes, FALSE); VSIFCloseL(fp); diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp index 61ee94970372..92590ca0b831 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp @@ -9200,8 +9200,8 @@ static CPLString GPKG_GDAL_GetMemFileFromBlob(sqlite3_value **argv) int nBytes = sqlite3_value_bytes(argv[0]); const GByte *pabyBLOB = reinterpret_cast(sqlite3_value_blob(argv[0])); - CPLString osMemFileName; - osMemFileName.Printf("/vsimem/GPKG_GDAL_GetMemFileFromBlob_%p", argv); + const CPLString osMemFileName( + VSIMemGenerateHiddenFilename("GPKG_GDAL_GetMemFileFromBlob")); VSILFILE *fp = VSIFileFromMemBuffer( osMemFileName.c_str(), const_cast(pabyBLOB), nBytes, FALSE); VSIFCloseL(fp); From 18ccc5ce7c414614c87c4a777bd21060b0cd7035 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:42:32 +0200 Subject: [PATCH 090/710] VSIMemGenerateHiddenFilename(): use in PMTiles driver --- ogr/ogrsf_frmts/pmtiles/ogrpmtilesdataset.cpp | 3 ++- .../pmtiles/ogrpmtilesvectorlayer.cpp | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ogr/ogrsf_frmts/pmtiles/ogrpmtilesdataset.cpp b/ogr/ogrsf_frmts/pmtiles/ogrpmtilesdataset.cpp index ab82a4d9bfe5..53fa0db2510c 100644 --- a/ogr/ogrsf_frmts/pmtiles/ogrpmtilesdataset.cpp +++ b/ogr/ogrsf_frmts/pmtiles/ogrpmtilesdataset.cpp @@ -239,7 +239,8 @@ bool OGRPMTilesDataset::Open(GDALOpenInfo *poOpenInfo) CPLDebugOnly("PMTiles", "Metadata = %s", posMetadata->c_str()); m_osMetadata = *posMetadata; - m_osMetadataFilename = CPLSPrintf("/vsimem/pmtiles/metadata_%p.json", this); + m_osMetadataFilename = + VSIMemGenerateHiddenFilename("pmtiles_metadata.json"); VSIFCloseL(VSIFileFromMemBuffer(m_osMetadataFilename.c_str(), reinterpret_cast(&m_osMetadata[0]), m_osMetadata.size(), false)); diff --git a/ogr/ogrsf_frmts/pmtiles/ogrpmtilesvectorlayer.cpp b/ogr/ogrsf_frmts/pmtiles/ogrpmtilesvectorlayer.cpp index c6eea3ae3bc3..6bd80339f7f2 100644 --- a/ogr/ogrsf_frmts/pmtiles/ogrpmtilesvectorlayer.cpp +++ b/ogr/ogrsf_frmts/pmtiles/ogrpmtilesvectorlayer.cpp @@ -155,8 +155,8 @@ OGRwkbGeometryType OGRPMTilesVectorLayer::GuessGeometryType( } osTileData = *posStr; - std::string osTmpFilename = - CPLSPrintf("/vsimem/mvt_%p_%u_%u.pbf", poDS, sTile.x, sTile.y); + const std::string osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("pmtiles_%u_%u.pbf", sTile.x, sTile.y)); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), reinterpret_cast(&osTileData[0]), osTileData.size(), false)); @@ -226,8 +226,8 @@ GIntBig OGRPMTilesVectorLayer::GetTotalFeatureCount() const } osTileData = *posStr; - std::string osTmpFilename = CPLSPrintf( - "/vsimem/mvt_%p_%u_%u_getfeaturecount.pbf", this, sTile.x, sTile.y); + const std::string osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("pmtiles_%u_%u_getfeaturecount.pbf", sTile.x, sTile.y)); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), reinterpret_cast(&osTileData[0]), osTileData.size(), false)); @@ -300,8 +300,8 @@ OGRFeature *OGRPMTilesVectorLayer::GetFeature(GIntBig nFID) } std::string osTileData = *posStr; - std::string osTmpFilename = CPLSPrintf( - "/vsimem/mvt_%p_%u_%u_getfeature.pbf", this, sTile.x, sTile.y); + const std::string osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("pmtiles_getfeature_%u_%u.pbf", sTile.x, sTile.y)); VSIFCloseL(VSIFileFromMemBuffer(osTmpFilename.c_str(), reinterpret_cast(&osTileData[0]), osTileData.size(), false)); @@ -408,8 +408,8 @@ std::unique_ptr OGRPMTilesVectorLayer::GetNextSrcFeature() } m_poTileDS.reset(); - const std::string osTmpFilename = - CPLSPrintf("/vsimem/mvt_%p_%u_%u.pbf", this, sTile.x, sTile.y); + const std::string osTmpFilename = VSIMemGenerateHiddenFilename( + CPLSPrintf("pmtiles_%u_%u.pbf", sTile.x, sTile.y)); VSIFCloseL(VSIFileFromMemBuffer( osTmpFilename.c_str(), reinterpret_cast(&m_osTileData[0]), From f290b882259f968bb4922a6a80bb7fe1cb150452 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:44:26 +0200 Subject: [PATCH 091/710] VSIMemGenerateHiddenFilename(): use in MVT driver --- ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp b/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp index 7b37c500dd48..de3166fac2f3 100644 --- a/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp +++ b/ogr/ogrsf_frmts/mvt/ogrmvtdataset.cpp @@ -2804,7 +2804,7 @@ GDALDataset *OGRMVTDataset::OpenDirectory(GDALOpenInfo *poOpenInfo) OGRMVTDataset *poDS = new OGRMVTDataset(nullptr); const CPLString osMetadataMemFilename = - CPLSPrintf("/vsimem/%p_metadata.json", poDS); + VSIMemGenerateHiddenFilename("mvt_metadata.json"); if (!LoadMetadata(osMetadataFile, osMetadataContent, oVectorLayers, oTileStatLayers, oBounds, poDS->m_poSRS, poDS->m_dfTopXOrigin, poDS->m_dfTopYOrigin, @@ -4484,7 +4484,8 @@ static void GZIPCompress(std::string &oTileBuffer) { if (!oTileBuffer.empty()) { - CPLString osTmpFilename(CPLSPrintf("/vsimem/%p.gz", &oTileBuffer)); + const CPLString osTmpFilename( + VSIMemGenerateHiddenFilename("mvt_temp.gz")); CPLString osTmpGZipFilename("/vsigzip/" + osTmpFilename); VSILFILE *fpGZip = VSIFOpenL(osTmpGZipFilename, "wb"); if (fpGZip) From 3480bd85aaa0b3de5d501abea0e42f0e6277f347 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:47:11 +0200 Subject: [PATCH 092/710] VSIMemGenerateHiddenFilename(): use in GPSBabel driver --- ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldatasource.cpp | 2 +- ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldriver.cpp | 3 ++- ogr/ogrsf_frmts/gpsbabel/ogrgpsbabelwritedatasource.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldatasource.cpp b/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldatasource.cpp index f03210037ac3..5f06d4d61653 100644 --- a/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldatasource.cpp +++ b/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldatasource.cpp @@ -261,7 +261,7 @@ int OGRGPSBabelDataSource::Open(const char *pszDatasourceName, if (pszOptionUseTempFile && CPLTestBool(pszOptionUseTempFile)) osTmpFileName = CPLGenerateTempFilename(nullptr); else - osTmpFileName.Printf("/vsimem/ogrgpsbabeldatasource_%p", this); + osTmpFileName = VSIMemGenerateHiddenFilename("gpsbabel"); bool bRet = false; if (IsSpecialFile(pszFilename)) diff --git a/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldriver.cpp b/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldriver.cpp index eca3cc717701..3eeb68d2c384 100644 --- a/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldriver.cpp +++ b/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabeldriver.cpp @@ -115,7 +115,8 @@ OGRGPSBabelDriverIdentifyInternal(GDALOpenInfo *poOpenInfo, { CPLErrorStateBackuper oErrorStateBackuper(CPLQuietErrorHandler); const char *const apszArgs[] = {"gpsbabel", "-V", nullptr}; - CPLString osTmpFileName("/vsimem/gpsbabel_tmp.tmp"); + const CPLString osTmpFileName = + VSIMemGenerateHiddenFilename("gpsbabel"); VSILFILE *tmpfp = VSIFOpenL(osTmpFileName, "wb"); bGPSBabelFound = CPLSpawn(apszArgs, nullptr, tmpfp, FALSE) == 0; VSIFCloseL(tmpfp); diff --git a/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabelwritedatasource.cpp b/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabelwritedatasource.cpp index f9bae27b9fc8..bea9d5a41fd6 100644 --- a/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabelwritedatasource.cpp +++ b/ogr/ogrsf_frmts/gpsbabel/ogrgpsbabelwritedatasource.cpp @@ -193,7 +193,7 @@ int OGRGPSBabelWriteDataSource::Create(const char *pszNameIn, if (pszOptionUseTempFile && CPLTestBool(pszOptionUseTempFile)) osTmpFileName = CPLGenerateTempFilename(nullptr); else - osTmpFileName.Printf("/vsimem/ogrgpsbabeldatasource_%p", this); + osTmpFileName = VSIMemGenerateHiddenFilename("gpsbabel"); poGPXDS = poGPXDriver->Create(osTmpFileName.c_str(), 0, 0, 0, GDT_Unknown, papszOptions); From 2f0b254f451ba57821b3f43503ae94a8b5eb5235 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:48:04 +0200 Subject: [PATCH 093/710] VSIMemGenerateHiddenFilename(): use in OSM driver --- ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp index 6dfe592d232d..b3c2976516dd 100644 --- a/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp +++ b/ogr/ogrsf_frmts/osm/ogrosmdatasource.cpp @@ -2885,7 +2885,7 @@ int OGROSMDataSource::Open(const char *pszFilename, } m_bInMemoryNodesFile = true; - m_osNodesFilename.Printf("/vsimem/osm_temp_nodes_%p", this); + m_osNodesFilename = VSIMemGenerateHiddenFilename("osm_temp_nodes"); m_fpNodes = VSIFOpenL(m_osNodesFilename, "wb+"); if (m_fpNodes == nullptr) { @@ -2971,7 +2971,7 @@ bool OGROSMDataSource::CreateTempDB() } else { - m_osTmpDBName.Printf("/vsimem/osm_temp_%p.sqlite", this); + m_osTmpDBName = VSIMemGenerateHiddenFilename("osm_temp.sqlite"); // On 32 bit, the virtual memory space is scarce, so we need to // reserve it right now. Will not hurt on 64 bit either. From 75affb4c74cdfed2548387f94eee6f3e12ab3272 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 18:53:26 +0200 Subject: [PATCH 094/710] VSIMemGenerateHiddenFilename(): use in SQLite driver --- .../sqlite/ogrsqliteexecutesql.cpp | 9 ++---- ogr/ogrsf_frmts/sqlite/ogrsqlitevfs.cpp | 28 ++++++++----------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqliteexecutesql.cpp b/ogr/ogrsf_frmts/sqlite/ogrsqliteexecutesql.cpp index 879b335b5e0e..6e74a85440f7 100644 --- a/ogr/ogrsf_frmts/sqlite/ogrsqliteexecutesql.cpp +++ b/ogr/ogrsf_frmts/sqlite/ogrsqliteexecutesql.cpp @@ -851,9 +851,7 @@ OGRLayer *OGRSQLiteExecuteSQL(GDALDataset *poDS, const char *pszStatement, } char *pszTmpDBName = static_cast(CPLMalloc(256)); - char szPtr[32]; - snprintf(szPtr, sizeof(szPtr), "%p", pszTmpDBName); - snprintf(pszTmpDBName, 256, "/vsimem/ogr2sqlite/temp_%s.db", szPtr); + snprintf(pszTmpDBName, 256, "%s", VSIMemGenerateHiddenFilename("temp.db")); OGRSQLiteDataSource *poSQLiteDS = nullptr; bool bSpatialiteDB = false; @@ -881,9 +879,8 @@ OGRLayer *OGRSQLiteExecuteSQL(GDALDataset *poDS, const char *pszStatement, { bTried = true; char *pszCachedFilename = static_cast(CPLMalloc(256)); - snprintf(szPtr, sizeof(szPtr), "%p", pszCachedFilename); - snprintf(pszCachedFilename, 256, - "/vsimem/ogr2sqlite/reference_%s.db", szPtr); + snprintf(pszCachedFilename, 256, "%s", + VSIMemGenerateHiddenFilename("reference.db")); char **papszOptions = CSLAddString(nullptr, "SPATIALITE=YES"); OGRSQLiteDataSource *poCachedDS = new OGRSQLiteDataSource(); const int nRet = diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitevfs.cpp b/ogr/ogrsf_frmts/sqlite/ogrsqlitevfs.cpp index a7a0b6a3aa83..8674380598ee 100644 --- a/ogr/ogrsf_frmts/sqlite/ogrsqlitevfs.cpp +++ b/ogr/ogrsf_frmts/sqlite/ogrsqlitevfs.cpp @@ -33,7 +33,6 @@ #include #include -#include "cpl_atomic_ops.h" #include "cpl_conv.h" #include "cpl_error.h" #include "cpl_string.h" @@ -54,7 +53,6 @@ typedef struct sqlite3_vfs *pDefaultVFS; pfnNotifyFileOpenedType pfn; void *pfnUserData; - int nCounter; } OGRSQLiteVFSAppDataStruct; #define GET_UNDERLYING_VFS(pVFS) \ @@ -243,22 +241,19 @@ static const sqlite3_io_methods OGRSQLiteIOMethods = { nullptr, // xUnfetch }; -static int OGRSQLiteVFSOpen(sqlite3_vfs *pVFS, const char *zName, +static int OGRSQLiteVFSOpen(sqlite3_vfs *pVFS, const char *zNameIn, sqlite3_file *pFile, int flags, int *pOutFlags) { #ifdef DEBUG_IO - CPLDebug("SQLITE", "OGRSQLiteVFSOpen(%s, %d)", zName ? zName : "(null)", + CPLDebug("SQLITE", "OGRSQLiteVFSOpen(%s, %d)", zNameIn ? zNameIn : "(null)", flags); #endif OGRSQLiteVFSAppDataStruct *pAppData = static_cast(pVFS->pAppData); - if (zName == nullptr) - { - zName = CPLSPrintf("/vsimem/sqlite/%p_%d", pVFS, - CPLAtomicInc(&(pAppData->nCounter))); - } + const std::string osName( + zNameIn ? zNameIn : VSIMemGenerateHiddenFilename("sqlitevfs")); OGRSQLiteFileStruct *pMyFile = reinterpret_cast(pFile); @@ -266,17 +261,17 @@ static int OGRSQLiteVFSOpen(sqlite3_vfs *pVFS, const char *zName, pMyFile->bDeleteOnClose = FALSE; pMyFile->pszFilename = nullptr; if (flags & SQLITE_OPEN_READONLY) - pMyFile->fp = VSIFOpenL(zName, "rb"); + pMyFile->fp = VSIFOpenL(osName.c_str(), "rb"); else if (flags & SQLITE_OPEN_CREATE) { VSIStatBufL sStatBufL; - if (VSIStatExL(zName, &sStatBufL, VSI_STAT_EXISTS_FLAG) == 0) - pMyFile->fp = VSIFOpenL(zName, "rb+"); + if (VSIStatExL(osName.c_str(), &sStatBufL, VSI_STAT_EXISTS_FLAG) == 0) + pMyFile->fp = VSIFOpenL(osName.c_str(), "rb+"); else - pMyFile->fp = VSIFOpenL(zName, "wb+"); + pMyFile->fp = VSIFOpenL(osName.c_str(), "wb+"); } else if (flags & SQLITE_OPEN_READWRITE) - pMyFile->fp = VSIFOpenL(zName, "rb+"); + pMyFile->fp = VSIFOpenL(osName.c_str(), "rb+"); else pMyFile->fp = nullptr; @@ -290,12 +285,12 @@ static int OGRSQLiteVFSOpen(sqlite3_vfs *pVFS, const char *zName, pfnNotifyFileOpenedType pfn = pAppData->pfn; if (pfn) { - pfn(pAppData->pfnUserData, zName, pMyFile->fp); + pfn(pAppData->pfnUserData, osName.c_str(), pMyFile->fp); } pMyFile->pMethods = &OGRSQLiteIOMethods; pMyFile->bDeleteOnClose = (flags & SQLITE_OPEN_DELETEONCLOSE); - pMyFile->pszFilename = CPLStrdup(zName); + pMyFile->pszFilename = CPLStrdup(osName.c_str()); if (pOutFlags != nullptr) *pOutFlags = flags; @@ -514,7 +509,6 @@ sqlite3_vfs *OGRSQLiteCreateVFS(pfnNotifyFileOpenedType pfn, void *pfnUserData) pVFSAppData->pDefaultVFS = pDefaultVFS; pVFSAppData->pfn = pfn; pVFSAppData->pfnUserData = pfnUserData; - pVFSAppData->nCounter = 0; pMyVFS->iVersion = 2; pMyVFS->szOsFile = sizeof(OGRSQLiteFileStruct); From fc925d50d067f9d529423a89c39a1c7e9aa705c9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 19:02:12 +0200 Subject: [PATCH 095/710] VSIMemGenerateHiddenFilename(): use in GML driver --- ogr/ogrsf_frmts/gml/ogr_gml.h | 1 + ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ogr/ogrsf_frmts/gml/ogr_gml.h b/ogr/ogrsf_frmts/gml/ogr_gml.h index 84aa6d54c72b..1a0e89c94409 100644 --- a/ogr/ogrsf_frmts/gml/ogr_gml.h +++ b/ogr/ogrsf_frmts/gml/ogr_gml.h @@ -150,6 +150,7 @@ class OGRGMLDataSource final : public OGRDataSource // input related parameters. CPLString osFilename{}; CPLString osXSDFilename{}; + bool m_bUnlinkXSDFilename = false; IGMLReader *poReader; bool bOutIsTempFile; diff --git a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp index 3c57a20cb8bb..8bee7da6d9f9 100644 --- a/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp +++ b/ogr/ogrsf_frmts/gml/ogrgmldatasource.cpp @@ -269,9 +269,10 @@ OGRGMLDataSource::~OGRGMLDataSource() delete poStoredGMLFeature; - if (osXSDFilename.compare(CPLSPrintf("/vsimem/tmp_gml_xsd_%p.xsd", this)) == - 0) + if (m_bUnlinkXSDFilename) + { VSIUnlink(osXSDFilename); + } } /************************************************************************/ @@ -1102,8 +1103,10 @@ bool OGRGMLDataSource::Open(GDALOpenInfo *poOpenInfo) psResult->pabyData != nullptr) { bHasFoundXSD = true; - osXSDFilename = CPLSPrintf( - "/vsimem/tmp_gml_xsd_%p.xsd", this); + m_bUnlinkXSDFilename = true; + osXSDFilename = + VSIMemGenerateHiddenFilename( + "tmp_ogr_gml.xsd"); VSILFILE *fpMem = VSIFileFromMemBuffer( osXSDFilename, psResult->pabyData, psResult->nDataLen, TRUE); From de372d81c599e4cc79b867530dc11e881d8f1c38 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 19:05:11 +0200 Subject: [PATCH 096/710] VSIMemGenerateHiddenFilename(): use in PLScenes driver --- ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp b/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp index 78c0eb3f670c..7ad724680d37 100644 --- a/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp +++ b/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp @@ -626,7 +626,11 @@ GDALDataset *OGRPLScenesDataV1Dataset::OpenRasterScene(GDALOpenInfo *poOpenInfo, { // Set a dummy name so that PAM goes here CPLPushErrorHandler(CPLQuietErrorHandler); - poOutDS->SetDescription("/vsimem/tmp/ogrplscenesDataV1"); + + const std::string osTmpFilename = + VSIMemGenerateHiddenFilename("ogrplscenesDataV1"); + + poOutDS->SetDescription(osTmpFilename.c_str()); /* Attach scene metadata. */ poLayer->SetAttributeFilter( @@ -663,8 +667,9 @@ GDALDataset *OGRPLScenesDataV1Dataset::OpenRasterScene(GDALOpenInfo *poOpenInfo, delete poFeat; poOutDS->FlushCache(false); - VSIUnlink("/vsimem/tmp/ogrplscenesDataV1"); - VSIUnlink("/vsimem/tmp/ogrplscenesDataV1.aux.xml"); + VSIUnlink(osTmpFilename.c_str()); + VSIUnlink( + std::string(osTmpFilename).append(".aux.xml").c_str()); CPLPopErrorHandler(); } } From 9119a488a7210a2e6f6a642e0caa4c50f0fb8931 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Sep 2024 19:05:55 +0200 Subject: [PATCH 097/710] VSIMemGenerateHiddenFilename(): use in ogr_geocoding --- ogr/ogr_geocoding.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ogr/ogr_geocoding.cpp b/ogr/ogr_geocoding.cpp index 412d7d7557c1..c9821807a6b0 100644 --- a/ogr/ogr_geocoding.cpp +++ b/ogr/ogr_geocoding.cpp @@ -471,8 +471,9 @@ static OGRLayer *OGRGeocodeGetCacheLayer(OGRGeocodingSessionH hSession, (EQUAL(osExt, "SQLITE") || EQUAL(osExt, "CSV"))) { CPLFree(hSession->pszCacheFilename); - hSession->pszCacheFilename = CPLStrdup(CPLSPrintf( - "/vsimem/%s.%s", CACHE_LAYER_NAME, osExt.c_str())); + hSession->pszCacheFilename = + CPLStrdup(VSIMemGenerateHiddenFilename(CPLSPrintf( + "%s.%s", CACHE_LAYER_NAME, osExt.c_str()))); CPLDebug("OGR", "Switch geocode cache file to %s", hSession->pszCacheFilename); poDS = reinterpret_cast( From 79577b1ab31b0b447ef0265071495fa4c8c688d5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 20:07:28 +0200 Subject: [PATCH 098/710] S102: add support for IHO S102 v3.0 specification (#10779) --- ..._v3.0_with_QualityOfBathymetryCoverage.xml | 1 + autotest/gdrivers/data/s102/generate_test.py | 68 ++++++--- ...2_v3.0_with_QualityOfBathymetryCoverage.h5 | Bin 0 -> 17944 bytes autotest/gdrivers/s102.py | 32 +++-- doc/source/drivers/raster/s102.rst | 15 +- frmts/hdf5/hdf5multidim.cpp | 23 ++- frmts/hdf5/s102dataset.cpp | 133 ++++++++++++------ 7 files changed, 188 insertions(+), 84 deletions(-) create mode 100644 autotest/gdrivers/data/s102/MD_test_s102_v3.0_with_QualityOfBathymetryCoverage.xml create mode 100644 autotest/gdrivers/data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5 diff --git a/autotest/gdrivers/data/s102/MD_test_s102_v3.0_with_QualityOfBathymetryCoverage.xml b/autotest/gdrivers/data/s102/MD_test_s102_v3.0_with_QualityOfBathymetryCoverage.xml new file mode 100644 index 000000000000..7d5c8daa45e2 --- /dev/null +++ b/autotest/gdrivers/data/s102/MD_test_s102_v3.0_with_QualityOfBathymetryCoverage.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/autotest/gdrivers/data/s102/generate_test.py b/autotest/gdrivers/data/s102/generate_test.py index c4d3f81a8ed3..c87dbc755d86 100755 --- a/autotest/gdrivers/data/s102/generate_test.py +++ b/autotest/gdrivers/data/s102/generate_test.py @@ -35,7 +35,13 @@ import numpy as np -def generate(filename, version, with_QualityOfSurvey=False): +def generate( + filename, + version, + with_QualityOfSurvey=False, + with_QualityOfCoverage=False, + use_compound_type_for_Quality=False, +): f = h5py.File(os.path.join(os.path.dirname(__file__), f"{filename}.h5"), "w") BathymetryCoverage = f.create_group("BathymetryCoverage") BathymetryCoverage_01 = BathymetryCoverage.create_group("BathymetryCoverage.01") @@ -71,7 +77,7 @@ def generate(filename, version, with_QualityOfSurvey=False): f.attrs["issueDate"] = "2023-12-31" f.attrs["geographicIdentifier"] = "Somewhere" f.attrs["verticalDatum"] = np.int16(12) - if version == "INT.IHO.S-102.2.2": + if version >= "INT.IHO.S-102.2.2": f.attrs["horizontalCRS"] = np.int32(4326) f.attrs["verticalCS"] = np.int32(6498) # Depth, metres, orientation down f.attrs["verticalCoordinateBase"] = np.int32(2) @@ -90,9 +96,12 @@ def generate(filename, version, with_QualityOfSurvey=False): b"" ) - if with_QualityOfSurvey: - QualityOfSurvey = f.create_group("QualityOfSurvey") - QualityOfSurvey_01 = QualityOfSurvey.create_group("QualityOfSurvey.01") + if with_QualityOfSurvey or with_QualityOfCoverage: + quality_name = ( + "QualityOfSurvey" if with_QualityOfSurvey else "QualityOfBathymetryCoverage" + ) + qualityGroup = f.create_group(quality_name) + quality01Group = qualityGroup.create_group(quality_name + ".01") for attr_name in ( "gridOriginLongitude", @@ -102,16 +111,26 @@ def generate(filename, version, with_QualityOfSurvey=False): "numPointsLongitudinal", "numPointsLatitudinal", ): - QualityOfSurvey_01.attrs[attr_name] = BathymetryCoverage_01.attrs[attr_name] - - Group_001 = QualityOfSurvey_01.create_group("Group_001") - - values = Group_001.create_dataset("values", (2, 3), dtype=np.uint32) - data = np.array( - [0, 1, 2, 1000000, 3, 2], - dtype=np.uint32, - ).reshape(values.shape) - values[...] = data + quality01Group.attrs[attr_name] = BathymetryCoverage_01.attrs[attr_name] + + Group_001 = quality01Group.create_group("Group_001") + + if use_compound_type_for_Quality: + # As found in product 102DE00CA22_UNC_MD.H5 + data_struct_type = np.dtype([("iD", "u4")]) + values = Group_001.create_dataset("values", (2, 3), dtype=data_struct_type) + data = np.array( + [(0), (1), (2), (1000000), (3), (2)], + dtype=data_struct_type, + ).reshape(values.shape) + values[...] = data + else: + values = Group_001.create_dataset("values", (2, 3), dtype=np.uint32) + data = np.array( + [0, 1, 2, 1000000, 3, 2], + dtype=np.uint32, + ).reshape(values.shape) + values[...] = data featureAttributeTable_struct_type = np.dtype( [ @@ -121,7 +140,7 @@ def generate(filename, version, with_QualityOfSurvey=False): ] ) - featureAttributeTable = QualityOfSurvey.create_dataset( + featureAttributeTable = qualityGroup.create_dataset( "featureAttributeTable", (5,), dtype=featureAttributeTable_struct_type ) @@ -137,7 +156,7 @@ def generate(filename, version, with_QualityOfSurvey=False): ) featureAttributeTable[...] = data - GroupFQualityOfSurvey_struct_type = np.dtype( + GroupFQuality_struct_type = np.dtype( [ ("code", "S16"), ("name", "S16"), @@ -149,12 +168,12 @@ def generate(filename, version, with_QualityOfSurvey=False): ("closure", "S16"), ] ) - GroupFQualityOfSurvey = group_f.create_dataset( - "QualityOfSurvey", (1,), dtype=GroupFQualityOfSurvey_struct_type + GroupFQuality = group_f.create_dataset( + quality_name, (1,), dtype=GroupFQuality_struct_type ) - GroupFQualityOfSurvey[...] = np.array( + GroupFQuality[...] = np.array( [("id", "", "", "0", "H5T_INTEGER", "1", "", "geSemiInterval")], - dtype=GroupFQualityOfSurvey_struct_type, + dtype=GroupFQuality_struct_type, ) @@ -166,3 +185,10 @@ def generate(filename, version, with_QualityOfSurvey=False): "INT.IHO.S-102.2.2", with_QualityOfSurvey=True, ) + +generate( + "test_s102_v3.0_with_QualityOfBathymetryCoverage", + "INT.IHO.S-102.3.0.0", + with_QualityOfCoverage=True, + use_compound_type_for_Quality=True, +) diff --git a/autotest/gdrivers/data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5 b/autotest/gdrivers/data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8877ede735bb384a786e469c20ea7d59cc80ab02 GIT binary patch literal 17944 zcmeHOJ!~5{6nV~;-6VDr6v!3m;0}=$Cuvg@tuOwGWgv}R%Pq1L_E|bh2S|6q z>6BPq>e1airky-y%-AubQ4~ecDcv#^9y@kRNxt_iiSb!-+6=MD;dmq;zkB@dkw-rA ze$39?8SKB@4>F`u@D2>ATfV%caPbc2aGJ|k>BgdXi{i&Ucw@mJ$g+!+KScPOS1@CU zA=~9G7VI+4-CLNkpcmNgY_eO%)9u(M+@~Ml4a@COIz{6) zkHb3??Q*-6-n&6S$0V`@gRSELf5)C*#s=;YQJuztzXwsC-^Veb$26l$2J~@&=SAD} zeP9^;US(9@mwDU|9TqrZjN^c`Vhm&NVa<l&)%e9Pu&7UfhCr4;%-!FJaTOU5*1; znqctzIQGPu2DUMvj{`j4d_>;|7JlHC>t3zd|Fc~1nz23(WKJUxj|2B9y`kyxnvXlS z)hy`4F`0<7U`(~u>cI?Nme6zy^mH$5srnRvxuB?EvI@ zme(n=F54fkhnhGSG@Mqgl5u(@s0c~>GrGUJlVA4P4C1w~YB0}Sk3R#tKW2=YJVyH+ z%s{vMhx+P>>+3P^BW%-hX^1*Xv)}fse(Tq2b#KOvX*&}xS(fT!vY%lUv|R3IazlUn zLwuUd8-PWXu;2oY1728++-k|s&lkKP_Sbzck^-G@rF)ohWgi>>+>x^2jOxc?&o%b* zZds^WuZZ#?p%(U4sVVC!>tWFAm%_;ZJ`7^FJiU|?BC`HLnl zfQih+^{dw=u3jg5;pem4J{MNJhb8eeY(A~@LChL3h!uKzKD&_4UA>l>NMBE9(iv5N z90Pnua&ulFcF$w4V6Qc7w;qSFSBuAsg~%6gxHevJV|TnJAjXR%xFQA8)rLJ5gt1+V zBR?OP8}@p=EUq~rY3IXAwd}h={IPgOA3f(2iocps04ox6<$=&^<)I&!Ru1WI>5WQ@ z*WlS10y2?dz%XDKFbo(53GHFkl!k3>XI90tVP`y+yk6e-yg^rW%EXdOpro zy}a<<Ku#Wwv+r<7j^anTpl4W9HDFrM@w=y>8NzAK#t5=(|o z`bmEP<4GSul7Ck}XWYrZYtzB$F{y+JCyoK#zv~bKshjOo9OiJ6hjG_E;n8EeTEXLnP8EaH07=Mp!nTn(Hq>y^WY%K5X{rF^|w oRr0G{%IC{rO{|}x+NFH59rVAC5SwjC`Z$B9UE4%Vq6-H81y1;`+W-In literal 0 HcmV?d00001 diff --git a/autotest/gdrivers/s102.py b/autotest/gdrivers/s102.py index 3e3010202319..90e7374ee2a1 100755 --- a/autotest/gdrivers/s102.py +++ b/autotest/gdrivers/s102.py @@ -211,34 +211,40 @@ def test_s102_multidim(): ############################################################################### -def test_s102_QualityOfSurvey(): +@pytest.mark.parametrize( + "filename,quality_group_name", + [ + ("data/s102/test_s102_v2.2_with_QualityOfSurvey.h5", "QualityOfSurvey"), + ( + "data/s102/test_s102_v3.0_with_QualityOfBathymetryCoverage.h5", + "QualityOfBathymetryCoverage", + ), + ], +) +def test_s102_QualityOfSurvey(filename, quality_group_name): - ds = gdal.Open("data/s102/test_s102_v2.2_with_QualityOfSurvey.h5") + ds = gdal.Open(filename) assert ds.GetSubDatasets() == [ ( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":BathymetryCoverage', + f'S102:"{filename}":BathymetryCoverage', "Bathymetric gridded data", ), ( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":QualityOfSurvey', - "Georeferenced metadata QualityOfSurvey", + f'S102:"{filename}":{quality_group_name}', + f"Georeferenced metadata {quality_group_name}", ), ] with pytest.raises(Exception, match="Unsupported subdataset component"): - gdal.Open('S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":invalid') + gdal.Open(f'S102:"{filename}":invalid') - ds = gdal.Open( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":BathymetryCoverage' - ) + ds = gdal.Open(f'S102:"{filename}":BathymetryCoverage') assert len(ds.GetSubDatasets()) == 0 assert ds.RasterCount == 2 assert ds.RasterXSize == 3 assert ds.RasterYSize == 2 - ds = gdal.Open( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":QualityOfSurvey' - ) + ds = gdal.Open(f'S102:"{filename}":{quality_group_name}') assert len(ds.GetSubDatasets()) == 0 assert ds.RasterCount == 1 assert ds.RasterXSize == 3 @@ -278,7 +284,7 @@ def test_s102_QualityOfSurvey(): assert rat.GetValueAsString(4, 2) == "e" ds = gdal.OpenEx( - 'S102:"data/s102/test_s102_v2.2_with_QualityOfSurvey.h5":QualityOfSurvey', + f'S102:"{filename}":{quality_group_name}', open_options=["NORTH_UP=NO"], ) assert ds.GetGeoTransform() == pytest.approx((1.8, 0.4, 0.0, 47.75, 0.0, 0.5)) diff --git a/doc/source/drivers/raster/s102.rst b/doc/source/drivers/raster/s102.rst index 85ccbd6e397c..02663798c46d 100644 --- a/doc/source/drivers/raster/s102.rst +++ b/doc/source/drivers/raster/s102.rst @@ -11,7 +11,7 @@ S102 -- S-102 Bathymetric Surface Product .. versionadded:: 3.8 This driver provides read-only support for bathymetry data in the S-102 format, -which is a specific product profile in an HDF5 file +which is a specific product profile in an HDF5 file. S-102 files have two image bands representing depth (band 1), uncertainty (band 2) values for each cell in a raster grid area. @@ -25,6 +25,9 @@ Georeferencing is reported. Nodata, minimum and maximum values for each band are also reported. +Supported versions of the specification are S-102 v2.1, v2.2 and v3.0 +(support for v3.0 spatial metadata added in GDAL 3.10) + Driver capabilities ------------------- @@ -63,12 +66,16 @@ The following open options are supported: Spatial metadata support ------------------------ -Starting with GDAL 3.9, GDAL can handle QualityOfSurvey spatial metadata. +Starting with GDAL 3.9, GDAL can handle QualityOfSurvey +(or QualityOfBathymetryCoverage in S102 v3.0) spatial metadata. When such spatial metadata is present, the subdataset list will include -a name of the form ``S102:"{filename}":QualityOfSurvey`` +a name of the form ``S102:"{filename}":QualityOfSurvey`` ( +or ``S102:"{filename}":QualityOfBathymetryCoverage`` in S102 v3.0) -The ``/QualityOfSurvey/featureAttributeTable`` dataset is exposed as a +The ``/QualityOfSurvey/featureAttributeTable`` +(``/QualityOfBathymetryCoverage/featureAttributeTable`` in S102 v3.0) +dataset is exposed as a GDAL Raster Attribute Table associated to the GDAL raster band. The pixel values of the raster match the ``id`` column of the Raster Attribute Table. diff --git a/frmts/hdf5/hdf5multidim.cpp b/frmts/hdf5/hdf5multidim.cpp index f5a0704c458f..5d8b2a1d64cc 100644 --- a/frmts/hdf5/hdf5multidim.cpp +++ b/frmts/hdf5/hdf5multidim.cpp @@ -1070,17 +1070,28 @@ HDF5Array::HDF5Array(const std::string &osParentName, const std::string &osName, } // Special case for S102 QualityOfSurvey nodata value that is typically at 0 - if (GetFullName() == - "/QualityOfSurvey/QualityOfSurvey.01/Group_001/values" && - m_dt.GetClass() == GEDTC_NUMERIC && - m_dt.GetNumericDataType() == GDT_UInt32) + const bool bIsQualityOfSurvey = + (GetFullName() == + "/QualityOfSurvey/QualityOfSurvey.01/Group_001/values"); + const bool bIsQualityOfBathymetryCoverage = + (GetFullName() == "/QualityOfBathymetryCoverage/" + "QualityOfBathymetryCoverage.01/Group_001/values"); + if ((bIsQualityOfSurvey || bIsQualityOfBathymetryCoverage) && + ((m_dt.GetClass() == GEDTC_NUMERIC && + m_dt.GetNumericDataType() == GDT_UInt32) || + (m_dt.GetClass() == GEDTC_COMPOUND && + m_dt.GetComponents().size() == 1 && + m_dt.GetComponents()[0]->GetType().GetClass() == GEDTC_NUMERIC && + m_dt.GetComponents()[0]->GetType().GetNumericDataType() == + GDT_UInt32))) { if (auto poRootGroup = HDF5Array::GetRootGroup()) { if (const auto poGroupF = poRootGroup->OpenGroup("Group_F")) { - const auto poGroupFArray = - poGroupF->OpenMDArray("QualityOfSurvey"); + const auto poGroupFArray = poGroupF->OpenMDArray( + bIsQualityOfSurvey ? "QualityOfSurvey" + : "QualityOfBathymetryCoverage"); if (poGroupFArray && poGroupFArray->GetDataType().GetClass() == GEDTC_COMPOUND && poGroupFArray->GetDataType().GetComponents().size() == 8 && diff --git a/frmts/hdf5/s102dataset.cpp b/frmts/hdf5/s102dataset.cpp index fb84cc49cbb1..563ca20ce1f2 100644 --- a/frmts/hdf5/s102dataset.cpp +++ b/frmts/hdf5/s102dataset.cpp @@ -46,8 +46,8 @@ class S102Dataset final : public S100BaseDataset { - bool OpenQualityOfSurvey(GDALOpenInfo *poOpenInfo, - const std::shared_ptr &poRootGroup); + bool OpenQuality(GDALOpenInfo *poOpenInfo, + const std::shared_ptr &poRootGroup); public: explicit S102Dataset(const std::string &osFilename) @@ -169,7 +169,7 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) std::string osFilename(poOpenInfo->pszFilename); bool bIsSubdataset = false; - bool bIsQualityOfSurvey = false; + bool bIsQuality = false; if (STARTS_WITH(poOpenInfo->pszFilename, "S102:")) { const CPLStringList aosTokens( @@ -188,9 +188,10 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) { // Default dataset } - else if (EQUAL(aosTokens[2], "QualityOfSurvey")) + else if (EQUAL(aosTokens[2], "QualityOfSurvey") || // < v3 + EQUAL(aosTokens[2], "QualityOfBathymetryCoverage")) // v3 { - bIsQualityOfSurvey = true; + bIsQuality = true; } else { @@ -220,9 +221,9 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) const bool bNorthUp = CPLTestBool( CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, "NORTH_UP", "YES")); - if (bIsQualityOfSurvey) + if (bIsQuality) { - if (!poDS->OpenQualityOfSurvey(poOpenInfo, poRootGroup)) + if (!poDS->OpenQuality(poOpenInfo, poRootGroup)) return nullptr; // Setup/check for pam .aux.xml. @@ -376,12 +377,21 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) poDS->GDALDataset::SetMetadataItem(GDALMD_AREA_OR_POINT, GDALMD_AOP_POINT); - auto poGroupQualityOfSurvey = poRootGroup->OpenGroup("QualityOfSurvey"); - if (!bIsSubdataset && poGroupQualityOfSurvey) + auto poGroupQuality = poRootGroup->OpenGroup("QualityOfSurvey"); + const bool bIsNamedQualityOfSurvey = poGroupQuality != nullptr; + if (!bIsNamedQualityOfSurvey) { - auto poGroupQualityOfSurvey01 = - poGroupQualityOfSurvey->OpenGroup("QualityOfSurvey.01"); - if (poGroupQualityOfSurvey01) + // S102 v3 now uses QualityOfBathymetryCoverage instead of QualityOfSurvey + poGroupQuality = poRootGroup->OpenGroup("QualityOfBathymetryCoverage"); + } + if (!bIsSubdataset && poGroupQuality) + { + const char *pszNameOfQualityGroup = bIsNamedQualityOfSurvey + ? "QualityOfSurvey" + : "QualityOfBathymetryCoverage"; + auto poGroupQuality01 = poGroupQuality->OpenGroup( + CPLSPrintf("%s.01", pszNameOfQualityGroup)); + if (poGroupQuality01) { poDS->GDALDataset::SetMetadataItem( "SUBDATASET_1_NAME", @@ -393,10 +403,12 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) poDS->GDALDataset::SetMetadataItem( "SUBDATASET_2_NAME", - CPLSPrintf("S102:\"%s\":QualityOfSurvey", osFilename.c_str()), + CPLSPrintf("S102:\"%s\":%s", osFilename.c_str(), + pszNameOfQualityGroup), "SUBDATASETS"); poDS->GDALDataset::SetMetadataItem( - "SUBDATASET_2_DESC", "Georeferenced metadata QualityOfSurvey", + "SUBDATASET_2_DESC", + CPLSPrintf("Georeferenced metadata %s", pszNameOfQualityGroup), "SUBDATASETS"); } } @@ -412,34 +424,45 @@ GDALDataset *S102Dataset::Open(GDALOpenInfo *poOpenInfo) } /************************************************************************/ -/* OpenQualityOfSurvey() */ +/* OpenQuality() */ /************************************************************************/ -bool S102Dataset::OpenQualityOfSurvey( - GDALOpenInfo *poOpenInfo, const std::shared_ptr &poRootGroup) +bool S102Dataset::OpenQuality(GDALOpenInfo *poOpenInfo, + const std::shared_ptr &poRootGroup) { const bool bNorthUp = CPLTestBool( CSLFetchNameValueDef(poOpenInfo->papszOpenOptions, "NORTH_UP", "YES")); - auto poGroupQualityOfSurvey = poRootGroup->OpenGroup("QualityOfSurvey"); - if (!poGroupQualityOfSurvey) + const char *pszNameOfQualityGroup = "QualityOfSurvey"; + auto poGroupQuality = poRootGroup->OpenGroup(pszNameOfQualityGroup); + if (!poGroupQuality) { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find group /QualityOfSurvey"); - return false; + pszNameOfQualityGroup = "QualityOfBathymetryCoverage"; + poGroupQuality = poRootGroup->OpenGroup(pszNameOfQualityGroup); + if (!poGroupQuality) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot find group /QualityOfSurvey or " + "/QualityOfBathymetryCoverage"); + return false; + } } - auto poGroupQualityOfSurvey01 = - poGroupQualityOfSurvey->OpenGroup("QualityOfSurvey.01"); - if (!poGroupQualityOfSurvey01) + const std::string osQuality01Name = + std::string(pszNameOfQualityGroup).append(".01"); + const std::string osQuality01FullName = std::string("/") + .append(pszNameOfQualityGroup) + .append("/") + .append(osQuality01Name); + auto poGroupQuality01 = poGroupQuality->OpenGroup(osQuality01Name); + if (!poGroupQuality01) { - CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find group /QualityOfSurvey/QualityOfSurvey.01"); + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find group %s", + osQuality01FullName.c_str()); return false; } - if (auto poStartSequence = - poGroupQualityOfSurvey01->GetAttribute("startSequence")) + if (auto poStartSequence = poGroupQuality01->GetAttribute("startSequence")) { const char *pszStartSequence = poStartSequence->ReadAsString(); if (pszStartSequence && !EQUAL(pszStartSequence, "0,0")) @@ -452,15 +475,14 @@ bool S102Dataset::OpenQualityOfSurvey( } // Compute geotransform - m_bHasGT = S100GetGeoTransform(poGroupQualityOfSurvey01.get(), - m_adfGeoTransform, bNorthUp); + m_bHasGT = S100GetGeoTransform(poGroupQuality01.get(), m_adfGeoTransform, + bNorthUp); - auto poGroup001 = poGroupQualityOfSurvey01->OpenGroup("Group_001"); + auto poGroup001 = poGroupQuality01->OpenGroup("Group_001"); if (!poGroup001) { - CPLError( - CE_Failure, CPLE_AppDefined, - "Cannot find group /QualityOfSurvey/QualityOfSurvey.01/Group_001"); + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find group %s/Group_001", + osQuality01FullName.c_str()); return false; } @@ -469,14 +491,42 @@ bool S102Dataset::OpenQualityOfSurvey( { CPLError(CE_Failure, CPLE_AppDefined, "Cannot find array " - "/QualityOfSurvey/QualityOfSurvey.01/Group_001/values"); + "%s/Group_001/values", + osQuality01FullName.c_str()); return false; } { const auto &oType = poValuesArray->GetDataType(); - if (oType.GetClass() != GEDTC_NUMERIC && - oType.GetNumericDataType() != GDT_UInt32) + if (oType.GetClass() == GEDTC_NUMERIC && + oType.GetNumericDataType() == GDT_UInt32) + { + // ok + } + else if (oType.GetClass() == GEDTC_COMPOUND && + oType.GetComponents().size() == 1 && + oType.GetComponents()[0]->GetType().GetClass() == + GEDTC_NUMERIC && + oType.GetComponents()[0]->GetType().GetNumericDataType() == + GDT_UInt32) + { + // seen in a S102 v3 product (102DE00CA22_UNC_MD.H5), although + // I believe this is non-conformant. + + // Escape potentials single-quote and double-quote with back-slash + const auto osEscapedCompName = + CPLString(oType.GetComponents()[0]->GetName()) + .replaceAll("\\", "\\\\") + .replaceAll("'", "\\'") + .replaceAll("\"", "\\\""); + + // Gets a view with that single component extracted. + poValuesArray = poValuesArray->GetView( + std::string("['").append(osEscapedCompName).append("']")); + if (!poValuesArray) + return false; + } + else { CPLError(CE_Failure, CPLE_NotSupported, "Unsupported data type for %s", @@ -494,11 +544,12 @@ bool S102Dataset::OpenQualityOfSurvey( } auto poFeatureAttributeTable = - poGroupQualityOfSurvey->OpenMDArray("featureAttributeTable"); + poGroupQuality->OpenMDArray("featureAttributeTable"); if (!poFeatureAttributeTable) { CPLError(CE_Failure, CPLE_AppDefined, - "Cannot find array /QualityOfSurvey/featureAttributeTable"); + "Cannot find array /%s/featureAttributeTable", + pszNameOfQualityGroup); return false; } @@ -527,6 +578,8 @@ bool S102Dataset::OpenQualityOfSurvey( auto poDS = std::unique_ptr(poValuesArray->AsClassicDataset(1, 0)); + if (!poDS) + return false; nRasterXSize = poDS->GetRasterXSize(); nRasterYSize = poDS->GetRasterYSize(); From 596f7d26fb10d27e0c12b9135c534f99f9f94420 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 19:30:51 +0200 Subject: [PATCH 099/710] VRTSourcedRasterBand::IRasterIO(): initialize output buffer to nodata value if VRT band type is Int64 or UInt64 --- autotest/gcore/vrt_read.py | 106 ++++++++++++++++++----------- frmts/vrt/vrtsourcedrasterband.cpp | 24 ++++++- 2 files changed, 91 insertions(+), 39 deletions(-) diff --git a/autotest/gcore/vrt_read.py b/autotest/gcore/vrt_read.py index 2c4d319f4e33..9ef974eac620 100755 --- a/autotest/gcore/vrt_read.py +++ b/autotest/gcore/vrt_read.py @@ -2493,30 +2493,68 @@ def test_vrt_read_top_and_bottom_strips_average(): @pytest.mark.parametrize( - "input_datatype", [gdal.GDT_Byte, gdal.GDT_UInt16, gdal.GDT_Int16] -) -@pytest.mark.parametrize("vrt_type", ["Byte", "UInt16", "Int16"]) -@pytest.mark.parametrize("nodata", [0, 254]) -@pytest.mark.parametrize( - "request_type", [gdal.GDT_Byte, gdal.GDT_UInt16, gdal.GDT_Int16] + "input_datatype,vrt_type,nodata,vrt_nodata,request_type", + [ + (gdal.GDT_Byte, "Byte", 0, 255, gdal.GDT_Byte), + (gdal.GDT_Byte, "Byte", 254, 255, gdal.GDT_Byte), + (gdal.GDT_Byte, "Int8", 254, 255, gdal.GDT_Byte), + (gdal.GDT_Byte, "Byte", 254, 127, gdal.GDT_Int8), + (gdal.GDT_Byte, "UInt16", 254, 255, gdal.GDT_Byte), + (gdal.GDT_Byte, "Byte", 254, 255, gdal.GDT_UInt16), + (gdal.GDT_Int8, "Int8", 0, 127, gdal.GDT_Int8), + (gdal.GDT_Int8, "Int16", 0, 127, gdal.GDT_Int8), + (gdal.GDT_UInt16, "UInt16", 0, 65535, gdal.GDT_UInt16), + (gdal.GDT_Int16, "Int16", 0, 32767, gdal.GDT_Int16), + (gdal.GDT_UInt32, "UInt32", 0, (1 << 31) - 1, gdal.GDT_UInt32), + (gdal.GDT_Int32, "Int32", 0, (1 << 30) - 1, gdal.GDT_Int32), + (gdal.GDT_Int32, "Float32", 0, (1 << 30) - 1, gdal.GDT_Float64), + (gdal.GDT_UInt64, "UInt64", 0, (1 << 63) - 1, gdal.GDT_UInt64), + (gdal.GDT_Int64, "Int64", 0, (1 << 62) - 1, gdal.GDT_Int64), + (gdal.GDT_Int64, "Float32", 0, (1 << 62), gdal.GDT_Int64), + (gdal.GDT_Float32, "Float32", 0, 1.5, gdal.GDT_Float32), + (gdal.GDT_Float32, "Float32", 0, 1.5, gdal.GDT_Float64), + (gdal.GDT_Float32, "Float64", 0, 1.5, gdal.GDT_Float32), + (gdal.GDT_Float32, "Float64", 0, 1.5, gdal.GDT_Float64), + (gdal.GDT_Float64, "Float64", 0, 1.5, gdal.GDT_Float64), + (gdal.GDT_Float64, "Float32", 0, 1.5, gdal.GDT_Float64), + ], ) def test_vrt_read_complex_source_nodata( - tmp_vsimem, input_datatype, vrt_type, nodata, request_type + tmp_vsimem, input_datatype, vrt_type, nodata, vrt_nodata, request_type ): - - if input_datatype == gdal.GDT_Byte: - array_type = "B" - elif input_datatype == gdal.GDT_UInt16: - array_type = "H" - elif input_datatype == gdal.GDT_Int16: - array_type = "h" + def get_array_type(dt): + m = { + gdal.GDT_Byte: "B", + gdal.GDT_Int8: "b", + gdal.GDT_UInt16: "H", + gdal.GDT_Int16: "h", + gdal.GDT_UInt32: "I", + gdal.GDT_Int32: "i", + gdal.GDT_UInt64: "Q", + gdal.GDT_Int64: "q", + gdal.GDT_Float32: "f", + gdal.GDT_Float64: "d", + } + return m[dt] + + if input_datatype in (gdal.GDT_Float32, gdal.GDT_Float64): + input_val = 1.75 + if vrt_type in ("Float32", "Float64") and request_type in ( + gdal.GDT_Float32, + gdal.GDT_Float64, + ): + expected_val = input_val + else: + expected_val = math.round(input_val) else: - assert False + input_val = 1 + expected_val = input_val + input_data = array.array( - array_type, + get_array_type(input_datatype), [ nodata, - 1, + input_val, 2, 3, nodata, # EOL @@ -2553,7 +2591,7 @@ def test_vrt_read_complex_source_nodata( ds.Close() complex_xml = f""" - 255 + {vrt_nodata} {input_filename} 1 @@ -2563,24 +2601,16 @@ def test_vrt_read_complex_source_nodata( """ vrt_ds = gdal.Open(complex_xml) - if request_type == gdal.GDT_Byte: - array_request_type = "B" - elif request_type == gdal.GDT_UInt16: - array_request_type = "H" - elif request_type == gdal.GDT_Int16: - array_request_type = "h" - else: - assert False got_data = vrt_ds.ReadRaster(buf_type=request_type) - got_data = struct.unpack(array_request_type * (5 * 6), got_data) + got_data = struct.unpack(get_array_type(request_type) * (5 * 6), got_data) assert got_data == ( - 255, - 1, + vrt_nodata, + expected_val, 2, 3, - 255, # EOL + vrt_nodata, # EOL 4, - 255, + vrt_nodata, 5, 6, 7, # EOL @@ -2591,18 +2621,18 @@ def test_vrt_read_complex_source_nodata( 20, # EOL 8, 9, - 255, + vrt_nodata, 10, 11, # EOL - 255, - 255, - 255, - 255, - 255, # EOL + vrt_nodata, + vrt_nodata, + vrt_nodata, + vrt_nodata, + vrt_nodata, # EOL 12, 13, 14, - 255, + vrt_nodata, 15, # EOL ) diff --git a/frmts/vrt/vrtsourcedrasterband.cpp b/frmts/vrt/vrtsourcedrasterband.cpp index 5e493e05ccf8..14184affe434 100644 --- a/frmts/vrt/vrtsourcedrasterband.cpp +++ b/frmts/vrt/vrtsourcedrasterband.cpp @@ -420,7 +420,9 @@ CPLErr VRTSourcedRasterBand::IRasterIO( // Do nothing } else if (nPixelSpace == GDALGetDataTypeSizeBytes(eBufType) && - (!m_bNoDataValueSet || m_dfNoDataValue == 0.0)) + !(m_bNoDataValueSet && m_dfNoDataValue != 0.0) && + !(m_bNoDataSetAsInt64 && m_nNoDataValueInt64 != 0) && + !(m_bNoDataSetAsUInt64 && m_nNoDataValueUInt64 != 0)) { if (nLineSpace == nBufXSize * nPixelSpace) { @@ -436,6 +438,26 @@ CPLErr VRTSourcedRasterBand::IRasterIO( } } } + else if (m_bNoDataSetAsInt64) + { + for (int iLine = 0; iLine < nBufYSize; iLine++) + { + GDALCopyWords(&m_nNoDataValueInt64, GDT_Int64, 0, + static_cast(pData) + + static_cast(nLineSpace) * iLine, + eBufType, static_cast(nPixelSpace), nBufXSize); + } + } + else if (m_bNoDataSetAsUInt64) + { + for (int iLine = 0; iLine < nBufYSize; iLine++) + { + GDALCopyWords(&m_nNoDataValueUInt64, GDT_UInt64, 0, + static_cast(pData) + + static_cast(nLineSpace) * iLine, + eBufType, static_cast(nPixelSpace), nBufXSize); + } + } else { double dfWriteValue = 0.0; From 47bc43b8c5bd39a473d6deea3676203126b91165 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 19:33:24 +0200 Subject: [PATCH 100/710] VRTComplexSource::RasterIO(): use double working data type for more source or VRT data types --- frmts/vrt/vrtsources.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frmts/vrt/vrtsources.cpp b/frmts/vrt/vrtsources.cpp index e9a72f05d448..26a5e468d58a 100644 --- a/frmts/vrt/vrtsources.cpp +++ b/frmts/vrt/vrtsources.cpp @@ -2986,10 +2986,10 @@ CPLErr VRTComplexSource::RasterIO(GDALDataType eVRTBandDataType, int nXOff, GByte *const pabyOut = static_cast(pData) + nPixelSpace * nOutXOff + static_cast(nLineSpace) * nOutYOff; + const auto eSourceType = poSourceBand->GetRasterDataType(); if (m_nProcessingFlags == PROCESSING_FLAG_NODATA) { // Optimization if doing only nodata processing - const auto eSourceType = poSourceBand->GetRasterDataType(); if (eSourceType == GDT_Byte) { if (!GDALIsValueInRange(m_dfNoDataValue)) @@ -3043,7 +3043,10 @@ CPLErr VRTComplexSource::RasterIO(GDALDataType eVRTBandDataType, int nXOff, // For Int32, float32 isn't sufficiently precise as working data type if (eVRTBandDataType == GDT_CInt32 || eVRTBandDataType == GDT_CFloat64 || eVRTBandDataType == GDT_Int32 || eVRTBandDataType == GDT_UInt32 || - eVRTBandDataType == GDT_Float64) + eVRTBandDataType == GDT_Int64 || eVRTBandDataType == GDT_UInt64 || + eVRTBandDataType == GDT_Float64 || eSourceType == GDT_Int32 || + eSourceType == GDT_UInt32 || eSourceType == GDT_Int64 || + eSourceType == GDT_UInt64) { eErr = RasterIOInternal( poSourceBand, eVRTBandDataType, nReqXOff, nReqYOff, nReqXSize, From 99a17e54d64a6f252ad2691afeabc038be626ae0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 19:34:04 +0200 Subject: [PATCH 101/710] VRTComplexSource::RasterIO(): speed-up pixel copy in more cases Refs #10809 --- frmts/vrt/vrtsources.cpp | 80 +++++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 9 deletions(-) diff --git a/frmts/vrt/vrtsources.cpp b/frmts/vrt/vrtsources.cpp index 26a5e468d58a..bc7a8f6b9a37 100644 --- a/frmts/vrt/vrtsources.cpp +++ b/frmts/vrt/vrtsources.cpp @@ -3080,6 +3080,52 @@ static inline bool hasZeroByte(uint32_t v) return (((v)-0x01010101U) & ~(v)&0x80808080U) != 0; } +/************************************************************************/ +/* CopyWord() */ +/************************************************************************/ + +template +static void CopyWord(const SrcType *pSrcVal, GDALDataType eSrcType, + void *pDstVal, GDALDataType eDstType) +{ + switch (eDstType) + { + case GDT_Byte: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_Int8: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_UInt16: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_Int16: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_UInt32: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_Int32: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_UInt64: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_Int64: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_Float32: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + case GDT_Float64: + GDALCopyWord(*pSrcVal, *static_cast(pDstVal)); + break; + default: + GDALCopyWords(pSrcVal, eSrcType, 0, pDstVal, eDstType, 0, 1); + break; + } +} + /************************************************************************/ /* RasterIOProcessNoData() */ /************************************************************************/ @@ -3235,8 +3281,8 @@ CPLErr VRTComplexSource::RasterIOProcessNoData( { if (paSrcData[idxBuffer] != nNoDataValue) { - GDALCopyWords(&paSrcData[idxBuffer], eSourceType, 0, - pDstLocation, eBufType, 0, 1); + CopyWord(&paSrcData[idxBuffer], eSourceType, pDstLocation, + eBufType); } } } @@ -3256,8 +3302,8 @@ CPLErr VRTComplexSource::RasterIOProcessNoData( { // Convert first to the VRTRasterBand data type // to get its clamping, before outputting to buffer data type - GDALCopyWords(&paSrcData[idxBuffer], eSourceType, 0, - abyTemp, eVRTBandDataType, 0, 1); + CopyWord(&paSrcData[idxBuffer], eSourceType, abyTemp, + eVRTBandDataType); GDALCopyWords(abyTemp, eVRTBandDataType, 0, pDstLocation, eBufType, 0, 1); } @@ -3415,6 +3461,12 @@ CPLErr VRTComplexSource::RasterIOInternal( /* Selectively copy into output buffer with nodata masking, */ /* and/or scaling. */ /* -------------------------------------------------------------------- */ + + const bool bTwoStepDataTypeConversion = + CPL_TO_BOOL( + GDALDataTypeIsConversionLossy(eWrkDataType, eVRTBandDataType)) && + !CPL_TO_BOOL(GDALDataTypeIsConversionLossy(eVRTBandDataType, eBufType)); + size_t idxBuffer = 0; for (int iY = 0; iY < nOutYSize; iY++) { @@ -3551,18 +3603,28 @@ CPLErr VRTComplexSource::RasterIOInternal( 255.0f, std::max(0.0f, static_cast(afResult[0]) + 0.5f))); } - else if (eBufType == eVRTBandDataType) + else if (!bTwoStepDataTypeConversion) + { + CopyWord(afResult, eWrkDataType, pDstLocation, + eBufType); + } + else if (eVRTBandDataType == GDT_Float32 && eBufType == GDT_Float64) { - GDALCopyWords(afResult, eWrkDataType, 0, pDstLocation, eBufType, - 0, 1); + // Particular case of the below 2-step conversion. + // Helps a bit for some geolocation based warping with Sentinel3 + // data where the longitude/latitude arrays are Int32 bands, + // rescaled in VRT as Float32 and requested as Float64 + float fVal; + GDALCopyWord(afResult[0], fVal); + *reinterpret_cast(pDstLocation) = fVal; } else { GByte abyTemp[2 * sizeof(double)]; // Convert first to the VRTRasterBand data type // to get its clamping, before outputting to buffer data type - GDALCopyWords(afResult, eWrkDataType, 0, abyTemp, - eVRTBandDataType, 0, 1); + CopyWord(afResult, eWrkDataType, abyTemp, + eVRTBandDataType); GDALCopyWords(abyTemp, eVRTBandDataType, 0, pDstLocation, eBufType, 0, 1); } From b8e608f78a2945d42750e535149666534c67698d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 19:35:34 +0200 Subject: [PATCH 102/710] GDALCreateGeoLocTransformer(): fix inverted logic to decide for a debug message Refs #10809 --- alg/gdalgeoloc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alg/gdalgeoloc.cpp b/alg/gdalgeoloc.cpp index d053169304de..2fca72750ab5 100644 --- a/alg/gdalgeoloc.cpp +++ b/alg/gdalgeoloc.cpp @@ -2007,7 +2007,7 @@ void *GDALCreateGeoLocTransformerEx(GDALDatasetH hBaseDS, else { psTransform->bUseArray = nXSize < 16 * 1000 * 1000 / nYSize; - if (psTransform->bUseArray) + if (!psTransform->bUseArray) { CPLDebug("GEOLOC", "Using temporary GTiff backing to store backmap, because " From a93a76cd4e91322e74cf321a7cd7f0c9ec763d05 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 19:37:12 +0200 Subject: [PATCH 103/710] GDALCreateGeoLocTransformer(): increase threshold to use GTiff geoloc working datasets to 24 megapixels Fixes #10809 --- alg/gdalgeoloc.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/alg/gdalgeoloc.cpp b/alg/gdalgeoloc.cpp index 2fca72750ab5..3d82718c8234 100644 --- a/alg/gdalgeoloc.cpp +++ b/alg/gdalgeoloc.cpp @@ -2006,14 +2006,20 @@ void *GDALCreateGeoLocTransformerEx(GDALDatasetH hBaseDS, psTransform->bUseArray = !CPLTestBool(pszUseTempDatasets); else { - psTransform->bUseArray = nXSize < 16 * 1000 * 1000 / nYSize; + constexpr int MEGAPIXEL_LIMIT = 24; + psTransform->bUseArray = + nXSize < MEGAPIXEL_LIMIT * 1000 * 1000 / nYSize; if (!psTransform->bUseArray) { CPLDebug("GEOLOC", "Using temporary GTiff backing to store backmap, because " - "geoloc arrays exceed 16 megapixels. You can set the " + "geoloc arrays require %d megapixels, exceeding the %d " + "megapixels limit. You can set the " "GDAL_GEOLOC_USE_TEMP_DATASETS configuration option to " - "NO to force RAM storage of backmap"); + "NO to force RAM storage of backmap", + static_cast(static_cast(nXSize) * nYSize / + (1000 * 1000)), + MEGAPIXEL_LIMIT); } } From 642f7ccf71208d9fbc292185bc3243a38914592f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 19:37:59 +0200 Subject: [PATCH 104/710] GDALGeoLocDatasetAccessors: use smaller, but more, cached tiles Fixes #10809 --- alg/gdalgeoloc_dataset_accessor.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/alg/gdalgeoloc_dataset_accessor.h b/alg/gdalgeoloc_dataset_accessor.h index d78fd0334289..432346f20e77 100644 --- a/alg/gdalgeoloc_dataset_accessor.h +++ b/alg/gdalgeoloc_dataset_accessor.h @@ -53,13 +53,14 @@ class GDALGeoLocDatasetAccessors bool LoadGeoloc(bool bIsRegularGrid); public: - static constexpr int TILE_SIZE = 1024; - - GDALCachedPixelAccessor geolocXAccessor; - GDALCachedPixelAccessor geolocYAccessor; - GDALCachedPixelAccessor backMapXAccessor; - GDALCachedPixelAccessor backMapYAccessor; - GDALCachedPixelAccessor backMapWeightAccessor; + static constexpr int TILE_SIZE = 256; + static constexpr int TILE_COUNT = 64; + + GDALCachedPixelAccessor geolocXAccessor; + GDALCachedPixelAccessor geolocYAccessor; + GDALCachedPixelAccessor backMapXAccessor; + GDALCachedPixelAccessor backMapYAccessor; + GDALCachedPixelAccessor backMapWeightAccessor; explicit GDALGeoLocDatasetAccessors(GDALGeoLocTransformInfo *psTransform) : m_psTransform(psTransform), geolocXAccessor(nullptr), From 37777a309413da0491d1948bbcbf14d3a971d61c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 17 Sep 2024 00:57:45 +0200 Subject: [PATCH 105/710] Parquet: fix crash when using SetIgnoredFields() + SetSpatialFilter() on GEOMETRY_ENCODING=GEOARROW layers with a covering bounding box Fixes qgis/QGIS#58086 --- autotest/ogr/ogr_parquet.py | 2 + .../arrow_common/ograrrowlayer.hpp | 113 +++++++++--------- ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp | 3 +- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/autotest/ogr/ogr_parquet.py b/autotest/ogr/ogr_parquet.py index 2563010535ac..0079455069cb 100755 --- a/autotest/ogr/ogr_parquet.py +++ b/autotest/ogr/ogr_parquet.py @@ -3870,6 +3870,8 @@ def check(lyr): lyr = ds.GetLayer(0) lyr.SetIgnoredFields(["foo"]) check(lyr) + lyr.SetSpatialFilter(geom) + assert lyr.GetFeatureCount() == (3 if geom.GetGeometryCount() > 1 else 2) ds = ogr.Open(filename_to_open) lyr = ds.GetLayer(0) diff --git a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp index 084a7273acea..be491b9faab5 100644 --- a/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp +++ b/ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp @@ -3919,62 +3919,65 @@ OGRArrowLayer::SetBatch(const std::shared_ptr &poBatch) { const int idx = m_bIgnoredFields ? oIter->second.iArrayIdx : oIter->second.iArrowCol; - CPLAssert(idx >= 0); - CPLAssert(static_cast(idx) < m_poBatchColumns.size()); - m_poArrayBBOX = m_poBatchColumns[idx].get(); - CPLAssert(m_poArrayBBOX->type_id() == arrow::Type::STRUCT); - const auto castArray = - static_cast(m_poArrayBBOX); - const auto &subArrays = castArray->fields(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldXMin) < - subArrays.size()); - const auto xminArray = - subArrays[oIter->second.iArrowSubfieldXMin].get(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldYMin) < - subArrays.size()); - const auto yminArray = - subArrays[oIter->second.iArrowSubfieldYMin].get(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldXMax) < - subArrays.size()); - const auto xmaxArray = - subArrays[oIter->second.iArrowSubfieldXMax].get(); - CPLAssert( - static_cast(oIter->second.iArrowSubfieldYMax) < - subArrays.size()); - const auto ymaxArray = - subArrays[oIter->second.iArrowSubfieldYMax].get(); - if (oIter->second.bIsFloat) + if (idx >= 0) { - CPLAssert(xminArray->type_id() == arrow::Type::FLOAT); - m_poArrayXMinFloat = - static_cast(xminArray); - CPLAssert(yminArray->type_id() == arrow::Type::FLOAT); - m_poArrayYMinFloat = - static_cast(yminArray); - CPLAssert(xmaxArray->type_id() == arrow::Type::FLOAT); - m_poArrayXMaxFloat = - static_cast(xmaxArray); - CPLAssert(ymaxArray->type_id() == arrow::Type::FLOAT); - m_poArrayYMaxFloat = - static_cast(ymaxArray); - } - else - { - CPLAssert(xminArray->type_id() == arrow::Type::DOUBLE); - m_poArrayXMinDouble = - static_cast(xminArray); - CPLAssert(yminArray->type_id() == arrow::Type::DOUBLE); - m_poArrayYMinDouble = - static_cast(yminArray); - CPLAssert(xmaxArray->type_id() == arrow::Type::DOUBLE); - m_poArrayXMaxDouble = - static_cast(xmaxArray); - CPLAssert(ymaxArray->type_id() == arrow::Type::DOUBLE); - m_poArrayYMaxDouble = - static_cast(ymaxArray); + CPLAssert(static_cast(idx) < + m_poBatchColumns.size()); + m_poArrayBBOX = m_poBatchColumns[idx].get(); + CPLAssert(m_poArrayBBOX->type_id() == arrow::Type::STRUCT); + const auto castArray = + static_cast(m_poArrayBBOX); + const auto &subArrays = castArray->fields(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldXMin) < + subArrays.size()); + const auto xminArray = + subArrays[oIter->second.iArrowSubfieldXMin].get(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldYMin) < + subArrays.size()); + const auto yminArray = + subArrays[oIter->second.iArrowSubfieldYMin].get(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldXMax) < + subArrays.size()); + const auto xmaxArray = + subArrays[oIter->second.iArrowSubfieldXMax].get(); + CPLAssert( + static_cast(oIter->second.iArrowSubfieldYMax) < + subArrays.size()); + const auto ymaxArray = + subArrays[oIter->second.iArrowSubfieldYMax].get(); + if (oIter->second.bIsFloat) + { + CPLAssert(xminArray->type_id() == arrow::Type::FLOAT); + m_poArrayXMinFloat = + static_cast(xminArray); + CPLAssert(yminArray->type_id() == arrow::Type::FLOAT); + m_poArrayYMinFloat = + static_cast(yminArray); + CPLAssert(xmaxArray->type_id() == arrow::Type::FLOAT); + m_poArrayXMaxFloat = + static_cast(xmaxArray); + CPLAssert(ymaxArray->type_id() == arrow::Type::FLOAT); + m_poArrayYMaxFloat = + static_cast(ymaxArray); + } + else + { + CPLAssert(xminArray->type_id() == arrow::Type::DOUBLE); + m_poArrayXMinDouble = + static_cast(xminArray); + CPLAssert(yminArray->type_id() == arrow::Type::DOUBLE); + m_poArrayYMinDouble = + static_cast(yminArray); + CPLAssert(xmaxArray->type_id() == arrow::Type::DOUBLE); + m_poArrayXMaxDouble = + static_cast(xmaxArray); + CPLAssert(ymaxArray->type_id() == arrow::Type::DOUBLE); + m_poArrayYMaxDouble = + static_cast(ymaxArray); + } } } } diff --git a/ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp b/ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp index d001813141e0..16cdf040723d 100644 --- a/ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp +++ b/ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp @@ -1980,8 +1980,7 @@ OGRErr OGRParquetLayer::SetIgnoredFields(CSLConstList papszFields) m_oMapGeomFieldIndexToGeomColBBOXParquet.find(i); if (oIter != m_oMapGeomFieldIndexToGeomColBBOX.end() && oIterParquet != - m_oMapGeomFieldIndexToGeomColBBOXParquet.end() && - !OGRArrowIsGeoArrowStruct(m_aeGeomEncoding[i])) + m_oMapGeomFieldIndexToGeomColBBOXParquet.end()) { oIter->second.iArrayIdx = nBatchColumns++; m_anRequestedParquetColumns.insert( From eff841f1294c5d1f606501c08049104ff37c1e8f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 17 Sep 2024 18:56:51 +0200 Subject: [PATCH 106/710] /vsitar/: fix support of /vsitar/ of /vsitar/ Fixes #10821 --- autotest/gcore/data/tar_of_tar_gzip.tar | Bin 0 -> 10240 bytes autotest/gcore/vsifile.py | 9 +++++++++ port/cpl_vsil_gzip.cpp | 8 +++++--- 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 autotest/gcore/data/tar_of_tar_gzip.tar diff --git a/autotest/gcore/data/tar_of_tar_gzip.tar b/autotest/gcore/data/tar_of_tar_gzip.tar new file mode 100644 index 0000000000000000000000000000000000000000..ed18401a2bc5a28e5819161eaeba64436925c9e9 GIT binary patch literal 10240 zcmYeQEJ=+o$xPELNi5P!uVSDDFaQEGGZO|NWoT|{fJ}p=3=M#Ch9+j_M#jd51_tH~ z28Ko^<|YgZ2DEStx|yZLK%XlpFr=2H=Ap|W^9ji#s~tjY`ECw~Ynk6p-rp@8C~*9~ z+Q$tXT@U6eiE)QTWt8cwy=BeuGikLK*ml|7rM93+m{nF=*5fl*cX)?RS+rh}>!~Sb zO5f9e=tsQz{u3}4aZ)0=G0``ohj3TEH49z51p+I`FMR8{zIf%mtaV%M}Y_8gr3ne_?x zjiY;AlEYR77^QHuM@##P`E=j<-L+&x$O1N2n>j^%>;;)a4Xk`$4eHSlS zZlCqDp})I#=I-7 Date: Tue, 17 Sep 2024 17:00:41 +0200 Subject: [PATCH 107/710] gdalwarp: fix crash/infinite loop when using -tr one a 1x1 blank raster (3.8.0 regression) Fixes regression introduced by https://github.com/OSGeo/gdal/pull/7911 Test case: https://lists.osgeo.org/pipermail/gdal-dev/2024-September/059512.html --- apps/gdalwarp_lib.cpp | 18 +++++++++++++++- autotest/utilities/test_gdalwarp_lib.py | 28 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index bf4ef59c1d08..53fa14271b03 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -4428,10 +4428,14 @@ static GDALDatasetH GDALWarpCreateOutput( nPixels = static_cast((psOptions->dfMaxX - psOptions->dfMinX + (psOptions->dfXRes / 2.0)) / psOptions->dfXRes); + if (nPixels == 0) + nPixels = 1; nLines = static_cast( (std::fabs(psOptions->dfMaxY - psOptions->dfMinY) + (psOptions->dfYRes / 2.0)) / psOptions->dfYRes); + if (nLines == 0) + nLines = 1; adfDstGeoTransform[0] = psOptions->dfMinX; adfDstGeoTransform[3] = psOptions->dfMaxY; adfDstGeoTransform[1] = psOptions->dfXRes; @@ -4445,7 +4449,7 @@ static GDALDatasetH GDALWarpCreateOutput( { // Try to detect if the edge of the raster would be blank // Cf https://github.com/OSGeo/gdal/issues/7905 - while (true) + while (nPixels > 1 || nLines > 1) { UpdateGeoTransformandAndPixelLines(); @@ -4547,18 +4551,30 @@ static GDALDatasetH GDALWarpCreateOutput( if (bTopBlankLine) { + if (psOptions->dfMaxY - psOptions->dfMinY <= + 2 * psOptions->dfYRes) + break; psOptions->dfMaxY -= psOptions->dfYRes; } if (bBottomBlankLine) { + if (psOptions->dfMaxY - psOptions->dfMinY <= + 2 * psOptions->dfYRes) + break; psOptions->dfMinY += psOptions->dfYRes; } if (bLeftBlankCol) { + if (psOptions->dfMaxX - psOptions->dfMinX <= + 2 * psOptions->dfXRes) + break; psOptions->dfMinX += psOptions->dfXRes; } if (bRightBlankCol) { + if (psOptions->dfMaxX - psOptions->dfMinX <= + 2 * psOptions->dfXRes) + break; psOptions->dfMaxX -= psOptions->dfXRes; } } diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index d7a4869d646d..2d55e07c212a 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -4259,3 +4259,31 @@ def test_gdalwarp_lib_minus_180_plus_180_to_span_over_180(tmp_vsimem, extra_colu ) == src_ds.GetRasterBand(1).ReadRaster( 0, 0, src_ds.RasterXSize // 2, src_ds.RasterYSize ) + + +############################################################################### +# Test bugfix for https://lists.osgeo.org/pipermail/gdal-dev/2024-September/059512.html + + +@pytest.mark.parametrize("with_tap", [True, False]) +def test_gdalwarp_lib_blank_edge_one_by_one(with_tap): + + src_ds = gdal.GetDriverByName("MEM").Create("", 1, 1) + src_ds.SetGeoTransform([6.8688, 0.0009, 0, 51.3747, 0, -0.0009]) + srs = osr.SpatialReference() + srs.SetFromUserInput("WGS84") + srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + src_ds.SetSpatialRef(srs) + options = "-f MEM -tr 1000 1000 -t_srs EPSG:32631" + if with_tap: + options += " -tap" + out_ds = gdal.Warp("", src_ds, options=options) + assert out_ds.RasterXSize == 1 + assert out_ds.RasterYSize == 1 + gt = out_ds.GetGeoTransform() + if with_tap: + assert gt == pytest.approx((769000.0, 1000.0, 0.0, 5699000.0, 0.0, -1000.0)) + else: + assert gt == pytest.approx( + (769234.6506516202, 1000.0, 0.0, 5698603.782217737, 0.0, -1000.0) + ) From ec3f277237eb5f7a4ceb96e8084ddbd32b3f652c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 17 Sep 2024 18:11:43 +0200 Subject: [PATCH 108/710] MakeValid(METHOD=STRUCTURE): make sure to return a MULTIPOLYGON if input is MULTIPOLYGON Fixes #10819 --- autotest/ogr/ogr_geom.py | 66 +++++++++++++++++++++++++++++++--------- ogr/ogrgeometry.cpp | 21 ++++++++++++- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/autotest/ogr/ogr_geom.py b/autotest/ogr/ogr_geom.py index 935176d7c4f2..67911355b88c 100755 --- a/autotest/ogr/ogr_geom.py +++ b/autotest/ogr/ogr_geom.py @@ -3986,21 +3986,57 @@ def test_ogr_geom_makevalid(): g, "MULTIPOLYGON (((0 0,5 5,10 0,0 0)),((5 5,0 10,10 10,5 5)))" ) - if ( - ogr.GetGEOSVersionMajor() * 10000 - + ogr.GetGEOSVersionMinor() * 100 - + ogr.GetGEOSVersionMicro() - >= 31000 - ): - g = ogr.CreateGeometryFromWkt( - "POLYGON ((0 0,0 10,10 10,10 0,0 0),(5 5,15 10,15 0,5 5))" - ) - # Only since GEOS 3.10 - g = g.MakeValid(["METHOD=STRUCTURE"]) - if g is not None: - ogrtest.check_feature_geometry( - g, "POLYGON ((0 10,10 10,10.0 7.5,5 5,10.0 2.5,10 0,0 0,0 10))" - ) + +############################################################################### + + +@pytest.mark.require_geos(3, 10, 0) +def test_ogr_geom_makevalid_structure(): + + g = ogr.CreateGeometryFromWkt( + "POLYGON ((0 0,0 10,10 10,10 0,0 0),(5 5,15 10,15 0,5 5))" + ) + g = g.MakeValid(["METHOD=STRUCTURE"]) + ogrtest.check_feature_geometry( + g, "POLYGON ((0 10,10 10,10.0 7.5,5 5,10.0 2.5,10 0,0 0,0 10))" + ) + + # Already valid multi-polygon made of a single-part + g = ogr.CreateGeometryFromWkt("MULTIPOLYGON (((0 0,1 0,1 1,0 1,0 0)))") + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,1 0,1 1,0 1,0 0)))" + or g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,0 1,1 1,1 0,0 0)))" + ) + + # Already valid multi-polygon made of a single-part, with duplicated point + g = ogr.CreateGeometryFromWkt("MULTIPOLYGON (((0 0,1 0,1 0,1 1,0 1,0 0)))") + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,1 0,1 1,0 1,0 0)))" + or g.ExportToIsoWkt() == "MULTIPOLYGON (((0 0,0 1,1 1,1 0,0 0)))" + ) + + # Already valid multi-polygon made of a single-part + g = ogr.CreateGeometryFromWkt( + "MULTIPOLYGON Z (((0 0 10,1 0 10,1 1 10,0 1 10,0 0 10)))" + ) + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "MULTIPOLYGON Z (((0 0 10,1 0 10,1 1 10,0 1 10,0 0 10)))" + or g.ExportToIsoWkt() + == "MULTIPOLYGON Z (((0 0 10,0 1 10,1 1 10,1 0 10,0 0 10)))" + ) + + # Already valid geometry collection + g = ogr.CreateGeometryFromWkt( + "GEOMETRYCOLLECTION (POLYGON ((0 0,1 0,1 1,0 1,0 0)))" + ) + g = g.MakeValid(["METHOD=STRUCTURE"]) + assert ( + g.ExportToIsoWkt() == "GEOMETRYCOLLECTION (POLYGON ((0 0,1 0,1 1,0 1,0 0)))" + or g.ExportToIsoWkt() == "GEOMETRYCOLLECTION (POLYGON ((0 0,0 1,1 1,1 0,0 0)))" + ) ############################################################################### diff --git a/ogr/ogrgeometry.cpp b/ogr/ogrgeometry.cpp index 287f20370bff..46dd0bd1264c 100644 --- a/ogr/ogrgeometry.cpp +++ b/ogr/ogrgeometry.cpp @@ -3862,7 +3862,12 @@ static OGRBoolean OGRGEOSBooleanPredicate( /** * \brief Attempts to make an invalid geometry valid without losing vertices. * - * Already-valid geometries are cloned without further intervention. + * Already-valid geometries are cloned without further intervention + * for default MODE=LINEWORK. Already-valid geometries with MODE=STRUCTURE + * may be subject to non-significant transformations, such as duplicated point + * removal, change in ring winding order, etc. (before GDAL 3.10, single-part + * geometry collections could be returned a single geometry. GDAL 3.10 + * returns the same type of geometry). * * Running OGRGeometryFactory::removeLowerDimensionSubGeoms() as a * post-processing step is often desired. @@ -3973,6 +3978,20 @@ OGRGeometry *OGRGeometry::MakeValid(CSLConstList papszOptions) const poOGRProduct = OGRGeometryRebuildCurves(this, nullptr, poOGRProduct); GEOSGeom_destroy_r(hGEOSCtxt, hGEOSRet); + +#if GEOS_VERSION_MAJOR > 3 || \ + (GEOS_VERSION_MAJOR == 3 && GEOS_VERSION_MINOR >= 10) + // METHOD=STRUCTURE is not guaranteed to return a multiple geometry + // if the input is a multiple geometry + if (poOGRProduct && bStructureMethod && + OGR_GT_IsSubClassOf(getGeometryType(), wkbGeometryCollection) && + !OGR_GT_IsSubClassOf(poOGRProduct->getGeometryType(), + wkbGeometryCollection)) + { + poOGRProduct = OGRGeometryFactory::forceTo(poOGRProduct, + getGeometryType()); + } +#endif } } freeGEOSContext(hGEOSCtxt); From 262ffc2002107f8bcd9bac9ce6066d9efbe6cb74 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 17 Sep 2024 21:13:22 +0200 Subject: [PATCH 109/710] viewshed_executor.cpp: use correct mutex --- alg/viewshed/viewshed_executor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alg/viewshed/viewshed_executor.cpp b/alg/viewshed/viewshed_executor.cpp index 01a9d5e32658..83c9669b9346 100644 --- a/alg/viewshed/viewshed_executor.cpp +++ b/alg/viewshed/viewshed_executor.cpp @@ -136,7 +136,7 @@ ViewshedExecutor::ViewshedExecutor(GDALRasterBand &srcBand, // calculate the height adjustment factor. double ViewshedExecutor::calcHeightAdjFactor() { - std::lock_guard g(iMutex); + std::lock_guard g(oMutex); const OGRSpatialReference *poDstSRS = m_dstBand.GetDataset()->GetSpatialRef(); From 0c4395b53ebd480c5032b319e6672d22d322b99f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Sep 2024 01:02:30 +0200 Subject: [PATCH 110/710] CPLWorkerThreadPool: fix locking in GetNextJob() to make 'valgrind --tool=helgrind autotest/cpp/gdal_unit_test --gtest_filter=test_cpl.CPLWorkerThreadPool' happy (master only) Related to #10825 --- port/cpl_worker_thread_pool.cpp | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/port/cpl_worker_thread_pool.cpp b/port/cpl_worker_thread_pool.cpp index 6136b0febb03..35f2f916c780 100644 --- a/port/cpl_worker_thread_pool.cpp +++ b/port/cpl_worker_thread_pool.cpp @@ -186,11 +186,8 @@ bool CPLWorkerThreadPool::SubmitJob(std::function task) if (static_cast(aWT.size()) < m_nMaxThreads) { // CPLDebug("CPL", "Starting new thread..."); - std::unique_ptr wt(new CPLWorkerThread); - wt->pfnInitFunc = nullptr; - wt->pInitData = nullptr; + auto wt = std::make_unique(); wt->poTP = this; - wt->bMarkedAsWaiting = false; //ABELL - Why should this fail? And this is a *pool* thread, not necessarily // tied to the submitted job. The submitted job still needs to run, even if // this fails. If we can't create a thread, should the entire pool become invalid? @@ -279,10 +276,7 @@ bool CPLWorkerThreadPool::SubmitJobs(CPLThreadFunc pfnFunc, if (static_cast(aWT.size()) < m_nMaxThreads) { std::unique_ptr wt(new CPLWorkerThread); - wt->pfnInitFunc = nullptr; - wt->pInitData = nullptr; wt->poTP = this; - wt->bMarkedAsWaiting = false; wt->hThread = CPLCreateJoinableThread(WorkerThreadFunction, wt.get()); if (wt->hThread == nullptr) @@ -425,10 +419,6 @@ bool CPLWorkerThreadPool::Setup(int nThreads, CPLThreadFunc pfnInitFunc, wt->pfnInitFunc = pfnInitFunc; wt->pInitData = pasInitData ? pasInitData[i] : nullptr; wt->poTP = this; - { - std::lock_guard oGuard(wt->m_mutex); - wt->bMarkedAsWaiting = false; - } wt->hThread = CPLCreateJoinableThread(WorkerThreadFunction, wt.get()); if (wt->hThread == nullptr) { @@ -529,10 +519,7 @@ CPLWorkerThreadPool::GetNextJob(CPLWorkerThread *psWorkerThread) std::unique_lock oGuardThisThread(psWorkerThread->m_mutex); // coverity[uninit_use_in_call] oGuard.unlock(); - while (psWorkerThread->bMarkedAsWaiting && eState != CPLWTS_STOP) - { - psWorkerThread->m_cv.wait(oGuardThisThread); - } + psWorkerThread->m_cv.wait(oGuardThisThread); } } From 8b9376f6f19dd81370d9f13290ff9cdf576f45b4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Sep 2024 02:48:52 +0200 Subject: [PATCH 111/710] viewshed: add explicit ~Cumulative() destructor, to avoid redefinition of NotifyQueue symbols, leading to mutex corruption Fixes #10825 --- alg/viewshed/combiner.h | 5 ++++- alg/viewshed/cumulative.cpp | 4 ++++ alg/viewshed/cumulative.h | 13 ++++++++++++- alg/viewshed/notifyqueue.h | 8 +++++++- alg/viewshed/progress.h | 5 ++++- alg/viewshed/util.h | 5 ++++- alg/viewshed/viewshed.cpp | 6 ++++++ alg/viewshed/viewshed.h | 25 ++++++++++++++----------- alg/viewshed/viewshed_types.h | 5 ++++- 9 files changed, 59 insertions(+), 17 deletions(-) diff --git a/alg/viewshed/combiner.h b/alg/viewshed/combiner.h index 4597a8fd8d9d..0e6ce9787a48 100644 --- a/alg/viewshed/combiner.h +++ b/alg/viewshed/combiner.h @@ -20,7 +20,8 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ -#pragma once +#ifndef VIEWSHED_COMBINER_H_INCLUDED +#define VIEWSHED_COMBINER_H_INCLUDED #include "cumulative.h" #include "viewshed_types.h" @@ -67,3 +68,5 @@ class Combiner } // namespace viewshed } // namespace gdal + +#endif diff --git a/alg/viewshed/cumulative.cpp b/alg/viewshed/cumulative.cpp index 79d48c68c4d0..606ca23c8215 100644 --- a/alg/viewshed/cumulative.cpp +++ b/alg/viewshed/cumulative.cpp @@ -44,6 +44,10 @@ Cumulative::Cumulative(const Options &opts) : m_opts(opts) { } +/// Destructor +/// +Cumulative::~Cumulative() = default; + /// Compute the cumulative viewshed of a raster band. /// /// @param srcFilename Source filename. diff --git a/alg/viewshed/cumulative.h b/alg/viewshed/cumulative.h index 1d93a0896368..c9d2f8a3f958 100644 --- a/alg/viewshed/cumulative.h +++ b/alg/viewshed/cumulative.h @@ -20,7 +20,8 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ -#pragma once +#ifndef VIEWSHED_CUMULATIVE_H_INCLUDED +#define VIEWSHED_CUMULATIVE_H_INCLUDED #include #include @@ -41,6 +42,11 @@ class Cumulative { public: CPL_DLL explicit Cumulative(const Options &opts); + // We define an explicit destructor, whose implementation is in libgdal, + // otherwise with gcc 9.4 of Ubuntu 20.04 in debug mode, this would need to + // redefinition of the NotifyQueue class in both libgdal and gdal_viewshed, + // leading to weird things related to mutex. + CPL_DLL ~Cumulative(); CPL_DLL bool run(const std::string &srcFilename, GDALProgressFunc pfnProgress = GDALDummyProgress, void *pProgressArg = nullptr); @@ -70,7 +76,12 @@ class Cumulative void rollupRasters(); void scaleOutput(); bool writeOutput(DatasetPtr pDstDS); + + Cumulative(const Cumulative &) = delete; + Cumulative &operator=(const Cumulative &) = delete; }; } // namespace viewshed } // namespace gdal + +#endif diff --git a/alg/viewshed/notifyqueue.h b/alg/viewshed/notifyqueue.h index 6ddeed8ec79d..e0e3f9da3725 100644 --- a/alg/viewshed/notifyqueue.h +++ b/alg/viewshed/notifyqueue.h @@ -19,7 +19,11 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. ****************************************************************************/ -#pragma once + +#ifndef VIEWSHED_NOTIFYQUEUE_H_INCLUDED +#define VIEWSHED_NOTIFYQUEUE_H_INCLUDED + +#include "cpl_port.h" #include #include @@ -134,3 +138,5 @@ template class NotifyQueue } // namespace viewshed } // namespace gdal + +#endif diff --git a/alg/viewshed/progress.h b/alg/viewshed/progress.h index d0afaa588a97..0eaee7d449c8 100644 --- a/alg/viewshed/progress.h +++ b/alg/viewshed/progress.h @@ -20,7 +20,8 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ -#pragma once +#ifndef VIEWSHED_PROGRESS_H_INCLUDED +#define VIEWSHED_PROGRESS_H_INCLUDED #include #include @@ -54,3 +55,5 @@ class Progress } // namespace viewshed } // namespace gdal + +#endif diff --git a/alg/viewshed/util.h b/alg/viewshed/util.h index 281bd382608f..3ebf6a9203dc 100644 --- a/alg/viewshed/util.h +++ b/alg/viewshed/util.h @@ -20,7 +20,8 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ -#pragma once +#ifndef VIEWSHED_UTIL_H_INCLUDED +#define VIEWSHED_UTIL_H_INCLUDED #include "viewshed_types.h" @@ -36,3 +37,5 @@ DatasetPtr createOutputDataset(GDALRasterBand &srcBand, const Options &opts, } // namespace viewshed } // namespace gdal + +#endif diff --git a/alg/viewshed/viewshed.cpp b/alg/viewshed/viewshed.cpp index fb0008ed7460..df5638bb70ac 100644 --- a/alg/viewshed/viewshed.cpp +++ b/alg/viewshed/viewshed.cpp @@ -234,6 +234,12 @@ bool getTransforms(GDALRasterBand &band, double *pFwdTransform, } // unnamed namespace +Viewshed::Viewshed(const Options &opts) : oOpts{opts} +{ +} + +Viewshed::~Viewshed() = default; + /// Calculate the extent of the output raster in terms of the input raster and /// save the input raster extent. /// diff --git a/alg/viewshed/viewshed.h b/alg/viewshed/viewshed.h index 9109772d214d..996276596dd9 100644 --- a/alg/viewshed/viewshed.h +++ b/alg/viewshed/viewshed.h @@ -27,7 +27,8 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ -#pragma once +#ifndef VIEWSHED_H_INCLUDED +#define VIEWSHED_H_INCLUDED #include #include @@ -59,13 +60,10 @@ class Viewshed * * @param opts Options to use when calculating viewshed. */ - CPL_DLL explicit Viewshed(const Options &opts) - : oOpts{opts}, oOutExtent{}, oCurExtent{}, poDstDS{}, pSrcBand{} - { - } + CPL_DLL explicit Viewshed(const Options &opts); - Viewshed(const Viewshed &) = delete; - Viewshed &operator=(const Viewshed &) = delete; + /** Destructor */ + CPL_DLL ~Viewshed(); CPL_DLL bool run(GDALRasterBandH hBand, GDALProgressFunc pfnProgress = GDALDummyProgress, @@ -83,10 +81,10 @@ class Viewshed private: Options oOpts; - Window oOutExtent; - Window oCurExtent; - DatasetPtr poDstDS; - GDALRasterBand *pSrcBand; + Window oOutExtent{}; + Window oCurExtent{}; + DatasetPtr poDstDS{}; + GDALRasterBand *pSrcBand = nullptr; DatasetPtr execute(int nX, int nY, const std::string &outFilename); void setOutput(double &dfResult, double &dfCellVal, double dfZ); @@ -96,7 +94,12 @@ class Viewshed std::vector &thisLineVal); bool calcExtents(int nX, int nY, const std::array &adfInvTransform); + + Viewshed(const Viewshed &) = delete; + Viewshed &operator=(const Viewshed &) = delete; }; } // namespace viewshed } // namespace gdal + +#endif diff --git a/alg/viewshed/viewshed_types.h b/alg/viewshed/viewshed_types.h index abdaae49a2fb..fa1f0b2b9ead 100644 --- a/alg/viewshed/viewshed_types.h +++ b/alg/viewshed/viewshed_types.h @@ -19,7 +19,8 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. ****************************************************************************/ -#pragma once +#ifndef VIEWSHED_TYPES_H_INCLUDED +#define VIEWSHED_TYPES_H_INCLUDED #include #include @@ -171,3 +172,5 @@ struct Window } // namespace viewshed } // namespace gdal + +#endif From 212543dcdde13df48a40ae9dbd9060a9103a1b81 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Sep 2024 10:32:31 +0200 Subject: [PATCH 112/710] Python bindings: silence SWIG 'detected a memory leak' message Previous workarounds (PR #8003) no longer worked with latest SWIG versions Refs #4907 --- swig/python/setup.py.in | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/swig/python/setup.py.in b/swig/python/setup.py.in index cb37ed5db7ba..b63bbcff9394 100644 --- a/swig/python/setup.py.in +++ b/swig/python/setup.py.in @@ -321,6 +321,14 @@ class gdal_ext(build_ext): # the Extension() instantiations below because we want to use the same # logic to resolve the location of gdal-config throughout. ext.extra_compile_args.extend(self.extra_cflags) + + # Work around "swig/python detected a memory leak" bug + # Cf https://github.com/swig/swig/issues/2638#issuecomment-2345002698 + if self.get_compiler() == 'msvc': + ext.extra_compile_args.append("/DSWIG_PYTHON_SILENT_MEMLEAK") + else: + ext.extra_compile_args.append("-DSWIG_PYTHON_SILENT_MEMLEAK") + return build_ext.build_extension(self, ext) From ce8b294ca5db1d952c2d28665fc02292f424a1f9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Sep 2024 10:58:35 +0200 Subject: [PATCH 113/710] LIBKML: dump feature when its geometry cannot be written Fixes #10829 Example ``` $ ogr2ogr -f LIBKML test.kml test.csv || echo "==> failed" Warning 1: Self-intersection at or near point 0.5 0.5 ERROR 6: Invalid polygon ERROR 1: Cannot translate feature: OGRFeature(test):-1 id (Integer) = 1 i (Integer) = -21121 WKT (String) = POLYGON((0 0,1 1,1 0,0 1,0 0)) POLYGON ((0 0,1 1,1 0,0 1,0 0)) ERROR 1: Unable to write feature 1 from layer test. ERROR 1: Terminating translation prematurely after failed translation of layer test (use -skipfailures to skip errors) ==> failed ``` --- ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp b/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp index 17c0ff7c96e5..91ee6c43cf7b 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp @@ -815,7 +815,12 @@ FeaturePtr feat2kml(OGRLIBKMLDataSource *poOgrDS, OGRLIBKMLLayer *poOgrLayer, { ElementPtr poKmlElement = geom2kml(poOgrGeom, -1, poKmlFactory); if (!poKmlElement) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot translate feature: %s", + poOgrFeat->DumpReadableAsString().c_str()); return nullptr; + } poKmlPlacemark->set_geometry(AsGeometry(std::move(poKmlElement))); } From a290af1bc6bfed7ff5e19b0d74f5288740c57261 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Sep 2024 22:16:16 +0200 Subject: [PATCH 114/710] gdalwarper.cpp: more details about OPTIMIZE_SIZE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and document XSCALE and YSCALE warping options Ref #10813 --- alg/gdalwarper.cpp | 20 +++++++++++++++++++- doc/source/spelling_wordlist.txt | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/alg/gdalwarper.cpp b/alg/gdalwarper.cpp index ca78650897b3..3f09b4b2409a 100644 --- a/alg/gdalwarper.cpp +++ b/alg/gdalwarper.cpp @@ -1167,6 +1167,20 @@ CPLErr GDALWarpDstAlphaMasker(void *pMaskFuncArg, int nBandCount, * will be selected, not just those whose center point falls within the * polygon. * + *
  • XSCALE: Ratio expressing the resampling factor (number of destination + * pixels per source pixel) along the target horizontal axis. + * The scale is used to determine the number of source pixels along the x-axis + * that are considered by the resampling algorithm. + * Equals to one for no resampling, below one for downsampling + * and above one for upsampling. This is automatically computed, for each + * processing chunk, and may thus vary among them, depending on the + * shape of output regions vs input regions. Such variations can be undesired + * in some situations. If the resampling factor can be considered as constant + * over the warped area, setting a constant value can lead to more reproducible + * pixel output.
    • + * + *
  • YSCALE: Same as XSCALE, but along the horizontal axis.
    • + * *
  • OPTIMIZE_SIZE: This defaults to FALSE, but may be set to TRUE * typically when writing to a compressed dataset (GeoTIFF with * COMPRESS creation option set for example) for achieving a smaller @@ -1176,7 +1190,11 @@ CPLErr GDALWarpDstAlphaMasker(void *pMaskFuncArg, int nBandCount, * of the file. However sticking to target block size may cause major * processing slowdown for some particular reprojections. Starting * with GDAL 3.8, OPTIMIZE_SIZE mode is automatically enabled when it is safe - * to do so.
    • + * to do so. + * As this parameter influences the shape of warping chunk, and by default the + * XSCALE and YSCALE parameters are computed per warping chunk, this parameter may + * influence the pixel output. + * * *
  • NUM_THREADS: (GDAL >= 1.10) Can be set to a numeric value or ALL_CPUS to * set the number of threads to use to parallelize the computation part of the diff --git a/doc/source/spelling_wordlist.txt b/doc/source/spelling_wordlist.txt index e1447916c7b9..991c9919ec42 100644 --- a/doc/source/spelling_wordlist.txt +++ b/doc/source/spelling_wordlist.txt @@ -3360,6 +3360,7 @@ UpdateFeature updateTime UpperLeftX UpperLeftY +upsampling upsert uri url From 300ceb821988e7bdaa6b138115e84ecad7bea279 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Sep 2024 17:22:28 +0200 Subject: [PATCH 115/710] EEDA/EEDAI drivers: add a VSI_PATH_FOR_AUTH open option to allow using a /vsigs/ path-specific GOOGLE_APPLICATION_CREDENTIALS option --- autotest/gdrivers/eedai.py | 16 +++++++++++++--- doc/source/drivers/raster/eedai.rst | 10 +++++++++- doc/source/drivers/vector/eeda.rst | 10 +++++++++- frmts/eeda/eedacommon.cpp | 14 +++++++++++--- frmts/eeda/eedadataset.cpp | 15 ++++++++++----- frmts/eeda/eedaidataset.cpp | 4 ++++ 6 files changed, 56 insertions(+), 13 deletions(-) diff --git a/autotest/gdrivers/eedai.py b/autotest/gdrivers/eedai.py index 39a15f90d5f8..3c32369f485e 100755 --- a/autotest/gdrivers/eedai.py +++ b/autotest/gdrivers/eedai.py @@ -302,7 +302,8 @@ def test_eedai_3(): # Test OAuth2 with GOOGLE_APPLICATION_CREDENTIALS -def test_eedai_GOOGLE_APPLICATION_CREDENTIALS(): +@pytest.mark.parametrize("use_vsi_path", [False, True]) +def test_eedai_GOOGLE_APPLICATION_CREDENTIALS(use_vsi_path): gdal.FileFromMemBuffer( "/vsimem/my.json", @@ -313,7 +314,12 @@ def test_eedai_GOOGLE_APPLICATION_CREDENTIALS(): ) gdal.SetConfigOption("EEDA_URL", "/vsimem/ee/") - gdal.SetConfigOption("GOOGLE_APPLICATION_CREDENTIALS", "/vsimem/my.json") + if use_vsi_path: + gdal.SetPathSpecificOption( + "/vsigs/to_test_eeda", "GOOGLE_APPLICATION_CREDENTIALS", "/vsimem/my.json" + ) + else: + gdal.SetConfigOption("GOOGLE_APPLICATION_CREDENTIALS", "/vsimem/my.json") gdal.SetConfigOption("EEDA_PRIVATE_KEY", None) gdal.SetConfigOption("EEDA_CLIENT_EMAIL", None) gdal.SetConfigOption("GO2A_AUD", "/vsimem/oauth2/v4/token") @@ -323,8 +329,11 @@ def test_eedai_GOOGLE_APPLICATION_CREDENTIALS(): '{ "access_token": "my_token", "token_type": "Bearer", "expires_in": 3600 }', ) + open_options = [] + if use_vsi_path: + open_options.append("VSI_PATH_FOR_AUTH=/vsigs/to_test_eeda") try: - ds = gdal.Open("EEDAI:image") + ds = gdal.OpenEx("EEDAI:image", open_options=open_options) assert ds is not None except RuntimeError: pass @@ -335,6 +344,7 @@ def test_eedai_GOOGLE_APPLICATION_CREDENTIALS(): gdal.SetConfigOption("GOOGLE_APPLICATION_CREDENTIALS", None) gdal.SetConfigOption("EEDA_PRIVATE_KEY", None) gdal.SetConfigOption("EEDA_CLIENT_EMAIL", None) + gdal.ClearPathSpecificOptions("/vsigs/to_test_eeda") if "CPLRSASHA256Sign() not implemented" in gdal.GetLastErrorMsg(): pytest.skip("CPLRSASHA256Sign() not implemented") diff --git a/doc/source/drivers/raster/eedai.rst b/doc/source/drivers/raster/eedai.rst index 7ce85962c03f..61337ed0d6c1 100644 --- a/doc/source/drivers/raster/eedai.rst +++ b/doc/source/drivers/raster/eedai.rst @@ -60,6 +60,12 @@ The following open options are available : Size of a GDAL block, which is the minimum unit to query pixels. +- .. oo:: VSI_PATH_FOR_AUTH + :since: 3.10 + + /vsigs/... path onto which a GOOGLE_APPLICATION_CREDENTIALS path specific + option is set + Authentication methods ---------------------- @@ -68,7 +74,9 @@ The following authentication methods can be used: - Authentication Bearer header passed through the :config:`EEDA_BEARER` or :config:`EEDA_BEARER_FILE` configuration options. - Service account private key file, through the - :config:`GOOGLE_APPLICATION_CREDENTIALS` configuration option. + :config:`GOOGLE_APPLICATION_CREDENTIALS` configuration option, or set + as a path-specific option whose value is set in the VSI_PATH_FOR_AUTH + open option. - OAuth2 Service Account authentication through the :config:`EEDA_PRIVATE_KEY`/ :config:`EEDA_PRIVATE_KEY_FILE` + :config:`EEDA_CLIENT_EMAIL` configuration options. - Finally if none of the above method succeeds, the code will check if diff --git a/doc/source/drivers/vector/eeda.rst b/doc/source/drivers/vector/eeda.rst index 90717adce691..8e754b2ea84b 100644 --- a/doc/source/drivers/vector/eeda.rst +++ b/doc/source/drivers/vector/eeda.rst @@ -40,6 +40,12 @@ The following open options are available: To specify the collection if not specified in the connection string. +- .. oo:: VSI_PATH_FOR_AUTH + :since: 3.10 + + /vsigs/... path onto which a GOOGLE_APPLICATION_CREDENTIALS path specific + option is set + Authentication methods ---------------------- @@ -48,7 +54,9 @@ The following authentication methods can be used: - Authentication Bearer header passed through the EEDA_BEARER or :config:`EEDA_BEARER_FILE` configuration options. - Service account private key file, through the - :config:`GOOGLE_APPLICATION_CREDENTIALS` configuration option. + :config:`GOOGLE_APPLICATION_CREDENTIALS` configuration option, or set + as a path-specific option whose value is set in the VSI_PATH_FOR_AUTH + open option. - OAuth2 Service Account authentication through the :config:`EEDA_PRIVATE_KEY`/ :config:`EEDA_PRIVATE_KEY_FILE` + diff --git a/frmts/eeda/eedacommon.cpp b/frmts/eeda/eedacommon.cpp index 0ba2c6a82492..3bface33bcb6 100644 --- a/frmts/eeda/eedacommon.cpp +++ b/frmts/eeda/eedacommon.cpp @@ -386,8 +386,15 @@ char **GDALEEDABaseDataset::GetBaseHTTPOptions() } } - CPLString osServiceAccountJson( - CPLGetConfigOption("GOOGLE_APPLICATION_CREDENTIALS", "")); + CPLString osServiceAccountJson; + const char *pszVSIPath = + CSLFetchNameValue(papszOpenOptions, "VSI_PATH_FOR_AUTH"); + if (pszVSIPath) + osServiceAccountJson = VSIGetPathSpecificOption( + pszVSIPath, "GOOGLE_APPLICATION_CREDENTIALS", ""); + if (osServiceAccountJson.empty()) + osServiceAccountJson = + CPLGetConfigOption("GOOGLE_APPLICATION_CREDENTIALS", ""); if (!osServiceAccountJson.empty()) { CPLJSONDocument oDoc; @@ -443,7 +450,8 @@ char **GDALEEDABaseDataset::GetBaseHTTPOptions() "Missing EEDA_BEARER, EEDA_BEARER_FILE or " "GOOGLE_APPLICATION_CREDENTIALS or " "EEDA_PRIVATE_KEY/EEDA_PRIVATE_KEY_FILE + " - "EEDA_CLIENT_EMAIL config option"); + "EEDA_CLIENT_EMAIL config option or " + "VSI_PATH_FOR_AUTH open option"); CSLDestroy(papszOptions); return nullptr; } diff --git a/frmts/eeda/eedadataset.cpp b/frmts/eeda/eedadataset.cpp index f41c4b4781d3..f0ed00d36b7e 100644 --- a/frmts/eeda/eedadataset.cpp +++ b/frmts/eeda/eedadataset.cpp @@ -1245,11 +1245,16 @@ void GDALRegister_EEDA() poDriver->SetMetadataItem(GDAL_DMD_LONGNAME, "Earth Engine Data API"); poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, "drivers/vector/eeda.html"); poDriver->SetMetadataItem(GDAL_DMD_CONNECTION_PREFIX, "EEDA:"); - poDriver->SetMetadataItem(GDAL_DMD_OPENOPTIONLIST, - "" - " "); + poDriver->SetMetadataItem( + GDAL_DMD_OPENOPTIONLIST, + "" + " "); poDriver->pfnOpen = GDALEEDAOpen; poDriver->pfnIdentify = GDALEEDAdentify; diff --git a/frmts/eeda/eedaidataset.cpp b/frmts/eeda/eedaidataset.cpp index e6df9d0d2a8a..e443c39ae3ff 100644 --- a/frmts/eeda/eedaidataset.cpp +++ b/frmts/eeda/eedaidataset.cpp @@ -1558,6 +1558,10 @@ void GDALRegister_EEDAI() " " "
  • GDAL_OF_VECTOR for vector drivers,
    • *
  • GDAL_OF_GNM for Geographic Network Model drivers.
    • * - * GDAL_OF_RASTER and GDAL_OF_MULTIDIM_RASTER are generally mutually + * GDAL_OF_RASTER and GDAL_OF_MULTIDIM_RASTER are generally mutually * exclusive. If none of the value is specified, GDAL_OF_RASTER | GDAL_OF_VECTOR - * | GDAL_OF_GNM is implied.
  • Access mode: GDAL_OF_READONLY - * (exclusive)or GDAL_OF_UPDATE.
  • Shared mode: GDAL_OF_SHARED. If set, + * | GDAL_OF_GNM is implied. + *
    • + *
  • Access mode: GDAL_OF_READONLY (exclusive)or GDAL_OF_UPDATE. + *
    • + *
  • Shared mode: GDAL_OF_SHARED. If set, * it allows the sharing of GDALDataset handles for a dataset with other callers * that have set GDAL_OF_SHARED. In particular, GDALOpenEx() will first consult * its list of currently open and shared GDALDataset's, and if the * GetDescription() name for one exactly matches the pszFilename passed to * GDALOpenEx() it will be referenced and returned, if GDALOpenEx() is called - * from the same thread.
  • Verbose error: GDAL_OF_VERBOSE_ERROR. If set, + * from the same thread. + *
    • + *
  • Thread safe mode: GDAL_OF_THREAD_SAFE (added in 3.10). + * This must be use in combination with GDAL_OF_RASTER, and is mutually + * exclusive with GDAL_OF_UPDATE, GDAL_OF_VECTOR, GDAL_OF_MULTIDIM_RASTER or + * GDAL_OF_GNM. + *
    • + *
  • Verbose error: GDAL_OF_VERBOSE_ERROR. If set, * a failed attempt to open the file will lead to an error message to be - * reported.
    • + * reported. + * * * * @param papszAllowedDrivers NULL to consider all candidate drivers, or a NULL @@ -3581,6 +3592,33 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, { VALIDATE_POINTER1(pszFilename, "GDALOpen", nullptr); + // Do some sanity checks on incompatible flags with thread-safe mode. + if ((nOpenFlags & GDAL_OF_THREAD_SAFE) != 0) + { + const struct + { + int nFlag; + const char *pszFlagName; + } asFlags[] = { + {GDAL_OF_UPDATE, "GDAL_OF_UPDATE"}, + {GDAL_OF_VECTOR, "GDAL_OF_VECTOR"}, + {GDAL_OF_MULTIDIM_RASTER, "GDAL_OF_MULTIDIM_RASTER"}, + {GDAL_OF_GNM, "GDAL_OF_GNM"}, + }; + + for (const auto &asFlag : asFlags) + { + if ((nOpenFlags & asFlag.nFlag) != 0) + { + CPLError(CE_Failure, CPLE_IllegalArg, + "GDAL_OF_THREAD_SAFE and %s are mutually " + "exclusive", + asFlag.pszFlagName); + return nullptr; + } + } + } + // If no driver kind is specified, assume all are to be probed. if ((nOpenFlags & GDAL_OF_KIND_MASK) == 0) nOpenFlags |= GDAL_OF_KIND_MASK & ~GDAL_OF_MULTIDIM_RASTER; @@ -3872,7 +3910,12 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, } else { - if (!(nOpenFlags & GDAL_OF_INTERNAL)) + // For thread-safe opening, currently poDS is what will be + // the "master" dataset owned by the thread-safe dataset + // returned to the user, hence we do not register it as a + // visible one in the open dataset list, or mark it as shared. + if (!(nOpenFlags & GDAL_OF_INTERNAL) && + !(nOpenFlags & GDAL_OF_THREAD_SAFE)) { poDS->AddToDatasetOpenList(); } @@ -3881,7 +3924,8 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, CSLDestroy(poDS->papszOpenOptions); poDS->papszOpenOptions = CSLDuplicate(papszOpenOptions); poDS->nOpenFlags = nOpenFlags; - poDS->MarkAsShared(); + if (!(nOpenFlags & GDAL_OF_THREAD_SAFE)) + poDS->MarkAsShared(); } } } @@ -3895,8 +3939,11 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, "and description (%s)", pszFilename, poDS->GetDescription()); } - else + else if (!(nOpenFlags & GDAL_OF_THREAD_SAFE)) { + // For thread-safe opening, currently poDS is what will be + // the "master" dataset owned by the thread-safe dataset + // returned to the user, hence we do not or mark it as shared. poDS->MarkAsShared(); } } @@ -3914,6 +3961,29 @@ GDALDatasetH CPL_STDCALL GDALOpenEx(const char *pszFilename, } #endif + if (poDS) + { + poDS->m_bCanBeReopened = true; + + if ((nOpenFlags & GDAL_OF_THREAD_SAFE) != 0) + { + poDS = + GDALGetThreadSafeDataset( + std::unique_ptr(poDS), GDAL_OF_RASTER) + .release(); + if (poDS) + { + poDS->m_bCanBeReopened = true; + poDS->poDriver = poDriver; + poDS->nOpenFlags = nOpenFlags; + if (!(nOpenFlags & GDAL_OF_INTERNAL)) + poDS->AddToDatasetOpenList(); + if (nOpenFlags & GDAL_OF_SHARED) + poDS->MarkAsShared(); + } + } + } + return poDS; } @@ -8180,7 +8250,8 @@ bool GDALDataset::SetQueryLoggerFunc(CPL_UNUSED GDALQueryLoggerFunc callback, int GDALDataset::EnterReadWrite(GDALRWFlag eRWFlag) { - if (m_poPrivate == nullptr) + if (m_poPrivate == nullptr || + IsThreadSafe(GDAL_OF_RASTER | (nOpenFlags & GDAL_OF_UPDATE))) return FALSE; if (m_poPrivate->poParentDataset) @@ -10136,3 +10207,75 @@ CPLErr GDALDatasetReadCompressedData(GDALDatasetH hDS, const char *pszFormat, pszFormat, nXOff, nYOff, nXSize, nYSize, nBandCount, panBandList, ppBuffer, pnBufferSize, ppszDetailedFormat); } + +/************************************************************************/ +/* CanBeCloned() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress + +/** This method is called by GDALThreadSafeDataset::Create() to determine if + * it is possible to create a thread-safe wrapper for a dataset, which involves + * the ability to Clone() it. + * + * Implementations of this method must be thread-safe. + * + * @param nScopeFlags Combination of GDAL_OF_RASTER, GDAL_OF_VECTOR, etc. flags, + * expressing the intended use for thread-safety. + * Currently, the only valid scope is in the base + * implementation is GDAL_OF_RASTER. + * @param bCanShareState Determines if cloned datasets are allowed to share + * state with the dataset they have been cloned from. + * If set to true, the dataset from which they have been + * cloned from must remain opened during the lifetime of + * its clones. + * @return true if the Clone() method is expecte to succeed with the same values + * of nScopeFlags and bCanShareState. + */ +bool GDALDataset::CanBeCloned(int nScopeFlags, + [[maybe_unused]] bool bCanShareState) const +{ + return m_bCanBeReopened && nScopeFlags == GDAL_OF_RASTER; +} + +//! @endcond + +/************************************************************************/ +/* Clone() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress + +/** This method "clones" the current dataset, that is it returns a new instance + * that is opened on the same underlying "file". + * + * The base implementation uses GDALDataset::Open() to re-open the dataset. + * The MEM driver has a specialized implementation that returns a new instance, + * but which shares the same memory buffer as this. + * + * Implementations of this method must be thread-safe. + * + * @param nScopeFlags Combination of GDAL_OF_RASTER, GDAL_OF_VECTOR, etc. flags, + * expressing the intended use for thread-safety. + * Currently, the only valid scope is in the base + * implementation is GDAL_OF_RASTER. + * @param bCanShareState Determines if cloned datasets are allowed to share + * state with the dataset they have been cloned from. + * If set to true, the dataset from which they have been + * cloned from must remain opened during the lifetime of + * its clones. + * @return a new instance, or nullptr in case of error. + */ +std::unique_ptr +GDALDataset::Clone(int nScopeFlags, [[maybe_unused]] bool bCanShareState) const +{ + CPLStringList aosAllowedDrivers; + if (poDriver) + aosAllowedDrivers.AddString(poDriver->GetDescription()); + return std::unique_ptr(GDALDataset::Open( + GetDescription(), + nScopeFlags | GDAL_OF_INTERNAL | GDAL_OF_VERBOSE_ERROR, + aosAllowedDrivers.List(), papszOpenOptions)); +} + +//! @endcond diff --git a/gcore/gdaldriver.cpp b/gcore/gdaldriver.cpp index c1de9b891886..f801283bdc4a 100644 --- a/gcore/gdaldriver.cpp +++ b/gcore/gdaldriver.cpp @@ -123,7 +123,13 @@ GDALDataset *GDALDriver::Open(GDALOpenInfo *poOpenInfo, bool bSetOpenOptions) if (poDS) { - poDS->nOpenFlags = poOpenInfo->nOpenFlags & ~GDAL_OF_FROM_GDALOPEN; + // Only set GDAL_OF_THREAD_SAFE if the driver itself has set it in + // poDS->nOpenFlags + int nOpenFlags = poOpenInfo->nOpenFlags & + ~(GDAL_OF_FROM_GDALOPEN | GDAL_OF_THREAD_SAFE); + if (poDS->nOpenFlags & GDAL_OF_THREAD_SAFE) + nOpenFlags |= GDAL_OF_THREAD_SAFE; + poDS->nOpenFlags = nOpenFlags; if (strlen(poDS->GetDescription()) == 0) poDS->SetDescription(poOpenInfo->pszFilename); diff --git a/gcore/gdalpamrasterband.cpp b/gcore/gdalpamrasterband.cpp index 670a4d3b38a4..adf8a0ddc4ec 100644 --- a/gcore/gdalpamrasterband.cpp +++ b/gcore/gdalpamrasterband.cpp @@ -50,6 +50,61 @@ #include "gdal_priv.h" #include "gdal_rat.h" +/************************************************************************/ +/* CopyFrom() */ +/************************************************************************/ + +//! @cond Doxygen_Suppress + +void GDALRasterBandPamInfo::CopyFrom(const GDALRasterBandPamInfo &sOther) +{ + bNoDataValueSet = sOther.bNoDataValueSet; + bNoDataValueSetAsInt64 = sOther.bNoDataValueSetAsInt64; + bNoDataValueSetAsUInt64 = sOther.bNoDataValueSetAsUInt64; + + dfNoDataValue = sOther.dfNoDataValue; + nNoDataValueInt64 = sOther.nNoDataValueInt64; + nNoDataValueUInt64 = sOther.nNoDataValueUInt64; + + delete poColorTable; + poColorTable = sOther.poColorTable + ? new GDALColorTable(*(sOther.poColorTable)) + : nullptr; + + eColorInterp = sOther.eColorInterp; + + CPLFree(pszUnitType); + pszUnitType = sOther.pszUnitType ? CPLStrdup(sOther.pszUnitType) : nullptr; + + CSLDestroy(papszCategoryNames); + papszCategoryNames = CSLDuplicate(sOther.papszCategoryNames); + + dfOffset = sOther.dfOffset; + dfScale = sOther.dfScale; + + bHaveMinMax = sOther.bHaveMinMax; + dfMin = sOther.dfMin; + dfMax = sOther.dfMax; + + bHaveStats = sOther.bHaveStats; + dfMean = sOther.dfMean; + dfStdDev = sOther.dfStdDev; + + if (psSavedHistograms) + CPLDestroyXMLNode(psSavedHistograms); + psSavedHistograms = sOther.psSavedHistograms + ? CPLCloneXMLTree(sOther.psSavedHistograms) + : nullptr; + + delete poDefaultRAT; + poDefaultRAT = sOther.poDefaultRAT ? sOther.poDefaultRAT->Clone() : nullptr; + + bOffsetSet = sOther.bOffsetSet; + bScaleSet = sOther.bScaleSet; +} + +//! @endcond + /************************************************************************/ /* GDALPamRasterBand() */ /************************************************************************/ diff --git a/gcore/gdalthreadsafedataset.cpp b/gcore/gdalthreadsafedataset.cpp new file mode 100644 index 000000000000..3b2c8156f249 --- /dev/null +++ b/gcore/gdalthreadsafedataset.cpp @@ -0,0 +1,1169 @@ +/****************************************************************************** + * + * Project: GDAL Core + * Purpose: Base class for thread safe dataset + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2024, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#ifndef DOXYGEN_SKIP + +#include "cpl_mem_cache.h" +#include "gdal_proxy.h" +#include "gdal_rat.h" +#include "gdal_priv.h" + +#include +#include +#include +#include +#include + +/** Design notes of this file. + * + * This file is at the core of the "RFC 101 - Raster dataset read-only thread-safety". + * Please consult it for high level understanding. + * + * 3 classes are involved: + * - GDALThreadSafeDataset whose instances are returned to the user, and can + * use them in a thread-safe way. + * - GDALThreadSafeRasterBand whose instances are created (and owned) by a + * GDALThreadSafeDataset instance, and returned to the user, which can use + * them in a thread-safe way. + * - GDALThreadLocalDatasetCache which is an internal class, which holds the + * thread-local datasets. + */ + +/************************************************************************/ +/* GDALThreadLocalDatasetCache */ +/************************************************************************/ + +class GDALThreadSafeDataset; + +/** This class is instantiated once per thread that uses a + * GDALThreadSafeDataset instance. It holds mostly a cache that maps a + * GDALThreadSafeDataset* pointer to the corresponding per-thread dataset. + */ +class GDALThreadLocalDatasetCache +{ + private: + /** Least-recently-used based cache that maps a GDALThreadSafeDataset* + * instance to the corresponding per-thread dataset. + * It should be noted as this a LRU cache, entries might get evicted when + * its capacity is reached (64 datasets), which might be undesirable. + * Hence it is doubled with m_oMapReferencedDS for datasets that are in + * active used by a thread. + * + * This cache is created as a unique_ptr, and not a standard object, for + * delicate reasons related to application termination, where we might + * want to leak the memory associated to it, to avoid the dataset it + * references from being closed, after GDAL has been "closed" (typically + * GDALDestroyDriverManager() has been called), which would otherwise lead + * to crashes. + */ + std::unique_ptr>> + m_poCache{}; + + GDALThreadLocalDatasetCache(const GDALThreadLocalDatasetCache &) = delete; + GDALThreadLocalDatasetCache & + operator=(const GDALThreadLocalDatasetCache &) = delete; + + public: + GDALThreadLocalDatasetCache(); + ~GDALThreadLocalDatasetCache(); + + /** Thread-id of the thread that instantiated this object. Used only for + * CPLDebug() purposes + */ + GIntBig m_nThreadID = 0; + + /** Mutex that protects access to m_oCache. There is "competition" around + * access to m_oCache since the destructor of a GDALThreadSafeDataset + * instance needs to evict entries corresponding to itself from all + * GDALThreadLocalDatasetCache instances. + */ + std::mutex m_oMutex{}; + + /** This is a reference to *(m_poCache.get()). + */ + lru11::Cache> + &m_oCache; + + /** Pair of shared_ptr with associated thread-local config + * options that were valid in the calling thread at the time + * GDALThreadLocalDatasetCache::RefUnderlyingDataset() was called, so they + * can be restored at UnrefUnderlyingDataset() time. + */ + struct SharedPtrDatasetThreadLocalConfigOptionsPair + { + std::shared_ptr poDS; + CPLStringList aosTLConfigOptions; + + SharedPtrDatasetThreadLocalConfigOptionsPair( + const std::shared_ptr &poDSIn, + CPLStringList &&aosTLConfigOptionsIn) + : poDS(poDSIn), aosTLConfigOptions(std::move(aosTLConfigOptionsIn)) + { + } + }; + + /** Maps a GDALThreadSafeDataset* + * instance to the corresponding per-thread dataset. Insertion into this + * map is done by GDALThreadLocalDatasetCache::RefUnderlyingDataset() and + * removal by UnrefUnderlyingDataset(). In most all use cases, the size of + * this map should be 0 or 1 (not clear if it could be more, that would + * involve RefUnderlyingDataset() being called in nested ways by the same + * thread, but it doesn't hurt from being robust to that potential situation) + */ + std::map + m_oMapReferencedDS{}; + + /** Maps a GDALRasterBand* returned by GDALThreadSafeRasterBand::RefUnderlyingDataset() + * to the (thread-local) dataset that owns it (that is a dataset returned + * by RefUnderlyingDataset(). The size of his map should be 0 or 1 in + * most cases. + */ + std::map m_oMapReferencedDSFromBand{}; +}; + +/************************************************************************/ +/* GDALThreadSafeDataset */ +/************************************************************************/ + +/** Global variable used to determine if the singleton of GlobalCache + * owned by GDALThreadSafeDataset is valid. + * This is needed to avoid issues at process termination where the order + * of destruction between static global instances and TLS instances can be + * tricky. + */ +static bool bGlobalCacheValid = false; + +/** Thread-safe GDALDataset class. + * + * That class delegates all calls to its members to per-thread GDALDataset + * instances. + */ +class GDALThreadSafeDataset final : public GDALProxyDataset +{ + public: + GDALThreadSafeDataset(std::unique_ptr poPrototypeDSUniquePtr, + GDALDataset *poPrototypeDS); + ~GDALThreadSafeDataset() override; + + static std::unique_ptr + Create(std::unique_ptr poPrototypeDS, int nScopeFlags); + + static GDALDataset *Create(GDALDataset *poPrototypeDS, int nScopeFlags); + + protected: + GDALDataset *RefUnderlyingDataset() const override; + + void + UnrefUnderlyingDataset(GDALDataset *poUnderlyingDataset) const override; + + int CloseDependentDatasets() override; + + private: + friend class GDALThreadSafeRasterBand; + friend class GDALThreadLocalDatasetCache; + + /** Mutex that protects accesses to m_poPrototypeDS */ + std::mutex m_oPrototypeDSMutex{}; + + /** "Prototype" dataset, that is the dataset that was passed to the + * GDALThreadSafeDataset constructor. All calls on to it should be on + * const methods, and should be protected by m_oPrototypeDSMutex (except + * during GDALThreadSafeDataset instance construction) + */ + const GDALDataset *m_poPrototypeDS = nullptr; + + /** Unique pointer for m_poPrototypeDS in the cases where GDALThreadSafeDataset + * has been passed a unique pointer */ + std::unique_ptr m_poPrototypeDSUniquePtr{}; + + /** Thread-local config options at the time where GDALThreadSafeDataset + * has been constructed. + */ + const CPLStringList m_aosThreadLocalConfigOptions{}; + + /** Structure that references all GDALThreadLocalDatasetCache* instances. + */ + struct GlobalCache + { + /** Mutex that protect access to oSetOfCache */ + std::mutex oMutex{}; + + /** Set of GDALThreadLocalDatasetCache* instances. That is it has + * one entry per thread that has used at least once a + * GDALThreadLocalDatasetCache/GDALThreadSafeRasterBand + */ + std::set oSetOfCache{}; + + GlobalCache() + { + bGlobalCacheValid = true; + } + + ~GlobalCache() + { + bGlobalCacheValid = false; + } + }; + + /** Returns a singleton for a GlobalCache instance that references all + * GDALThreadLocalDatasetCache* instances. + */ + static GlobalCache &GetSetOfCache() + { + static GlobalCache cache; + return cache; + } + + /** Thread-local dataset cache. */ + static thread_local std::unique_ptr tl_poCache; + + void UnrefUnderlyingDataset(GDALDataset *poUnderlyingDataset, + GDALThreadLocalDatasetCache *poCache) const; + + GDALThreadSafeDataset(const GDALThreadSafeDataset &) = delete; + GDALThreadSafeDataset &operator=(const GDALThreadSafeDataset &) = delete; +}; + +/************************************************************************/ +/* GDALThreadSafeRasterBand */ +/************************************************************************/ + +/** Thread-safe GDALRasterBand class. + * + * That class delegates all calls to its members to per-thread GDALDataset + * instances. + */ +class GDALThreadSafeRasterBand final : public GDALProxyRasterBand +{ + public: + GDALThreadSafeRasterBand(GDALThreadSafeDataset *poTSDS, + GDALDataset *poParentDS, int nBandIn, + GDALRasterBand *poPrototypeBand, + int nBaseBandOfMaskBand, int nOvrIdx); + + GDALRasterBand *GetMaskBand() override; + int GetOverviewCount() override; + GDALRasterBand *GetOverview(int idx) override; + GDALRasterBand *GetRasterSampleOverview(GUIntBig nDesiredSamples) override; + + GDALRasterAttributeTable *GetDefaultRAT() override; + + protected: + GDALRasterBand *RefUnderlyingRasterBand(bool bForceOpen) const override; + void UnrefUnderlyingRasterBand( + GDALRasterBand *poUnderlyingRasterBand) const override; + + private: + /** Pointer to the thread-safe dataset from which this band has been + *created */ + GDALThreadSafeDataset *m_poTSDS = nullptr; + + /** Pointer to the "prototype" raster band that corresponds to us. + * All calls to m_poPrototypeBand should be protected by + * GDALThreadSafeDataset:m_oPrototypeDSMutex. + */ + const GDALRasterBand *m_poPrototypeBand = nullptr; + + /** 0 for standard bands, otherwise > 0 value that indicates that this + * band is a mask band and m_nBaseBandOfMaskBand is then the number + * of the band that is the parent of the mask band. + */ + const int m_nBaseBandOfMaskBand; + + /** 0 for standard bands, otherwise >= 0 value that indicates that this + * band is an overview band and m_nOvrIdx is then the index of the overview. + */ + const int m_nOvrIdx; + + /** Mask band associated with this band. */ + std::unique_ptr m_poMaskBand{}; + + /** List of overviews associated with this band. */ + std::vector> m_apoOverviews{}; + + GDALThreadSafeRasterBand(const GDALThreadSafeRasterBand &) = delete; + GDALThreadSafeRasterBand & + operator=(const GDALThreadSafeRasterBand &) = delete; +}; + +/************************************************************************/ +/* Global variables initialization. */ +/************************************************************************/ + +/** Instantiation of the TLS cache of datasets */ +thread_local std::unique_ptr + GDALThreadSafeDataset::tl_poCache; + +/************************************************************************/ +/* GDALThreadLocalDatasetCache() */ +/************************************************************************/ + +/** Constructor of GDALThreadLocalDatasetCache. This is called implicitly + * when GDALThreadSafeDataset::tl_poCache is called the first time by a + * thread. + */ +GDALThreadLocalDatasetCache::GDALThreadLocalDatasetCache() + : m_poCache(std::make_unique>>()), + m_nThreadID(CPLGetPID()), m_oCache(*m_poCache.get()) +{ + CPLDebug("GDAL", + "Registering thread-safe dataset cache for thread " CPL_FRMT_GIB, + m_nThreadID); + + // We reference ourselves to the GDALThreadSafeDataset set-of-cache singleton + auto &oSetOfCache = GDALThreadSafeDataset::GetSetOfCache(); + std::lock_guard oLock(oSetOfCache.oMutex); + oSetOfCache.oSetOfCache.insert(this); +} + +/************************************************************************/ +/* ~GDALThreadLocalDatasetCache() */ +/************************************************************************/ + +/** Destructor of GDALThreadLocalDatasetCache. This is called implicitly when a + * thread is terminated. + */ +GDALThreadLocalDatasetCache::~GDALThreadLocalDatasetCache() +{ + // If GDAL has been de-initialized explicitly (ie GDALDestroyDriverManager() + // has been called), or we are during process termination, do not try to + // free m_poCache at all, which would cause the datasets its owned to be + // destroyed, which will generally lead to crashes in those situations where + // GDAL has been de-initialized. + const bool bDriverManagerDestroyed = *GDALGetphDMMutex() == nullptr; + if (bDriverManagerDestroyed || !bGlobalCacheValid) + { + // Leak datasets when GDAL has been de-initialized + if (!m_poCache->empty()) + CPL_IGNORE_RET_VAL(m_poCache.release()); + return; + } + + // Unreference ourselves from the GDALThreadSafeDataset set-of-cache singleton + CPLDebug("GDAL", + "Unregistering thread-safe dataset cache for thread " CPL_FRMT_GIB, + m_nThreadID); + { + auto &oSetOfCache = GDALThreadSafeDataset::GetSetOfCache(); + std::lock_guard oLock(oSetOfCache.oMutex); + oSetOfCache.oSetOfCache.erase(this); + } + + // Below code is just for debugging purposes and show which internal + // thread-local datasets are released at thread termination. + const auto lambda = + [this](const lru11::KeyValuePair> &kv) + { + CPLDebug("GDAL", + "~GDALThreadLocalDatasetCache(): GDALClose(%s, this=%p) " + "for thread " CPL_FRMT_GIB, + kv.value->GetDescription(), kv.value.get(), m_nThreadID); + }; + m_oCache.cwalk(lambda); +} + +/************************************************************************/ +/* GDALThreadSafeDataset() */ +/************************************************************************/ + +/** Constructor of GDALThreadSafeDataset. + * It may be called with poPrototypeDSUniquePtr set to null, in the situations + * where GDALThreadSafeDataset doesn't own the prototype dataset. + * poPrototypeDS should always be not-null, and if poPrototypeDSUniquePtr is + * not null, then poPrototypeDS should be equal to poPrototypeDSUniquePtr.get() + */ +GDALThreadSafeDataset::GDALThreadSafeDataset( + std::unique_ptr poPrototypeDSUniquePtr, + GDALDataset *poPrototypeDS) + : m_poPrototypeDS(poPrototypeDS), + m_aosThreadLocalConfigOptions(CPLGetThreadLocalConfigOptions()) +{ + CPLAssert(poPrototypeDS != nullptr); + if (poPrototypeDSUniquePtr) + { + CPLAssert(poPrototypeDS == poPrototypeDSUniquePtr.get()); + } + + // Replicate the characteristics of the prototype dataset onto ourselves + nRasterXSize = poPrototypeDS->GetRasterXSize(); + nRasterYSize = poPrototypeDS->GetRasterYSize(); + for (int i = 1; i <= poPrototypeDS->GetRasterCount(); ++i) + { + SetBand(i, std::make_unique( + this, this, i, poPrototypeDS->GetRasterBand(i), 0, -1)); + } + nOpenFlags = GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE; + SetDescription(poPrototypeDS->GetDescription()); + papszOpenOptions = CSLDuplicate(poPrototypeDS->GetOpenOptions()); + + m_poPrototypeDSUniquePtr = std::move(poPrototypeDSUniquePtr); + + // In the case where we are constructed without owning the prototype + // dataset, let's increase its reference counter though. + if (!m_poPrototypeDSUniquePtr) + const_cast(m_poPrototypeDS)->Reference(); +} + +/************************************************************************/ +/* Create() */ +/************************************************************************/ + +/** Utility method used by GDALGetThreadSafeDataset() to construct a + * GDALThreadSafeDataset instance in the case where the GDALThreadSafeDataset + * instance owns the prototype dataset. + */ + +/* static */ std::unique_ptr +GDALThreadSafeDataset::Create(std::unique_ptr poPrototypeDS, + int nScopeFlags) +{ + if (nScopeFlags != GDAL_OF_RASTER) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Only nScopeFlags == " + "GDAL_OF_RASTER is supported"); + return nullptr; + } + if (poPrototypeDS->IsThreadSafe(nScopeFlags)) + { + return poPrototypeDS; + } + if (!poPrototypeDS->CanBeCloned(nScopeFlags, /* bCanShareState = */ true)) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Source dataset cannot be " + "cloned"); + return nullptr; + } + auto poPrototypeDSRaw = poPrototypeDS.get(); + return std::make_unique(std::move(poPrototypeDS), + poPrototypeDSRaw); +} + +/************************************************************************/ +/* Create() */ +/************************************************************************/ + +/** Utility method used by GDALGetThreadSafeDataset() to construct a + * GDALThreadSafeDataset instance in the case where the GDALThreadSafeDataset + * instance does not own the prototype dataset. + */ + +/* static */ GDALDataset * +GDALThreadSafeDataset::Create(GDALDataset *poPrototypeDS, int nScopeFlags) +{ + if (nScopeFlags != GDAL_OF_RASTER) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Only nScopeFlags == " + "GDAL_OF_RASTER is supported"); + return nullptr; + } + if (poPrototypeDS->IsThreadSafe(nScopeFlags)) + { + poPrototypeDS->Reference(); + return poPrototypeDS; + } + if (!poPrototypeDS->CanBeCloned(nScopeFlags, /* bCanShareState = */ true)) + { + CPLError(CE_Failure, CPLE_NotSupported, + "GDALGetThreadSafeDataset(): Source dataset cannot be " + "cloned"); + return nullptr; + } + return std::make_unique(nullptr, poPrototypeDS) + .release(); +} + +/************************************************************************/ +/* ~GDALThreadSafeDataset() */ +/************************************************************************/ + +GDALThreadSafeDataset::~GDALThreadSafeDataset() +{ + // Collect TLS datasets in a vector, and free them after releasing + // g_nInDestructorCounter to limit contention + std::vector, GIntBig>> aoDSToFree; + { + auto &oSetOfCache = GetSetOfCache(); + std::lock_guard oLock(oSetOfCache.oMutex); + for (auto *poCache : oSetOfCache.oSetOfCache) + { + std::unique_lock oLockCache(poCache->m_oMutex); + std::shared_ptr poDS; + if (poCache->m_oCache.tryGet(this, poDS)) + { + aoDSToFree.emplace_back(std::move(poDS), poCache->m_nThreadID); + poCache->m_oCache.remove(this); + } + } + } + + for (const auto &oEntry : aoDSToFree) + { + CPLDebug("GDAL", + "~GDALThreadSafeDataset(): GDALClose(%s, this=%p) for " + "thread " CPL_FRMT_GIB, + GetDescription(), oEntry.first.get(), oEntry.second); + } + // Actually release TLS datasets + aoDSToFree.clear(); + + GDALThreadSafeDataset::CloseDependentDatasets(); +} + +/************************************************************************/ +/* CloseDependentDatasets() */ +/************************************************************************/ + +/** Implements GDALDataset::CloseDependentDatasets() + * + * Takes care of releasing the prototype dataset. + * + * As implied by the contract of CloseDependentDatasets(), returns true if + * the prototype dataset has actually been released (or false if + * CloseDependentDatasets() has already been closed) + */ +int GDALThreadSafeDataset::CloseDependentDatasets() +{ + int bRet = false; + if (m_poPrototypeDSUniquePtr) + { + bRet = true; + } + else if (m_poPrototypeDS) + { + if (const_cast(m_poPrototypeDS)->ReleaseRef()) + { + bRet = true; + } + } + + m_poPrototypeDSUniquePtr.reset(); + m_poPrototypeDS = nullptr; + + return bRet; +} + +/************************************************************************/ +/* RefUnderlyingDataset() */ +/************************************************************************/ + +/** Implements GDALProxyDataset::RefUnderlyingDataset. + * + * This method is called by all virtual methods of GDALDataset overridden by + * RefUnderlyingDataset() when it delegates the calls to the underlying + * dataset. + * + * Our implementation takes care of opening a thread-local dataset, on the + * same underlying dataset of m_poPrototypeDS, if needed, and to insert it + * into a cache for fast later uses by the same thread. + */ +GDALDataset *GDALThreadSafeDataset::RefUnderlyingDataset() const +{ + // Back-up thread-local config options at the time we are called + CPLStringList aosTLConfigOptionsBackup(CPLGetThreadLocalConfigOptions()); + + // Now merge the thread-local config options at the time where this + // instance has been created with the current ones. + const CPLStringList aosMerged( + CSLMerge(CSLDuplicate(m_aosThreadLocalConfigOptions.List()), + aosTLConfigOptionsBackup.List())); + + // And make that merged list active + CPLSetThreadLocalConfigOptions(aosMerged.List()); + + std::shared_ptr poTLSDS; + + // Get the thread-local dataset cache for this thread. + GDALThreadLocalDatasetCache *poCache = tl_poCache.get(); + if (!poCache) + { + auto poCacheUniquePtr = std::make_unique(); + poCache = poCacheUniquePtr.get(); + tl_poCache = std::move(poCacheUniquePtr); + } + + // Check if there's an entry in this cache for our current GDALThreadSafeDataset + // instance. + std::unique_lock oLock(poCache->m_oMutex); + if (poCache->m_oCache.tryGet(this, poTLSDS)) + { + // If so, return it, but before returning, make sure to creates a + // "hard" reference to the thread-local dataset, in case it would + // get evicted from poCache->m_oCache (by other threads that would + // access lots of datasets in between) + CPLAssert(!cpl::contains(poCache->m_oMapReferencedDS, this)); + auto poDSRet = poTLSDS.get(); + poCache->m_oMapReferencedDS.insert( + {this, GDALThreadLocalDatasetCache:: + SharedPtrDatasetThreadLocalConfigOptionsPair( + poTLSDS, std::move(aosTLConfigOptionsBackup))}); + return poDSRet; + } + + // "Clone" the prototype dataset, which in 99% of the cases, involves + // doing a GDALDataset::Open() call to re-open it. Do that by temporarily + // dropping the lock that protects poCache->m_oCache. + oLock.unlock(); + poTLSDS = m_poPrototypeDS->Clone(GDAL_OF_RASTER, /* bCanShareState=*/true); + if (poTLSDS) + { + CPLDebug("GDAL", "GDALOpen(%s, this=%p) for thread " CPL_FRMT_GIB, + GetDescription(), poTLSDS.get(), CPLGetPID()); + + // Check that the re-openeded dataset has the same characteristics + // as "this" / m_poPrototypeDS + if (poTLSDS->GetRasterXSize() != nRasterXSize || + poTLSDS->GetRasterYSize() != nRasterYSize || + poTLSDS->GetRasterCount() != nBands) + { + poTLSDS.reset(); + CPLError(CE_Failure, CPLE_AppDefined, + "Re-opened dataset for %s does not share the same " + "characteristics has the master dataset", + GetDescription()); + } + } + + // Re-acquire the lok + oLock.lock(); + + // In case of failed closing, restore the thread-local config options that + // were valid at the beginning of this method, and return in error. + if (!poTLSDS) + { + CPLSetThreadLocalConfigOptions(aosTLConfigOptionsBackup.List()); + return nullptr; + } + + // We have managed to get a thread-local dataset. Insert it into the + // LRU cache and the m_oMapReferencedDS map that holds strong references. + auto poDSRet = poTLSDS.get(); + { + poCache->m_oCache.insert(this, poTLSDS); + CPLAssert(!cpl::contains(poCache->m_oMapReferencedDS, this)); + poCache->m_oMapReferencedDS.insert( + {this, GDALThreadLocalDatasetCache:: + SharedPtrDatasetThreadLocalConfigOptionsPair( + poTLSDS, std::move(aosTLConfigOptionsBackup))}); + } + return poDSRet; +} + +/************************************************************************/ +/* UnrefUnderlyingDataset() */ +/************************************************************************/ + +/** Implements GDALProxyDataset::UnrefUnderlyingDataset. + * + * This is called by GDALProxyDataset overridden methods of GDALDataset, when + * they no longer need to access the underlying dataset. + * + * This method actually delegates most of the work to the other + * UnrefUnderlyingDataset() method that takes an explicit GDALThreadLocalDatasetCache* + * instance. + */ +void GDALThreadSafeDataset::UnrefUnderlyingDataset( + GDALDataset *poUnderlyingDataset) const +{ + GDALThreadLocalDatasetCache *poCache = tl_poCache.get(); + CPLAssert(poCache); + std::unique_lock oLock(poCache->m_oMutex); + UnrefUnderlyingDataset(poUnderlyingDataset, poCache); +} + +/************************************************************************/ +/* UnrefUnderlyingDataset() */ +/************************************************************************/ + +/** Takes care of removing the strong reference to a thread-local dataset + * from the TLS cache of datasets. + */ +void GDALThreadSafeDataset::UnrefUnderlyingDataset( + [[maybe_unused]] GDALDataset *poUnderlyingDataset, + GDALThreadLocalDatasetCache *poCache) const +{ + auto oIter = poCache->m_oMapReferencedDS.find(this); + CPLAssert(oIter != poCache->m_oMapReferencedDS.end()); + CPLAssert(oIter->second.poDS.get() == poUnderlyingDataset); + CPLSetThreadLocalConfigOptions(oIter->second.aosTLConfigOptions.List()); + poCache->m_oMapReferencedDS.erase(oIter); +} + +/************************************************************************/ +/* GDALThreadSafeRasterBand() */ +/************************************************************************/ + +GDALThreadSafeRasterBand::GDALThreadSafeRasterBand( + GDALThreadSafeDataset *poTSDS, GDALDataset *poParentDS, int nBandIn, + GDALRasterBand *poPrototypeBand, int nBaseBandOfMaskBand, int nOvrIdx) + : m_poTSDS(poTSDS), m_poPrototypeBand(poPrototypeBand), + m_nBaseBandOfMaskBand(nBaseBandOfMaskBand), m_nOvrIdx(nOvrIdx) +{ + // Replicates characteristics of the prototype band. + poDS = poParentDS; + nBand = nBandIn; + eDataType = poPrototypeBand->GetRasterDataType(); + nRasterXSize = poPrototypeBand->GetXSize(); + nRasterYSize = poPrototypeBand->GetYSize(); + poPrototypeBand->GetBlockSize(&nBlockXSize, &nBlockYSize); + + if (nBandIn > 0) + { + // For regular bands instantiates a (thread-safe) mask band and + // as many overviews as needed. + + m_poMaskBand = std::make_unique( + poTSDS, nullptr, 0, poPrototypeBand->GetMaskBand(), nBandIn, + nOvrIdx); + if (nOvrIdx < 0) + { + const int nOvrCount = poPrototypeBand->GetOverviewCount(); + for (int iOvrIdx = 0; iOvrIdx < nOvrCount; ++iOvrIdx) + { + m_apoOverviews.emplace_back( + std::make_unique( + poTSDS, nullptr, nBandIn, + poPrototypeBand->GetOverview(iOvrIdx), + nBaseBandOfMaskBand, iOvrIdx)); + } + } + } + else if (nBaseBandOfMaskBand > 0) + { + // If we are a mask band, nstanciates a (thread-safe) mask band of + // ourselves, but with the trick of negating nBaseBandOfMaskBand to + // avoid infinite recursion... + m_poMaskBand = std::make_unique( + poTSDS, nullptr, 0, poPrototypeBand->GetMaskBand(), + -nBaseBandOfMaskBand, nOvrIdx); + } +} + +/************************************************************************/ +/* RefUnderlyingRasterBand() */ +/************************************************************************/ + +/** Implements GDALProxyRasterBand::RefUnderlyingDataset. + * + * This method is called by all virtual methods of GDALRasterBand overridden by + * RefUnderlyingRasterBand() when it delegates the calls to the underlying + * band. + */ +GDALRasterBand * +GDALThreadSafeRasterBand::RefUnderlyingRasterBand(bool /*bForceOpen*/) const +{ + // Get a thread-local dataset + auto poTLDS = m_poTSDS->RefUnderlyingDataset(); + if (!poTLDS) + return nullptr; + + // Get corresponding thread-local band. If m_nBaseBandOfMaskBand is not + // zero, then the base band is indicated into it, otherwise use nBand. + const int nTLSBandIdx = + m_nBaseBandOfMaskBand ? std::abs(m_nBaseBandOfMaskBand) : nBand; + auto poTLRasterBand = poTLDS->GetRasterBand(nTLSBandIdx); + if (!poTLRasterBand) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): " + "GetRasterBand(%d) failed", + nTLSBandIdx); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + + // Get the overview level if needed. + if (m_nOvrIdx >= 0) + { + poTLRasterBand = poTLRasterBand->GetOverview(m_nOvrIdx); + if (!poTLRasterBand) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): " + "GetOverview(%d) failed", + m_nOvrIdx); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + } + + // Get the mask band (or the mask band of the mask band) if needed. + if (m_nBaseBandOfMaskBand) + { + poTLRasterBand = poTLRasterBand->GetMaskBand(); + if (m_nBaseBandOfMaskBand < 0) + poTLRasterBand = poTLRasterBand->GetMaskBand(); + } + + // Check that the thread-local band characteristics are identical to the + // ones of the prototype band. + if (m_poPrototypeBand->GetXSize() != poTLRasterBand->GetXSize() || + m_poPrototypeBand->GetYSize() != poTLRasterBand->GetYSize() || + m_poPrototypeBand->GetRasterDataType() != + poTLRasterBand->GetRasterDataType() + // m_poPrototypeBand->GetMaskFlags() is not thread-safe + // || m_poPrototypeBand->GetMaskFlags() != poTLRasterBand->GetMaskFlags() + ) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): TLS " + "band has not expected characteristics"); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + int nThisBlockXSize; + int nThisBlockYSize; + int nTLSBlockXSize; + int nTLSBlockYSize; + m_poPrototypeBand->GetBlockSize(&nThisBlockXSize, &nThisBlockYSize); + poTLRasterBand->GetBlockSize(&nTLSBlockXSize, &nTLSBlockYSize); + if (nThisBlockXSize != nTLSBlockXSize || nThisBlockYSize != nTLSBlockYSize) + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::RefUnderlyingRasterBand(): TLS " + "band has not expected characteristics"); + m_poTSDS->UnrefUnderlyingDataset(poTLDS); + return nullptr; + } + + // Registers the association between the thread-local band and the + // thread-local dataset + { + GDALThreadLocalDatasetCache *poCache = + GDALThreadSafeDataset::tl_poCache.get(); + CPLAssert(poCache); + std::unique_lock oLock(poCache->m_oMutex); + CPLAssert(!cpl::contains(poCache->m_oMapReferencedDSFromBand, + poTLRasterBand)); + poCache->m_oMapReferencedDSFromBand[poTLRasterBand] = poTLDS; + } + // CPLDebug("GDAL", "%p->RefUnderlyingRasterBand() return %p", this, poTLRasterBand); + return poTLRasterBand; +} + +/************************************************************************/ +/* UnrefUnderlyingRasterBand() */ +/************************************************************************/ + +/** Implements GDALProxyRasterBand::UnrefUnderlyingRasterBand. + * + * This is called by GDALProxyRasterBand overridden methods of GDALRasterBand, + * when they no longer need to access the underlying dataset. + */ +void GDALThreadSafeRasterBand::UnrefUnderlyingRasterBand( + GDALRasterBand *poUnderlyingRasterBand) const +{ + // CPLDebug("GDAL", "%p->UnrefUnderlyingRasterBand(%p)", this, poUnderlyingRasterBand); + + // Unregisters the association between the thread-local band and the + // thread-local dataset + { + GDALThreadLocalDatasetCache *poCache = + GDALThreadSafeDataset::tl_poCache.get(); + CPLAssert(poCache); + std::unique_lock oLock(poCache->m_oMutex); + auto oIter = + poCache->m_oMapReferencedDSFromBand.find(poUnderlyingRasterBand); + CPLAssert(oIter != poCache->m_oMapReferencedDSFromBand.end()); + + m_poTSDS->UnrefUnderlyingDataset(oIter->second, poCache); + poCache->m_oMapReferencedDSFromBand.erase(oIter); + } +} + +/************************************************************************/ +/* GetMaskBand() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetMaskBand + */ +GDALRasterBand *GDALThreadSafeRasterBand::GetMaskBand() +{ + return m_poMaskBand ? m_poMaskBand.get() : this; +} + +/************************************************************************/ +/* GetOverviewCount() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetOverviewCount + */ +int GDALThreadSafeRasterBand::GetOverviewCount() +{ + return static_cast(m_apoOverviews.size()); +} + +/************************************************************************/ +/* GetOverview() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetOverview + */ +GDALRasterBand *GDALThreadSafeRasterBand::GetOverview(int nIdx) +{ + if (nIdx < 0 || nIdx >= static_cast(m_apoOverviews.size())) + return nullptr; + return m_apoOverviews[nIdx].get(); +} + +/************************************************************************/ +/* GetRasterSampleOverview() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetRasterSampleOverview + */ +GDALRasterBand * +GDALThreadSafeRasterBand::GetRasterSampleOverview(GUIntBig nDesiredSamples) + +{ + // Call the base implementation, and do not forward to proxy + return GDALRasterBand::GetRasterSampleOverview(nDesiredSamples); +} + +/************************************************************************/ +/* GetDefaultRAT() */ +/************************************************************************/ + +/** Implements GDALRasterBand::GetDefaultRAT + * + * This is a bit tricky to do as GDALRasterAttributeTable has virtual methods + * with potential (non thread-safe) side-effects. The clean solution would be + * to implement a GDALThreadSafeRAT wrapper class, but this is a bit too much + * effort. So for now, we check if the RAT returned by the prototype band is + * an instance of GDALDefaultRasterAttributeTable. If it is, given that this + * class has thread-safe getters, we can directly return it. + * Otherwise return in error. + */ +GDALRasterAttributeTable *GDALThreadSafeRasterBand::GetDefaultRAT() +{ + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + const auto poRAT = + const_cast(m_poPrototypeBand)->GetDefaultRAT(); + if (!poRAT) + return nullptr; + + if (dynamic_cast(poRAT)) + return poRAT; + + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::GetDefaultRAT() not supporting a " + "non-GDALDefaultRasterAttributeTable implementation"); + return nullptr; +} + +#endif // DOXYGEN_SKIP + +/************************************************************************/ +/* GDALDataset::IsThreadSafe() */ +/************************************************************************/ + +/** Return whether this dataset, and its related objects (typically raster + * bands), can be called for the intended scope. + * + * Note that in the current implementation, nScopeFlags should be set to + * GDAL_OF_RASTER, as thread-safety is limited to read-only operations and + * excludes operations on vector layers (OGRLayer) or multidimensional API + * (GDALGroup, GDALMDArray, etc.) + * + * This is the same as the C function GDALDatasetIsThreadSafe(). + * + * @since 3.10 + */ +bool GDALDataset::IsThreadSafe(int nScopeFlags) const +{ + return (nOpenFlags & GDAL_OF_THREAD_SAFE) != 0 && + nScopeFlags == GDAL_OF_RASTER && (nOpenFlags & GDAL_OF_RASTER) != 0; +} + +/************************************************************************/ +/* GDALDatasetIsThreadSafe() */ +/************************************************************************/ + +/** Return whether this dataset, and its related objects (typically raster + * bands), can be called for the intended scope. + * + * Note that in the current implementation, nScopeFlags should be set to + * GDAL_OF_RASTER, as thread-safety is limited to read-only operations and + * excludes operations on vector layers (OGRLayer) or multidimensional API + * (GDALGroup, GDALMDArray, etc.) + * + * This is the same as the C++ method GDALDataset::IsThreadSafe(). + * + * @param hDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * @param papszOptions Options. None currently. + * + * @since 3.10 + */ +bool GDALDatasetIsThreadSafe(GDALDatasetH hDS, int nScopeFlags, + CSLConstList papszOptions) +{ + VALIDATE_POINTER1(hDS, __func__, false); + + CPL_IGNORE_RET_VAL(papszOptions); + + return GDALDataset::FromHandle(hDS)->IsThreadSafe(nScopeFlags); +} + +/************************************************************************/ +/* GDALGetThreadSafeDataset() */ +/************************************************************************/ + +/** Return a thread-safe dataset. + * + * In the general case, this thread-safe dataset will open a + * behind-the-scenes per-thread dataset (re-using the name and open options of poDS), + * the first time a thread calls a method on the thread-safe dataset, and will + * transparently redirect calls from the calling thread to this behind-the-scenes + * per-thread dataset. Hence there is an initial setup cost per thread. + * Datasets of the MEM driver cannot be opened by name, but this function will + * take care of "cloning" them, using the same backing memory, when needed. + * + * Ownership of the passed dataset is transferred to the thread-safe dataset. + * + * The function may also return the passed dataset if it is already thread-safe. + * + * @param poDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * + * @return a new thread-safe dataset, or nullptr in case of error. + * + * @since 3.10 + */ +std::unique_ptr +GDALGetThreadSafeDataset(std::unique_ptr poDS, int nScopeFlags) +{ + return GDALThreadSafeDataset::Create(std::move(poDS), nScopeFlags); +} + +/************************************************************************/ +/* GDALGetThreadSafeDataset() */ +/************************************************************************/ + +/** Return a thread-safe dataset. + * + * In the general case, this thread-safe dataset will open a + * behind-the-scenes per-thread dataset (re-using the name and open options of poDS), + * the first time a thread calls a method on the thread-safe dataset, and will + * transparently redirect calls from the calling thread to this behind-the-scenes + * per-thread dataset. Hence there is an initial setup cost per thread. + * Datasets of the MEM driver cannot be opened by name, but this function will + * take care of "cloning" them, using the same backing memory, when needed. + * + * The life-time of the passed dataset must be longer than the one of + * the returned thread-safe dataset. + * + * Note that this function does increase the reference count on poDS while + * it is being used + * + * The function may also return the passed dataset if it is already thread-safe. + * A non-nullptr returned dataset must be released with ReleaseRef(). + * + * Patterns like the following one are valid: + * \code{.cpp} + * auto poDS = GDALDataset::Open(...); + * auto poThreadSafeDS = GDALGetThreadSafeDataset(poDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE); + * poDS->ReleaseRef(); + * if (poThreadSafeDS ) + * { + * // ... do something with poThreadSafeDS ... + * poThreadSafeDS->ReleaseRef(); + * } + * \endcode + * + * @param poDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * + * @return a new thread-safe dataset or poDS, or nullptr in case of error. + + * @since 3.10 + */ +GDALDataset *GDALGetThreadSafeDataset(GDALDataset *poDS, int nScopeFlags) +{ + return GDALThreadSafeDataset::Create(poDS, nScopeFlags); +} + +/************************************************************************/ +/* GDALGetThreadSafeDataset() */ +/************************************************************************/ + +/** Return a thread-safe dataset. + * + * In the general case, this thread-safe dataset will open a + * behind-the-scenes per-thread dataset (re-using the name and open options of hDS), + * the first time a thread calls a method on the thread-safe dataset, and will + * transparently redirect calls from the calling thread to this behind-the-scenes + * per-thread dataset. Hence there is an initial setup cost per thread. + * Datasets of the MEM driver cannot be opened by name, but this function will + * take care of "cloning" them, using the same backing memory, when needed. + * + * The life-time of the passed dataset must be longer than the one of + * the returned thread-safe dataset. + * + * Note that this function does increase the reference count on poDS while + * it is being used + * + * The function may also return the passed dataset if it is already thread-safe. + * A non-nullptr returned dataset must be released with GDALReleaseDataset(). + * + * \code{.cpp} + * hDS = GDALOpenEx(...); + * hThreadSafeDS = GDALGetThreadSafeDataset(hDS, GDAL_OF_RASTER | GDAL_OF_THREAD_SAFE, NULL); + * GDALReleaseDataset(hDS); + * if( hThreadSafeDS ) + * { + * // ... do something with hThreadSafeDS ... + * GDALReleaseDataset(hThreadSafeDS); + * } + * \endcode + * + * @param hDS Source dataset + * @param nScopeFlags Intended scope of use. + * Only GDAL_OF_RASTER is supported currently. + * @param papszOptions Options. None currently. + * + * @since 3.10 + */ +GDALDatasetH GDALGetThreadSafeDataset(GDALDatasetH hDS, int nScopeFlags, + CSLConstList papszOptions) +{ + VALIDATE_POINTER1(hDS, __func__, nullptr); + + CPL_IGNORE_RET_VAL(papszOptions); + return GDALDataset::ToHandle( + GDALGetThreadSafeDataset(GDALDataset::FromHandle(hDS), nScopeFlags)); +} diff --git a/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp b/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp index d6ce19133af9..3e34c28334f7 100644 --- a/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp +++ b/ogr/ogrsf_frmts/sqlite/ogrsqlitedatasource.cpp @@ -2165,7 +2165,7 @@ bool OGRSQLiteDataSource::Open(GDALOpenInfo *poOpenInfo) const char *pszNewName = poOpenInfo->pszFilename; CPLAssert(m_apoLayers.empty()); eAccess = poOpenInfo->eAccess; - nOpenFlags = poOpenInfo->nOpenFlags; + nOpenFlags = poOpenInfo->nOpenFlags & ~GDAL_OF_THREAD_SAFE; SetDescription(pszNewName); if (m_pszFilename == nullptr) diff --git a/port/cpl_mem_cache.h b/port/cpl_mem_cache.h index 6fecca4b239f..0e021b762748 100644 --- a/port/cpl_mem_cache.h +++ b/port/cpl_mem_cache.h @@ -86,6 +86,10 @@ template struct KeyValuePair KeyValuePair(const K &k, V &&v) : key(k), value(std::move(v)) { } + + private: + KeyValuePair(const KeyValuePair &) = delete; + KeyValuePair &operator=(const KeyValuePair &) = delete; }; /** diff --git a/swig/include/Band.i b/swig/include/Band.i index 5766f0e0c6cc..000a897f457f 100644 --- a/swig/include/Band.i +++ b/swig/include/Band.i @@ -295,6 +295,10 @@ public: return (GDALRasterBandShadow*) GDALGetOverview( self, i ); } + GDALRasterBandShadow *GetSampleOverview(GUIntBig nDesiredSamples) { + return (GDALRasterBandShadow*) GDALGetRasterSampleOverview( self, nDesiredSamples ); + } + #if defined (SWIGJAVA) int Checksum( int xoff, int yoff, int xsize, int ysize) { return GDALChecksumImage( self, xoff, yoff, xsize, ysize ); diff --git a/swig/include/Dataset.i b/swig/include/Dataset.i index 2f3b2e9b6ab7..a128a66992f9 100644 --- a/swig/include/Dataset.i +++ b/swig/include/Dataset.i @@ -300,6 +300,17 @@ public: return (GDALRasterBandShadow*) GDALGetRasterBand( self, nBand ); } + bool IsThreadSafe(int nScopeFlags) + { + return GDALDatasetIsThreadSafe(self, nScopeFlags, nullptr); + } + +%newobject GetThreadSafeDataset; + GDALDatasetShadow* GetThreadSafeDataset(int nScopeFlags) + { + return GDALGetThreadSafeDataset(self, nScopeFlags, nullptr); + } + %newobject GetRootGroup; GDALGroupHS* GetRootGroup() { return GDALDatasetGetRootGroup(self); diff --git a/swig/include/gdalconst.i b/swig/include/gdalconst.i index 4a664f959e78..c9902d6f1293 100644 --- a/swig/include/gdalconst.i +++ b/swig/include/gdalconst.i @@ -197,6 +197,7 @@ %constant OF_UPDATE = GDAL_OF_UPDATE; %constant OF_SHARED = GDAL_OF_SHARED; %constant OF_VERBOSE_ERROR = GDAL_OF_VERBOSE_ERROR; +%constant OF_THREAD_SAFE = GDAL_OF_THREAD_SAFE; #if !defined(SWIGCSHARP) && !defined(SWIGJAVA) diff --git a/swig/include/java/gdal_java.i b/swig/include/java/gdal_java.i index a357385f37e0..c3ed14adfcbd 100644 --- a/swig/include/java/gdal_java.i +++ b/swig/include/java/gdal_java.i @@ -1312,7 +1312,8 @@ import org.gdal.gdalconst.gdalconstConstants; // Add a Java reference to prevent premature garbage collection and resulting use // of dangling C++ pointer. Intended for methods that return pointers or // references to a member variable. -%typemap(javaout) GDALRasterBandShadow* GetRasterBand, +%typemap(javaout) GDALDatasetShadow* GetThreadSafeDataset, + GDALRasterBandShadow* GetRasterBand, GDALRasterBandShadow* GetOverview, GDALRasterBandShadow* GetMaskBand, GDALColorTableShadow* GetColorTable, diff --git a/swig/include/python/gdal_python.i b/swig/include/python/gdal_python.i index f6fc9feeea18..fe09bada8620 100644 --- a/swig/include/python/gdal_python.i +++ b/swig/include/python/gdal_python.i @@ -1614,6 +1614,16 @@ CPLErr ReadRaster1( double xoff, double yoff, double xsize, double ysize, return get(value) %} +%feature("pythonappend") GetThreadSafeDataset %{ + if val: + val._parent_ds = self + + import weakref + if not hasattr(self, '_child_references'): + self._child_references = weakref.WeakSet() + self._child_references.add(val) +%} + %feature("pythonprepend") Close %{ self._invalidate_children() %} From 80f785d91ae4b45a6f7f73c89898a00d241bfbd8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 09:35:01 +0200 Subject: [PATCH 188/710] OGRSpatialReference: add a AssignAndSetThreadSafe() method --- ogr/ogr_spatialref.h | 2 + ogr/ogrspatialreference.cpp | 340 +++++++++++++++++++++++++++++++++++- 2 files changed, 338 insertions(+), 4 deletions(-) diff --git a/ogr/ogr_spatialref.h b/ogr/ogr_spatialref.h index 91d8be2ba575..f1a3b7c91ba0 100644 --- a/ogr/ogr_spatialref.h +++ b/ogr/ogr_spatialref.h @@ -194,6 +194,8 @@ class CPL_DLL OGRSpatialReference OGRSpatialReference &operator=(const OGRSpatialReference &); OGRSpatialReference &operator=(OGRSpatialReference &&); + OGRSpatialReference &AssignAndSetThreadSafe(const OGRSpatialReference &); + int Reference(); int Dereference(); int GetReferenceCount() const; diff --git a/ogr/ogrspatialreference.cpp b/ogr/ogrspatialreference.cpp index 52a5cd487423..fc9bec8b49de 100644 --- a/ogr/ogrspatialreference.cpp +++ b/ogr/ogrspatialreference.cpp @@ -107,6 +107,7 @@ struct OGRSpatialReference::Private std::vector m_wktImportErrors{}; CPLString m_osAreaName{}; + bool m_bIsThreadSafe = false; bool m_bNodesChanged = false; bool m_bNodesWKT2 = false; OGR_SRSNode *m_poRoot = nullptr; @@ -133,7 +134,7 @@ struct OGRSpatialReference::Private std::shared_ptr m_poListener{}; - std::mutex m_mutex{}; + std::recursive_mutex m_mutex{}; OSRAxisMappingStrategy m_axisMappingStrategy = OAMS_AUTHORITY_COMPLIANT; std::vector m_axisMapping{1, 2, 3}; @@ -145,6 +146,11 @@ struct OGRSpatialReference::Private Private(const Private &) = delete; Private &operator=(const Private &) = delete; + void SetThreadSafe() + { + m_bIsThreadSafe = true; + } + void clear(); void setPjCRS(PJ *pj_crsIn, bool doRefreshAxisMapping = true); void setRoot(OGR_SRSNode *poRoot); @@ -172,8 +178,42 @@ struct OGRSpatialReference::Private const char *nullifyTargetKeyIfPossible(const char *pszTargetKey); void refreshAxisMapping(); + + // This structures enables locking during calls to OGRSpatialReference + // public methods. Locking is only needed for instances of + // OGRSpatialReference that have been asked to be thread-safe at + // construction. + // The lock is not just for a single call to OGRSpatialReference::Private, + // but for the series of calls done by a OGRSpatialReference method. + // We need a recursive mutex, because some OGRSpatialReference methods + // may call other ones. + struct OptionalLockGuard + { + Private &m_private; + + explicit OptionalLockGuard(Private *p) : m_private(*p) + { + if (m_private.m_bIsThreadSafe) + m_private.m_mutex.lock(); + } + + ~OptionalLockGuard() + { + if (m_private.m_bIsThreadSafe) + m_private.m_mutex.unlock(); + } + }; + + inline OptionalLockGuard GetOptionalLockGuard() + { + return OptionalLockGuard(this); + } }; +#define TAKE_OPTIONAL_LOCK() \ + auto lock = d->GetOptionalLockGuard(); \ + CPL_IGNORE_RET_VAL(lock) + static OSRAxisMappingStrategy GetDefaultAxisMappingStrategy() { const char *pszDefaultAMS = @@ -989,6 +1029,28 @@ OGRSpatialReference::operator=(OGRSpatialReference &&oSource) return *this; } +/************************************************************************/ +/* AssignAndSetThreadSafe() */ +/************************************************************************/ + +/** Assignment method, with thread-safety. + * + * Same as an assignment operator, but asking also that the *this instance + * becomes thread-safe. + * + * @param oSource SRS to assign to *this + * @return *this + * @since 3.10 + */ + +OGRSpatialReference & +OGRSpatialReference::AssignAndSetThreadSafe(const OGRSpatialReference &oSource) +{ + *this = oSource; + d->SetThreadSafe(); + return *this; +} + /************************************************************************/ /* Reference() */ /************************************************************************/ @@ -1117,6 +1179,8 @@ void OSRRelease(OGRSpatialReferenceH hSRS) OGR_SRSNode *OGRSpatialReference::GetRoot() { + TAKE_OPTIONAL_LOCK(); + if (!d->m_poRoot) { d->refreshRootFromProjObj(); @@ -1126,6 +1190,8 @@ OGR_SRSNode *OGRSpatialReference::GetRoot() const OGR_SRSNode *OGRSpatialReference::GetRoot() const { + TAKE_OPTIONAL_LOCK(); + if (!d->m_poRoot) { d->refreshRootFromProjObj(); @@ -1320,6 +1386,8 @@ const char *CPL_STDCALL OSRGetAttrValue(OGRSpatialReferenceH hSRS, const char *OGRSpatialReference::GetName() const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) return nullptr; @@ -1378,6 +1446,8 @@ OGRSpatialReference *OGRSpatialReference::Clone() const { OGRSpatialReference *poNewRef = new OGRSpatialReference(); + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (d->m_pj_crs != nullptr) poNewRef->d->setPjCRS(proj_clone(d->getPROJContext(), d->m_pj_crs)); @@ -1613,7 +1683,7 @@ OGRErr OGRSpatialReference::exportToWkt(char **ppszResult, // In the past calling this method was thread-safe, even if we never // guaranteed it. Now proj_as_wkt() will cache the result internally, // so this is no longer thread-safe. - std::lock_guard oLock(d->m_mutex); + std::lock_guard oLock(d->m_mutex); d->refreshProjObj(); if (!d->m_pj_crs) @@ -1949,6 +2019,8 @@ OGRErr OSRExportToWktEx(OGRSpatialReferenceH hSRS, char **ppszReturn, OGRErr OGRSpatialReference::exportToPROJJSON( char **ppszResult, CPL_UNUSED const char *const *papszOptions) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) { @@ -2114,6 +2186,8 @@ OGRErr OGRSpatialReference::importFromWkt(const char **ppszInput, CSLConstList papszOptions) { + TAKE_OPTIONAL_LOCK(); + if (!ppszInput || !*ppszInput) return OGRERR_FAILURE; @@ -2297,6 +2371,8 @@ OGRErr OGRSpatialReference::importFromWkt(const char *pszInput) OGRErr OGRSpatialReference::Validate() const { + TAKE_OPTIONAL_LOCK(); + for (const auto &str : d->m_wktImportErrors) { CPLDebug("OGRSpatialReference::Validate", "%s", str.c_str()); @@ -2380,6 +2456,8 @@ OGRErr OGRSpatialReference::SetNode(const char *pszNodePath, const char *pszNewNodeValue) { + TAKE_OPTIONAL_LOCK(); + char **papszPathTokens = CSLTokenizeStringComplex(pszNodePath, "|", TRUE, FALSE); @@ -2517,6 +2595,8 @@ OGRErr OGRSpatialReference::SetAngularUnits(const char *pszUnitsName, double dfInRadians) { + TAKE_OPTIONAL_LOCK(); + d->bNormInfoSet = FALSE; d->refreshProjObj(); @@ -2582,6 +2662,8 @@ OGRErr OSRSetAngularUnits(OGRSpatialReferenceH hSRS, const char *pszUnits, double OGRSpatialReference::GetAngularUnits(const char **ppszName) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_osAngularUnits.empty()) @@ -2720,6 +2802,8 @@ OGRErr OGRSpatialReference::SetLinearUnitsAndUpdateParameters( const char *pszUnitCode) { + TAKE_OPTIONAL_LOCK(); + if (dfInMeters <= 0.0) return OGRERR_FAILURE; @@ -2855,6 +2939,8 @@ OGRErr OGRSpatialReference::SetTargetLinearUnits(const char *pszTargetKey, const char *pszUnitCode) { + TAKE_OPTIONAL_LOCK(); + if (dfInMeters <= 0.0) return OGRERR_FAILURE; @@ -3044,6 +3130,8 @@ double OGRSpatialReference::GetTargetLinearUnits(const char *pszTargetKey, const char **ppszName) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); pszTargetKey = d->nullifyTargetKeyIfPossible(pszTargetKey); @@ -3271,6 +3359,8 @@ double OSRGetTargetLinearUnits(OGRSpatialReferenceH hSRS, double OGRSpatialReference::GetPrimeMeridian(const char **ppszName) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_osPrimeMeridianName.empty()) @@ -3408,6 +3498,8 @@ OGRErr OGRSpatialReference::SetGeogCS( double dfConvertToRadians) { + TAKE_OPTIONAL_LOCK(); + d->bNormInfoSet = FALSE; d->m_osAngularUnits.clear(); d->m_dfAngularUnitToRadian = 0.0; @@ -3516,6 +3608,8 @@ OGRErr OSRSetGeogCS(OGRSpatialReferenceH hSRS, const char *pszGeogName, OGRErr OGRSpatialReference::SetWellKnownGeogCS(const char *pszName) { + TAKE_OPTIONAL_LOCK(); + /* -------------------------------------------------------------------- */ /* Check for EPSG authority numbers. */ /* -------------------------------------------------------------------- */ @@ -3658,6 +3752,8 @@ OGRErr OSRSetWellKnownGeogCS(OGRSpatialReferenceH hSRS, const char *pszName) OGRErr OGRSpatialReference::CopyGeogCSFrom(const OGRSpatialReference *poSrcSRS) { + TAKE_OPTIONAL_LOCK(); + d->bNormInfoSet = FALSE; d->m_osAngularUnits.clear(); d->m_dfAngularUnitToRadian = 0.0; @@ -3907,6 +4003,8 @@ OGRErr OGRSpatialReference::SetFromUserInput(const char *pszDefinition) OGRErr OGRSpatialReference::SetFromUserInput(const char *pszDefinition, CSLConstList papszOptions) { + TAKE_OPTIONAL_LOCK(); + // Skip leading white space while (isspace(static_cast(*pszDefinition))) pszDefinition++; @@ -4274,6 +4372,8 @@ OGRErr OSRSetFromUserInputEx(OGRSpatialReferenceH hSRS, const char *pszDef, OGRErr OGRSpatialReference::importFromUrl(const char *pszUrl) { + TAKE_OPTIONAL_LOCK(); + if (!STARTS_WITH_CI(pszUrl, "http://") && !STARTS_WITH_CI(pszUrl, "https://")) { @@ -4484,6 +4584,8 @@ OGRErr OGRSpatialReference::importFromURNPart(const char *pszAuthority, OGRErr OGRSpatialReference::importFromURN(const char *pszURN) { + TAKE_OPTIONAL_LOCK(); + #if PROJ_AT_LEAST_VERSION(8, 1, 0) // PROJ 8.2.0 has support for IAU codes now. @@ -4672,6 +4774,8 @@ OGRErr OGRSpatialReference::importFromURN(const char *pszURN) OGRErr OGRSpatialReference::importFromCRSURL(const char *pszURL) { + TAKE_OPTIONAL_LOCK(); + #if PROJ_AT_LEAST_VERSION(8, 1, 0) if (strlen(pszURL) >= 10000) { @@ -4853,6 +4957,8 @@ OGRErr OGRSpatialReference::importFromCRSURL(const char *pszURL) OGRErr OGRSpatialReference::importFromWMSAUTO(const char *pszDefinition) { + TAKE_OPTIONAL_LOCK(); + #if PROJ_AT_LEAST_VERSION(8, 1, 0) if (strlen(pszDefinition) >= 10000) { @@ -5010,6 +5116,8 @@ OGRErr OGRSpatialReference::importFromWMSAUTO(const char *pszDefinition) double OGRSpatialReference::GetSemiMajor(OGRErr *pnErr) const { + TAKE_OPTIONAL_LOCK(); + if (pnErr != nullptr) *pnErr = OGRERR_FAILURE; @@ -5071,6 +5179,8 @@ double OSRGetSemiMajor(OGRSpatialReferenceH hSRS, OGRErr *pnErr) double OGRSpatialReference::GetInvFlattening(OGRErr *pnErr) const { + TAKE_OPTIONAL_LOCK(); + if (pnErr != nullptr) *pnErr = OGRERR_FAILURE; @@ -5230,6 +5340,8 @@ double OSRGetSemiMinor(OGRSpatialReferenceH hSRS, OGRErr *pnErr) OGRErr OGRSpatialReference::SetLocalCS(const char *pszName) { + TAKE_OPTIONAL_LOCK(); + if (d->m_pjType == PJ_TYPE_UNKNOWN || d->m_pjType == PJ_TYPE_ENGINEERING_CRS) { @@ -5288,6 +5400,8 @@ OGRErr OSRSetLocalCS(OGRSpatialReferenceH hSRS, const char *pszName) OGRErr OGRSpatialReference::SetGeocCS(const char *pszName) { + TAKE_OPTIONAL_LOCK(); + OGRErr eErr = OGRERR_NONE; d->refreshProjObj(); d->demoteFromBoundCRS(); @@ -5391,6 +5505,8 @@ OGRErr OGRSpatialReference::SetVertCS(const char *pszVertCSName, int nVertDatumType) { + TAKE_OPTIONAL_LOCK(); + CPL_IGNORE_RET_VAL(nVertDatumType); d->refreshProjObj(); @@ -5463,6 +5579,8 @@ OGRErr OGRSpatialReference::SetCompoundCS(const char *pszName, const OGRSpatialReference *poVertSRS) { + TAKE_OPTIONAL_LOCK(); + /* -------------------------------------------------------------------- */ /* Verify these are legal horizontal and vertical coordinate */ /* systems. */ @@ -5537,6 +5655,8 @@ OGRErr OSRSetCompoundCS(OGRSpatialReferenceH hSRS, const char *pszName, OGRErr OGRSpatialReference::SetProjCS(const char *pszName) { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->demoteFromBoundCRS(); if (d->m_pjType == PJ_TYPE_PROJECTED_CRS) @@ -5597,6 +5717,8 @@ OGRErr OSRSetProjCS(OGRSpatialReferenceH hSRS, const char *pszName) OGRErr OGRSpatialReference::SetProjection(const char *pszProjection) { + TAKE_OPTIONAL_LOCK(); + OGR_SRSNode *poGeogCS = nullptr; if (GetRoot() != nullptr && EQUAL(d->m_poRoot->GetValue(), "GEOGCS")) @@ -5666,6 +5788,8 @@ OGRSpatialReference::GetWKT2ProjectionMethod(const char **ppszMethodName, const char **ppszMethodAuthName, const char **ppszMethodCode) const { + TAKE_OPTIONAL_LOCK(); + auto conv = proj_crs_get_coordoperation(d->getPROJContext(), d->m_pj_crs); if (!conv) return OGRERR_FAILURE; @@ -5717,6 +5841,8 @@ OGRErr OGRSpatialReference::SetProjParm(const char *pszParamName, double dfValue) { + TAKE_OPTIONAL_LOCK(); + OGR_SRSNode *poPROJCS = GetAttrNode("PROJCS"); if (poPROJCS == nullptr) @@ -5789,6 +5915,8 @@ int OGRSpatialReference::FindProjParm(const char *pszParameter, const OGR_SRSNode *poPROJCS) const { + TAKE_OPTIONAL_LOCK(); + if (poPROJCS == nullptr) poPROJCS = GetAttrNode("PROJCS"); @@ -5883,6 +6011,8 @@ double OGRSpatialReference::GetProjParm(const char *pszName, OGRErr *pnErr) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); GetRoot(); // force update of d->m_bNodesWKT2 @@ -5969,6 +6099,8 @@ double OGRSpatialReference::GetNormProjParm(const char *pszName, OGRErr *pnErr) const { + TAKE_OPTIONAL_LOCK(); + GetNormInfo(); OGRErr nError = OGRERR_NONE; @@ -6032,6 +6164,8 @@ double OSRGetNormProjParm(OGRSpatialReferenceH hSRS, const char *pszName, OGRErr OGRSpatialReference::SetNormProjParm(const char *pszName, double dfValue) { + TAKE_OPTIONAL_LOCK(); + GetNormInfo(); if (d->dfToDegrees != 0.0 && @@ -6074,6 +6208,8 @@ OGRErr OGRSpatialReference::SetTM(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_transverse_mercator( d->getPROJContext(), dfCenterLat, dfCenterLong, dfScale, @@ -6106,6 +6242,8 @@ OGRErr OGRSpatialReference::SetTMVariant(const char *pszVariantName, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + SetProjection(pszVariantName); SetNormProjParm(SRS_PP_LATITUDE_OF_ORIGIN, dfCenterLat); SetNormProjParm(SRS_PP_CENTRAL_MERIDIAN, dfCenterLong); @@ -6141,6 +6279,8 @@ OGRErr OGRSpatialReference::SetTMSO(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + auto conv = proj_create_conversion_transverse_mercator_south_oriented( d->getPROJContext(), dfCenterLat, dfCenterLong, dfScale, dfFalseEasting, dfFalseNorthing, nullptr, 0.0, nullptr, 0.0); @@ -6194,6 +6334,8 @@ OGRErr OGRSpatialReference::SetTPED(double dfLat1, double dfLong1, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_two_point_equidistant( d->getPROJContext(), dfLat1, dfLong1, dfLat2, dfLong2, @@ -6224,6 +6366,8 @@ OGRErr OGRSpatialReference::SetTMG(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_tunisia_mapping_grid( d->getPROJContext(), dfCenterLat, dfCenterLong, dfFalseEasting, @@ -6255,6 +6399,8 @@ OGRErr OGRSpatialReference::SetACEA(double dfStdP1, double dfStdP2, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + // Note different order of parameters. The one in PROJ is conformant with // EPSG return d->replaceConversionAndUnref( @@ -6286,6 +6432,8 @@ OGRErr OGRSpatialReference::SetAE(double dfCenterLat, double dfCenterLong, double dfFalseEasting, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_azimuthal_equidistant( d->getPROJContext(), dfCenterLat, dfCenterLong, dfFalseEasting, @@ -6316,6 +6464,8 @@ OGRErr OGRSpatialReference::SetBonne(double dfStdP1, double dfCentralMeridian, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_bonne( d->getPROJContext(), dfStdP1, dfCentralMeridian, dfFalseEasting, dfFalseNorthing, nullptr, 0.0, nullptr, 0.0)); @@ -6345,6 +6495,8 @@ OGRErr OGRSpatialReference::SetCEA(double dfStdP1, double dfCentralMeridian, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_lambert_cylindrical_equal_area( d->getPROJContext(), dfStdP1, dfCentralMeridian, dfFalseEasting, @@ -6374,6 +6526,8 @@ OGRErr OGRSpatialReference::SetCS(double dfCenterLat, double dfCenterLong, double dfFalseEasting, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_cassini_soldner( d->getPROJContext(), dfCenterLat, dfCenterLong, dfFalseEasting, dfFalseNorthing, nullptr, 0.0, nullptr, 0.0)); @@ -6403,6 +6557,8 @@ OGRErr OGRSpatialReference::SetEC(double dfStdP1, double dfStdP2, double dfFalseEasting, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + // Note: different order of arguments return d->replaceConversionAndUnref( proj_create_conversion_equidistant_conic( @@ -6435,6 +6591,8 @@ OGRErr OGRSpatialReference::SetEckert(int nVariation, // 1-6. double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + PJ *conv; if (nVariation == 1) { @@ -6563,6 +6721,8 @@ OGRErr OGRSpatialReference::SetEquirectangular(double dfCenterLat, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + if (dfCenterLat == 0.0) { return d->replaceConversionAndUnref( @@ -6608,6 +6768,8 @@ OGRErr OGRSpatialReference::SetEquirectangular2(double dfCenterLat, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + if (dfCenterLat == 0.0) { return d->replaceConversionAndUnref( @@ -6678,6 +6840,8 @@ OGRErr OGRSpatialReference::SetGH(double dfCentralMeridian, double dfFalseEasting, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_goode_homolosine( d->getPROJContext(), dfCentralMeridian, dfFalseEasting, dfFalseNorthing, nullptr, 0.0, nullptr, 0.0)); @@ -6704,6 +6868,8 @@ OGRErr OSRSetGH(OGRSpatialReferenceH hSRS, double dfCentralMeridian, OGRErr OGRSpatialReference::SetIGH() { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_interrupted_goode_homolosine( d->getPROJContext(), 0.0, 0.0, 0.0, nullptr, 0.0, nullptr, 0.0)); @@ -6731,6 +6897,8 @@ OGRErr OGRSpatialReference::SetGEOS(double dfCentralMeridian, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_geostationary_satellite_sweep_y( d->getPROJContext(), dfCentralMeridian, dfSatelliteHeight, @@ -6763,6 +6931,8 @@ OGRErr OGRSpatialReference::SetGaussSchreiberTMercator(double dfCenterLat, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_gauss_schreiber_transverse_mercator( d->getPROJContext(), dfCenterLat, dfCenterLong, dfScale, @@ -6794,6 +6964,8 @@ OGRErr OGRSpatialReference::SetGnomonic(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_gnomonic( d->getPROJContext(), dfCenterLat, dfCenterLong, dfFalseEasting, dfFalseNorthing, nullptr, 0.0, nullptr, 0.0)); @@ -6845,6 +7017,8 @@ OGRErr OGRSpatialReference::SetHOMAC(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_hotine_oblique_mercator_variant_b( d->getPROJContext(), dfCenterLat, dfCenterLong, dfAzimuth, @@ -6904,6 +7078,8 @@ OGRErr OGRSpatialReference::SetHOM(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_hotine_oblique_mercator_variant_a( d->getPROJContext(), dfCenterLat, dfCenterLong, dfAzimuth, @@ -6960,6 +7136,8 @@ OGRErr OGRSpatialReference::SetHOM2PNO(double dfCenterLat, double dfLat1, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_hotine_oblique_mercator_two_point_natural_origin( d->getPROJContext(), dfCenterLat, dfLat1, dfLong1, dfLat2, dfLong2, @@ -7013,6 +7191,8 @@ OGRErr OGRSpatialReference::SetLOM(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_laborde_oblique_mercator( d->getPROJContext(), dfCenterLat, dfCenterLong, dfAzimuth, dfScale, @@ -7029,6 +7209,8 @@ OGRErr OGRSpatialReference::SetIWMPolyconic(double dfLat1, double dfLat2, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_international_map_world_polyconic( d->getPROJContext(), dfCenterLong, dfLat1, dfLat2, dfFalseEasting, @@ -7066,6 +7248,8 @@ OGRErr OGRSpatialReference::SetKrovak(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_krovak_north_oriented( d->getPROJContext(), dfCenterLat, dfCenterLong, dfAzimuth, @@ -7099,6 +7283,8 @@ OGRErr OGRSpatialReference::SetLAEA(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + auto conv = proj_create_conversion_lambert_azimuthal_equal_area( d->getPROJContext(), dfCenterLat, dfCenterLong, dfFalseEasting, dfFalseNorthing, nullptr, 0.0, nullptr, 0.0); @@ -7157,6 +7343,8 @@ OGRErr OGRSpatialReference::SetLCC(double dfStdP1, double dfStdP2, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_lambert_conic_conformal_2sp( d->getPROJContext(), dfCenterLat, dfCenterLong, dfStdP1, dfStdP2, @@ -7187,6 +7375,8 @@ OGRErr OGRSpatialReference::SetLCC1SP(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_lambert_conic_conformal_1sp( d->getPROJContext(), dfCenterLat, dfCenterLong, dfScale, @@ -7218,6 +7408,8 @@ OGRErr OGRSpatialReference::SetLCCB(double dfStdP1, double dfStdP2, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_lambert_conic_conformal_2sp_belgium( d->getPROJContext(), dfCenterLat, dfCenterLong, dfStdP1, dfStdP2, @@ -7247,6 +7439,8 @@ OGRErr OGRSpatialReference::SetMC(double dfCenterLat, double dfCenterLong, double dfFalseEasting, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + (void)dfCenterLat; // ignored return d->replaceConversionAndUnref( @@ -7279,6 +7473,8 @@ OGRErr OGRSpatialReference::SetMercator(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + if (dfCenterLat != 0.0 && dfScale == 1.0) { // Not sure this is correct, but this is how it has been used @@ -7327,6 +7523,8 @@ OGRErr OGRSpatialReference::SetMercator2SP(double dfStdP1, double dfCenterLat, dfFalseNorthing, nullptr, 0, nullptr, 0)); } + TAKE_OPTIONAL_LOCK(); + SetProjection(SRS_PT_MERCATOR_2SP); SetNormProjParm(SRS_PP_STANDARD_PARALLEL_1, dfStdP1); @@ -7362,6 +7560,8 @@ OGRErr OGRSpatialReference::SetMollweide(double dfCentralMeridian, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_mollweide( d->getPROJContext(), dfCentralMeridian, dfFalseEasting, dfFalseNorthing, nullptr, 0, nullptr, 0)); @@ -7390,6 +7590,8 @@ OGRErr OGRSpatialReference::SetNZMG(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_new_zealand_mapping_grid( d->getPROJContext(), dfCenterLat, dfCenterLong, dfFalseEasting, @@ -7420,6 +7622,8 @@ OGRErr OGRSpatialReference::SetOS(double dfOriginLat, double dfCMeridian, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_oblique_stereographic( d->getPROJContext(), dfOriginLat, dfCMeridian, dfScale, @@ -7451,6 +7655,8 @@ OGRErr OGRSpatialReference::SetOrthographic(double dfCenterLat, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_orthographic( d->getPROJContext(), dfCenterLat, dfCenterLong, dfFalseEasting, dfFalseNorthing, nullptr, 0, nullptr, 0)); @@ -7481,6 +7687,8 @@ OGRErr OGRSpatialReference::SetPolyconic(double dfCenterLat, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + // note: it seems that by some definitions this should include a // scale_factor parameter. return d->replaceConversionAndUnref( @@ -7522,6 +7730,8 @@ OGRErr OGRSpatialReference::SetPS(double dfCenterLat, double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + PJ *conv; if (dfScale == 1.0 && std::abs(std::abs(dfCenterLat) - 90) > 1e-8) { @@ -7586,6 +7796,8 @@ OGRErr OGRSpatialReference::SetRobinson(double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_robinson( d->getPROJContext(), dfCenterLong, dfFalseEasting, dfFalseNorthing, nullptr, 0, nullptr, 0)); @@ -7614,6 +7826,8 @@ OGRErr OGRSpatialReference::SetSinusoidal(double dfCenterLong, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_sinusoidal( d->getPROJContext(), dfCenterLong, dfFalseEasting, dfFalseNorthing, nullptr, 0, nullptr, 0)); @@ -7643,6 +7857,8 @@ OGRErr OGRSpatialReference::SetStereographic(double dfOriginLat, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_stereographic( d->getPROJContext(), dfOriginLat, dfCMeridian, dfScale, dfFalseEasting, dfFalseNorthing, nullptr, 0, nullptr, 0)); @@ -7679,6 +7895,8 @@ OGRErr OGRSpatialReference::SetSOC(double dfLatitudeOfOrigin, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_hotine_oblique_mercator_variant_b( d->getPROJContext(), dfLatitudeOfOrigin, dfCentralMeridian, 90.0, @@ -7718,6 +7936,8 @@ OGRErr OGRSpatialReference::SetVDG(double dfCMeridian, double dfFalseEasting, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref(proj_create_conversion_van_der_grinten( d->getPROJContext(), dfCMeridian, dfFalseEasting, dfFalseNorthing, nullptr, 0, nullptr, 0)); @@ -7762,6 +7982,8 @@ OGRErr OSRSetVDG(OGRSpatialReferenceH hSRS, double dfCentralMeridian, OGRErr OGRSpatialReference::SetUTM(int nZone, int bNorth) { + TAKE_OPTIONAL_LOCK(); + if (nZone < 0 || nZone > 60) { CPLError(CE_Failure, CPLE_AppDefined, "Invalid zone: %d", nZone); @@ -7813,6 +8035,8 @@ OGRErr OSRSetUTM(OGRSpatialReferenceH hSRS, int nZone, int bNorth) int OGRSpatialReference::GetUTMZone(int *pbNorth) const { + TAKE_OPTIONAL_LOCK(); + if (IsProjected() && GetAxesCount() == 3) { OGRSpatialReference *poSRSTmp = Clone(); @@ -7883,6 +8107,8 @@ OGRErr OGRSpatialReference::SetWagner(int nVariation, // 1--7. double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + PJ *conv; if (nVariation == 1) { @@ -7957,6 +8183,8 @@ OGRErr OSRSetWagner(OGRSpatialReferenceH hSRS, int nVariation, OGRErr OGRSpatialReference::SetQSC(double dfCenterLat, double dfCenterLong) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_quadrilateralized_spherical_cube( d->getPROJContext(), dfCenterLat, dfCenterLong, 0.0, 0.0, nullptr, @@ -7984,6 +8212,8 @@ OGRErr OGRSpatialReference::SetSCH(double dfPegLat, double dfPegLong, double dfPegHeading, double dfPegHgt) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_spherical_cross_track_height( d->getPROJContext(), dfPegLat, dfPegLong, dfPegHeading, dfPegHgt, @@ -8011,6 +8241,8 @@ OGRErr OGRSpatialReference::SetVerticalPerspective( double dfTopoOriginLat, double dfTopoOriginLon, double dfTopoOriginHeight, double dfViewPointHeight, double dfFalseEasting, double dfFalseNorthing) { + TAKE_OPTIONAL_LOCK(); + return d->replaceConversionAndUnref( proj_create_conversion_vertical_perspective( d->getPROJContext(), dfTopoOriginLat, dfTopoOriginLon, @@ -8044,6 +8276,8 @@ OGRErr OGRSpatialReference::SetDerivedGeogCRSWithPoleRotationGRIBConvention( const char *pszCRSName, double dfSouthPoleLat, double dfSouthPoleLon, double dfAxisRotation) { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) return OGRERR_FAILURE; @@ -8068,6 +8302,8 @@ OGRErr OGRSpatialReference::SetDerivedGeogCRSWithPoleRotationNetCDFCFConvention( const char *pszCRSName, double dfGridNorthPoleLat, double dfGridNorthPoleLon, double dfNorthPoleGridLon) { + TAKE_OPTIONAL_LOCK(); + #if PROJ_VERSION_MAJOR > 8 || \ (PROJ_VERSION_MAJOR == 8 && PROJ_VERSION_MINOR >= 2) d->refreshProjObj(); @@ -8124,6 +8360,8 @@ OGRErr OGRSpatialReference::SetAuthority(const char *pszTargetKey, const char *pszAuthority, int nCode) { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); pszTargetKey = d->nullifyTargetKeyIfPossible(pszTargetKey); @@ -8254,6 +8492,8 @@ const char * OGRSpatialReference::GetAuthorityCode(const char *pszTargetKey) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); const char *pszInputTargetKey = pszTargetKey; pszTargetKey = d->nullifyTargetKeyIfPossible(pszTargetKey); @@ -8388,6 +8628,8 @@ const char * OGRSpatialReference::GetAuthorityName(const char *pszTargetKey) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); const char *pszInputTargetKey = pszTargetKey; pszTargetKey = d->nullifyTargetKeyIfPossible(pszTargetKey); @@ -8507,7 +8749,7 @@ const char *OSRGetAuthorityName(OGRSpatialReferenceH hSRS, * a compound CRS whose horizontal and vertical parts have a top-level * identifier. * - * @return a string to free with CPLFree(), or nulptr when no result can be + * @return a string to free with CPLFree(), or nullptr when no result can be * generated * * @since GDAL 3.5 @@ -8516,6 +8758,8 @@ const char *OSRGetAuthorityName(OGRSpatialReferenceH hSRS, char *OGRSpatialReference::GetOGCURN() const { + TAKE_OPTIONAL_LOCK(); + const char *pszAuthName = GetAuthorityName(nullptr); const char *pszAuthCode = GetAuthorityCode(nullptr); if (pszAuthName && pszAuthCode) @@ -8565,6 +8809,8 @@ char *OGRSpatialReference::GetOGCURN() const OGRErr OGRSpatialReference::StripVertical() { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->demoteFromBoundCRS(); if (!d->m_pj_crs || d->m_pjType != PJ_TYPE_COMPOUND_CRS) @@ -8668,6 +8914,8 @@ bool OGRSpatialReference::StripTOWGS84IfKnownDatumAndAllowed() bool OGRSpatialReference::StripTOWGS84IfKnownDatum() { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs || d->m_pjType != PJ_TYPE_BOUND_CRS) { @@ -8758,6 +9006,8 @@ bool OGRSpatialReference::StripTOWGS84IfKnownDatum() int OGRSpatialReference::IsCompound() const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->demoteFromBoundCRS(); bool isCompound = d->m_pjType == PJ_TYPE_COMPOUND_CRS; @@ -8799,6 +9049,8 @@ int OSRIsCompound(OGRSpatialReferenceH hSRS) int OGRSpatialReference::IsProjected() const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->demoteFromBoundCRS(); bool isProjected = d->m_pjType == PJ_TYPE_PROJECTED_CRS; @@ -8860,6 +9112,8 @@ int OSRIsProjected(OGRSpatialReferenceH hSRS) int OGRSpatialReference::IsGeocentric() const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->demoteFromBoundCRS(); bool isGeocentric = d->m_pjType == PJ_TYPE_GEOCENTRIC_CRS; @@ -8895,6 +9149,8 @@ int OSRIsGeocentric(OGRSpatialReferenceH hSRS) bool OGRSpatialReference::IsEmpty() const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); return d->m_pj_crs == nullptr; } @@ -8916,6 +9172,8 @@ bool OGRSpatialReference::IsEmpty() const int OGRSpatialReference::IsGeographic() const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->demoteFromBoundCRS(); bool isGeog = d->m_pjType == PJ_TYPE_GEOGRAPHIC_2D_CRS || @@ -8979,6 +9237,8 @@ int OSRIsGeographic(OGRSpatialReferenceH hSRS) int OGRSpatialReference::IsDerivedGeographic() const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->demoteFromBoundCRS(); const bool isGeog = d->m_pjType == PJ_TYPE_GEOGRAPHIC_2D_CRS || @@ -9022,6 +9282,7 @@ int OGRSpatialReference::IsDerivedProjected() const { #if PROJ_AT_LEAST_VERSION(9, 2, 0) + TAKE_OPTIONAL_LOCK(); d->refreshProjObj(); d->demoteFromBoundCRS(); const bool isDerivedProjected = @@ -9067,6 +9328,7 @@ int OSRIsDerivedProjected(OGRSpatialReferenceH hSRS) int OGRSpatialReference::IsLocal() const { + TAKE_OPTIONAL_LOCK(); d->refreshProjObj(); return d->m_pjType == PJ_TYPE_ENGINEERING_CRS; } @@ -9106,6 +9368,7 @@ int OSRIsLocal(OGRSpatialReferenceH hSRS) int OGRSpatialReference::IsVertical() const { + TAKE_OPTIONAL_LOCK(); d->refreshProjObj(); d->demoteFromBoundCRS(); bool isVertical = d->m_pjType == PJ_TYPE_VERTICAL_CRS; @@ -9173,6 +9436,7 @@ int OSRIsVertical(OGRSpatialReferenceH hSRS) bool OGRSpatialReference::IsDynamic() const { + TAKE_OPTIONAL_LOCK(); bool isDynamic = false; d->refreshProjObj(); d->demoteFromBoundCRS(); @@ -9282,6 +9546,7 @@ bool OGRSpatialReference::HasPointMotionOperation() const { #if PROJ_VERSION_MAJOR > 9 || \ (PROJ_VERSION_MAJOR == 9 && PROJ_VERSION_MINOR >= 4) + TAKE_OPTIONAL_LOCK(); d->refreshProjObj(); d->demoteFromBoundCRS(); auto ctxt = d->getPROJContext(); @@ -9330,6 +9595,7 @@ int OSRHasPointMotionOperation(OGRSpatialReferenceH hSRS) OGRSpatialReference *OGRSpatialReference::CloneGeogCS() const { + TAKE_OPTIONAL_LOCK(); d->refreshProjObj(); if (d->m_pj_crs) { @@ -9447,6 +9713,8 @@ int OGRSpatialReference::IsSameGeogCS(const OGRSpatialReference *poOther, const char *const *papszOptions) const { + TAKE_OPTIONAL_LOCK(); + CPL_IGNORE_RET_VAL(papszOptions); d->refreshProjObj(); @@ -9514,6 +9782,8 @@ int OSRIsSameGeogCS(OGRSpatialReferenceH hSRS1, OGRSpatialReferenceH hSRS2) int OGRSpatialReference::IsSameVertCS(const OGRSpatialReference *poOther) const { + TAKE_OPTIONAL_LOCK(); + /* -------------------------------------------------------------------- */ /* Does the datum name match? */ /* -------------------------------------------------------------------- */ @@ -9599,6 +9869,8 @@ int OGRSpatialReference::IsSame(const OGRSpatialReference *poOtherSRS, const char *const *papszOptions) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); poOtherSRS->d->refreshProjObj(); if (!d->m_pj_crs || !poOtherSRS->d->m_pj_crs) @@ -9718,6 +9990,8 @@ OGRSpatialReference *OGRSpatialReference::convertToOtherProjection( const char *pszTargetProjection, CPL_UNUSED const char *const *papszOptions) const { + TAKE_OPTIONAL_LOCK(); + if (pszTargetProjection == nullptr) return nullptr; int new_code; @@ -9923,6 +10197,8 @@ OGRSpatialReference::FindBestMatch(int nMinimumMatchConfidence, const char *pszPreferredAuthority, CSLConstList papszOptions) const { + TAKE_OPTIONAL_LOCK(); + CPL_IGNORE_RET_VAL(papszOptions); // ignored for now. if (nMinimumMatchConfidence == 0) @@ -10049,6 +10325,8 @@ OGRErr OGRSpatialReference::SetTOWGS84(double dfDX, double dfDY, double dfDZ, double dfPPM) { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (d->m_pj_crs == nullptr) { @@ -10220,6 +10498,8 @@ OGRErr OSRSetTOWGS84(OGRSpatialReferenceH hSRS, double dfDX, double dfDY, OGRErr OGRSpatialReference::GetTOWGS84(double *padfCoeff, int nCoeffCount) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (d->m_pjType != PJ_TYPE_BOUND_CRS) return OGRERR_FAILURE; @@ -10261,6 +10541,7 @@ OGRErr OSRGetTOWGS84(OGRSpatialReferenceH hSRS, double *padfCoeff, * @return TRUE or FALSE */ +/* static */ int OGRSpatialReference::IsAngularParameter(const char *pszParameterName) { @@ -10285,6 +10566,7 @@ int OGRSpatialReference::IsAngularParameter(const char *pszParameterName) * @return TRUE or FALSE */ +/* static */ int OGRSpatialReference::IsLongitudeParameter(const char *pszParameterName) { @@ -10304,6 +10586,8 @@ int OGRSpatialReference::IsLongitudeParameter(const char *pszParameterName) * * @return TRUE or FALSE */ + +/* static */ int OGRSpatialReference::IsLinearParameter(const char *pszParameterName) { @@ -10325,6 +10609,8 @@ int OGRSpatialReference::IsLinearParameter(const char *pszParameterName) void OGRSpatialReference::GetNormInfo() const { + TAKE_OPTIONAL_LOCK(); + if (d->bNormInfoSet) return; @@ -10362,6 +10648,8 @@ const char *OGRSpatialReference::GetExtension(const char *pszTargetKey, const char *pszDefault) const { + TAKE_OPTIONAL_LOCK(); + /* -------------------------------------------------------------------- */ /* Find the target node. */ /* -------------------------------------------------------------------- */ @@ -10410,6 +10698,8 @@ OGRErr OGRSpatialReference::SetExtension(const char *pszTargetKey, const char *pszValue) { + TAKE_OPTIONAL_LOCK(); + /* -------------------------------------------------------------------- */ /* Find the target node. */ /* -------------------------------------------------------------------- */ @@ -10491,6 +10781,8 @@ void OSRCleanup(void) */ int OGRSpatialReference::GetAxesCount() const { + TAKE_OPTIONAL_LOCK(); + int axisCount = 0; d->refreshProjObj(); if (d->m_pj_crs == nullptr) @@ -10588,6 +10880,8 @@ const char *OGRSpatialReference::GetAxis(const char *pszTargetKey, int iAxis, double *pdfConvUnit) const { + TAKE_OPTIONAL_LOCK(); + if (peOrientation != nullptr) *peOrientation = OAO_Other; if (pdfConvUnit != nullptr) @@ -10852,6 +11146,8 @@ OGRErr OGRSpatialReference::SetAxes(const char *pszTargetKey, OGRAxisOrientation eYAxisOrientation) { + TAKE_OPTIONAL_LOCK(); + /* -------------------------------------------------------------------- */ /* Find the target node. */ /* -------------------------------------------------------------------- */ @@ -11177,6 +11473,8 @@ OGRErr OSRImportFromProj4(OGRSpatialReferenceH hSRS, const char *pszProj4) OGRErr OGRSpatialReference::importFromProj4(const char *pszProj4) { + TAKE_OPTIONAL_LOCK(); + if (strlen(pszProj4) >= 10000) { CPLError(CE_Failure, CPLE_AppDefined, "Too long PROJ string"); @@ -11285,7 +11583,7 @@ OGRErr OGRSpatialReference::exportToProj4(char **ppszProj4) const // In the past calling this method was thread-safe, even if we never // guaranteed it. Now proj_as_proj_string() will cache the result // internally, so this is no longer thread-safe. - std::lock_guard oLock(d->m_mutex); + std::lock_guard oLock(d->m_mutex); d->refreshProjObj(); if (d->m_pj_crs == nullptr || d->m_pjType == PJ_TYPE_ENGINEERING_CRS) @@ -11380,6 +11678,8 @@ OGRErr OGRSpatialReference::exportToProj4(char **ppszProj4) const OGRErr OGRSpatialReference::morphToESRI() { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->setMorphToESRI(true); @@ -11430,6 +11730,8 @@ OGRErr OSRMorphToESRI(OGRSpatialReferenceH hSRS) OGRErr OGRSpatialReference::morphFromESRI() { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); d->setMorphToESRI(false); @@ -11489,6 +11791,8 @@ OGRSpatialReferenceH * OGRSpatialReference::FindMatches(char **papszOptions, int *pnEntries, int **ppanMatchConfidence) const { + TAKE_OPTIONAL_LOCK(); + CPL_IGNORE_RET_VAL(papszOptions); if (pnEntries) @@ -11629,6 +11933,8 @@ OGRSpatialReference::FindMatches(char **papszOptions, int *pnEntries, OGRErr OGRSpatialReference::importFromEPSGA(int nCode) { + TAKE_OPTIONAL_LOCK(); + Clear(); const char *pszUseNonDeprecated = @@ -11738,6 +12044,8 @@ OGRErr OGRSpatialReference::importFromEPSGA(int nCode) */ OGRErr OGRSpatialReference::AddGuessedTOWGS84() { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) return OGRERR_FAILURE; @@ -11869,6 +12177,8 @@ OGRErr CPL_STDCALL OSRImportFromEPSG(OGRSpatialReferenceH hSRS, int nCode) int OGRSpatialReference::EPSGTreatsAsLatLong() const { + TAKE_OPTIONAL_LOCK(); + if (!IsGeographic()) return FALSE; @@ -11978,6 +12288,8 @@ int OSREPSGTreatsAsLatLong(OGRSpatialReferenceH hSRS) int OGRSpatialReference::EPSGTreatsAsNorthingEasting() const { + TAKE_OPTIONAL_LOCK(); + if (!IsProjected()) return FALSE; @@ -12050,6 +12362,8 @@ OGRErr OGRSpatialReference::ImportFromESRIWisconsinWKT(const char *prjName, const char *unitsName, const char *crsName) { + TAKE_OPTIONAL_LOCK(); + if (centralMeridian < -93 || centralMeridian > -87) return OGRERR_FAILURE; if (latOfOrigin < 40 || latOfOrigin > 47) @@ -12201,6 +12515,8 @@ OGRErr OGRSpatialReference::ImportFromESRIWisconsinWKT(const char *prjName, */ OSRAxisMappingStrategy OGRSpatialReference::GetAxisMappingStrategy() const { + TAKE_OPTIONAL_LOCK(); + return d->m_axisMappingStrategy; } @@ -12239,6 +12555,8 @@ OSRAxisMappingStrategy OSRGetAxisMappingStrategy(OGRSpatialReferenceH hSRS) void OGRSpatialReference::SetAxisMappingStrategy( OSRAxisMappingStrategy strategy) { + TAKE_OPTIONAL_LOCK(); + d->m_axisMappingStrategy = strategy; d->refreshAxisMapping(); } @@ -12276,6 +12594,8 @@ void OSRSetAxisMappingStrategy(OGRSpatialReferenceH hSRS, */ const std::vector &OGRSpatialReference::GetDataAxisToSRSAxisMapping() const { + TAKE_OPTIONAL_LOCK(); + return d->m_axisMapping; } @@ -12343,6 +12663,8 @@ const int *OSRGetDataAxisToSRSAxisMapping(OGRSpatialReferenceH hSRS, OGRErr OGRSpatialReference::SetDataAxisToSRSAxisMapping( const std::vector &mapping) { + TAKE_OPTIONAL_LOCK(); + if (mapping.size() < 2) return OGRERR_FAILURE; d->m_axisMappingStrategy = OAMS_CUSTOM; @@ -12412,6 +12734,8 @@ bool OGRSpatialReference::GetAreaOfUse(double *pdfWestLongitudeDeg, double *pdfNorthLatitudeDeg, const char **ppszAreaName) const { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) { @@ -12575,6 +12899,8 @@ void OSRDestroyCRSInfoList(OSRCRSInfo **list) */ void OGRSpatialReference::UpdateCoordinateSystemFromGeogCRS() { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) return; @@ -12667,6 +12993,8 @@ void OGRSpatialReference::UpdateCoordinateSystemFromGeogCRS() */ OGRErr OGRSpatialReference::PromoteTo3D(const char *pszName) { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) return OGRERR_FAILURE; @@ -12708,6 +13036,8 @@ OGRErr OSRPromoteTo3D(OGRSpatialReferenceH hSRS, const char *pszName) */ OGRErr OGRSpatialReference::DemoteTo2D(const char *pszName) { + TAKE_OPTIONAL_LOCK(); + d->refreshProjObj(); if (!d->m_pj_crs) return OGRERR_FAILURE; @@ -12749,6 +13079,8 @@ OGRErr OSRDemoteTo2D(OGRSpatialReferenceH hSRS, const char *pszName) int OGRSpatialReference::GetEPSGGeogCS() const { + TAKE_OPTIONAL_LOCK(); + /* -------------------------------------------------------------------- */ /* Check axis order. */ /* -------------------------------------------------------------------- */ From b4999bb5e828e83462931a620770225a346a590b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 09:49:48 +0200 Subject: [PATCH 189/710] GDALDataset::GetProjectionRef() and GetGCPProjection(): add locking on thread-safe dataset --- gcore/gdaldataset.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/gcore/gdaldataset.cpp b/gcore/gdaldataset.cpp index ee91b2d19302..2f17d2d69757 100644 --- a/gcore/gdaldataset.cpp +++ b/gcore/gdaldataset.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,8 @@ class GDALDataset::Private GIntBig nTotalFeatures = TOTAL_FEATURES_NOT_INIT; OGRLayer *poCurrentLayer = nullptr; + std::mutex m_oMutexWKT{}; + char *m_pszWKTCached = nullptr; OGRSpatialReference *m_poSRSCached = nullptr; char *m_pszWKTGCPCached = nullptr; @@ -1166,6 +1169,11 @@ const char *GDALDataset::GetProjectionRef() const { return ""; } + + // If called on a thread-safe dataset, we might be called by several + // threads, so make sure our accesses to m_pszWKTCached are protected + // by a mutex. + std::lock_guard oLock(m_poPrivate->m_oMutexWKT); if (m_poPrivate->m_pszWKTCached && strcmp(pszWKT, m_poPrivate->m_pszWKTCached) == 0) { @@ -1831,6 +1839,11 @@ const char *GDALDataset::GetGCPProjection() { return ""; } + + // If called on a thread-safe dataset, we might be called by several + // threads, so make sure our accesses to m_pszWKTCached are protected + // by a mutex. + std::lock_guard oLock(m_poPrivate->m_oMutexWKT); if (m_poPrivate->m_pszWKTGCPCached && strcmp(pszWKT, m_poPrivate->m_pszWKTGCPCached) == 0) { From 580c6100872678e99a9c1a07954d9bc243c445d3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Sep 2024 11:00:25 +0200 Subject: [PATCH 190/710] GDALThreadSafeDataset: handle the case of methods that return non primitive types --- autotest/gcore/thread_test.py | 189 +++++++++++++++++++++++++++++++- gcore/gdalthreadsafedataset.cpp | 115 ++++++++++++++++++- 2 files changed, 302 insertions(+), 2 deletions(-) diff --git a/autotest/gcore/thread_test.py b/autotest/gcore/thread_test.py index f57281c00fc2..91e8d574b202 100755 --- a/autotest/gcore/thread_test.py +++ b/autotest/gcore/thread_test.py @@ -88,7 +88,7 @@ def verify_checksum(): res[0] = False assert False, (got_cs, expected_cs) - threads = [threading.Thread(target=verify_checksum)] + threads = [threading.Thread(target=verify_checksum) for i in range(2)] for t in threads: t.start() for t in threads: @@ -426,3 +426,190 @@ def test_thread_safe_unsupported_rat(): match="not supporting a non-GDALDefaultRasterAttributeTable implementation", ): ds.GetRasterBand(1).GetDefaultRAT() + + +def test_thread_safe_many_datasets(): + + tab_ds = [ + gdal.OpenEx( + "data/byte.tif" if (i % 3) < 2 else "data/utmsmall.tif", + gdal.OF_RASTER | gdal.OF_THREAD_SAFE, + ) + for i in range(100) + ] + + res = [True] + + def check(): + for _ in range(10): + for i, ds in enumerate(tab_ds): + if ds.GetRasterBand(1).Checksum() != (4672 if (i % 3) < 2 else 50054): + res[0] = False + + threads = [threading.Thread(target=check) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] + + +def test_thread_safe_BeginAsyncReader(): + + with gdal.OpenEx("data/byte.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + with pytest.raises(Exception, match="not supported"): + ds.BeginAsyncReader(0, 0, ds.RasterXSize, ds.RasterYSize) + + +def test_thread_safe_GetVirtualMem(): + + pytest.importorskip("numpy") + pytest.importorskip("osgeo.gdal_array") + + with gdal.OpenEx("data/byte.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + with pytest.raises(Exception, match="not supported"): + ds.GetRasterBand(1).GetVirtualMemAutoArray(gdal.GF_Read) + + +def test_thread_safe_GetMetadadata(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + with gdal.GetDriverByName("GTiff").Create(filename, 1, 1) as ds: + ds.SetMetadataItem("foo", "bar") + ds.GetRasterBand(1).SetMetadataItem("bar", "baz") + + with gdal.OpenEx(filename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + assert ds.GetMetadataItem("foo") == "bar" + assert ds.GetMetadataItem("not existing") is None + assert ds.GetMetadata() == {"foo": "bar"} + assert ds.GetMetadata("not existing") == {} + assert ds.GetRasterBand(1).GetMetadataItem("bar") == "baz" + assert ds.GetRasterBand(1).GetMetadataItem("not existing") is None + assert ds.GetRasterBand(1).GetMetadata() == {"bar": "baz"} + assert ds.GetRasterBand(1).GetMetadata("not existing") == {} + + +def test_thread_safe_GetUnitType(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + with gdal.GetDriverByName("GTiff").Create(filename, 1, 1) as ds: + ds.GetRasterBand(1).SetUnitType("foo") + + with gdal.OpenEx(filename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + assert ds.GetRasterBand(1).GetUnitType() == "foo" + + +def test_thread_safe_GetColorTable(tmp_vsimem): + + filename = str(tmp_vsimem / "test.tif") + with gdal.GetDriverByName("GTiff").Create(filename, 1, 1) as ds: + ct = gdal.ColorTable() + ct.SetColorEntry(0, (1, 2, 3, 255)) + ds.GetRasterBand(1).SetColorTable(ct) + + with gdal.OpenEx(filename, gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + res = [None] + + def thread_job(): + res[0] = ds.GetRasterBand(1).GetColorTable() + + t = threading.Thread(target=thread_job) + t.start() + t.join() + assert res[0] + assert res[0].GetColorEntry(0) == (1, 2, 3, 255) + ct = ds.GetRasterBand(1).GetColorTable() + assert ct.GetColorEntry(0) == (1, 2, 3, 255) + + +def test_thread_safe_GetSpatialRef(): + + with gdal.OpenEx("data/byte.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE) as ds: + + res = [True] + + def check(): + for i in range(100): + + if len(ds.GetGCPs()) != 0: + res[0] = False + assert False + + if ds.GetGCPSpatialRef(): + res[0] = False + assert False + + if ds.GetGCPProjection(): + res[0] = False + assert False + + srs = ds.GetSpatialRef() + if not srs: + res[0] = False + assert False + if not srs.IsProjected(): + res[0] = False + assert False + if "NAD27 / UTM zone 11N" not in srs.ExportToWkt(): + res[0] = False + assert False + + if "NAD27 / UTM zone 11N" not in ds.GetProjectionRef(): + res[0] = False + assert False + + threads = [threading.Thread(target=check) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] + + +def test_thread_safe_GetGCPs(): + + with gdal.OpenEx( + "data/byte_gcp_pixelispoint.tif", gdal.OF_RASTER | gdal.OF_THREAD_SAFE + ) as ds: + + res = [True] + + def check(): + for i in range(100): + + if len(ds.GetGCPs()) != 4: + res[0] = False + assert False + + gcp_srs = ds.GetGCPSpatialRef() + if gcp_srs is None: + res[0] = False + assert False + if not gcp_srs.IsGeographic(): + res[0] = False + assert False + if "unretrievable - using WGS84" not in gcp_srs.ExportToWkt(): + res[0] = False + assert False + + gcp_wkt = ds.GetGCPProjection() + if not gcp_wkt: + res[0] = False + assert False + if "unretrievable - using WGS84" not in gcp_wkt: + res[0] = False + assert False + + if ds.GetSpatialRef(): + res[0] = False + assert False + if ds.GetProjectionRef() != "": + res[0] = False + assert False + + threads = [threading.Thread(target=check) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + assert res[0] diff --git a/gcore/gdalthreadsafedataset.cpp b/gcore/gdalthreadsafedataset.cpp index 3b2c8156f249..2ce0de26d31c 100644 --- a/gcore/gdalthreadsafedataset.cpp +++ b/gcore/gdalthreadsafedataset.cpp @@ -177,6 +177,71 @@ class GDALThreadSafeDataset final : public GDALProxyDataset static GDALDataset *Create(GDALDataset *poPrototypeDS, int nScopeFlags); + /* All below public methods override GDALDataset methods, and instead of + * forwarding to a thread-local dataset, they act on the prototype dataset, + * because they return a non-trivial type, that could be invalidated + * otherwise if the thread-local dataset is evicted from the LRU cache. + */ + const OGRSpatialReference *GetSpatialRef() const override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + if (m_oSRS.IsEmpty()) + { + auto poSRS = m_poPrototypeDS->GetSpatialRef(); + if (poSRS) + { + m_oSRS.AssignAndSetThreadSafe(*poSRS); + } + } + return m_oSRS.IsEmpty() ? nullptr : &m_oSRS; + } + + const OGRSpatialReference *GetGCPSpatialRef() const override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + if (m_oGCPSRS.IsEmpty()) + { + auto poSRS = m_poPrototypeDS->GetGCPSpatialRef(); + if (poSRS) + { + m_oGCPSRS.AssignAndSetThreadSafe(*poSRS); + } + } + return m_oGCPSRS.IsEmpty() ? nullptr : &m_oGCPSRS; + } + + const GDAL_GCP *GetGCPs() override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + return const_cast(m_poPrototypeDS)->GetGCPs(); + } + + const char *GetMetadataItem(const char *pszName, + const char *pszDomain = "") override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + return const_cast(m_poPrototypeDS) + ->GetMetadataItem(pszName, pszDomain); + } + + char **GetMetadata(const char *pszDomain = "") override + { + std::lock_guard oGuard(m_oPrototypeDSMutex); + return const_cast(m_poPrototypeDS) + ->GetMetadata(pszDomain); + } + + /* End of methods that forward on the prototype dataset */ + + GDALAsyncReader *BeginAsyncReader(int, int, int, int, void *, int, int, + GDALDataType, int, int *, int, int, int, + char **) override + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeDataset::BeginAsyncReader() not supported"); + return nullptr; + } + protected: GDALDataset *RefUnderlyingDataset() const override; @@ -190,7 +255,7 @@ class GDALThreadSafeDataset final : public GDALProxyDataset friend class GDALThreadLocalDatasetCache; /** Mutex that protects accesses to m_poPrototypeDS */ - std::mutex m_oPrototypeDSMutex{}; + mutable std::mutex m_oPrototypeDSMutex{}; /** "Prototype" dataset, that is the dataset that was passed to the * GDALThreadSafeDataset constructor. All calls on to it should be on @@ -208,6 +273,12 @@ class GDALThreadSafeDataset final : public GDALProxyDataset */ const CPLStringList m_aosThreadLocalConfigOptions{}; + /** Cached value returned by GetSpatialRef() */ + mutable OGRSpatialReference m_oSRS{}; + + /** Cached value returned by GetGCPSpatialRef() */ + mutable OGRSpatialReference m_oGCPSRS{}; + /** Structure that references all GDALThreadLocalDatasetCache* instances. */ struct GlobalCache @@ -275,6 +346,48 @@ class GDALThreadSafeRasterBand final : public GDALProxyRasterBand GDALRasterAttributeTable *GetDefaultRAT() override; + /* All below public methods override GDALRasterBand methods, and instead of + * forwarding to a thread-local dataset, they act on the prototype band, + * because they return a non-trivial type, that could be invalidated + * otherwise if the thread-local dataset is evicted from the LRU cache. + */ + const char *GetMetadataItem(const char *pszName, + const char *pszDomain = "") override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand) + ->GetMetadataItem(pszName, pszDomain); + } + + char **GetMetadata(const char *pszDomain = "") override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand) + ->GetMetadata(pszDomain); + } + + const char *GetUnitType() override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand)->GetUnitType(); + } + + GDALColorTable *GetColorTable() override + { + std::lock_guard oGuard(m_poTSDS->m_oPrototypeDSMutex); + return const_cast(m_poPrototypeBand)->GetColorTable(); + } + + /* End of methods that forward on the prototype band */ + + CPLVirtualMem *GetVirtualMemAuto(GDALRWFlag, int *, GIntBig *, + char **) override + { + CPLError(CE_Failure, CPLE_AppDefined, + "GDALThreadSafeRasterBand::GetVirtualMemAuto() not supported"); + return nullptr; + } + protected: GDALRasterBand *RefUnderlyingRasterBand(bool bForceOpen) const override; void UnrefUnderlyingRasterBand( From 518c2e222e037ef5cc8cd2874ef04dbd6320d4d0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Sep 2024 23:04:07 +0200 Subject: [PATCH 191/710] LIBKML: report XML id attribute as 'id' field; implement SetFeature() and DeleteFeature() for regular layers Fixes qgis/QGIS#58780 --- apps/test_ogrsf.cpp | 3 +- autotest/ogr/ogr_libkml.py | 90 ++++++- doc/source/drivers/vector/libkml.rst | 7 + ogr/ogrsf_frmts/libkml/fieldconfig.h | 88 +++++++ ogr/ogrsf_frmts/libkml/ogr_libkml.h | 21 +- ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp | 15 +- ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.h | 4 +- ogr/ogrsf_frmts/libkml/ogrlibkmlfield.cpp | 38 +-- ogr/ogrsf_frmts/libkml/ogrlibkmlfield.h | 64 +---- ogr/ogrsf_frmts/libkml/ogrlibkmlgeometry.cpp | 6 +- ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp | 254 ++++++++++++++----- 11 files changed, 433 insertions(+), 157 deletions(-) create mode 100644 ogr/ogrsf_frmts/libkml/fieldconfig.h diff --git a/apps/test_ogrsf.cpp b/apps/test_ogrsf.cpp index ddb13d4dc22b..5f5e4a6a7134 100644 --- a/apps/test_ogrsf.cpp +++ b/apps/test_ogrsf.cpp @@ -2041,7 +2041,8 @@ static int TestOGRLayerRandomWrite(OGRLayer *poLayer) CPLString os_Id2; CPLString os_Id5; - const bool bHas_Id = poLayer->GetLayerDefn()->GetFieldIndex("_id") == 0; + const bool bHas_Id = poLayer->GetLayerDefn()->GetFieldIndex("_id") == 0 || + poLayer->GetLayerDefn()->GetFieldIndex("id") == 0; /* -------------------------------------------------------------------- */ /* Fetch five features. */ diff --git a/autotest/ogr/ogr_libkml.py b/autotest/ogr/ogr_libkml.py index 777058abd284..faa2ca2be036 100755 --- a/autotest/ogr/ogr_libkml.py +++ b/autotest/ogr/ogr_libkml.py @@ -29,7 +29,6 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### - import gdaltest import ogrtest import pytest @@ -289,7 +288,7 @@ def ogr_libkml_write(filename): lyr = ds.CreateLayer("test_wgs72", srs=srs) assert lyr.TestCapability(ogr.OLCSequentialWrite) == 1 - assert lyr.TestCapability(ogr.OLCRandomWrite) == 0 + assert lyr.TestCapability(ogr.OLCRandomWrite) == 1 dst_feat = ogr.Feature(lyr.GetLayerDefn()) dst_feat.SetGeometry(ogr.CreateGeometryFromWkt("POINT (2 49)")) @@ -530,6 +529,32 @@ def test_ogr_libkml_test_ogrsf(): ) +############################################################################### +# Run test_ogrsf + + +def test_ogr_libkml_test_ogrsf_write(tmp_path): + + test_filename = str(tmp_path / "test.kml") + gdal.VectorTranslate( + test_filename, "data/poly.shp", options="-s_srs EPSG:32631 -t_srs EPSG:4326" + ) + + import test_cli_utilities + + if test_cli_utilities.get_test_ogrsf_path() is None: + pytest.skip() + + ret = gdaltest.runexternal( + test_cli_utilities.get_test_ogrsf_path() + + f" --config OGR_SKIP KML {test_filename}" + ) + + assert "using driver `LIBKML'" in ret + assert "INFO" in ret + assert "ERROR" not in ret + + ############################################################################### # Test reading KML with only Placemark @@ -2253,3 +2278,64 @@ def test_ogr_libkml_write_geometries(input_wkt, expected_wkt, tmp_vsimem): assert f.GetGeometryRef().ExportToIsoWkt() == expected_wkt else: assert f is None + + +############################################################################### +# Test update of existing file + + +@pytest.mark.parametrize("custom_id", [False, True]) +def test_ogr_libkml_update_delete_existing_kml(tmp_vsimem, custom_id): + + filename = str(tmp_vsimem / "test.kml") + with ogr.GetDriverByName("LIBKML").CreateDataSource(filename) as ds: + lyr = ds.CreateLayer("test") + lyr.CreateField(ogr.FieldDefn("id")) + lyr.CreateField(ogr.FieldDefn("name")) + f = ogr.Feature(lyr.GetLayerDefn()) + if custom_id: + f["id"] = "feat1" + f["name"] = "name1" + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (1 2)")) + lyr.CreateFeature(f) + f = ogr.Feature(lyr.GetLayerDefn()) + if custom_id: + f["id"] = "feat2" + f["name"] = "name2" + f.SetGeometry(ogr.CreateGeometryFromWkt("POINT (3 4)")) + lyr.CreateFeature(f) + + with gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) as ds: + lyr = ds.GetLayer(0) + with pytest.raises(Exception, match="Non existing feature"): + lyr.DeleteFeature(0) + + with gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) as ds: + lyr = ds.GetLayer(0) + with pytest.raises(Exception, match="Non existing feature"): + lyr.DeleteFeature(3) + + with gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) as ds: + lyr = ds.GetLayer(0) + lyr.DeleteFeature(1) + assert lyr.GetFeatureCount() == 1 + lyr.ResetReading() + f = lyr.GetNextFeature() + assert f.GetFID() == 2 + if custom_id: + assert f["id"] == "feat2" + assert f["name"] == "name2" + f["name"] = "name2_updated" + lyr.SetFeature(f) + + with gdal.OpenEx(filename, gdal.OF_VECTOR | gdal.OF_UPDATE) as ds: + lyr = ds.GetLayer(0) + f = lyr.GetNextFeature() + if custom_id: + # FIDs are renumbered after feature update/deletion´if using + # custom KML ids + assert f.GetFID() == 1 + assert f["id"] == "feat2" + else: + assert f.GetFID() == 2 + assert f["name"] == "name2_updated" diff --git a/doc/source/drivers/vector/libkml.rst b/doc/source/drivers/vector/libkml.rst index ee7ac9114270..b1d9eb9e8d9c 100644 --- a/doc/source/drivers/vector/libkml.rst +++ b/doc/source/drivers/vector/libkml.rst @@ -587,6 +587,13 @@ For example, if you want a field called 'Cities' to map to the tag in KML, you can set a configuration option. Note these are independent of layer creation and dataset creation options' ``. +- .. config:: LIBKML_ID_FIELD + :default: id + :since: 3.10 + + Name of the string field that maps to the kml attribute + ` `__. + - .. config:: LIBKML_NAME_FIELD :default: name diff --git a/ogr/ogrsf_frmts/libkml/fieldconfig.h b/ogr/ogrsf_frmts/libkml/fieldconfig.h new file mode 100644 index 000000000000..ab641388e0d6 --- /dev/null +++ b/ogr/ogrsf_frmts/libkml/fieldconfig.h @@ -0,0 +1,88 @@ +/****************************************************************************** + * + * Project: KML Translator + * Purpose: Implements OGRLIBKMLDriver + * Author: Brian Case, rush at winkey dot org + * + ****************************************************************************** + * Copyright (c) 2010, Brian Case + * Copyright (c) 2014, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef OGRLIBKMLFIELDCONFIG_H_INCLUDED +#define OGRLIBKMLFIELDCONFIG_H_INCLUDED + +/******************************************************************************* + Function to fetch the field config options. +*******************************************************************************/ + +struct fieldconfig +{ + const char *idfield; + const char *namefield; + const char *descfield; + const char *tsfield; + const char *beginfield; + const char *endfield; + const char *altitudeModefield; + const char *tessellatefield; + const char *extrudefield; + const char *visibilityfield; + const char *drawOrderfield; + const char *iconfield; + const char *headingfield; + const char *tiltfield; + const char *rollfield; + const char *snippetfield; + const char *modelfield; + const char *scalexfield; + const char *scaleyfield; + const char *scalezfield; + const char *networklinkfield; + const char *networklink_refreshvisibility_field; + const char *networklink_flytoview_field; + const char *networklink_refreshMode_field; + const char *networklink_refreshInterval_field; + const char *networklink_viewRefreshMode_field; + const char *networklink_viewRefreshTime_field; + const char *networklink_viewBoundScale_field; + const char *networklink_viewFormat_field; + const char *networklink_httpQuery_field; + const char *camera_longitude_field; + const char *camera_latitude_field; + const char *camera_altitude_field; + const char *camera_altitudemode_field; + const char *photooverlayfield; + const char *leftfovfield; + const char *rightfovfield; + const char *bottomfovfield; + const char *topfovfield; + const char *nearfield; + const char *photooverlay_shape_field; + const char *imagepyramid_tilesize_field; + const char *imagepyramid_maxwidth_field; + const char *imagepyramid_maxheight_field; + const char *imagepyramid_gridorigin_field; +}; + +void get_fieldconfig(struct fieldconfig *oFC); + +#endif diff --git a/ogr/ogrsf_frmts/libkml/ogr_libkml.h b/ogr/ogrsf_frmts/libkml/ogr_libkml.h index c1c51a1b209f..4c3da1186d24 100644 --- a/ogr/ogrsf_frmts/libkml/ogr_libkml.h +++ b/ogr/ogrsf_frmts/libkml/ogr_libkml.h @@ -33,6 +33,7 @@ #include "ogrsf_frmts.h" #include "libkml_headers.h" +#include "fieldconfig.h" #include @@ -47,18 +48,20 @@ CPLString OGRLIBKMLGetSanitizedNCName(const char *pszName); class OGRLIBKMLLayer final : public OGRLayer, public OGRGetNextFeatureThroughRaw { - int bUpdate; + int bUpdate = false; - int nFeatures; - int iFeature; - long nFID; + int nFeatures = 0; + int iFeature = 0; + GIntBig nFID = 1; const char *m_pszName; const char *m_pszFileName; + std::string m_osSanitizedNCName; kmldom::ContainerPtr m_poKmlLayer; kmldom::ElementPtr m_poKmlLayerRoot; kmldom::UpdatePtr m_poKmlUpdate; + fieldconfig m_oFieldConfig; OGRLIBKMLDataSource *m_poOgrDS; OGRFeatureDefn *m_poOgrFeatureDefn; kmldom::SchemaPtr m_poKmlSchema; @@ -84,6 +87,11 @@ class OGRLIBKMLLayer final : public OGRLayer, bool m_bUpdateIsFolder; + bool m_bAllReadAtLeastOnce = false; + std::map m_oMapOGRIdToKmlId{}; + std::map m_oMapKmlIdToOGRId{}; + + void ScanAllFeatures(); OGRFeature *GetNextRawFeature(); public: @@ -161,6 +169,11 @@ class OGRLIBKMLLayer final : public OGRLayer, return m_pszFileName; } + const fieldconfig &GetFieldConfig() const + { + return m_oFieldConfig; + } + void SetLookAt(const char *pszLookatLongitude, const char *pszLookatLatitude, const char *pszLookatAltitude, const char *pszLookatHeading, const char *pszLookatTilt, diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp b/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp index 17c0ff7c96e5..ad1c6a6d20a6 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.cpp @@ -225,9 +225,7 @@ FeaturePtr feat2kml(OGRLIBKMLDataSource *poOgrDS, OGRLIBKMLLayer *poOgrLayer, int bUseSimpleField) { FeaturePtr poKmlFeature = nullptr; - - struct fieldconfig oFC; - get_fieldconfig(&oFC); + const auto &oFC = poOgrLayer->GetFieldConfig(); /***** geometry *****/ OGRGeometry *poOgrGeom = poOgrFeat->GetGeometryRef(); @@ -833,13 +831,13 @@ FeaturePtr feat2kml(OGRLIBKMLDataSource *poOgrDS, OGRLIBKMLLayer *poOgrLayer, /***** fields *****/ field2kml(poOgrFeat, poOgrLayer, poKmlFactory, poKmlFeature, - bUseSimpleField); + bUseSimpleField, oFC); return poKmlFeature; } OGRFeature *kml2feat(PlacemarkPtr poKmlPlacemark, OGRLIBKMLDataSource *poOgrDS, - OGRLayer *poOgrLayer, OGRFeatureDefn *poOgrFeatDefn, + OGRLIBKMLLayer *poOgrLayer, OGRFeatureDefn *poOgrFeatDefn, OGRSpatialReference *poOgrSRS) { OGRFeature *poOgrFeat = new OGRFeature(poOgrFeatDefn); @@ -872,14 +870,15 @@ OGRFeature *kml2feat(PlacemarkPtr poKmlPlacemark, OGRLIBKMLDataSource *poOgrDS, } /***** fields *****/ - kml2field(poOgrFeat, AsFeature(poKmlPlacemark)); + kml2field(poOgrFeat, AsFeature(poKmlPlacemark), + poOgrLayer->GetFieldConfig()); return poOgrFeat; } OGRFeature *kmlgroundoverlay2feat(GroundOverlayPtr poKmlOverlay, OGRLIBKMLDataSource * /* poOgrDS */, - OGRLayer * /* poOgrLayer */, + OGRLIBKMLLayer *poOgrLayer, OGRFeatureDefn *poOgrFeatDefn, OGRSpatialReference *poOgrSRS) { @@ -900,7 +899,7 @@ OGRFeature *kmlgroundoverlay2feat(GroundOverlayPtr poKmlOverlay, } /***** fields *****/ - kml2field(poOgrFeat, AsFeature(poKmlOverlay)); + kml2field(poOgrFeat, AsFeature(poKmlOverlay), poOgrLayer->GetFieldConfig()); return poOgrFeat; } diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.h b/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.h index d470c00746fb..a0a9ba0d6a22 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.h +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmlfeature.h @@ -45,13 +45,13 @@ kmldom::FeaturePtr feat2kml(OGRLIBKMLDataSource *poOgrDS, ******************************************************************************/ OGRFeature *kml2feat(kmldom::PlacemarkPtr poKmlPlacemark, - OGRLIBKMLDataSource *poOgrDS, OGRLayer *poOgrLayer, + OGRLIBKMLDataSource *poOgrDS, OGRLIBKMLLayer *poOgrLayer, OGRFeatureDefn *poOgrFeatDefn, OGRSpatialReference *poOgrSRS); OGRFeature *kmlgroundoverlay2feat(kmldom::GroundOverlayPtr poKmlOverlay, OGRLIBKMLDataSource *poOgrDS, - OGRLayer *poOgrLayer, + OGRLIBKMLLayer *poOgrLayer, OGRFeatureDefn *poOgrFeatDefn, OGRSpatialReference *poOgrSRS); diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.cpp b/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.cpp index ad0161f1393e..a525524a616d 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.cpp +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.cpp @@ -273,7 +273,7 @@ static char *OGRLIBKMLSanitizeUTF8String(const char *pszString) void field2kml(OGRFeature *poOgrFeat, OGRLIBKMLLayer *poOgrLayer, KmlFactory *poKmlFactory, FeaturePtr poKmlFeature, - int bUseSimpleFieldIn) + int bUseSimpleFieldIn, const fieldconfig &oFC) { const bool bUseSimpleField = CPL_TO_BOOL(bUseSimpleFieldIn); SchemaDataPtr poKmlSchemaData = nullptr; @@ -293,10 +293,6 @@ void field2kml(OGRFeature *poOgrFeat, OGRLIBKMLLayer *poOgrLayer, } } - /***** Get the field config *****/ - struct fieldconfig oFC; - get_fieldconfig(&oFC); - TimeSpanPtr poKmlTimeSpan = nullptr; const int nFields = poOgrFeat->GetFieldCount(); @@ -338,6 +334,13 @@ void field2kml(OGRFeature *poOgrFeat, OGRLIBKMLLayer *poOgrLayer, continue; } + /***** id *****/ + if (EQUAL(name, oFC.idfield)) + { + poKmlFeature->set_id(pszUTF8String); + CPLFree(pszUTF8String); + continue; + } /***** name *****/ if (EQUAL(name, oFC.namefield)) { @@ -1157,13 +1160,19 @@ static void kmldatetime2ogr(OGRFeature *poOgrFeat, const char *pszOGRField, function to read kml into ogr fields ******************************************************************************/ -void kml2field(OGRFeature *poOgrFeat, FeaturePtr poKmlFeature) +void kml2field(OGRFeature *poOgrFeat, FeaturePtr poKmlFeature, + const fieldconfig &oFC) { - /***** get the field config *****/ + /***** id *****/ - struct fieldconfig oFC; - get_fieldconfig(&oFC); + if (poKmlFeature->has_id()) + { + const std::string oKmlId = poKmlFeature->get_id(); + int iField = poOgrFeat->GetFieldIndex(oFC.idfield); + if (iField > -1) + poOgrFeat->SetField(iField, oKmlId.c_str()); + } /***** name *****/ if (poKmlFeature->has_name()) @@ -1552,15 +1561,13 @@ void kml2field(OGRFeature *poOgrFeat, FeaturePtr poKmlFeature) ******************************************************************************/ SimpleFieldPtr FieldDef2kml(const OGRFieldDefn *poOgrFieldDef, - KmlFactory *poKmlFactory, bool bApproxOK) + KmlFactory *poKmlFactory, bool bApproxOK, + const fieldconfig &oFC) { - /***** Get the field config. *****/ - struct fieldconfig oFC; - get_fieldconfig(&oFC); - const char *pszFieldName = poOgrFieldDef->GetNameRef(); - if (EQUAL(pszFieldName, oFC.namefield) || + if (EQUAL(pszFieldName, oFC.idfield) || + EQUAL(pszFieldName, oFC.namefield) || EQUAL(pszFieldName, oFC.descfield) || EQUAL(pszFieldName, oFC.tsfield) || EQUAL(pszFieldName, oFC.beginfield) || @@ -1724,6 +1731,7 @@ void kml2FeatureDef(SchemaPtr poKmlSchema, OGRFeatureDefn *poOgrFeatureDefn) void get_fieldconfig(struct fieldconfig *oFC) { + oFC->idfield = CPLGetConfigOption("LIBKML_ID_FIELD", "id"); oFC->namefield = CPLGetConfigOption("LIBKML_NAME_FIELD", "Name"); oFC->descfield = CPLGetConfigOption("LIBKML_DESCRIPTION_FIELD", "description"); diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.h b/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.h index 2fa81e36fa77..010337c6d9e8 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.h +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmlfield.h @@ -30,6 +30,8 @@ #ifndef OGRLIBKMLFIELD_H_INCLUDED #define OGRLIBKMLFIELD_H_INCLUDED +#include "fieldconfig.h" + /****************************************************************************** Function to output ogr fields in kml. @@ -55,13 +57,15 @@ void field2kml(OGRFeature *poOgrFeat, OGRLIBKMLLayer *poOgrLayer, kmldom::KmlFactory *poKmlFactory, - kmldom::FeaturePtr poKmlPlacemark, int bUseSimpleField); + kmldom::FeaturePtr poKmlPlacemark, int bUseSimpleField, + const fieldconfig &oFC); /****************************************************************************** Function to read kml into ogr fields. ******************************************************************************/ -void kml2field(OGRFeature *poOgrFeat, kmldom::FeaturePtr poKmlFeature); +void kml2field(OGRFeature *poOgrFeat, kmldom::FeaturePtr poKmlFeature, + const fieldconfig &oFC); /****************************************************************************** Function create a simplefield from a FieldDefn. @@ -69,7 +73,7 @@ void kml2field(OGRFeature *poOgrFeat, kmldom::FeaturePtr poKmlFeature); kmldom::SimpleFieldPtr FieldDef2kml(const OGRFieldDefn *poOgrFieldDef, kmldom::KmlFactory *poKmlFactory, - bool bApproxOK); + bool bApproxOK, const fieldconfig &oFC); /****************************************************************************** Function to add the simpleFields in a schema to a featuredefn. @@ -78,60 +82,6 @@ kmldom::SimpleFieldPtr FieldDef2kml(const OGRFieldDefn *poOgrFieldDef, void kml2FeatureDef(kmldom::SchemaPtr poKmlSchema, OGRFeatureDefn *poOgrFeatureDefn); -/******************************************************************************* - Function to fetch the field config options. -*******************************************************************************/ - -struct fieldconfig -{ - const char *namefield; - const char *descfield; - const char *tsfield; - const char *beginfield; - const char *endfield; - const char *altitudeModefield; - const char *tessellatefield; - const char *extrudefield; - const char *visibilityfield; - const char *drawOrderfield; - const char *iconfield; - const char *headingfield; - const char *tiltfield; - const char *rollfield; - const char *snippetfield; - const char *modelfield; - const char *scalexfield; - const char *scaleyfield; - const char *scalezfield; - const char *networklinkfield; - const char *networklink_refreshvisibility_field; - const char *networklink_flytoview_field; - const char *networklink_refreshMode_field; - const char *networklink_refreshInterval_field; - const char *networklink_viewRefreshMode_field; - const char *networklink_viewRefreshTime_field; - const char *networklink_viewBoundScale_field; - const char *networklink_viewFormat_field; - const char *networklink_httpQuery_field; - const char *camera_longitude_field; - const char *camera_latitude_field; - const char *camera_altitude_field; - const char *camera_altitudemode_field; - const char *photooverlayfield; - const char *leftfovfield; - const char *rightfovfield; - const char *bottomfovfield; - const char *topfovfield; - const char *nearfield; - const char *photooverlay_shape_field; - const char *imagepyramid_tilesize_field; - const char *imagepyramid_maxwidth_field; - const char *imagepyramid_maxheight_field; - const char *imagepyramid_gridorigin_field; -}; - -void get_fieldconfig(struct fieldconfig *oFC); - int kmlAltitudeModeFromString(const char *pszAltitudeMode, int &isGX); #endif /* OGRLIBKMLFIELD_H_INCLUDED */ diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmlgeometry.cpp b/ogr/ogrsf_frmts/libkml/ogrlibkmlgeometry.cpp index 7e083ea0dadc..ee55619eb41c 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmlgeometry.cpp +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmlgeometry.cpp @@ -347,8 +347,7 @@ ElementPtr geom2kml(OGRGeometry *poOgrGeom, int extra, KmlFactory *poKmlFactory) bool bError; { CPLErrorStateBackuper oErrorStateBackuper; - bError = !poOgrGeom->IsValid() || - CPLGetLastErrorType() != CE_None; + bError = !poOgrGeom->IsValid(); } if (bError) { @@ -390,8 +389,7 @@ ElementPtr geom2kml(OGRGeometry *poOgrGeom, int extra, KmlFactory *poKmlFactory) bool bError; { CPLErrorStateBackuper oErrorStateBackuper; - bError = !poOgrGeom->IsValid() || - CPLGetLastErrorType() != CE_None; + bError = !poOgrGeom->IsValid(); } if (bError) { diff --git a/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp b/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp index 0e455726e11b..1d8141f0ac42 100644 --- a/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp +++ b/ogr/ogrsf_frmts/libkml/ogrlibkmllayer.cpp @@ -122,8 +122,8 @@ OGRLIBKMLLayer::OGRLIBKMLLayer( const OGRSpatialReference *poSRSIn, OGRLIBKMLDataSource *poOgrDS, ElementPtr poKmlRoot, ContainerPtr poKmlContainer, UpdatePtr poKmlUpdate, const char *pszFileName, int bNew, int bUpdateIn) - : bUpdate(CPL_TO_BOOL(bUpdateIn)), nFeatures(0), iFeature(0), nFID(1), - m_pszName(CPLStrdup(pszLayerName)), m_pszFileName(CPLStrdup(pszFileName)), + : bUpdate(CPL_TO_BOOL(bUpdateIn)), m_pszName(CPLStrdup(pszLayerName)), + m_pszFileName(CPLStrdup(pszFileName)), m_poKmlLayer(std::move(poKmlContainer)), // Store the layers container. m_poKmlLayerRoot( std::move(poKmlRoot)), // Store the root element pointer. @@ -140,6 +140,8 @@ OGRLIBKMLLayer::OGRLIBKMLLayer( m_dfRegionMinX(200), m_dfRegionMinY(200), m_dfRegionMaxX(-200), m_dfRegionMaxY(-200), m_bUpdateIsFolder(false) { + get_fieldconfig(&m_oFieldConfig); + m_poStyleTable = nullptr; m_poOgrSRS->SetWellKnownGeogCS("WGS84"); @@ -176,6 +178,9 @@ OGRLIBKMLLayer::OGRLIBKMLLayer( } } + m_osSanitizedNCName = + OGRLIBKMLGetSanitizedNCName(m_poOgrFeatureDefn->GetName()); + SetDescription(m_poOgrFeatureDefn->GetName()); m_poOgrFeatureDefn->Reference(); m_poOgrFeatureDefn->SetGeomType(eGType); @@ -188,52 +193,56 @@ OGRLIBKMLLayer::OGRLIBKMLLayer( /***** get the number of features on the layer *****/ nFeatures = static_cast(m_poKmlLayer->get_feature_array_size()); - /***** get the field config *****/ - struct fieldconfig oFC; - get_fieldconfig(&oFC); + /***** id field *****/ + OGRFieldDefn oOgrFieldId(m_oFieldConfig.idfield, OFTString); + m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldId); /***** name field *****/ - OGRFieldDefn oOgrFieldName(oFC.namefield, OFTString); + OGRFieldDefn oOgrFieldName(m_oFieldConfig.namefield, OFTString); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldName); /***** description field *****/ - OGRFieldDefn oOgrFieldDesc(oFC.descfield, OFTString); + OGRFieldDefn oOgrFieldDesc(m_oFieldConfig.descfield, OFTString); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldDesc); /***** timestamp field *****/ - OGRFieldDefn oOgrFieldTs(oFC.tsfield, OFTDateTime); + OGRFieldDefn oOgrFieldTs(m_oFieldConfig.tsfield, OFTDateTime); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldTs); /***** timespan begin field *****/ - OGRFieldDefn oOgrFieldBegin(oFC.beginfield, OFTDateTime); + OGRFieldDefn oOgrFieldBegin(m_oFieldConfig.beginfield, OFTDateTime); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldBegin); /***** timespan end field *****/ - OGRFieldDefn oOgrFieldEnd(oFC.endfield, OFTDateTime); + OGRFieldDefn oOgrFieldEnd(m_oFieldConfig.endfield, OFTDateTime); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldEnd); /***** altitudeMode field *****/ - OGRFieldDefn oOgrFieldAltitudeMode(oFC.altitudeModefield, OFTString); + OGRFieldDefn oOgrFieldAltitudeMode(m_oFieldConfig.altitudeModefield, + OFTString); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldAltitudeMode); /***** tessellate field *****/ - OGRFieldDefn oOgrFieldTessellate(oFC.tessellatefield, OFTInteger); + OGRFieldDefn oOgrFieldTessellate(m_oFieldConfig.tessellatefield, + OFTInteger); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldTessellate); /***** extrude field *****/ - OGRFieldDefn oOgrFieldExtrude(oFC.extrudefield, OFTInteger); + OGRFieldDefn oOgrFieldExtrude(m_oFieldConfig.extrudefield, OFTInteger); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldExtrude); /***** visibility field *****/ - OGRFieldDefn oOgrFieldVisibility(oFC.visibilityfield, OFTInteger); + OGRFieldDefn oOgrFieldVisibility(m_oFieldConfig.visibilityfield, + OFTInteger); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldVisibility); /***** draw order field *****/ - OGRFieldDefn oOgrFieldDrawOrder(oFC.drawOrderfield, OFTInteger); + OGRFieldDefn oOgrFieldDrawOrder(m_oFieldConfig.drawOrderfield, + OFTInteger); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldDrawOrder); /***** icon field *****/ - OGRFieldDefn oOgrFieldIcon(oFC.iconfield, OFTString); + OGRFieldDefn oOgrFieldIcon(m_oFieldConfig.iconfield, OFTString); m_poOgrFeatureDefn->AddFieldDefn(&oOgrFieldIcon); /***** get the styles *****/ @@ -298,19 +307,22 @@ OGRLIBKMLLayer::OGRLIBKMLLayer( if (camera->has_heading() && !bHasHeading) { bHasHeading = true; - OGRFieldDefn oOgrField(oFC.headingfield, OFTReal); + OGRFieldDefn oOgrField(m_oFieldConfig.headingfield, + OFTReal); m_poOgrFeatureDefn->AddFieldDefn(&oOgrField); } if (camera->has_tilt() && !bHasTilt) { bHasTilt = true; - OGRFieldDefn oOgrField(oFC.tiltfield, OFTReal); + OGRFieldDefn oOgrField(m_oFieldConfig.tiltfield, + OFTReal); m_poOgrFeatureDefn->AddFieldDefn(&oOgrField); } if (camera->has_roll() && !bHasRoll) { bHasRoll = true; - OGRFieldDefn oOgrField(oFC.rollfield, OFTReal); + OGRFieldDefn oOgrField(m_oFieldConfig.rollfield, + OFTReal); m_poOgrFeatureDefn->AddFieldDefn(&oOgrField); } } @@ -384,7 +396,8 @@ OGRLIBKMLLayer::OGRLIBKMLLayer( if (!bHasSnippet && poKmlFeature->has_snippet()) { bHasSnippet = true; - OGRFieldDefn oOgrField(oFC.snippetfield, OFTString); + OGRFieldDefn oOgrField(m_oFieldConfig.snippetfield, + OFTString); m_poOgrFeatureDefn->AddFieldDefn(&oOgrField); } } @@ -430,14 +443,20 @@ OGRFeature *OGRLIBKMLLayer::GetNextRawFeature() /***** loop over the kml features to find the next placemark *****/ + std::string id; do { if (iFeature >= nFeatures) + { + m_bAllReadAtLeastOnce = true; break; + } /***** get the next kml feature in the container *****/ const FeaturePtr poKmlFeature = m_poKmlLayer->get_feature_array_at(iFeature++); + if (poKmlFeature->has_id()) + id = poKmlFeature->get_id(); /***** what type of kml feature in the container? *****/ switch (poKmlFeature->Type()) @@ -463,11 +482,61 @@ OGRFeature *OGRLIBKMLLayer::GetNextRawFeature() /***** set the FID on the ogr feature *****/ if (poOgrFeature) - poOgrFeature->SetFID(nFID++); + { + // If KML id is of the form "layername.number", use number as the FID + if (!id.empty() && id.size() > m_osSanitizedNCName.size() && + id[m_osSanitizedNCName.size()] == '.' && + STARTS_WITH(id.c_str(), m_osSanitizedNCName.c_str())) + { + auto iFID = + CPLAtoGIntBig(id.c_str() + m_osSanitizedNCName.size() + 1); + if (iFID > 0) + { + poOgrFeature->SetFID(iFID); + nFID = std::max(iFID + 1, nFID); + } + } + if (poOgrFeature->GetFID() < 0) + poOgrFeature->SetFID(nFID++); + + if (bUpdate && !id.empty()) + { + auto oIter = m_oMapKmlIdToOGRId.find(id); + if (oIter != m_oMapKmlIdToOGRId.end()) + { + poOgrFeature->SetFID(oIter->second); + } + else + { + m_oMapOGRIdToKmlId[poOgrFeature->GetFID()] = id; + m_oMapKmlIdToOGRId[id] = poOgrFeature->GetFID(); + } + } + } return poOgrFeature; } +/******************************************************************************/ +/* ScanAllFeatures() */ +/******************************************************************************/ + +void OGRLIBKMLLayer::ScanAllFeatures() +{ + if (!m_bAllReadAtLeastOnce) + { + const auto iFeatureBackup = iFeature; + const auto nFIDBackup = nFID; + while (iFeature < nFeatures && + std::unique_ptr(GetNextRawFeature())) + { + // do nothing + } + iFeature = iFeatureBackup; + nFID = nFIDBackup; + } +} + /****************************************************************************** Method to add a feature to a layer. @@ -483,6 +552,22 @@ OGRErr OGRLIBKMLLayer::ICreateFeature(OGRFeature *poOgrFeat) if (!bUpdate) return OGRERR_UNSUPPORTED_OPERATION; + const int idxIdField = + m_poOgrFeatureDefn->GetFieldIndex(m_oFieldConfig.idfield); + if (idxIdField >= 0 && poOgrFeat->IsFieldSet(idxIdField)) + { + ScanAllFeatures(); + + if (cpl::contains(m_oMapKmlIdToOGRId, + poOgrFeat->GetFieldAsString(idxIdField))) + { + CPLError(CE_Failure, CPLE_AppDefined, + "A feature with id %s already exists", + poOgrFeat->GetFieldAsString(idxIdField)); + return OGRERR_FAILURE; + } + } + OGRGeometry *poGeomBackup = nullptr; if (nullptr != m_poCT) { @@ -529,7 +614,7 @@ OGRErr OGRLIBKMLLayer::ICreateFeature(OGRFeature *poOgrFeat) poContainer = poKmlFactory->CreateFolder(); else poContainer = poKmlFactory->CreateDocument(); - poContainer->set_targetid(OGRLIBKMLGetSanitizedNCName(GetName())); + poContainer->set_targetid(m_osSanitizedNCName); poContainer->add_feature(poKmlFeature); poCreate->add_container(poContainer); m_poKmlUpdate->add_updateoperation(poCreate); @@ -540,10 +625,19 @@ OGRErr OGRLIBKMLLayer::ICreateFeature(OGRFeature *poOgrFeat) { nFeatures++; - const char *pszId = CPLSPrintf( - "%s.%d", OGRLIBKMLGetSanitizedNCName(GetName()).c_str(), nFeatures); - poOgrFeat->SetFID(nFeatures); - poKmlFeature->set_id(pszId); + if (poOgrFeat->GetFID() < 0) + { + poOgrFeat->SetFID(nFeatures); + } + if (!poKmlFeature->has_id()) + { + const char *pszId = + CPLSPrintf("%s." CPL_FRMT_GIB, m_osSanitizedNCName.c_str(), + poOgrFeat->GetFID()); + poKmlFeature->set_id(pszId); + } + m_oMapOGRIdToKmlId[poOgrFeat->GetFID()] = poKmlFeature->get_id(); + m_oMapKmlIdToOGRId[poKmlFeature->get_id()] = poOgrFeat->GetFID(); } else { @@ -560,12 +654,13 @@ OGRErr OGRLIBKMLLayer::ICreateFeature(OGRFeature *poOgrFeat) } else { - const char *pszId = - CPLSPrintf("%s." CPL_FRMT_GIB, - OGRLIBKMLGetSanitizedNCName(GetName()).c_str(), - poOgrFeat->GetFID()); - poOgrFeat->SetFID(nFeatures); - poKmlFeature->set_id(pszId); + if (!poKmlFeature->has_id()) + { + const char *pszId = + CPLSPrintf("%s." CPL_FRMT_GIB, m_osSanitizedNCName.c_str(), + poOgrFeat->GetFID()); + poKmlFeature->set_id(pszId); + } } } @@ -589,24 +684,33 @@ OGRErr OGRLIBKMLLayer::ICreateFeature(OGRFeature *poOgrFeat) OGRErr OGRLIBKMLLayer::ISetFeature(OGRFeature *poOgrFeat) { - if (!bUpdate || !m_poKmlUpdate) + if (!bUpdate) return OGRERR_UNSUPPORTED_OPERATION; if (poOgrFeat->GetFID() == OGRNullFID) - return OGRERR_FAILURE; - - FeaturePtr poKmlFeature = - feat2kml(m_poOgrDS, this, poOgrFeat, m_poOgrDS->GetKmlFactory(), - m_bUseSimpleField); + return OGRERR_NON_EXISTING_FEATURE; - const KmlFactory *poKmlFactory = m_poOgrDS->GetKmlFactory(); - const ChangePtr poChange = poKmlFactory->CreateChange(); - poChange->add_object(poKmlFeature); - m_poKmlUpdate->add_updateoperation(poChange); - - const char *pszId = CPLSPrintf( - "%s." CPL_FRMT_GIB, OGRLIBKMLGetSanitizedNCName(GetName()).c_str(), - poOgrFeat->GetFID()); - poKmlFeature->set_targetid(pszId); + if (m_poKmlUpdate) + { + FeaturePtr poKmlFeature = + feat2kml(m_poOgrDS, this, poOgrFeat, m_poOgrDS->GetKmlFactory(), + m_bUseSimpleField); + + const KmlFactory *poKmlFactory = m_poOgrDS->GetKmlFactory(); + const ChangePtr poChange = poKmlFactory->CreateChange(); + poChange->add_object(poKmlFeature); + m_poKmlUpdate->add_updateoperation(poChange); + + const char *pszId = + CPLSPrintf("%s." CPL_FRMT_GIB, m_osSanitizedNCName.c_str(), + poOgrFeat->GetFID()); + poKmlFeature->set_targetid(pszId); + } + else if (m_poKmlLayer) + { + if (DeleteFeature(poOgrFeat->GetFID()) != OGRERR_NONE) + return OGRERR_NON_EXISTING_FEATURE; + return ICreateFeature(poOgrFeat); + } /***** mark as updated *****/ m_poOgrDS->Updated(); @@ -628,19 +732,41 @@ OGRErr OGRLIBKMLLayer::ISetFeature(OGRFeature *poOgrFeat) OGRErr OGRLIBKMLLayer::DeleteFeature(GIntBig nFIDIn) { - if (!bUpdate || !m_poKmlUpdate) + if (!bUpdate) return OGRERR_UNSUPPORTED_OPERATION; - KmlFactory *poKmlFactory = m_poOgrDS->GetKmlFactory(); - DeletePtr poDelete = poKmlFactory->CreateDelete(); - m_poKmlUpdate->add_updateoperation(poDelete); - PlacemarkPtr poKmlPlacemark = poKmlFactory->CreatePlacemark(); - poDelete->add_feature(poKmlPlacemark); + if (m_poKmlUpdate) + { + KmlFactory *poKmlFactory = m_poOgrDS->GetKmlFactory(); + DeletePtr poDelete = poKmlFactory->CreateDelete(); + m_poKmlUpdate->add_updateoperation(poDelete); + PlacemarkPtr poKmlPlacemark = poKmlFactory->CreatePlacemark(); + poDelete->add_feature(poKmlPlacemark); + + const char *pszId = + CPLSPrintf("%s." CPL_FRMT_GIB, m_osSanitizedNCName.c_str(), nFIDIn); + poKmlPlacemark->set_targetid(pszId); + } + else if (m_poKmlLayer) + { + auto oIter = m_oMapOGRIdToKmlId.find(nFIDIn); + if (oIter == m_oMapOGRIdToKmlId.end()) + { + ScanAllFeatures(); - const char *pszId = - CPLSPrintf("%s." CPL_FRMT_GIB, - OGRLIBKMLGetSanitizedNCName(GetName()).c_str(), nFIDIn); - poKmlPlacemark->set_targetid(pszId); + oIter = m_oMapOGRIdToKmlId.find(nFIDIn); + if (oIter == m_oMapOGRIdToKmlId.end()) + return OGRERR_NON_EXISTING_FEATURE; + } + const auto &osKmlId = oIter->second; + if (!m_poKmlLayer->DeleteFeatureById(osKmlId)) + { + return OGRERR_NON_EXISTING_FEATURE; + } + nFeatures = static_cast(m_poKmlLayer->get_feature_array_size()); + m_oMapKmlIdToOGRId.erase(osKmlId); + m_oMapOGRIdToKmlId.erase(oIter); + } /***** mark as updated *****/ m_poOgrDS->Updated(); @@ -748,8 +874,9 @@ OGRErr OGRLIBKMLLayer::CreateField(const OGRFieldDefn *poField, int bApproxOK) { SimpleFieldPtr poKmlSimpleField = nullptr; - if ((poKmlSimpleField = FieldDef2kml( - poField, m_poOgrDS->GetKmlFactory(), CPL_TO_BOOL(bApproxOK)))) + if ((poKmlSimpleField = + FieldDef2kml(poField, m_poOgrDS->GetKmlFactory(), + CPL_TO_BOOL(bApproxOK), m_oFieldConfig))) { if (!m_poKmlSchema) { @@ -759,8 +886,7 @@ OGRErr OGRLIBKMLLayer::CreateField(const OGRFieldDefn *poField, int bApproxOK) m_poKmlSchema = poKmlFactory->CreateSchema(); /***** Set the id on the new schema *****/ - std::string oKmlSchemaID = - OGRLIBKMLGetSanitizedNCName(m_pszName); + std::string oKmlSchemaID = m_osSanitizedNCName; oKmlSchemaID.append(".schema"); m_poKmlSchema->set_id(oKmlSchemaID); } @@ -886,11 +1012,11 @@ int OGRLIBKMLLayer::TestCapability(const char *pszCap) // TODO(schwehr): The false statements are weird. if (EQUAL(pszCap, OLCRandomRead)) - result = FALSE; + result = TRUE; else if (EQUAL(pszCap, OLCSequentialWrite)) result = bUpdate; else if (EQUAL(pszCap, OLCRandomWrite)) - result = bUpdate && m_poKmlUpdate; + result = bUpdate; else if (EQUAL(pszCap, OLCFastFeatureCount)) result = FALSE; else if (EQUAL(pszCap, OLCFastSetNextByIndex)) @@ -898,7 +1024,7 @@ int OGRLIBKMLLayer::TestCapability(const char *pszCap) else if (EQUAL(pszCap, OLCCreateField)) result = bUpdate; else if (EQUAL(pszCap, OLCDeleteFeature)) - result = bUpdate && m_poKmlUpdate; + result = bUpdate; else if (EQUAL(pszCap, OLCStringsAsUTF8)) result = TRUE; else if (EQUAL(pszCap, OLCZGeometries)) From 63f2bf8244eeff10eeb823dc8eebd9d839b7c44b Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Sep 2024 20:19:05 +0200 Subject: [PATCH 192/710] PDF: remove use of OGRDataSourceH --- frmts/pdf/pdfcreatecopy.cpp | 14 ++++++++------ frmts/pdf/pdfcreatecopy.h | 2 +- frmts/pdf/pdfwritabledataset.cpp | 6 +++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/frmts/pdf/pdfcreatecopy.cpp b/frmts/pdf/pdfcreatecopy.cpp index 8c72aebf4501..94c5a9e55c0c 100644 --- a/frmts/pdf/pdfcreatecopy.cpp +++ b/frmts/pdf/pdfcreatecopy.cpp @@ -2178,13 +2178,15 @@ bool GDALPDFWriter::WriteOGRDataSource(const char *pszOGRDataSource, const char *pszOGRLinkField, int bWriteOGRAttributes) { - OGRDataSourceH hDS = OGROpen(pszOGRDataSource, 0, nullptr); + GDALDatasetH hDS = + GDALOpenEx(pszOGRDataSource, GDAL_OF_VECTOR | GDAL_OF_VERBOSE_ERROR, + nullptr, nullptr, nullptr); if (hDS == nullptr) return false; int iObj = 0; - int nLayers = OGR_DS_GetLayerCount(hDS); + int nLayers = GDALDatasetGetLayerCount(hDS); char **papszLayerNames = CSLTokenizeString2(pszOGRDisplayLayerNames, ",", 0); @@ -2193,7 +2195,7 @@ bool GDALPDFWriter::WriteOGRDataSource(const char *pszOGRDataSource, { CPLString osLayerName; if (CSLCount(papszLayerNames) < nLayers) - osLayerName = OGR_L_GetName(OGR_DS_GetLayer(hDS, iLayer)); + osLayerName = OGR_L_GetName(GDALDatasetGetLayer(hDS, iLayer)); else osLayerName = papszLayerNames[iLayer]; @@ -2201,7 +2203,7 @@ bool GDALPDFWriter::WriteOGRDataSource(const char *pszOGRDataSource, osLayerName, bWriteOGRAttributes, iObj); } - OGRReleaseDataSource(hDS); + GDALClose(hDS); CSLDestroy(papszLayerNames); @@ -2273,7 +2275,7 @@ void GDALPDFWriter::EndOGRLayer(GDALPDFLayerDesc &osVectorDesc) /* WriteOGRLayer() */ /************************************************************************/ -int GDALPDFWriter::WriteOGRLayer(OGRDataSourceH hDS, int iLayer, +int GDALPDFWriter::WriteOGRLayer(GDALDatasetH hDS, int iLayer, const char *pszOGRDisplayField, const char *pszOGRLinkField, const std::string &osLayerName, @@ -2286,7 +2288,7 @@ int GDALPDFWriter::WriteOGRLayer(OGRDataSourceH hDS, int iLayer, GDALPDFLayerDesc osVectorDesc = StartOGRLayer(osLayerName, bWriteOGRAttributes); - OGRLayerH hLyr = OGR_DS_GetLayer(hDS, iLayer); + OGRLayerH hLyr = GDALDatasetGetLayer(hDS, iLayer); const auto poLayerDefn = OGRLayer::FromHandle(hLyr)->GetLayerDefn(); for (int i = 0; i < poLayerDefn->GetFieldCount(); i++) diff --git a/frmts/pdf/pdfcreatecopy.h b/frmts/pdf/pdfcreatecopy.h index a4e4cfc2eb55..427abc961c09 100644 --- a/frmts/pdf/pdfcreatecopy.h +++ b/frmts/pdf/pdfcreatecopy.h @@ -377,7 +377,7 @@ class GDALPDFWriter final : public GDALPDFBaseWriter int bWriteOGRAttributes); void EndOGRLayer(GDALPDFLayerDesc &osVectorDesc); - int WriteOGRLayer(OGRDataSourceH hDS, int iLayer, + int WriteOGRLayer(GDALDatasetH hDS, int iLayer, const char *pszOGRDisplayField, const char *pszOGRLinkField, const std::string &osLayerName, int bWriteOGRAttributes, diff --git a/frmts/pdf/pdfwritabledataset.cpp b/frmts/pdf/pdfwritabledataset.cpp index 0c355dd44e02..9d402e51b979 100644 --- a/frmts/pdf/pdfwritabledataset.cpp +++ b/frmts/pdf/pdfwritabledataset.cpp @@ -379,9 +379,9 @@ OGRErr PDFWritableVectorDataset::SyncToDisk() else osLayerName = papszLayerNames[i]; - oWriter.WriteOGRLayer((OGRDataSourceH)this, i, pszOGRDisplayField, - pszOGRLinkField, osLayerName, bWriteOGRAttributes, - iObj); + oWriter.WriteOGRLayer(GDALDataset::ToHandle(this), i, + pszOGRDisplayField, pszOGRLinkField, osLayerName, + bWriteOGRAttributes, iObj); } CSLDestroy(papszLayerNames); From 9bb7aa798001d8d03c06d7d97741599f6272c5af Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 20 Sep 2024 02:31:41 +0200 Subject: [PATCH 193/710] PDF: enable -Weffc++ and -Wold-style-cast --- frmts/pdf/CMakeLists.txt | 1 + frmts/pdf/gdal_pdf.h | 20 ++- frmts/pdf/pdfcreatecopy.cpp | 212 +++++++++++++------------ frmts/pdf/pdfcreatecopy.h | 3 + frmts/pdf/pdfcreatefromcomposition.cpp | 32 ++-- frmts/pdf/pdfcreatefromcomposition.h | 8 +- frmts/pdf/pdfdataset.cpp | 201 +++++++++++++---------- frmts/pdf/pdfdrivercore.cpp | 2 +- frmts/pdf/pdfio.cpp | 36 +++-- frmts/pdf/pdfio.h | 24 +-- frmts/pdf/pdfobject.cpp | 116 +++++++------- frmts/pdf/pdfobject.h | 69 ++++---- frmts/pdf/pdfreadvectors.cpp | 53 ++++--- frmts/pdf/pdfwritabledataset.cpp | 4 +- 14 files changed, 432 insertions(+), 349 deletions(-) diff --git a/frmts/pdf/CMakeLists.txt b/frmts/pdf/CMakeLists.txt index 3eb90eef92e3..587b8bbc5ae3 100644 --- a/frmts/pdf/CMakeLists.txt +++ b/frmts/pdf/CMakeLists.txt @@ -19,6 +19,7 @@ add_gdal_driver( pdfwritabledataset.cpp pdfcreatefromcomposition.cpp CORE_SOURCES pdfdrivercore.cpp + STRONG_CXX_WFLAGS PLUGIN_CAPABLE NO_SHARED_SYMBOL_WITH_CORE) diff --git a/frmts/pdf/gdal_pdf.h b/frmts/pdf/gdal_pdf.h index d46ef09e96c9..80b64bff7c49 100644 --- a/frmts/pdf/gdal_pdf.h +++ b/frmts/pdf/gdal_pdf.h @@ -83,6 +83,8 @@ class OGRPDFLayer final : public OGRMemLayer int bGeomTypeSet; int bGeomTypeMixed; + CPL_DISALLOW_COPY_ASSIGN(OGRPDFLayer) + public: OGRPDFLayer(PDFDataset *poDS, const char *pszName, OGRSpatialReference *poSRS, OGRwkbGeometryType eGeomType); @@ -106,6 +108,8 @@ class OGRPDFWritableLayer final : public OGRMemLayer { PDFWritableVectorDataset *poDS; + CPL_DISALLOW_COPY_ASSIGN(OGRPDFWritableLayer) + public: OGRPDFWritableLayer(PDFWritableVectorDataset *poDS, const char *pszName, OGRSpatialReference *poSRS, @@ -161,13 +165,13 @@ typedef std::map TMapPdfiumPages; /************************************************************************/ // Structure for Mutex on File -typedef struct +struct TPdfiumDocumentStruct { - char *filename; - CPDF_Document *doc; - TMapPdfiumPages pages; - FPDF_FILEACCESS *psFileAccess; -} TPdfiumDocumentStruct; + char *filename = nullptr; + CPDF_Document *doc = nullptr; + TMapPdfiumPages pages{}; + FPDF_FILEACCESS *psFileAccess = nullptr; +}; #endif // ~ HAVE_PDFIUM @@ -266,6 +270,8 @@ class PDFDataset final : public GDALPamDataset void FindXMP(GDALPDFObject *poObj); void ParseInfo(GDALPDFObject *poObj); + CPL_DISALLOW_COPY_ASSIGN(PDFDataset) + #ifdef HAVE_POPPLER std::unique_ptr m_poCatalogObjectPoppler{}; #endif @@ -536,6 +542,8 @@ class PDFWritableVectorDataset final : public GDALDataset int bModified; + CPL_DISALLOW_COPY_ASSIGN(PDFWritableVectorDataset) + public: PDFWritableVectorDataset(); virtual ~PDFWritableVectorDataset(); diff --git a/frmts/pdf/pdfcreatecopy.cpp b/frmts/pdf/pdfcreatecopy.cpp index 94c5a9e55c0c..827e3e368948 100644 --- a/frmts/pdf/pdfcreatecopy.cpp +++ b/frmts/pdf/pdfcreatecopy.cpp @@ -195,7 +195,7 @@ int GDALPDFUpdateWriter::ParseTrailerAndXRef() /* Find startxref section */ VSIFSeekL(m_fp, nOffset, SEEK_SET); - int nRead = (int)VSIFReadL(szBuf, 1, 128, m_fp); + int nRead = static_cast(VSIFReadL(szBuf, 1, 128, m_fp)); szBuf[nRead] = 0; if (nRead < 9) return FALSE; @@ -244,7 +244,7 @@ int GDALPDFUpdateWriter::ParseTrailerAndXRef() } /* Read trailer content */ - nRead = (int)VSIFReadL(szBuf, 1, 1024, m_fp); + nRead = static_cast(VSIFReadL(szBuf, 1, 1024, m_fp)); szBuf[nRead] = 0; /* Find XRef size */ @@ -325,7 +325,7 @@ void GDALPDFUpdateWriter::UpdateProj(GDALDataset *poSrcDS, double dfDPI, int nPageGen) { m_bUpdateNeeded = true; - if ((int)m_asXRefEntries.size() < m_nLastXRefSize - 1) + if (static_cast(m_asXRefEntries.size()) < m_nLastXRefSize - 1) m_asXRefEntries.resize(m_nLastXRefSize - 1); GDALPDFObjectNum nViewportId; @@ -387,7 +387,7 @@ void GDALPDFUpdateWriter::UpdateProj(GDALDataset *poSrcDS, double dfDPI, void GDALPDFUpdateWriter::UpdateInfo(GDALDataset *poSrcDS) { m_bUpdateNeeded = true; - if ((int)m_asXRefEntries.size() < m_nLastXRefSize - 1) + if (static_cast(m_asXRefEntries.size()) < m_nLastXRefSize - 1) m_asXRefEntries.resize(m_nLastXRefSize - 1); auto nNewInfoId = SetInfo(poSrcDS, nullptr); @@ -414,7 +414,7 @@ void GDALPDFUpdateWriter::UpdateXMP(GDALDataset *poSrcDS, GDALPDFDictionaryRW *poCatalogDict) { m_bUpdateNeeded = true; - if ((int)m_asXRefEntries.size() < m_nLastXRefSize - 1) + if (static_cast(m_asXRefEntries.size()) < m_nLastXRefSize - 1) m_asXRefEntries.resize(m_nLastXRefSize - 1); CPLAssert(m_nCatalogId.toBool()); @@ -483,7 +483,8 @@ void GDALPDFBaseWriter::WriteXRefTableAndTrailer(bool bUpdate, m_asXRefEntries[i + nCount].bFree)) nCount++; - VSIFPrintfL(m_fp, "%d %d\n", (int)i + 1, (int)nCount); + VSIFPrintfL(m_fp, "%d %d\n", static_cast(i) + 1, + static_cast(nCount)); size_t iEnd = i + nCount; for (; i < iEnd; i++) { @@ -503,7 +504,8 @@ void GDALPDFBaseWriter::WriteXRefTableAndTrailer(bool bUpdate, } else { - VSIFPrintfL(m_fp, "%d %d\n", 0, (int)m_asXRefEntries.size() + 1); + VSIFPrintfL(m_fp, "%d %d\n", 0, + static_cast(m_asXRefEntries.size()) + 1); VSIFPrintfL(m_fp, "0000000000 65535 f \n"); for (size_t i = 0; i < m_asXRefEntries.size(); i++) { @@ -516,12 +518,12 @@ void GDALPDFBaseWriter::WriteXRefTableAndTrailer(bool bUpdate, VSIFPrintfL(m_fp, "trailer\n"); GDALPDFDictionaryRW oDict; - oDict.Add("Size", (int)m_asXRefEntries.size() + 1) + oDict.Add("Size", static_cast(m_asXRefEntries.size()) + 1) .Add("Root", m_nCatalogId, m_nCatalogGen); if (m_nInfoId.toBool()) oDict.Add("Info", m_nInfoId, m_nInfoGen); if (nLastStartXRef) - oDict.Add("Prev", (double)nLastStartXRef); + oDict.Add("Prev", static_cast(nLastStartXRef)); VSIFPrintfL(m_fp, "%s\n", oDict.Serialize().c_str()); VSIFPrintfL(m_fp, @@ -537,7 +539,7 @@ void GDALPDFBaseWriter::WriteXRefTableAndTrailer(bool bUpdate, void GDALPDFBaseWriter::StartObj(const GDALPDFObjectNum &nObjectId, int nGen) { CPLAssert(!m_bInWriteObj); - CPLAssert(nObjectId.toInt() - 1 < (int)m_asXRefEntries.size()); + CPLAssert(nObjectId.toInt() - 1 < static_cast(m_asXRefEntries.size())); CPLAssert(m_asXRefEntries[nObjectId.toInt() - 1].nOffset == 0); m_asXRefEntries[nObjectId.toInt() - 1].nOffset = VSIFTellL(m_fp); m_asXRefEntries[nObjectId.toInt() - 1].nGen = nGen; @@ -1702,7 +1704,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::SetXMP(GDALDataset *poSrcDS, GDALPDFDictionaryRW oDict; oDict.Add("Type", GDALPDFObjectRW::CreateName("Metadata")) .Add("Subtype", GDALPDFObjectRW::CreateName("XML")) - .Add("Length", (int)strlen(pszXMP)); + .Add("Length", static_cast(strlen(pszXMP))); VSIFPrintfL(m_fp, "%s\n", oDict.Serialize().c_str()); VSIFPrintfL(m_fp, "stream\n"); VSIFPrintfL(m_fp, "%s\n", pszXMP); @@ -1882,9 +1884,9 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteColorTable(GDALDataset *poSrcDS) for (int i = 0; i < nColors; i++) { const GDALColorEntry *poEntry = poCT->GetColorEntry(i); - pabyLookup[3 * i + 0] = (GByte)poEntry->c1; - pabyLookup[3 * i + 1] = (GByte)poEntry->c2; - pabyLookup[3 * i + 2] = (GByte)poEntry->c3; + pabyLookup[3 * i + 0] = static_cast(poEntry->c1); + pabyLookup[3 * i + 1] = static_cast(poEntry->c2); + pabyLookup[3 * i + 2] = static_cast(poEntry->c3); } VSIFWriteL(pabyLookup, 3 * nColors, 1, m_fp); VSIFPrintfL(m_fp, "\n"); @@ -1936,7 +1938,7 @@ bool GDALPDFWriter::WriteImagery(GDALDataset *poDS, const char *pszLayerName, int iImage = nBlockYOff * nXBlocks + nBlockXOff; void *pScaledData = GDALCreateScaledProgress( - iImage / (double)nBlocks, (iImage + 1) / (double)nBlocks, + iImage / double(nBlocks), (iImage + 1) / double(nBlocks), pfnProgress, pProgressData); int nX = nBlockXOff * nBlockXSize; int nY = nBlockYOff * nBlockYSize; @@ -2038,7 +2040,7 @@ bool GDALPDFWriter::WriteClippedImagery( int iImage = nBlockYOff * nXBlocks + nBlockXOff; void *pScaledData = GDALCreateScaledProgress( - iImage / (double)nBlocks, (iImage + 1) / (double)nBlocks, + iImage / double(nBlocks), (iImage + 1) / double(nBlocks), pfnProgress, pProgressData); int nX = nBlockXOff * nBlockXSize; @@ -2073,31 +2075,33 @@ bool GDALPDFWriter::WriteClippedImagery( /* Re-compute (x,y,width,height) subwindow of current raster * from */ /* the extent of the clipped block */ - nX = (int)((dfIntersectMinX - dfRasterMinX) / - adfGeoTransform[1] + - 0.5); + nX = static_cast((dfIntersectMinX - dfRasterMinX) / + adfGeoTransform[1] + + 0.5); if (adfGeoTransform[5] < 0) - nY = (int)((dfRasterMaxY - dfIntersectMaxY) / - (-adfGeoTransform[5]) + - 0.5); + nY = static_cast((dfRasterMaxY - dfIntersectMaxY) / + (-adfGeoTransform[5]) + + 0.5); else - nY = (int)((dfIntersectMinY - dfRasterMinY) / - adfGeoTransform[5] + - 0.5); - nReqWidth = (int)((dfIntersectMaxX - dfRasterMinX) / - adfGeoTransform[1] + - 0.5) - + nY = static_cast((dfIntersectMinY - dfRasterMinY) / + adfGeoTransform[5] + + 0.5); + nReqWidth = static_cast((dfIntersectMaxX - dfRasterMinX) / + adfGeoTransform[1] + + 0.5) - nX; if (adfGeoTransform[5] < 0) - nReqHeight = (int)((dfRasterMaxY - dfIntersectMinY) / - (-adfGeoTransform[5]) + - 0.5) - - nY; + nReqHeight = + static_cast((dfRasterMaxY - dfIntersectMinY) / + (-adfGeoTransform[5]) + + 0.5) - + nY; else - nReqHeight = (int)((dfIntersectMaxY - dfRasterMinY) / - adfGeoTransform[5] + - 0.5) - - nY; + nReqHeight = + static_cast((dfIntersectMaxY - dfRasterMinY) / + adfGeoTransform[5] + + 0.5) - + nY; if (nReqWidth > 0 && nReqHeight > 0) { @@ -2251,9 +2255,9 @@ void GDALPDFWriter::EndOGRLayer(GDALPDFLayerDesc &osVectorDesc) GDALPDFArrayRW *poArray = new GDALPDFArrayRW(); oDict.Add("K", poArray); - for (int i = 0; i < (int)osVectorDesc.aUserPropertiesIds.size(); i++) + for (const auto &prop : osVectorDesc.aUserPropertiesIds) { - poArray->Add(osVectorDesc.aUserPropertiesIds[i], 0); + poArray->Add(prop, 0); } if (!m_nStructTreeRootId.toBool()) @@ -2773,8 +2777,8 @@ void GDALPDFBaseWriter::GetObjectStyle( const GDALPDFImageDesc &oDesc = oMapSymbolFilenameToDesc[os.osSymbolId]; os.nImageSymbolId = oDesc.nImageId; - os.nImageWidth = (int)oDesc.dfXSize; - os.nImageHeight = (int)oDesc.dfYSize; + os.nImageWidth = static_cast(oDesc.dfXSize); + os.nImageHeight = static_cast(oDesc.dfYSize); } } } @@ -2848,14 +2852,14 @@ void GDALPDFBaseWriter::ComputeIntBBox( (os.nImageWidth >= os.nImageHeight) ? dfRadius * os.nImageHeight / os.nImageWidth : dfRadius; - bboxXMin = (int)floor(sEnvelope.MinX * adfMatrix[1] + adfMatrix[0] - - dfSemiWidth); - bboxYMin = (int)floor(sEnvelope.MinY * adfMatrix[3] + adfMatrix[2] - - dfSemiHeight); - bboxXMax = (int)ceil(sEnvelope.MaxX * adfMatrix[1] + adfMatrix[0] + - dfSemiWidth); - bboxYMax = (int)ceil(sEnvelope.MaxY * adfMatrix[3] + adfMatrix[2] + - dfSemiHeight); + bboxXMin = static_cast( + floor(sEnvelope.MinX * adfMatrix[1] + adfMatrix[0] - dfSemiWidth)); + bboxYMin = static_cast( + floor(sEnvelope.MinY * adfMatrix[3] + adfMatrix[2] - dfSemiHeight)); + bboxXMax = static_cast( + ceil(sEnvelope.MaxX * adfMatrix[1] + adfMatrix[0] + dfSemiWidth)); + bboxYMax = static_cast( + ceil(sEnvelope.MaxY * adfMatrix[3] + adfMatrix[2] + dfSemiHeight)); } else { @@ -2870,14 +2874,14 @@ void GDALPDFBaseWriter::ComputeIntBBox( else dfMargin += dfRadius; } - bboxXMin = - (int)floor(sEnvelope.MinX * adfMatrix[1] + adfMatrix[0] - dfMargin); - bboxYMin = - (int)floor(sEnvelope.MinY * adfMatrix[3] + adfMatrix[2] - dfMargin); - bboxXMax = - (int)ceil(sEnvelope.MaxX * adfMatrix[1] + adfMatrix[0] + dfMargin); - bboxYMax = - (int)ceil(sEnvelope.MaxY * adfMatrix[3] + adfMatrix[2] + dfMargin); + bboxXMin = static_cast( + floor(sEnvelope.MinX * adfMatrix[1] + adfMatrix[0] - dfMargin)); + bboxYMin = static_cast( + floor(sEnvelope.MinY * adfMatrix[3] + adfMatrix[2] - dfMargin)); + bboxXMax = static_cast( + ceil(sEnvelope.MaxX * adfMatrix[1] + adfMatrix[0] + dfMargin)); + bboxYMax = static_cast( + ceil(sEnvelope.MaxY * adfMatrix[3] + adfMatrix[2] + dfMargin)); } } @@ -3581,12 +3585,13 @@ int GDALPDFWriter::EndPage(const char *pszExtraImages, pszLinkVal = papszExtraImagesTokens[i] + 5; i++; } - GDALDataset *poImageDS = - (GDALDataset *)GDALOpen(pszImageFilename, GA_ReadOnly); + auto poImageDS = std::unique_ptr(GDALDataset::Open( + pszImageFilename, GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, + nullptr, nullptr, nullptr)); if (poImageDS) { auto nImageId = WriteBlock( - poImageDS, 0, 0, poImageDS->GetRasterXSize(), + poImageDS.get(), 0, 0, poImageDS->GetRasterXSize(), poImageDS->GetRasterYSize(), GDALPDFObjectNum(), COMPRESS_DEFAULT, 0, -1, nullptr, nullptr, nullptr); @@ -3645,8 +3650,6 @@ int GDALPDFWriter::EndPage(const char *pszExtraImages, EndObj(); } } - - GDALClose(poImageDS); } } CSLDestroy(papszExtraImagesTokens); @@ -3771,7 +3774,7 @@ int GDALPDFWriter::EndPage(const char *pszExtraImages, VSIFPrintfL(m_fp, "EMC\n"); } else - iObj += (int)oLayerDesc.aIds.size(); + iObj += static_cast(oLayerDesc.aIds.size()); } /* -------------------------------------------------------------- */ @@ -4020,7 +4023,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteMask(GDALDataset *poSrcDS, int nXOff, PDFCompressMethod eCompressMethod) { int nMaskSize = nReqXSize * nReqYSize; - GByte *pabyMask = (GByte *)VSIMalloc(nMaskSize); + GByte *pabyMask = static_cast(VSIMalloc(nMaskSize)); if (pabyMask == nullptr) return GDALPDFObjectNum(); @@ -4065,7 +4068,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteMask(GDALDataset *poSrcDS, int nXOff, { /* Translate to 1 bit */ int nReqXSize1 = (nReqXSize + 7) / 8; - GByte *pabyMask1 = (GByte *)VSICalloc(nReqXSize1, nReqYSize); + GByte *pabyMask1 = + static_cast(VSICalloc(nReqXSize1, nReqYSize)); if (pabyMask1 == nullptr) { CPLFree(pabyMask); @@ -4133,10 +4137,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( /* Test if we can directly copy original JPEG content */ /* if available */ - if (poSrcDS->GetDriver() != nullptr && - poSrcDS->GetDriver() == GDALGetDriverByName("VRT")) + if (VRTDataset *poVRTDS = dynamic_cast(poSrcDS)) { - VRTDataset *poVRTDS = (VRTDataset *)poSrcDS; poSrcDSToTest = poVRTDS->GetSingleSimpleSource(); } @@ -4152,7 +4154,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( CPLDebug("PDF", "Copying directly original JPEG file"); VSIFSeekL(fpSrc, 0, SEEK_END); - int nLength = (int)VSIFTellL(fpSrc); + const int nLength = static_cast(VSIFTellL(fpSrc)); VSIFSeekL(fpSrc, 0, SEEK_SET); auto nImageId = AllocNewObject(); @@ -4177,16 +4179,16 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( GByte abyBuffer[1024]; for (int i = 0; i < nLength; i += 1024) { - int nRead = (int)VSIFReadL(abyBuffer, 1, 1024, fpSrc); - if ((int)VSIFWriteL(abyBuffer, 1, nRead, m_fp) != nRead) + const auto nRead = VSIFReadL(abyBuffer, 1, 1024, fpSrc); + if (VSIFWriteL(abyBuffer, 1, nRead, m_fp) != nRead) { eErr = CE_Failure; break; } if (eErr == CE_None && pfnProgress != nullptr && - !pfnProgress((i + nRead) / (double)nLength, nullptr, - pProgressData)) + !pfnProgress(double(i + nRead) / double(nLength), + nullptr, pProgressData)) { CPLError(CE_Failure, CPLE_UserInterrupt, "User terminated CreateCopy()"); @@ -4228,7 +4230,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( poMEMDS.reset( MEMDataset::Create("", nReqXSize, nReqYSize, 0, GDT_Byte, nullptr)); - pabyMEMDSBuffer = (GByte *)VSIMalloc3(nReqXSize, nReqYSize, nBands); + pabyMEMDSBuffer = + static_cast(VSIMalloc3(nReqXSize, nReqYSize, nBands)); if (pabyMEMDSBuffer == nullptr) { return GDALPDFObjectNum(); @@ -4319,7 +4322,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( bool bEcwEncodeKeyRequiredButNotFound = false; if (eCompressMethod == COMPRESS_JPEG) { - poJPEGDriver = (GDALDriver *)GDALGetDriverByName("JPEG"); + poJPEGDriver = GetGDALDriverManager()->GetDriverByName("JPEG"); if (poJPEGDriver != nullptr && nJPEGQuality > 0) papszOptions = CSLAddString( papszOptions, CPLSPrintf("QUALITY=%d", nJPEGQuality)); @@ -4329,13 +4332,15 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( { if (pszJPEG2000_DRIVER == nullptr || EQUAL(pszJPEG2000_DRIVER, "JP2KAK")) - poJPEGDriver = (GDALDriver *)GDALGetDriverByName("JP2KAK"); + poJPEGDriver = + GetGDALDriverManager()->GetDriverByName("JP2KAK"); if (poJPEGDriver == nullptr) { if (pszJPEG2000_DRIVER == nullptr || EQUAL(pszJPEG2000_DRIVER, "JP2ECW")) { - poJPEGDriver = (GDALDriver *)GDALGetDriverByName("JP2ECW"); + poJPEGDriver = + GetGDALDriverManager()->GetDriverByName("JP2ECW"); if (poJPEGDriver && poJPEGDriver->GetMetadataItem( GDAL_DMD_CREATIONDATATYPES) == nullptr) @@ -4369,7 +4374,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( if (pszJPEG2000_DRIVER == nullptr || EQUAL(pszJPEG2000_DRIVER, "JP2OpenJPEG")) poJPEGDriver = - (GDALDriver *)GDALGetDriverByName("JP2OpenJPEG"); + GetGDALDriverManager()->GetDriverByName("JP2OpenJPEG"); if (poJPEGDriver) { papszOptions = CSLAddString(papszOptions, "GeoJP2=OFF"); @@ -4418,8 +4423,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( } else { - GByte *pabyLine = - (GByte *)CPLMalloc(static_cast(nReqXSize) * nBands); + GByte *pabyLine = static_cast( + CPLMalloc(static_cast(nReqXSize) * nBands)); for (int iLine = 0; iLine < nReqYSize; iLine++) { /* Get pixel interleaved data */ @@ -4438,7 +4443,8 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( for (int iPixel = 1; iPixel < nReqXSize; iPixel++) { int nCurValue = pabyLine[iPixel]; - pabyLine[iPixel] = (GByte)(nCurValue - nPrevValue); + pabyLine[iPixel] = + static_cast(nCurValue - nPrevValue); nPrevValue = nCurValue; } } @@ -4453,11 +4459,11 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( int nCurValueG = pabyLine[3 * iPixel + 1]; int nCurValueB = pabyLine[3 * iPixel + 2]; pabyLine[3 * iPixel + 0] = - (GByte)(nCurValueR - nPrevValueR); + static_cast(nCurValueR - nPrevValueR); pabyLine[3 * iPixel + 1] = - (GByte)(nCurValueG - nPrevValueG); + static_cast(nCurValueG - nPrevValueG); pabyLine[3 * iPixel + 2] = - (GByte)(nCurValueB - nPrevValueB); + static_cast(nCurValueB - nPrevValueB); nPrevValueR = nCurValueR; nPrevValueG = nCurValueG; nPrevValueB = nCurValueB; @@ -4473,7 +4479,7 @@ GDALPDFObjectNum GDALPDFBaseWriter::WriteBlock( } if (pfnProgress != nullptr && - !pfnProgress((iLine + 1) / (double)nReqYSize, nullptr, + !pfnProgress((iLine + 1) / double(nReqYSize), nullptr, pProgressData)) { CPLError(CE_Failure, CPLE_UserInterrupt, @@ -4552,11 +4558,12 @@ GDALPDFObjectNum GDALPDFWriter::WriteJavascriptFile(const char *pszJavascriptFile) { GDALPDFObjectNum nId; - char *pszJavascriptToFree = (char *)CPLMalloc(65536); + char *pszJavascriptToFree = static_cast(CPLMalloc(65536)); VSILFILE *fpJS = VSIFOpenL(pszJavascriptFile, "rb"); if (fpJS != nullptr) { - int nRead = (int)VSIFReadL(pszJavascriptToFree, 1, 65536, fpJS); + const int nRead = + static_cast(VSIFReadL(pszJavascriptToFree, 1, 65536, fpJS)); if (nRead < 65536) { pszJavascriptToFree[nRead] = '\0'; @@ -4579,7 +4586,7 @@ void GDALPDFWriter::WritePages() GDALPDFDictionaryRW oDict; GDALPDFArrayRW *poKids = new GDALPDFArrayRW(); oDict.Add("Type", GDALPDFObjectRW::CreateName("Pages")) - .Add("Count", (int)m_asPageId.size()) + .Add("Count", static_cast(m_asPageId.size())) .Add("Kids", poKids); for (size_t i = 0; i < m_asPageId.size(); i++) @@ -4752,9 +4759,11 @@ static int GDALPDFGetJPEGQuality(char **papszOptions) class GDALPDFClippingDataset final : public GDALDataset { - GDALDataset *poSrcDS; + GDALDataset *poSrcDS = nullptr; double adfGeoTransform[6]; + CPL_DISALLOW_COPY_ASSIGN(GDALPDFClippingDataset) + public: GDALPDFClippingDataset(GDALDataset *poSrcDSIn, double adfClippingExtent[4]) : poSrcDS(poSrcDSIn) @@ -4768,9 +4777,11 @@ class GDALPDFClippingDataset final : public GDALDataset : adfClippingExtent[1]; adfGeoTransform[4] = 0.0; adfGeoTransform[5] = adfSrcGeoTransform[5]; - nRasterXSize = (int)((adfClippingExtent[2] - adfClippingExtent[0]) / + nRasterXSize = + static_cast((adfClippingExtent[2] - adfClippingExtent[0]) / adfSrcGeoTransform[1]); - nRasterYSize = (int)((adfClippingExtent[3] - adfClippingExtent[1]) / + nRasterYSize = + static_cast((adfClippingExtent[3] - adfClippingExtent[1]) / fabs(adfSrcGeoTransform[5])); } @@ -5009,14 +5020,14 @@ GDALDataset *GDALPDFCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, { if (dfWidthInUserUnit >= dfHeightInUserUnit) { - dfDPI = ceil((double)nWidth / + dfDPI = ceil(double(nWidth) / (MAXIMUM_SIZE_IN_UNITS - (sMargins.nLeft + sMargins.nRight)) / USER_UNIT_IN_INCH); } else { - dfDPI = ceil((double)nHeight / + dfDPI = ceil(double(nHeight) / (MAXIMUM_SIZE_IN_UNITS - (sMargins.nBottom + sMargins.nTop)) / USER_UNIT_IN_INCH); @@ -5024,7 +5035,7 @@ GDALDataset *GDALPDFCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, CPLDebug("PDF", "Adjusting DPI to %d so that page dimension in " "user units remain in what is accepted by Acrobat", - (int)dfDPI); + static_cast(dfDPI)); } } else @@ -5032,9 +5043,9 @@ GDALDataset *GDALPDFCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, CPLError(CE_Warning, CPLE_AppDefined, "The page dimension in user units is %d x %d whereas the " "maximum allowed by Acrobat is %d x %d", - (int)(dfWidthInUserUnit + 0.5), - (int)(dfHeightInUserUnit + 0.5), MAXIMUM_SIZE_IN_UNITS, - MAXIMUM_SIZE_IN_UNITS); + static_cast(dfWidthInUserUnit + 0.5), + static_cast(dfHeightInUserUnit + 0.5), + MAXIMUM_SIZE_IN_UNITS, MAXIMUM_SIZE_IN_UNITS); } } @@ -5206,8 +5217,9 @@ GDALDataset *GDALPDFCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, for (int i = 0; bRet && bUseExtraRasters && papszExtraRasters[i] != nullptr; i++) { - GDALDataset *poDS = - (GDALDataset *)GDALOpen(papszExtraRasters[i], GA_ReadOnly); + auto poDS = std::unique_ptr(GDALDataset::Open( + papszExtraRasters[i], GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, + nullptr, nullptr, nullptr)); if (poDS != nullptr) { double adfGeoTransform[6]; @@ -5264,15 +5276,13 @@ GDALDataset *GDALPDFCreateCopy(const char *pszFilename, GDALDataset *poSrcDS, if (bUseRaster) { bRet = oWriter.WriteClippedImagery( - poDS, + poDS.get(), bUseExtraRastersLayerName ? papszExtraRastersLayerName[i] : nullptr, eCompressMethod, nPredictor, nJPEGQuality, pszJPEG2000_DRIVER, nBlockXSize, nBlockYSize, nullptr, nullptr); } - - GDALClose(poDS); } } diff --git a/frmts/pdf/pdfcreatecopy.h b/frmts/pdf/pdfcreatecopy.h index 427abc961c09..49cfac8edeb0 100644 --- a/frmts/pdf/pdfcreatecopy.h +++ b/frmts/pdf/pdfcreatecopy.h @@ -304,6 +304,9 @@ class GDALPDFBaseWriter const char *pszSUBJECT, const char *pszTITLE, const char *pszKEYWORDS); GDALPDFObjectNum SetXMP(GDALDataset *poSrcDS, const char *pszXMP); + + private: + CPL_DISALLOW_COPY_ASSIGN(GDALPDFBaseWriter) }; class GDALPDFUpdateWriter final : public GDALPDFBaseWriter diff --git a/frmts/pdf/pdfcreatefromcomposition.cpp b/frmts/pdf/pdfcreatefromcomposition.cpp index f7eb2a9deab6..c6491903929f 100644 --- a/frmts/pdf/pdfcreatefromcomposition.cpp +++ b/frmts/pdf/pdfcreatefromcomposition.cpp @@ -121,7 +121,7 @@ void GDALPDFComposerWriter::WritePages() GDALPDFDictionaryRW oDict; GDALPDFArrayRW *poKids = new GDALPDFArrayRW(); oDict.Add("Type", GDALPDFObjectRW::CreateName("Pages")) - .Add("Count", (int)m_asPageId.size()) + .Add("Count", static_cast(m_asPageId.size())) .Add("Kids", poKids); for (size_t i = 0; i < m_asPageId.size(); i++) @@ -1498,20 +1498,22 @@ bool GDALPDFComposerWriter::WriteRaster(const CPLXMLNode *psNode, /* Re-compute (x,y,width,height) subwindow of current raster * from */ /* the extent of the clipped block */ - nX = (int)((dfIntersectMinX - dfRasterMinX) / - adfRasterGT[1] + - 0.5); - nY = (int)((dfRasterMaxY - dfIntersectMaxY) / - (-adfRasterGT[5]) + - 0.5); - nReqWidth = (int)((dfIntersectMaxX - dfRasterMinX) / - adfRasterGT[1] + - 0.5) - - nX; - nReqHeight = (int)((dfRasterMaxY - dfIntersectMinY) / - (-adfRasterGT[5]) + - 0.5) - - nY; + nX = static_cast((dfIntersectMinX - dfRasterMinX) / + adfRasterGT[1] + + 0.5); + nY = static_cast((dfRasterMaxY - dfIntersectMaxY) / + (-adfRasterGT[5]) + + 0.5); + nReqWidth = + static_cast((dfIntersectMaxX - dfRasterMinX) / + adfRasterGT[1] + + 0.5) - + nX; + nReqHeight = + static_cast((dfRasterMaxY - dfIntersectMinY) / + (-adfRasterGT[5]) + + 0.5) - + nY; if (nReqWidth > 0 && nReqHeight > 0) { diff --git a/frmts/pdf/pdfcreatefromcomposition.h b/frmts/pdf/pdfcreatefromcomposition.h index 7cf0af9d6fe2..8be01d7689fd 100644 --- a/frmts/pdf/pdfcreatefromcomposition.h +++ b/frmts/pdf/pdfcreatefromcomposition.h @@ -79,16 +79,16 @@ class GDALPDFComposerWriter final : public GDALPDFBaseWriter double m_adfGT[6]{0, 1, 0, 0, 0, 1}; }; - std::vector m_anParentElements; - std::vector m_anFeatureLayerId; - std::map m_oMapPageIdToObjectNum; + std::vector m_anParentElements{}; + std::vector m_anFeatureLayerId{}; + std::map m_oMapPageIdToObjectNum{}; struct PageContext { double m_dfWidthInUserUnit = 0; double m_dfHeightInUserUnit = 0; CPLString m_osDrawingStream{}; - std::vector m_anFeatureUserProperties; + std::vector m_anFeatureUserProperties{}; int m_nMCID = 0; PDFCompressMethod m_eStreamCompressMethod = COMPRESS_DEFLATE; std::map m_oXObjects{}; diff --git a/frmts/pdf/pdfdataset.cpp b/frmts/pdf/pdfdataset.cpp index 845057fd68a9..f94a18798786 100644 --- a/frmts/pdf/pdfdataset.cpp +++ b/frmts/pdf/pdfdataset.cpp @@ -53,6 +53,7 @@ #include #include +#include #include #ifdef HAVE_PDFIUM @@ -282,20 +283,21 @@ class GDALPDFOutputDev : public SplashOutputDev class GDALPDFDumper { private: - FILE *f; - int nDepthLimit; - std::set aoSetObjectExplored; - int bDumpParent; + FILE *f = nullptr; + const int nDepthLimit; + std::set aoSetObjectExplored{}; + const bool bDumpParent; void DumpSimplified(GDALPDFObject *poObj); + CPL_DISALLOW_COPY_ASSIGN(GDALPDFDumper) + public: GDALPDFDumper(const char *pszFilename, const char *pszDumpFile, int nDepthLimitIn = -1) - : nDepthLimit(nDepthLimitIn) + : nDepthLimit(nDepthLimitIn), + bDumpParent(CPLGetConfigOption("PDF_DUMP_PARENT", "FALSE")) { - bDumpParent = - CPLTestBool(CPLGetConfigOption("PDF_DUMP_PARENT", "FALSE")); if (strcmp(pszDumpFile, "stderr") == 0) f = stderr; else if (EQUAL(pszDumpFile, "YES")) @@ -601,11 +603,11 @@ void PDFDataset::InitOverviews() GDALColorInterp PDFRasterBand::GetColorInterpretation() { - PDFDataset *poGDS = (PDFDataset *)poDS; + PDFDataset *poGDS = cpl::down_cast(poDS); if (poGDS->nBands == 1) return GCI_GrayIndex; else - return (GDALColorInterp)(GCI_RedBand + (nBand - 1)); + return static_cast(GCI_RedBand + (nBand - 1)); } /************************************************************************/ @@ -660,7 +662,7 @@ CPLErr PDFRasterBand::IReadBlockFromTile(int nBlockXOff, int nBlockYOff, void *pImage) { - PDFDataset *poGDS = (PDFDataset *)poDS; + PDFDataset *poGDS = cpl::down_cast(poDS); int nReqXSize = nBlockXSize; int nReqYSize = nBlockYSize; @@ -693,14 +695,14 @@ CPLErr PDFRasterBand::IReadBlockFromTile(int nBlockXOff, int nBlockYOff, GDALPDFObject *poColorSpace = poSMaskDict->Get("ColorSpace"); GDALPDFObject *poBitsPerComponent = poSMaskDict->Get("BitsPerComponent"); - int nBits = 0; + double dfBits = 0; if (poBitsPerComponent) - nBits = (int)Get(poBitsPerComponent); + dfBits = Get(poBitsPerComponent); if (poWidth && Get(poWidth) == nReqXSize && poHeight && Get(poHeight) == nReqYSize && poColorSpace && poColorSpace->GetType() == PDFObjectType_Name && poColorSpace->GetName() == "DeviceGray" && - (nBits == 1 || nBits == 8)) + (dfBits == 1 || dfBits == 8)) { GDALPDFStream *poStream = poSMask->GetStream(); GByte *pabyStream = nullptr; @@ -708,15 +710,15 @@ CPLErr PDFRasterBand::IReadBlockFromTile(int nBlockXOff, int nBlockYOff, if (poStream == nullptr) return CE_Failure; - pabyStream = (GByte *)poStream->GetBytes(); + pabyStream = reinterpret_cast(poStream->GetBytes()); if (pabyStream == nullptr) return CE_Failure; const int nReqXSize1 = (nReqXSize + 7) / 8; - if ((nBits == 8 && + if ((dfBits == 8 && static_cast(poStream->GetLength()) != static_cast(nReqXSize) * nReqYSize) || - (nBits == 1 && + (dfBits == 1 && static_cast(poStream->GetLength()) != static_cast(nReqXSize1) * nReqYSize)) { @@ -724,14 +726,14 @@ CPLErr PDFRasterBand::IReadBlockFromTile(int nBlockXOff, int nBlockYOff, return CE_Failure; } - GByte *pabyData = (GByte *)pImage; + GByte *pabyData = static_cast(pImage); if (nReqXSize != nBlockXSize || nReqYSize != nBlockYSize) { memset(pabyData, 0, static_cast(nBlockXSize) * nBlockYSize); } - if (nBits == 8) + if (dfBits == 8) { for (int j = 0; j < nReqYSize; j++) { @@ -781,7 +783,7 @@ CPLErr PDFRasterBand::IReadBlockFromTile(int nBlockXOff, int nBlockYOff, { poGDS->m_bTried = true; poGDS->m_pabyCachedData = - (GByte *)VSIMalloc3(3, nBlockXSize, nBlockYSize); + static_cast(VSIMalloc3(3, nBlockXSize, nBlockYSize)); } if (poGDS->m_pabyCachedData == nullptr) return CE_Failure; @@ -792,7 +794,7 @@ CPLErr PDFRasterBand::IReadBlockFromTile(int nBlockXOff, int nBlockYOff, if (poStream == nullptr) return CE_Failure; - pabyStream = (GByte *)poStream->GetBytes(); + pabyStream = reinterpret_cast(poStream->GetBytes()); if (pabyStream == nullptr) return CE_Failure; @@ -810,7 +812,7 @@ CPLErr PDFRasterBand::IReadBlockFromTile(int nBlockXOff, int nBlockYOff, poGDS->m_nLastBlockYOff = nBlockYOff; } - GByte *pabyData = (GByte *)pImage; + GByte *pabyData = static_cast(pImage); if (nBand != 4 && (nReqXSize != nBlockXSize || nReqYSize != nBlockYSize)) { memset(pabyData, 0, static_cast(nBlockXSize) * nBlockYSize); @@ -863,7 +865,7 @@ PDFRasterBand::GetSuggestedBlockAccessPattern() const CPLErr PDFRasterBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) { - PDFDataset *poGDS = (PDFDataset *)poDS; + PDFDataset *poGDS = cpl::down_cast(poDS); if (!poGDS->m_aiTiles.empty()) { @@ -1044,13 +1046,14 @@ struct cmp_str static int GDALPdfiumGetBlock(void *param, unsigned long position, unsigned char *pBuf, unsigned long size) { - VSILFILE *fp = (VSILFILE *)param; + VSILFILE *fp = static_cast(param); VSIFSeekL(fp, position, SEEK_SET); return VSIFReadL(pBuf, size, 1, fp) == 1; } // List of all PDF datasets -typedef std::map TMapPdfiumDatasets; +typedef std::map + TMapPdfiumDatasets; static TMapPdfiumDatasets g_mPdfiumDatasets; /** @@ -1084,7 +1087,7 @@ static int LoadPdfiumDocumentPage(const char *pszFilename, } TMapPdfiumDatasets::iterator it; - it = g_mPdfiumDatasets.find((char *)pszFilename); + it = g_mPdfiumDatasets.find(pszFilename); TPdfiumDocumentStruct *poDoc = nullptr; // Load new document if missing if (it == g_mPdfiumDatasets.end()) @@ -1098,17 +1101,20 @@ static int LoadPdfiumDocumentPage(const char *pszFilename, return FALSE; } VSIFSeekL(fp, 0, SEEK_END); - unsigned long nFileLen = (unsigned long)VSIFTellL(fp); - if (nFileLen != VSIFTellL(fp)) + const auto nFileLen64 = VSIFTellL(fp); + if constexpr (LONG_MAX < std::numeric_limits::max()) { - VSIFCloseL(fp); - CPLReleaseMutex(g_oPdfiumLoadDocMutex); - return FALSE; + if (nFileLen64 > LONG_MAX) + { + VSIFCloseL(fp); + CPLReleaseMutex(g_oPdfiumLoadDocMutex); + return FALSE; + } } FPDF_FILEACCESS *psFileAccess = new FPDF_FILEACCESS; psFileAccess->m_Param = fp; - psFileAccess->m_FileLen = nFileLen; + psFileAccess->m_FileLen = static_cast(nFileLen64); psFileAccess->m_GetBlock = GDALPdfiumGetBlock; CPDF_Document *docPdfium = CPDFDocumentFromFPDFDocument( FPDF_LoadCustomDocument(psFileAccess, nullptr)); @@ -1330,7 +1336,7 @@ static int UnloadPdfiumDocumentPage(TPdfiumDocumentStruct **doc, FPDF_CloseDocument(FPDFDocumentFromCPDFDocument(pDoc->doc)); g_mPdfiumDatasets.erase(pDoc->filename); CPLFree(pDoc->filename); - VSIFCloseL((VSILFILE *)pDoc->psFileAccess->m_Param); + VSIFCloseL(static_cast(pDoc->psFileAccess->m_Param)); delete pDoc->psFileAccess; delete pDoc; pDoc = nullptr; @@ -1420,6 +1426,8 @@ class GDALPDFiumOCContext : public CPDF_OCContextInterface PDFDataset *m_poDS; RetainPtr m_DefaultOCContext; + CPL_DISALLOW_COPY_ASSIGN(GDALPDFiumOCContext) + public: GDALPDFiumOCContext(PDFDataset *poDS, CPDF_Document *pDoc, CPDF_OCContext::UsageType usage) @@ -1458,6 +1466,8 @@ class GDALPDFiumRenderDeviceDriver : public RenderDeviceDriverIface int bEnableBitmap; int bTemporaryEnableVectorForTextStroking; + CPL_DISALLOW_COPY_ASSIGN(GDALPDFiumRenderDeviceDriver) + public: GDALPDFiumRenderDeviceDriver( std::unique_ptr &&poParent, @@ -2021,7 +2031,8 @@ CPLErr PDFDataset::ReadPixels(int nReqXOff, int nReqYOff, int nReqXSize, GByte *pabyDataB = pabyData + 2 * nBandSpace; GByte *pabyDataA = pabyData + 3 * nBandSpace; GByte *pabySrc = poBitmap->getDataPtr(); - GByte *pabyAlphaSrc = (GByte *)poBitmap->getAlphaPtr(); + GByte *pabyAlphaSrc = + reinterpret_cast(poBitmap->getAlphaPtr()); int i, j; for (j = 0; j < nReqYSize; j++) { @@ -2135,8 +2146,8 @@ CPLErr PDFDataset::ReadPixels(int nReqXOff, int nReqYOff, int nReqXSize, if (nRet == 0) { - GDALDataset *poDS = - (GDALDataset *)GDALOpen(osTmpFilename, GA_ReadOnly); + auto poDS = std::unique_ptr(GDALDataset::Open( + osTmpFilename, GDAL_OF_RASTER, nullptr, nullptr, nullptr)); if (poDS) { if (poDS->GetRasterCount() == 3) @@ -2146,7 +2157,6 @@ CPLErr PDFDataset::ReadPixels(int nReqXOff, int nReqYOff, int nReqXSize, GDT_Byte, 3, nullptr, nPixelSpace, nLineSpace, nBandSpace, nullptr); } - delete poDS; } } else @@ -2243,7 +2253,8 @@ CPLErr PDFDataset::ReadPixels(int nReqXOff, int nReqYOff, int nReqXSize, CPLDebug("PDF", "PDFDataset::ReadPixels(%d, %d, %d, %d, scaleFactor=%d)", nReqXOff, nReqYOff, nReqXSize, nReqYSize, - 1 << ((PDFRasterBand *)GetRasterBand(1))->nResolutionLevel); + 1 << cpl::down_cast(GetRasterBand(1)) + ->nResolutionLevel); CPLDebug("PDF", "FPDF_RenderPageBitmap(%d, %d, %d, %d)", -nReqXOff, -nReqYOff, nRasterXSize, nRasterYSize); @@ -2325,10 +2336,10 @@ PDFImageRasterBand::PDFImageRasterBand(PDFDataset *poDSIn, int nBandIn) /* IReadBlock() */ /************************************************************************/ -CPLErr PDFImageRasterBand::IReadBlock(int CPL_UNUSED nBlockXOff, int nBlockYOff, +CPLErr PDFImageRasterBand::IReadBlock(int /* nBlockXOff */, int nBlockYOff, void *pImage) { - PDFDataset *poGDS = (PDFDataset *)poDS; + PDFDataset *poGDS = cpl::down_cast(poDS); CPLAssert(poGDS->m_poImageObj != nullptr); if (!poGDS->m_bTried) @@ -2337,8 +2348,8 @@ CPLErr PDFImageRasterBand::IReadBlock(int CPL_UNUSED nBlockXOff, int nBlockYOff, poGDS->m_bTried = true; if (nBands == 3) { - poGDS->m_pabyCachedData = - (GByte *)VSIMalloc3(nBands, nRasterXSize, nRasterYSize); + poGDS->m_pabyCachedData = static_cast( + VSIMalloc3(nBands, nRasterXSize, nRasterYSize)); if (poGDS->m_pabyCachedData == nullptr) return CE_Failure; } @@ -2349,7 +2360,8 @@ CPLErr PDFImageRasterBand::IReadBlock(int CPL_UNUSED nBlockXOff, int nBlockYOff, if (poStream == nullptr || static_cast(poStream->GetLength()) != static_cast(nBands) * nRasterXSize * nRasterYSize || - (pabyStream = (GByte *)poStream->GetBytes()) == nullptr) + (pabyStream = reinterpret_cast(poStream->GetBytes())) == + nullptr) { VSIFree(poGDS->m_pabyCachedData); poGDS->m_pabyCachedData = nullptr; @@ -2706,7 +2718,7 @@ CPLErr PDFDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, if (bReadPixels) return ReadPixels(nXOff, nYOff, nXSize, nYSize, nPixelSpace, nLineSpace, - nBandSpace, (GByte *)pData); + nBandSpace, static_cast(pData)); if (nBufXSize != nXSize || nBufYSize != nYSize || eBufType != GDT_Byte) { @@ -2731,7 +2743,7 @@ CPLErr PDFRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg) { - PDFDataset *poGDS = (PDFDataset *)poDS; + PDFDataset *poGDS = cpl::down_cast(poDS); // Try to pass the request to the most appropriate overview dataset. if (nBufXSize < nXSize && nBufYSize < nYSize) @@ -3079,10 +3091,10 @@ int PDFDataset::CheckTiledRaster() double dfDrawHeight = m_asTiles[i].adfCM[3] * dfUserUnit; double dfX = m_asTiles[i].adfCM[4] * dfUserUnit; double dfY = m_asTiles[i].adfCM[5] * dfUserUnit; - int nX = (int)(dfX + 0.1); - int nY = (int)(dfY + 0.1); - int nWidth = (int)(m_asTiles[i].dfWidth + 1e-8); - int nHeight = (int)(m_asTiles[i].dfHeight + 1e-8); + int nX = static_cast(dfX + 0.1); + int nY = static_cast(dfY + 0.1); + int nWidth = static_cast(m_asTiles[i].dfWidth + 1e-8); + int nHeight = static_cast(m_asTiles[i].dfHeight + 1e-8); GDALPDFDictionary *poImageDict = m_asTiles[i].poImage->GetDictionary(); GDALPDFObject *poBitsPerComponent = @@ -3096,7 +3108,7 @@ int PDFDataset::CheckTiledRaster() poFilter->GetName() == "JPXDecode") { CPLDebug("PDF", "Tile %d : Incompatible image for tiled reading", - (int)i); + static_cast(i)); return FALSE; } @@ -3107,7 +3119,7 @@ int PDFDataset::CheckTiledRaster() poColorSpace->GetName() != "DeviceGray")) { CPLDebug("PDF", "Tile %d : Incompatible image for tiled reading", - (int)i); + static_cast(i)); return FALSE; } @@ -3118,8 +3130,8 @@ int PDFDataset::CheckTiledRaster() fabs(nX - dfX) > 1e-1 || fabs(nY - dfY) > 1e-1 || nX < 0 || nY < 0 || nX + nWidth > nRasterXSize || nY >= nRasterYSize) { - CPLDebug("PDF", "Tile %d : %f %f %f %f %f %f", (int)i, dfX, dfY, - dfDrawWidth, dfDrawHeight, m_asTiles[i].dfWidth, + CPLDebug("PDF", "Tile %d : %f %f %f %f %f %f", static_cast(i), + dfX, dfY, dfDrawWidth, dfDrawHeight, m_asTiles[i].dfWidth, m_asTiles[i].dfHeight); return FALSE; } @@ -3142,10 +3154,10 @@ int PDFDataset::CheckTiledRaster() { double dfX = m_asTiles[i].adfCM[4] * dfUserUnit; double dfY = m_asTiles[i].adfCM[5] * dfUserUnit; - int nX = (int)(dfX + 0.1); - int nY = (int)(dfY + 0.1); - int nWidth = (int)(m_asTiles[i].dfWidth + 1e-8); - int nHeight = (int)(m_asTiles[i].dfHeight + 1e-8); + int nX = static_cast(dfX + 0.1); + int nY = static_cast(dfY + 0.1); + int nWidth = static_cast(m_asTiles[i].dfWidth + 1e-8); + int nHeight = static_cast(m_asTiles[i].dfHeight + 1e-8); int bOK = TRUE; int nBlockXOff = nX / l_nBlockXSize; if ((nX % l_nBlockXSize) != 0) @@ -3162,8 +3174,8 @@ int PDFDataset::CheckTiledRaster() if (!bOK) { - CPLDebug("PDF", "Tile %d : %d %d %d %d", (int)i, nX, nY, nWidth, - nHeight); + CPLDebug("PDF", "Tile %d : %d %d %d %d", static_cast(i), nX, + nY, nWidth, nHeight); return FALSE; } } @@ -3174,12 +3186,12 @@ int PDFDataset::CheckTiledRaster() { double dfX = m_asTiles[i].adfCM[4] * dfUserUnit; double dfY = m_asTiles[i].adfCM[5] * dfUserUnit; - int nHeight = (int)(m_asTiles[i].dfHeight + 1e-8); - int nX = (int)(dfX + 0.1); - int nY = nRasterYSize - ((int)(dfY + 0.1) + nHeight); + int nHeight = static_cast(m_asTiles[i].dfHeight + 1e-8); + int nX = static_cast(dfX + 0.1); + int nY = nRasterYSize - (static_cast(dfY + 0.1) + nHeight); int nBlockXOff = nX / l_nBlockXSize; int nBlockYOff = nY / l_nBlockYSize; - m_aiTiles[nBlockYOff * nXBlocks + nBlockXOff] = (int)i; + m_aiTiles[nBlockYOff * nXBlocks + nBlockXOff] = static_cast(i); } this->m_nBlockXSize = l_nBlockXSize; @@ -3799,7 +3811,7 @@ void PDFDataset::FindLayersPoppler(int iPageOfInterest) if (ocg != nullptr && ocg->getName() != nullptr) { const char *pszLayerName = - (const char *)ocg->getName()->c_str(); + reinterpret_cast(ocg->getName()->c_str()); AddLayer(pszLayerName, 0); m_oLayerOCGListPoppler.push_back( std::make_pair(CPLString(pszLayerName), ocg)); @@ -4782,9 +4794,8 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) Ref *poPageRef = poCatalogPoppler->getPageRef(iPage); if (poPageRef != nullptr) { - ((GDALPDFObjectPoppler *)poPageObj) - ->SetRefNumAndGen(GDALPDFObjectNum(poPageRef->num), - poPageRef->gen); + cpl::down_cast(poPageObj)->SetRefNumAndGen( + GDALPDFObjectNum(poPageRef->num), poPageRef->gen); } } #endif // ~ HAVE_POPPLER @@ -5105,8 +5116,16 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) poDS->m_dfPageHeight = dfY2 - dfY1; // CPLDebug("PDF", "left=%f right=%f bottom=%f top=%f", dfX1, dfX2, dfY1, // dfY2); - poDS->nRasterXSize = (int)floor((dfX2 - dfX1) * dfUserUnit + 0.5); - poDS->nRasterYSize = (int)floor((dfY2 - dfY1) * dfUserUnit + 0.5); + const double dfXSize = floor((dfX2 - dfX1) * dfUserUnit + 0.5); + const double dfYSize = floor((dfY2 - dfY1) * dfUserUnit + 0.5); + if (!(dfXSize >= 0 && dfXSize <= INT_MAX && dfYSize >= 0 && + dfYSize <= INT_MAX)) + { + delete poDS; + return nullptr; + } + poDS->nRasterXSize = static_cast(dfXSize); + poDS->nRasterYSize = static_cast(dfYSize); if (!GDALCheckDatasetDimensions(poDS->nRasterXSize, poDS->nRasterYSize)) { @@ -5395,22 +5414,22 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) if ((fabs(poDS->m_adfGeoTransform[0]) > 1e5 || fabs(poDS->m_adfGeoTransform[3]) > 1e5) && fabs(poDS->m_adfGeoTransform[0] - - (int)floor(poDS->m_adfGeoTransform[0] + 0.5)) < + std::round(poDS->m_adfGeoTransform[0])) < 1e-6 * fabs(poDS->m_adfGeoTransform[0]) && fabs(poDS->m_adfGeoTransform[1] - - (int)floor(poDS->m_adfGeoTransform[1] + 0.5)) < + std::round(poDS->m_adfGeoTransform[1])) < 1e-3 * fabs(poDS->m_adfGeoTransform[1]) && fabs(poDS->m_adfGeoTransform[3] - - (int)floor(poDS->m_adfGeoTransform[3] + 0.5)) < + std::round(poDS->m_adfGeoTransform[3])) < 1e-6 * fabs(poDS->m_adfGeoTransform[3]) && fabs(poDS->m_adfGeoTransform[5] - - (int)floor(poDS->m_adfGeoTransform[5] + 0.5)) < + std::round(poDS->m_adfGeoTransform[5])) < 1e-3 * fabs(poDS->m_adfGeoTransform[5])) { for (int i = 0; i < 6; i++) { poDS->m_adfGeoTransform[i] = - (int)floor(poDS->m_adfGeoTransform[i] + 0.5); + std::round(poDS->m_adfGeoTransform[i]); } } } @@ -6016,7 +6035,8 @@ int PDFDataset::ParseLGIDictDictSecondPass(GDALPDFDictionary *poLGIDict) CPLTestBool(CPLGetConfigOption("PDF_REPORT_GCPS", "NO"))) { m_nGCPCount = 0; - m_pasGCPList = (GDAL_GCP *)CPLCalloc(sizeof(GDAL_GCP), nLength); + m_pasGCPList = + static_cast(CPLCalloc(sizeof(GDAL_GCP), nLength)); for (i = 0; i < nLength; i++) { @@ -6359,12 +6379,16 @@ int PDFDataset::ParseProjDict(GDALPDFDictionary *poProjDict) else if (EQUAL(osProjectionType, "UT")) /* UTM */ { - int nZone = (int)Get(poProjDict, "Zone"); - int bNorth = EQUAL(osHemisphere, "N"); - if (bIsWGS84) - oSRS.importFromEPSG(((bNorth) ? 32600 : 32700) + nZone); - else - oSRS.SetUTM(nZone, bNorth); + const double dfZone = Get(poProjDict, "Zone"); + if (dfZone >= 1 && dfZone <= 60) + { + int nZone = static_cast(dfZone); + int bNorth = EQUAL(osHemisphere, "N"); + if (bIsWGS84) + oSRS.importFromEPSG(((bNorth) ? 32600 : 32700) + nZone); + else + oSRS.SetUTM(nZone, bNorth); + } } else if (EQUAL(osProjectionType, @@ -6379,8 +6403,12 @@ int PDFDataset::ParseProjDict(GDALPDFDictionary *poProjDict) else if (EQUAL(osProjectionType, "SPCS")) /* State Plane */ { - int nZone = (int)Get(poProjDict, "Zone"); - oSRS.SetStatePlane(nZone, bIsNAD83); + const double dfZone = Get(poProjDict, "Zone"); + if (dfZone >= 0 && dfZone <= INT_MAX) + { + int nZone = static_cast(dfZone); + oSRS.SetStatePlane(nZone, bIsNAD83); + } } else if (EQUAL(osProjectionType, "AC")) /* Albers Equal Area Conic */ @@ -6613,11 +6641,12 @@ int PDFDataset::ParseProjDict(GDALPDFDictionary *poProjDict) double dfScale = Get(poProjDict, "ScaleFactor"); double dfFalseEasting = Get(poProjDict, "FalseEasting"); double dfFalseNorthing = Get(poProjDict, "FalseNorthing"); - if (dfCenterLat == 0.0 && dfScale == 0.9996 && - dfFalseEasting == 500000 && + if (dfCenterLat == 0.0 && dfScale == 0.9996 && dfCenterLong >= -180 && + dfCenterLong <= 180 && dfFalseEasting == 500000 && (dfFalseNorthing == 0.0 || dfFalseNorthing == 10000000.0)) { - int nZone = (int)floor((dfCenterLong + 180.0) / 6.0) + 1; + const int nZone = + static_cast(floor((dfCenterLong + 180.0) / 6.0) + 1); int bNorth = dfFalseNorthing == 0; if (bIsWGS84) oSRS.importFromEPSG(((bNorth) ? 32600 : 32700) + nZone); @@ -7631,7 +7660,7 @@ static void GDALPDFUnloadDriver(CPL_UNUSED GDALDriver *poDriver) FPDF_CloseDocument(FPDFDocumentFromCPDFDocument(pDoc->doc)); CPLFree(pDoc->filename); - VSIFCloseL((VSILFILE *)pDoc->psFileAccess->m_Param); + VSIFCloseL(static_cast(pDoc->psFileAccess->m_Param)); delete pDoc->psFileAccess; pDoc->pages.clear(); diff --git a/frmts/pdf/pdfdrivercore.cpp b/frmts/pdf/pdfdrivercore.cpp index 8f8e1ac054ef..b7693e3b7f52 100644 --- a/frmts/pdf/pdfdrivercore.cpp +++ b/frmts/pdf/pdfdrivercore.cpp @@ -103,7 +103,7 @@ int PDFDatasetIdentify(GDALOpenInfo *poOpenInfo) if (poOpenInfo->nHeaderBytes < 128) return FALSE; - return STARTS_WITH((const char *)poOpenInfo->pabyHeader, "%PDF"); + return memcmp(poOpenInfo->pabyHeader, "%PDF", 4) == 0; } /************************************************************************/ diff --git a/frmts/pdf/pdfio.cpp b/frmts/pdf/pdfio.cpp index f4e2cf4023f6..7526edd98367 100644 --- a/frmts/pdf/pdfio.cpp +++ b/frmts/pdf/pdfio.cpp @@ -48,10 +48,9 @@ static vsi_l_offset VSIPDFFileStreamGetSize(VSILFILE *f) VSIPDFFileStream::VSIPDFFileStream(VSILFILE *fIn, const char *pszFilename, Object &&dictA) - : BaseStream(std::move(dictA), (Goffset)VSIPDFFileStreamGetSize(fIn)), - poParent(nullptr), poFilename(new GooString(pszFilename)), f(fIn), - nStart(0), bLimited(false), nLength(0), nCurrentPos(VSI_L_OFFSET_MAX), - bHasSavedPos(FALSE), nSavedPos(0), nPosInBuffer(-1), nBufferLength(-1) + : BaseStream(std::move(dictA), + static_cast(VSIPDFFileStreamGetSize(fIn))), + poParent(nullptr), poFilename(new GooString(pszFilename)), f(fIn) { } @@ -62,10 +61,9 @@ VSIPDFFileStream::VSIPDFFileStream(VSILFILE *fIn, const char *pszFilename, VSIPDFFileStream::VSIPDFFileStream(VSIPDFFileStream *poParentIn, vsi_l_offset startA, bool limitedA, vsi_l_offset lengthA, Object &&dictA) - : BaseStream(std::move(dictA), (Goffset)lengthA), poParent(poParentIn), - poFilename(poParentIn->poFilename), f(poParentIn->f), nStart(startA), - bLimited(limitedA), nLength(lengthA), nCurrentPos(VSI_L_OFFSET_MAX), - bHasSavedPos(FALSE), nSavedPos(0), nPosInBuffer(-1), nBufferLength(-1) + : BaseStream(std::move(dictA), static_cast(lengthA)), + poParent(poParentIn), poFilename(poParentIn->poFilename), + f(poParentIn->f), nStart(startA), bLimited(limitedA), nLength(lengthA) { } @@ -117,7 +115,7 @@ Goffset VSIPDFFileStream::getPos() Goffset VSIPDFFileStream::getStart() { - return (Goffset)nStart; + return static_cast(nStart); } /************************************************************************/ @@ -154,12 +152,12 @@ int VSIPDFFileStream::FillBuffer() if (!bLimited) nToRead = BUFFER_SIZE; else if (nCurrentPos + BUFFER_SIZE > nStart + nLength) - nToRead = (int)(nStart + nLength - nCurrentPos); + nToRead = static_cast(nStart + nLength - nCurrentPos); else nToRead = BUFFER_SIZE; if (nToRead < 0) return FALSE; - nBufferLength = (int)VSIFReadL(abyBuffer, 1, nToRead, f); + nBufferLength = static_cast(VSIFReadL(abyBuffer, 1, nToRead, f)); if (nBufferLength == 0) return FALSE; @@ -173,7 +171,8 @@ int VSIPDFFileStream::FillBuffer() // with allocation and liberation of VSIPDFFileStream as PDFDoc::str member. if (nCurrentPos == 0 || nCurrentPos == VSI_L_OFFSET_MAX) { - for (int i = 0; i < nBufferLength - (int)strlen("/Linearized "); i++) + for (int i = 0; + i < nBufferLength - static_cast(strlen("/Linearized ")); i++) { if (memcmp(abyBuffer + i, "/Linearized ", strlen("/Linearized ")) == 0) @@ -279,7 +278,10 @@ void VSIPDFFileStream::unfilteredReset() void VSIPDFFileStream::close() { if (bHasSavedPos) - VSIFSeekL(f, nCurrentPos = nSavedPos, SEEK_SET); + { + nCurrentPos = nSavedPos; + VSIFSeekL(f, nCurrentPos, SEEK_SET); + } bHasSavedPos = FALSE; nSavedPos = 0; } @@ -305,7 +307,7 @@ void VSIPDFFileStream::setPos(Goffset pos, int dir) VSIFSeekL(f, nStart + nLength, SEEK_SET); } vsi_l_offset size = VSIFTellL(f); - vsi_l_offset newpos = (vsi_l_offset)pos; + vsi_l_offset newpos = static_cast(pos); if (newpos > size) newpos = size; VSIFSeekL(f, nCurrentPos = size - newpos, SEEK_SET); @@ -321,7 +323,8 @@ void VSIPDFFileStream::setPos(Goffset pos, int dir) void VSIPDFFileStream::moveStart(Goffset delta) { nStart += delta; - VSIFSeekL(f, nCurrentPos = nStart, SEEK_SET); + nCurrentPos = nStart; + VSIFSeekL(f, nCurrentPos, SEEK_SET); nPosInBuffer = -1; nBufferLength = -1; } @@ -349,7 +352,8 @@ int VSIPDFFileStream::getChars(int nChars, unsigned char *buffer) { if (!bLimited && nToRead > BUFFER_SIZE) { - int nJustRead = (int)VSIFReadL(buffer + nRead, 1, nToRead, f); + int nJustRead = + static_cast(VSIFReadL(buffer + nRead, 1, nToRead, f)); nPosInBuffer = -1; nBufferLength = -1; nCurrentPos += nJustRead; diff --git a/frmts/pdf/pdfio.h b/frmts/pdf/pdfio.h index 61feb88d5fe9..af10a8145ee3 100644 --- a/frmts/pdf/pdfio.h +++ b/frmts/pdf/pdfio.h @@ -77,24 +77,26 @@ class VSIPDFFileStream final : public BaseStream virtual bool hasGetChars() override; virtual int getChars(int nChars, unsigned char *buffer) override; - VSIPDFFileStream *poParent; - GooString *poFilename; - VSILFILE *f; - vsi_l_offset nStart; - bool bLimited; - vsi_l_offset nLength; + VSIPDFFileStream *poParent = nullptr; + GooString *poFilename = nullptr; + VSILFILE *f = nullptr; + vsi_l_offset nStart = 0; + bool bLimited = false; + vsi_l_offset nLength = 0; - vsi_l_offset nCurrentPos; - int bHasSavedPos; - vsi_l_offset nSavedPos; + vsi_l_offset nCurrentPos = VSI_L_OFFSET_MAX; + int bHasSavedPos = FALSE; + vsi_l_offset nSavedPos = 0; GByte abyBuffer[BUFFER_SIZE]; - int nPosInBuffer; - int nBufferLength; + int nPosInBuffer = -1; + int nBufferLength = -1; bool bFoundLinearizedHint = false; int FillBuffer(); + + CPL_DISALLOW_COPY_ASSIGN(VSIPDFFileStream) }; #endif // PDFIO_H_INCLUDED diff --git a/frmts/pdf/pdfobject.cpp b/frmts/pdf/pdfobject.cpp index 68f8f017aad8..d209e920181a 100644 --- a/frmts/pdf/pdfobject.cpp +++ b/frmts/pdf/pdfobject.cpp @@ -47,9 +47,9 @@ double ROUND_TO_INT_IF_CLOSE(double x, double eps) { if (eps == 0.0) eps = fabs(x) < 1 ? 1e-10 : 1e-8; - int nClosestInt = (int)floor(x + 0.5); - if (fabs(x - nClosestInt) < eps) - return nClosestInt; + const double dfRounded = std::round(x); + if (fabs(x - dfRounded) < eps) + return dfRounded; else return x; } @@ -60,10 +60,9 @@ double ROUND_TO_INT_IF_CLOSE(double x, double eps) static CPLString GDALPDFGetPDFString(const char *pszStr) { - GByte *pabyData = (GByte *)pszStr; - size_t i; + const GByte *pabyData = reinterpret_cast(pszStr); GByte ch; - for (i = 0; (ch = pabyData[i]) != '\0'; i++) + for (size_t i = 0; (ch = pabyData[i]) != '\0'; i++) { if (ch < 32 || ch > 127 || ch == '(' || ch == ')' || ch == '\\' || ch == '%' || ch == '#') @@ -80,7 +79,7 @@ static CPLString GDALPDFGetPDFString(const char *pszStr) wchar_t *pwszDest = CPLRecodeToWChar(pszStr, CPL_ENC_UTF8, CPL_ENC_UCS2); osStr = "= 0x10000 /* && pwszDest[i] <= 0x10FFFF */) @@ -97,8 +96,9 @@ static CPLString GDALPDFGetPDFString(const char *pszStr) else #endif { - osStr += CPLSPrintf("%02X", (int)(pwszDest[i] >> 8) & 0xff); - osStr += CPLSPrintf("%02X", (int)(pwszDest[i]) & 0xff); + osStr += + CPLSPrintf("%02X", static_cast(pwszDest[i] >> 8) & 0xff); + osStr += CPLSPrintf("%02X", static_cast(pwszDest[i]) & 0xff); } } osStr += ">"; @@ -132,7 +132,7 @@ static std::string GDALPDFGetUTF8StringFromBytes(const GByte *pabySrc, "GDALPDFGetUTF8StringFromBytes(): %s", e.what()); return osStr; } - osStr.assign((const char *)pabySrc, nLen); + osStr.assign(reinterpret_cast(pabySrc), nLen); const char *pszStr = osStr.c_str(); if (CPLIsUTF8(pszStr, -1)) return osStr; @@ -298,8 +298,15 @@ void GDALPDFObject::Serialize(CPLString &osStr, bool bEmitRef) char szReal[512]; double dfRealNonRounded = GetReal(); double dfReal = ROUND_TO_INT_IF_CLOSE(dfRealNonRounded); - if (dfReal == (double)(GIntBig)dfReal) - snprintf(szReal, sizeof(szReal), CPL_FRMT_GIB, (GIntBig)dfReal); + if (dfReal >= + static_cast(std::numeric_limits::min()) && + dfReal <= + static_cast(std::numeric_limits::max()) && + dfReal == static_cast(static_cast(dfReal))) + { + snprintf(szReal, sizeof(szReal), CPL_FRMT_GIB, + static_cast(dfReal)); + } else if (CanRepresentRealAsString()) { /* Used for OGC BP numeric values */ @@ -315,8 +322,8 @@ void GDALPDFObject::Serialize(CPLString &osStr, bool bEmitRef) char *pszDot = strchr(szReal, '.'); if (pszDot) { - int iDot = (int)(pszDot - szReal); - int nLen = (int)strlen(szReal); + int iDot = static_cast(pszDot - szReal); + int nLen = static_cast(strlen(szReal)); for (int i = nLen - 1; i > iDot; i--) { if (szReal[i] == '0') @@ -548,9 +555,7 @@ GDALPDFArrayRW *GDALPDFArray::Clone() /* ~GDALPDFStream() */ /************************************************************************/ -GDALPDFStream::~GDALPDFStream() -{ -} +GDALPDFStream::~GDALPDFStream() = default; /************************************************************************/ /* ==================================================================== */ @@ -562,11 +567,7 @@ GDALPDFStream::~GDALPDFStream() /* GDALPDFObjectRW() */ /************************************************************************/ -GDALPDFObjectRW::GDALPDFObjectRW(GDALPDFObjectType eType) - : m_eType(eType), m_nVal(0), m_dfVal(0.0), - // m_osVal - m_poDict(nullptr), m_poArray(nullptr), m_nNum(0), m_nGen(0), - m_bCanRepresentRealAsString(FALSE) +GDALPDFObjectRW::GDALPDFObjectRW(GDALPDFObjectType eType) : m_eType(eType) { } @@ -821,9 +822,7 @@ int GDALPDFObjectRW::GetRefGen() /* GDALPDFDictionaryRW() */ /************************************************************************/ -GDALPDFDictionaryRW::GDALPDFDictionaryRW() -{ -} +GDALPDFDictionaryRW::GDALPDFDictionaryRW() = default; /************************************************************************/ /* ~GDALPDFDictionaryRW() */ @@ -972,7 +971,9 @@ class GDALPDFDictionaryPoppler : public GDALPDFDictionary { private: Dict *m_poDict; - std::map m_map; + std::map m_map{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFDictionaryPoppler) public: GDALPDFDictionaryPoppler(Dict *poDict) : m_poDict(poDict) @@ -995,7 +996,9 @@ class GDALPDFArrayPoppler : public GDALPDFArray { private: Array *m_poArray; - std::vector> m_v; + std::vector> m_v{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFArrayPoppler) public: GDALPDFArrayPoppler(Array *poArray) : m_poArray(poArray) @@ -1019,15 +1022,13 @@ class GDALPDFStreamPoppler : public GDALPDFStream Stream *m_poStream; int64_t m_nRawLength = -1; + CPL_DISALLOW_COPY_ASSIGN(GDALPDFStreamPoppler) + public: GDALPDFStreamPoppler(Stream *poStream) : m_poStream(poStream) { } - virtual ~GDALPDFStreamPoppler() - { - } - virtual int64_t GetLength(int64_t nMaxSize = 0) override; virtual char *GetBytes() override; @@ -1292,7 +1293,7 @@ GDALPDFObject *GDALPDFDictionaryPoppler::Get(const char *pszKey) if (oIter != m_map.end()) return oIter->second; - auto &&o(m_poDict->lookupNF(((char *)pszKey))); + auto &&o(m_poDict->lookupNF(pszKey)); if (!o.isNull()) { GDALPDFObjectNum nRefNum; @@ -1301,7 +1302,7 @@ GDALPDFObject *GDALPDFDictionaryPoppler::Get(const char *pszKey) { nRefNum = o.getRefNum(); nRefGen = o.getRefGen(); - Object o2(m_poDict->lookup((char *)pszKey)); + Object o2(m_poDict->lookup(pszKey)); if (!o2.isNull()) { GDALPDFObjectPoppler *poObj = @@ -1333,7 +1334,7 @@ std::map &GDALPDFDictionaryPoppler::GetValues() int nLength = m_poDict->getLength(); for (i = 0; i < nLength; i++) { - const char *pszKey = (const char *)m_poDict->getKey(i); + const char *pszKey = m_poDict->getKey(i); Get(pszKey); } return m_map; @@ -1449,7 +1450,7 @@ static char *GooStringToCharStart(GooString &gstr) auto nLength = gstr.getLength(); if (nLength) { - char *pszContent = (char *)VSI_MALLOC_VERBOSE(nLength + 1); + char *pszContent = static_cast(VSI_MALLOC_VERBOSE(nLength + 1)); if (pszContent) { const char *srcStr = gstr.c_str(); @@ -1536,7 +1537,9 @@ class GDALPDFDictionaryPodofo : public GDALPDFDictionary private: const PoDoFo::PdfDictionary *m_poDict; const PoDoFo::PdfVecObjects &m_poObjects; - std::map m_map; + std::map m_map{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFDictionaryPodofo) public: GDALPDFDictionaryPodofo(const PoDoFo::PdfDictionary *poDict, @@ -1562,7 +1565,9 @@ class GDALPDFArrayPodofo : public GDALPDFArray private: const PoDoFo::PdfArray *m_poArray; const PoDoFo::PdfVecObjects &m_poObjects; - std::vector> m_v; + std::vector> m_v{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFArrayPodofo) public: GDALPDFArrayPodofo(const PoDoFo::PdfArray *poArray, @@ -1591,6 +1596,8 @@ class GDALPDFStreamPodofo : public GDALPDFStream const PoDoFo::PdfStream *m_pStream; #endif + CPL_DISALLOW_COPY_ASSIGN(GDALPDFStreamPodofo) + public: GDALPDFStreamPodofo( #if PODOFO_VERSION_MAJOR > 0 || \ @@ -1627,8 +1634,7 @@ class GDALPDFStreamPodofo : public GDALPDFStream GDALPDFObjectPodofo::GDALPDFObjectPodofo(const PoDoFo::PdfObject *po, const PoDoFo::PdfVecObjects &poObjects) - : m_po(po), m_poObjects(poObjects), m_poDict(nullptr), m_poArray(nullptr), - m_poStream(nullptr) + : m_po(po), m_poObjects(poObjects) { try { @@ -2076,7 +2082,7 @@ char *GDALPDFStreamPodofo::GetBytes() e.what()); return nullptr; } - char *pszContent = (char *)VSI_MALLOC_VERBOSE(str.size() + 1); + char *pszContent = static_cast(VSI_MALLOC_VERBOSE(str.size() + 1)); if (!pszContent) { return nullptr; @@ -2095,7 +2101,7 @@ char *GDALPDFStreamPodofo::GetBytes() { return nullptr; } - char *pszContent = (char *)VSI_MALLOC_VERBOSE(nLen + 1); + char *pszContent = static_cast(VSI_MALLOC_VERBOSE(nLen + 1)); if (!pszContent) { PoDoFo::podofo_free(pBuffer); @@ -2157,7 +2163,7 @@ char *GDALPDFStreamPodofo::GetRawBytes() e.what()); return nullptr; } - char *pszContent = (char *)VSI_MALLOC_VERBOSE(str.size() + 1); + char *pszContent = static_cast(VSI_MALLOC_VERBOSE(str.size() + 1)); if (!pszContent) { return nullptr; @@ -2176,7 +2182,7 @@ char *GDALPDFStreamPodofo::GetRawBytes() { return nullptr; } - char *pszContent = (char *)VSI_MALLOC_VERBOSE(nLen + 1); + char *pszContent = static_cast(VSI_MALLOC_VERBOSE(nLen + 1)); if (!pszContent) { PoDoFo::podofo_free(pBuffer); @@ -2203,7 +2209,7 @@ class GDALPDFDictionaryPdfium : public GDALPDFDictionary { private: RetainPtr m_poDict; - std::map m_map; + std::map m_map{}; public: GDALPDFDictionaryPdfium(RetainPtr poDict) @@ -2227,7 +2233,9 @@ class GDALPDFArrayPdfium : public GDALPDFArray { private: const CPDF_Array *m_poArray; - std::vector> m_v; + std::vector> m_v{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFArrayPdfium) public: GDALPDFArrayPdfium(const CPDF_Array *poArray) : m_poArray(poArray) @@ -2262,10 +2270,6 @@ class GDALPDFStreamPdfium : public GDALPDFStream { } - virtual ~GDALPDFStreamPdfium() - { - } - virtual int64_t GetLength(int64_t nMaxSize = 0) override; virtual char *GetBytes() override; @@ -2284,8 +2288,7 @@ class GDALPDFStreamPdfium : public GDALPDFStream /************************************************************************/ GDALPDFObjectPdfium::GDALPDFObjectPdfium(RetainPtr obj) - : m_obj(std::move(obj)), m_poDict(nullptr), m_poArray(nullptr), - m_poStream(nullptr) + : m_obj(std::move(obj)) { CPLAssert(m_obj != nullptr); } @@ -2401,7 +2404,7 @@ int GDALPDFObjectPdfium::GetInt() static double CPLRoundToMoreLikelyDouble(float f) { - if ((float)(int)f == f) + if (std::round(f) == f) return f; char szBuffer[80]; @@ -2417,7 +2420,7 @@ static double CPLRoundToMoreLikelyDouble(float f) { pszDot[2] = 0; double d2 = CPLAtof(szBuffer) + 0.01; - float f2 = (float)d2; + float f2 = static_cast(d2); if (f == f2 || nextafterf(f, f + 1.0f) == f2 || nextafterf(f, f - 1.0f) == f2) d = d2; @@ -2426,7 +2429,7 @@ static double CPLRoundToMoreLikelyDouble(float f) { pszDot[2] = 0; double d2 = CPLAtof(szBuffer); - float f2 = (float)d2; + float f2 = static_cast(d2); if (f == f2 || nextafterf(f, f + 1.0f) == f2 || nextafterf(f, f - 1.0f) == f2) d = d2; @@ -2701,7 +2704,7 @@ char *GDALPDFStreamPdfium::GetBytes() size_t nLength = static_cast(GetLength()); if (nLength == 0) return nullptr; - char *pszContent = (char *)VSI_MALLOC_VERBOSE(sizeof(char) * (nLength + 1)); + char *pszContent = static_cast(VSI_MALLOC_VERBOSE(nLength + 1)); if (!pszContent) return nullptr; memcpy(pszContent, m_pData.get(), nLength); @@ -2756,7 +2759,8 @@ char *GDALPDFStreamPdfium::GetRawBytes() size_t nLength = static_cast(GetRawLength()); if (nLength == 0) return nullptr; - char *pszContent = (char *)VSI_MALLOC_VERBOSE(sizeof(char) * (nLength + 1)); + char *pszContent = + static_cast(VSI_MALLOC_VERBOSE(sizeof(char) * (nLength + 1))); if (!pszContent) return nullptr; memcpy(pszContent, m_pRawData.get(), nLength); diff --git a/frmts/pdf/pdfobject.h b/frmts/pdf/pdfobject.h index 362eed4e1b06..c6ffedd5943f 100644 --- a/frmts/pdf/pdfobject.h +++ b/frmts/pdf/pdfobject.h @@ -214,19 +214,21 @@ class GDALPDFStream class GDALPDFObjectRW : public GDALPDFObject { private: - GDALPDFObjectType m_eType; - int m_nVal; - double m_dfVal; - CPLString m_osVal; - GDALPDFDictionaryRW *m_poDict; - GDALPDFArrayRW *m_poArray; - GDALPDFObjectNum m_nNum; - int m_nGen; - int m_bCanRepresentRealAsString; + const GDALPDFObjectType m_eType; + int m_nVal = 0; + double m_dfVal = 0; + CPLString m_osVal{}; + GDALPDFDictionaryRW *m_poDict = nullptr; + GDALPDFArrayRW *m_poArray = nullptr; + GDALPDFObjectNum m_nNum{}; + int m_nGen = 0; + int m_bCanRepresentRealAsString = FALSE; int m_nPrecision = 16; explicit GDALPDFObjectRW(GDALPDFObjectType eType); + CPL_DISALLOW_COPY_ASSIGN(GDALPDFObjectRW) + protected: virtual const char *GetTypeNameNative() override; @@ -273,7 +275,9 @@ class GDALPDFObjectRW : public GDALPDFObject class GDALPDFDictionaryRW : public GDALPDFDictionary { private: - std::map m_map; + std::map m_map{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFDictionaryRW) public: GDALPDFDictionaryRW(); @@ -322,7 +326,9 @@ class GDALPDFDictionaryRW : public GDALPDFDictionary class GDALPDFArrayRW : public GDALPDFArray { private: - std::vector m_array; + std::vector m_array{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFArrayRW) public: GDALPDFArrayRW(); @@ -379,21 +385,22 @@ class GDALPDFObjectPoppler : public GDALPDFObject { private: Object *m_po; - int m_bDestroy; - GDALPDFDictionary *m_poDict; - GDALPDFArray *m_poArray; - GDALPDFStream *m_poStream; - std::string osStr; - GDALPDFObjectNum m_nRefNum; - int m_nRefGen; + const bool m_bDestroy; + GDALPDFDictionary *m_poDict = nullptr; + GDALPDFArray *m_poArray = nullptr; + GDALPDFStream *m_poStream = nullptr; + std::string osStr{}; + GDALPDFObjectNum m_nRefNum{}; + int m_nRefGen = 0; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFObjectPoppler) protected: virtual const char *GetTypeNameNative() override; public: - GDALPDFObjectPoppler(Object *po, int bDestroy) - : m_po(po), m_bDestroy(bDestroy), m_poDict(nullptr), m_poArray(nullptr), - m_poStream(nullptr), m_nRefNum(0), m_nRefGen(0) + GDALPDFObjectPoppler(Object *po, bool bDestroy) + : m_po(po), m_bDestroy(bDestroy) { } @@ -425,10 +432,12 @@ class GDALPDFObjectPodofo : public GDALPDFObject private: const PoDoFo::PdfObject *m_po; const PoDoFo::PdfVecObjects &m_poObjects; - GDALPDFDictionary *m_poDict; - GDALPDFArray *m_poArray; - GDALPDFStream *m_poStream; - std::string osStr; + GDALPDFDictionary *m_poDict = nullptr; + GDALPDFArray *m_poArray = nullptr; + GDALPDFStream *m_poStream = nullptr; + std::string osStr{}; + + CPL_DISALLOW_COPY_ASSIGN(GDALPDFObjectPodofo) protected: virtual const char *GetTypeNameNative() override; @@ -460,13 +469,15 @@ class GDALPDFObjectPdfium : public GDALPDFObject { private: RetainPtr m_obj; - GDALPDFDictionary *m_poDict; - GDALPDFArray *m_poArray; - GDALPDFStream *m_poStream; - std::string osStr; + GDALPDFDictionary *m_poDict = nullptr; + GDALPDFArray *m_poArray = nullptr; + GDALPDFStream *m_poStream = nullptr; + std::string osStr{}; GDALPDFObjectPdfium(RetainPtr obj); + CPL_DISALLOW_COPY_ASSIGN(GDALPDFObjectPdfium) + protected: virtual const char *GetTypeNameNative() override; diff --git a/frmts/pdf/pdfreadvectors.cpp b/frmts/pdf/pdfreadvectors.cpp index 83eedece4df3..3ac2cd77b3f8 100644 --- a/frmts/pdf/pdfreadvectors.cpp +++ b/frmts/pdf/pdfreadvectors.cpp @@ -461,10 +461,10 @@ void PDFDataset::PDFCoordsToSRSCoords(double x, double y, double &X, double &Y) Y = m_adfGeoTransform[3] + x * m_adfGeoTransform[4] + y * m_adfGeoTransform[5]; - if (fabs(X - (int)floor(X + 0.5)) < 1e-8) - X = (int)floor(X + 0.5); - if (fabs(Y - (int)floor(Y + 0.5)) < 1e-8) - Y = (int)floor(Y + 0.5); + if (fabs(X - std::round(X)) < 1e-8) + X = std::round(X); + if (fabs(Y - std::round(Y)) < 1e-8) + Y = std::round(Y); } /************************************************************************/ @@ -1500,9 +1500,12 @@ OGRGeometry *PDFDataset::ParseContent( { poFeature->SetStyleString(CPLSPrintf( "PEN(c:#%02X%02X%02X)", - (int)(oGS.adfStrokeColor[0] * 255 + 0.5), - (int)(oGS.adfStrokeColor[1] * 255 + 0.5), - (int)(oGS.adfStrokeColor[2] * 255 + 0.5))); + static_cast( + oGS.adfStrokeColor[0] * 255 + 0.5), + static_cast( + oGS.adfStrokeColor[1] * 255 + 0.5), + static_cast( + oGS.adfStrokeColor[2] * 255 + 0.5))); } else if (eType == wkbPolygon || eType == wkbMultiPolygon) @@ -1510,12 +1513,18 @@ OGRGeometry *PDFDataset::ParseContent( poFeature->SetStyleString(CPLSPrintf( "PEN(c:#%02X%02X%02X);BRUSH(fc:#%02X%02X%" "02X)", - (int)(oGS.adfStrokeColor[0] * 255 + 0.5), - (int)(oGS.adfStrokeColor[1] * 255 + 0.5), - (int)(oGS.adfStrokeColor[2] * 255 + 0.5), - (int)(oGS.adfFillColor[0] * 255 + 0.5), - (int)(oGS.adfFillColor[1] * 255 + 0.5), - (int)(oGS.adfFillColor[2] * 255 + 0.5))); + static_cast( + oGS.adfStrokeColor[0] * 255 + 0.5), + static_cast( + oGS.adfStrokeColor[1] * 255 + 0.5), + static_cast( + oGS.adfStrokeColor[2] * 255 + 0.5), + static_cast(oGS.adfFillColor[0] * 255 + + 0.5), + static_cast(oGS.adfFillColor[1] * 255 + + 0.5), + static_cast(oGS.adfFillColor[2] * 255 + + 0.5))); } } poGeom->assignSpatialReference( @@ -1712,8 +1721,8 @@ OGRGeometry *PDFDataset::BuildGeometry(std::vector &oCoords, poPoly->addRingDirectly(poLS); poLS = nullptr; - papoPoly = (OGRGeometry **)CPLRealloc( - papoPoly, (nPolys + 1) * sizeof(OGRGeometry *)); + papoPoly = static_cast(CPLRealloc( + papoPoly, (nPolys + 1) * sizeof(OGRGeometry *))); papoPoly[nPolys++] = poPoly; } delete poLS; @@ -1783,8 +1792,8 @@ OGRGeometry *PDFDataset::BuildGeometry(std::vector &oCoords, poPoly->addRingDirectly(poLS); poLS = nullptr; - papoPoly = (OGRGeometry **)CPLRealloc( - papoPoly, (nPolys + 1) * sizeof(OGRGeometry *)); + papoPoly = static_cast(CPLRealloc( + papoPoly, (nPolys + 1) * sizeof(OGRGeometry *))); papoPoly[nPolys++] = poPoly; } else @@ -1900,7 +1909,7 @@ void PDFDataset::ExploreContents(GDALPDFObject *poObj, if (!pszStr) return; - const char *pszMCID = (const char *)pszStr; + const char *pszMCID = pszStr; while ((pszMCID = strstr(pszMCID, "/MCID")) != nullptr) { const char *pszBDC = strstr(pszMCID, "BDC"); @@ -1964,7 +1973,7 @@ void PDFDataset::ExploreContentsNonStructuredInternal( { GDALPDFArray *poArray = poContents->GetArray(); char *pszConcatStr = nullptr; - int nConcatLen = 0; + size_t nConcatLen = 0; for (int i = 0; i < poArray->GetLength(); i++) { GDALPDFObject *poObj = poArray->Get(i); @@ -1977,9 +1986,9 @@ void PDFDataset::ExploreContentsNonStructuredInternal( char *pszStr = poStream->GetBytes(); if (!pszStr) break; - int nLen = (int)strlen(pszStr); - char *pszConcatStrNew = - (char *)CPLRealloc(pszConcatStr, nConcatLen + nLen + 1); + const size_t nLen = strlen(pszStr); + char *pszConcatStrNew = static_cast( + CPLRealloc(pszConcatStr, nConcatLen + nLen + 1)); if (pszConcatStrNew == nullptr) { CPLFree(pszStr); diff --git a/frmts/pdf/pdfwritabledataset.cpp b/frmts/pdf/pdfwritabledataset.cpp index 9d402e51b979..2760c0974ba5 100644 --- a/frmts/pdf/pdfwritabledataset.cpp +++ b/frmts/pdf/pdfwritabledataset.cpp @@ -123,8 +123,8 @@ PDFWritableVectorDataset::ICreateLayer(const char *pszLayerName, if (poSRSClone) poSRSClone->Release(); - papoLayers = - (OGRLayer **)CPLRealloc(papoLayers, (nLayers + 1) * sizeof(OGRLayer *)); + papoLayers = static_cast( + CPLRealloc(papoLayers, (nLayers + 1) * sizeof(OGRLayer *))); papoLayers[nLayers] = poLayer; nLayers++; From 60fc9c50f8eed313b3574ca8f4a8b83524dabc4e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 20 Sep 2024 03:18:18 +0200 Subject: [PATCH 194/710] PDF: rename ROUND_TO_INT_IF_CLOSE() function --- frmts/pdf/pdfdataset.cpp | 21 +++++++++------------ frmts/pdf/pdfobject.cpp | 6 +++--- frmts/pdf/pdfobject.h | 2 +- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/frmts/pdf/pdfdataset.cpp b/frmts/pdf/pdfdataset.cpp index f94a18798786..b09331f4f971 100644 --- a/frmts/pdf/pdfdataset.cpp +++ b/frmts/pdf/pdfdataset.cpp @@ -3026,9 +3026,9 @@ static int GDALPDFParseStreamContent(const char *pszContent, INT_MAX && dfHeight / dfScaleY * DEFAULT_DPI < INT_MAX) { - double dfDPI_X = ROUND_TO_INT_IF_CLOSE( + double dfDPI_X = ROUND_IF_CLOSE( dfWidth / dfScaleX * DEFAULT_DPI, 1e-3); - double dfDPI_Y = ROUND_TO_INT_IF_CLOSE( + double dfDPI_Y = ROUND_IF_CLOSE( dfHeight / dfScaleY * DEFAULT_DPI, 1e-3); // CPLDebug("PDF", "Image %s, width = %.16g, @@ -3463,8 +3463,7 @@ void PDFDataset::GuessDPI(GDALPDFDictionary *poPageDict, int *pnBands) (poUserUnit->GetType() == PDFObjectType_Int || poUserUnit->GetType() == PDFObjectType_Real)) { - m_dfDPI = - ROUND_TO_INT_IF_CLOSE(Get(poUserUnit) * DEFAULT_DPI, 1e-5); + m_dfDPI = ROUND_IF_CLOSE(Get(poUserUnit) * DEFAULT_DPI, 1e-5); CPLDebug("PDF", "Found UserUnit in Page --> DPI = %.16g", m_dfDPI); SetMetadataItem("DPI", CPLSPrintf("%.16g", m_dfDPI)); } @@ -5399,13 +5398,11 @@ PDFDataset *PDFDataset::Open(GDALOpenInfo *poOpenInfo) ? 1e-5 : 1e-8; poDS->m_adfGeoTransform[0] = - ROUND_TO_INT_IF_CLOSE(poDS->m_adfGeoTransform[0], dfEps); - poDS->m_adfGeoTransform[1] = - ROUND_TO_INT_IF_CLOSE(poDS->m_adfGeoTransform[1]); + ROUND_IF_CLOSE(poDS->m_adfGeoTransform[0], dfEps); + poDS->m_adfGeoTransform[1] = ROUND_IF_CLOSE(poDS->m_adfGeoTransform[1]); poDS->m_adfGeoTransform[3] = - ROUND_TO_INT_IF_CLOSE(poDS->m_adfGeoTransform[3], dfEps); - poDS->m_adfGeoTransform[5] = - ROUND_TO_INT_IF_CLOSE(poDS->m_adfGeoTransform[5]); + ROUND_IF_CLOSE(poDS->m_adfGeoTransform[3], dfEps); + poDS->m_adfGeoTransform[5] = ROUND_IF_CLOSE(poDS->m_adfGeoTransform[5]); if (bUseLib.test(PDFLIB_PDFIUM)) { @@ -7209,8 +7206,8 @@ int PDFDataset::ParseMeasure(GDALPDFObject *poMeasure, double dfMediaBoxWidth, } } - x = ROUND_TO_INT_IF_CLOSE(x); - y = ROUND_TO_INT_IF_CLOSE(y); + x = ROUND_IF_CLOSE(x); + y = ROUND_IF_CLOSE(y); asGCPS[i].dfGCPX = x; asGCPS[i].dfGCPY = y; diff --git a/frmts/pdf/pdfobject.cpp b/frmts/pdf/pdfobject.cpp index d209e920181a..21c2659d6cfa 100644 --- a/frmts/pdf/pdfobject.cpp +++ b/frmts/pdf/pdfobject.cpp @@ -40,10 +40,10 @@ #include "pdfobject.h" /************************************************************************/ -/* ROUND_TO_INT_IF_CLOSE() */ +/* ROUND_IF_CLOSE() */ /************************************************************************/ -double ROUND_TO_INT_IF_CLOSE(double x, double eps) +double ROUND_IF_CLOSE(double x, double eps) { if (eps == 0.0) eps = fabs(x) < 1 ? 1e-10 : 1e-8; @@ -297,7 +297,7 @@ void GDALPDFObject::Serialize(CPLString &osStr, bool bEmitRef) { char szReal[512]; double dfRealNonRounded = GetReal(); - double dfReal = ROUND_TO_INT_IF_CLOSE(dfRealNonRounded); + double dfReal = ROUND_IF_CLOSE(dfRealNonRounded); if (dfReal >= static_cast(std::numeric_limits::min()) && dfReal <= diff --git a/frmts/pdf/pdfobject.h b/frmts/pdf/pdfobject.h index c6ffedd5943f..0021cd6d4e11 100644 --- a/frmts/pdf/pdfobject.h +++ b/frmts/pdf/pdfobject.h @@ -46,7 +46,7 @@ #define DEFAULT_DPI (72.0) #define USER_UNIT_IN_INCH (1.0 / DEFAULT_DPI) -double ROUND_TO_INT_IF_CLOSE(double x, double eps = 0); +double ROUND_IF_CLOSE(double x, double eps = 0); typedef enum { From d5ed93649195137d906003469d95b5c9764f23b3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 20 Sep 2024 03:35:19 +0200 Subject: [PATCH 195/710] PDF: avoid 'Non closed ring detected' warning when reading neatlines from OGC Best Practice encoding --- autotest/gdrivers/pdf.py | 2 ++ frmts/pdf/pdfdataset.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/autotest/gdrivers/pdf.py b/autotest/gdrivers/pdf.py index 7db4786de764..283f9507a7ed 100755 --- a/autotest/gdrivers/pdf.py +++ b/autotest/gdrivers/pdf.py @@ -374,12 +374,14 @@ def test_pdf_ogcbp(poppler_or_pdfium_or_podofo): tst = gdaltest.GDALTest( "PDF", "byte.tif", 1, None, options=["GEO_ENCODING=OGC_BP"] ) + gdal.ErrorReset() tst.testCreateCopy( check_minmax=0, check_gt=1, check_srs=True, check_checksum_not_null=pdf_checksum_available(), ) + assert gdal.GetLastErrorMsg() == "" ############################################################################### diff --git a/frmts/pdf/pdfdataset.cpp b/frmts/pdf/pdfdataset.cpp index 845057fd68a9..721681643fbb 100644 --- a/frmts/pdf/pdfdataset.cpp +++ b/frmts/pdf/pdfdataset.cpp @@ -5951,6 +5951,7 @@ int PDFDataset::ParseLGIDictDictFirstPass(GDALPDFDictionary *poLGIDict, poRing->addPoint(dfX, dfY); } } + poRing->closeRings(); m_poNeatLine->addRingDirectly(poRing); } From 14faa95720568407d1da60407c2dc7afe7ed67be Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 20 Sep 2024 14:57:49 +0200 Subject: [PATCH 196/710] appslib: enable -Weffc++ --- apps/CMakeLists.txt | 2 +- apps/gdal_footprint_lib.cpp | 4 +++- apps/gdal_grid_lib.cpp | 2 ++ apps/gdal_utils_priv.h | 10 +++++----- apps/gdalargumentparser.h | 11 ++++++++++- apps/gdalbuildvrt_lib.cpp | 2 ++ apps/gdaldem_lib.cpp | 8 ++++++-- apps/gdalinfo_lib.cpp | 2 +- apps/gdaltindex_lib.cpp | 2 ++ apps/gdalwarp_lib.cpp | 2 ++ apps/ogr2ogr_lib.cpp | 35 +++++++++++++++++++++-------------- apps/ogrinfo_lib.cpp | 2 +- 12 files changed, 56 insertions(+), 26 deletions(-) diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 4ce116ae1b01..3b4b5386dcb4 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -25,7 +25,7 @@ add_library( add_dependencies(appslib generate_gdal_version_h) target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) gdal_standard_includes(appslib) -target_compile_options(appslib PRIVATE ${GDAL_CXX_WARNING_FLAGS} ${WFLAG_OLD_STYLE_CAST}) +target_compile_options(appslib PRIVATE ${GDAL_CXX_WARNING_FLAGS} ${WFLAG_OLD_STYLE_CAST} ${WFLAG_EFFCXX}) target_include_directories( appslib PRIVATE $ $ $ $) diff --git a/apps/gdal_footprint_lib.cpp b/apps/gdal_footprint_lib.cpp index bbf5ae51b98b..2567212fad92 100644 --- a/apps/gdal_footprint_lib.cpp +++ b/apps/gdal_footprint_lib.cpp @@ -123,7 +123,7 @@ struct GDALFootprintOptions /*! Whether to force writing absolute paths in location field. */ bool bAbsolutePath = false; - std::string osSrcNoData; + std::string osSrcNoData{}; }; static std::unique_ptr GDALFootprintAppOptionsGetParser( @@ -295,6 +295,8 @@ class GDALFootprintMaskBand final : public GDALRasterBand { GDALRasterBand *m_poSrcBand = nullptr; + CPL_DISALLOW_COPY_ASSIGN(GDALFootprintMaskBand) + public: explicit GDALFootprintMaskBand(GDALRasterBand *poSrcBand) : m_poSrcBand(poSrcBand) diff --git a/apps/gdal_grid_lib.cpp b/apps/gdal_grid_lib.cpp index 1531f6d1ebca..304f61117149 100644 --- a/apps/gdal_grid_lib.cpp +++ b/apps/gdal_grid_lib.cpp @@ -116,6 +116,8 @@ struct GDALGridOptions &l_pOptions); pOptions.reset(l_pOptions); } + + CPL_DISALLOW_COPY_ASSIGN(GDALGridOptions) }; /************************************************************************/ diff --git a/apps/gdal_utils_priv.h b/apps/gdal_utils_priv.h index 7814c26babeb..aa509a6c36df 100644 --- a/apps/gdal_utils_priv.h +++ b/apps/gdal_utils_priv.h @@ -58,10 +58,10 @@ struct GDALInfoOptionsForBinary struct GDALDEMProcessingOptionsForBinary { - std::string osProcessing; - std::string osSrcFilename; - std::string osColorFilename; - std::string osDstFilename; + std::string osProcessing{}; + std::string osSrcFilename{}; + std::string osColorFilename{}; + std::string osDstFilename{}; bool bQuiet = false; }; @@ -83,7 +83,7 @@ struct GDALVectorTranslateOptionsForBinary std::string osDestDataSource{}; bool bQuiet = false; CPLStringList aosOpenOptions{}; - std::string osFormat; + std::string osFormat{}; GDALVectorTranslateAccessMode eAccessMode = ACCESS_CREATION; bool bShowUsageIfError = false; diff --git a/apps/gdalargumentparser.h b/apps/gdalargumentparser.h index d72b69083f98..9f4df4f80eb5 100644 --- a/apps/gdalargumentparser.h +++ b/apps/gdalargumentparser.h @@ -45,8 +45,17 @@ #pragma warning(disable : 4702) #endif +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif + #include "argparse/argparse.hpp" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + #ifdef _MSC_VER #pragma warning(pop) #endif @@ -156,7 +165,7 @@ class GDALArgumentParser : public ArgumentParser private: std::map::iterator find_argument(const std::string &name); - std::vector> aoSubparsers; + std::vector> aoSubparsers{}; std::string m_osExtraUsageHint{}; }; diff --git a/apps/gdalbuildvrt_lib.cpp b/apps/gdalbuildvrt_lib.cpp index 31e720fcc866..ee51213dfebd 100644 --- a/apps/gdalbuildvrt_lib.cpp +++ b/apps/gdalbuildvrt_lib.cpp @@ -279,6 +279,8 @@ class VRTBuilder void CreateVRTSeparate(VRTDatasetH hVRTDS); void CreateVRTNonSeparate(VRTDatasetH hVRTDS); + CPL_DISALLOW_COPY_ASSIGN(VRTBuilder) + public: VRTBuilder(bool bStrictIn, const char *pszOutputFilename, int nInputFiles, const char *const *ppszInputFilenames, GDALDatasetH *pahSrcDSIn, diff --git a/apps/gdaldem_lib.cpp b/apps/gdaldem_lib.cpp index c42bd152ec5f..3bce49466324 100644 --- a/apps/gdaldem_lib.cpp +++ b/apps/gdaldem_lib.cpp @@ -158,7 +158,7 @@ using namespace gdal::GDALDEM; struct GDALDEMProcessingOptions { /*! output format. Use the short format name. */ - std::string osFormat; + std::string osFormat{}; /*! the progress function to use */ GDALProgressFunc pfnProgress = nullptr; @@ -184,7 +184,7 @@ struct GDALDEMProcessingOptions bool bCombined = false; bool bIgor = false; bool bMultiDirectional = false; - CPLStringList aosCreationOptions; + CPLStringList aosCreationOptions{}; int nBand = 1; }; @@ -1972,6 +1972,8 @@ class GDALColorReliefDataset : public GDALDataset int nCurBlockXOff; int nCurBlockYOff; + CPL_DISALLOW_COPY_ASSIGN(GDALColorReliefDataset) + public: GDALColorReliefDataset(GDALDatasetH hSrcDS, GDALRasterBandH hSrcBand, const char *pszColorFilename, @@ -2565,6 +2567,8 @@ template class GDALGeneric3x3Dataset : public GDALDataset int nCurLine; bool bComputeAtEdges; + CPL_DISALLOW_COPY_ASSIGN(GDALGeneric3x3Dataset) + public: GDALGeneric3x3Dataset(GDALDatasetH hSrcDS, GDALRasterBandH hSrcBand, GDALDataType eDstDataType, int bDstHasNoData, diff --git a/apps/gdalinfo_lib.cpp b/apps/gdalinfo_lib.cpp index e61ef39ff8e5..63d86d11acf1 100644 --- a/apps/gdalinfo_lib.cpp +++ b/apps/gdalinfo_lib.cpp @@ -135,7 +135,7 @@ struct GDALInfoOptions /*! report metadata for the specified domains. "all" can be used to report metadata in all domains. */ - CPLStringList aosExtraMDDomains; + CPLStringList aosExtraMDDomains{}; /*! WKT format used for SRS */ std::string osWKTFormat = "WKT2"; diff --git a/apps/gdaltindex_lib.cpp b/apps/gdaltindex_lib.cpp index 7101a7d6ba0b..e544afd636bd 100644 --- a/apps/gdaltindex_lib.cpp +++ b/apps/gdaltindex_lib.cpp @@ -387,6 +387,8 @@ struct GDALTileIndexTileIterator int iCurSrc = 0; VSIDIR *psDir = nullptr; + CPL_DISALLOW_COPY_ASSIGN(GDALTileIndexTileIterator) + GDALTileIndexTileIterator(const GDALTileIndexOptions *psOptionsIn, int nSrcCountIn, const char *const *papszSrcDSNamesIn) diff --git a/apps/gdalwarp_lib.cpp b/apps/gdalwarp_lib.cpp index 53fa14271b03..84c400887eb6 100644 --- a/apps/gdalwarp_lib.cpp +++ b/apps/gdalwarp_lib.cpp @@ -4868,6 +4868,8 @@ static GDALDatasetH GDALWarpCreateOutput( class CutlineTransformer : public OGRCoordinateTransformation { + CPL_DISALLOW_COPY_ASSIGN(CutlineTransformer) + public: void *hSrcImageTransformer = nullptr; diff --git a/apps/ogr2ogr_lib.cpp b/apps/ogr2ogr_lib.cpp index 28866b20c7d5..911882beeb7f 100644 --- a/apps/ogr2ogr_lib.cpp +++ b/apps/ogr2ogr_lib.cpp @@ -133,6 +133,8 @@ struct CopyableGCPs CPLFree(pasGCPs); } } + + CopyableGCPs &operator=(const CopyableGCPs &) = delete; }; } // namespace gdal::ogr2ogr_lib @@ -433,7 +435,7 @@ struct GDALVectorTranslateOptions whose geometry intersects the extents will be selected. The geometries will not be clipped unless GDALVectorTranslateOptions::bClipSrc is true. */ - std::shared_ptr poSpatialFilter; + std::shared_ptr poSpatialFilter{}; /*! the progress function to use */ GDALProgressFunc pfnProgress = nullptr; @@ -743,15 +745,17 @@ typedef struct class OGRSplitListFieldLayer : public OGRLayer { - OGRLayer *poSrcLayer; - OGRFeatureDefn *poFeatureDefn; - ListFieldDesc *pasListFields; - int nListFieldCount; - int nMaxSplitListSubFields; + OGRLayer *poSrcLayer = nullptr; + OGRFeatureDefn *poFeatureDefn = nullptr; + ListFieldDesc *pasListFields = nullptr; + int nListFieldCount = 0; + const int nMaxSplitListSubFields; std::unique_ptr TranslateFeature(std::unique_ptr poSrcFeature); + CPL_DISALLOW_COPY_ASSIGN(OGRSplitListFieldLayer) + public: OGRSplitListFieldLayer(OGRLayer *poSrcLayer, int nMaxSplitListSubFields); virtual ~OGRSplitListFieldLayer(); @@ -826,8 +830,7 @@ class OGRSplitListFieldLayer : public OGRLayer OGRSplitListFieldLayer::OGRSplitListFieldLayer(OGRLayer *poSrcLayerIn, int nMaxSplitListSubFieldsIn) - : poSrcLayer(poSrcLayerIn), poFeatureDefn(nullptr), pasListFields(nullptr), - nListFieldCount(0), + : poSrcLayer(poSrcLayerIn), nMaxSplitListSubFields( nMaxSplitListSubFieldsIn < 0 ? INT_MAX : nMaxSplitListSubFieldsIn) { @@ -1569,15 +1572,17 @@ class GDALVectorTranslateWrappedDataset : public GDALDataset std::unique_ptr m_poDriverToFree{}; GDALDataset *m_poBase = nullptr; OGRSpatialReference *m_poOutputSRS = nullptr; - bool m_bTransform = false; + const bool m_bTransform = false; std::vector> m_apoLayers{}; - std::vector> m_apoHiddenLayers; + std::vector> m_apoHiddenLayers{}; GDALVectorTranslateWrappedDataset(GDALDataset *poBase, OGRSpatialReference *poOutputSRS, bool bTransform); + CPL_DISALLOW_COPY_ASSIGN(GDALVectorTranslateWrappedDataset) + public: virtual int GetLayerCount() override { @@ -1605,6 +1610,8 @@ class GDALVectorTranslateWrappedLayer : public OGRLayerDecorator std::unique_ptr TranslateFeature(std::unique_ptr poSrcFeat); + CPL_DISALLOW_COPY_ASSIGN(GDALVectorTranslateWrappedLayer) + public: virtual ~GDALVectorTranslateWrappedLayer(); @@ -1849,12 +1856,12 @@ void GDALVectorTranslateWrappedDataset::ReleaseResultSet(OGRLayer *poResultsSet) class OGR2OGRSpatialReferenceHolder { - OGRSpatialReference *m_poSRS; + OGRSpatialReference *m_poSRS = nullptr; + + CPL_DISALLOW_COPY_ASSIGN(OGR2OGRSpatialReferenceHolder) public: - OGR2OGRSpatialReferenceHolder() : m_poSRS(nullptr) - { - } + OGR2OGRSpatialReferenceHolder() = default; ~OGR2OGRSpatialReferenceHolder() { diff --git a/apps/ogrinfo_lib.cpp b/apps/ogrinfo_lib.cpp index b227370ddbb9..03c91a7f3b9b 100644 --- a/apps/ogrinfo_lib.cpp +++ b/apps/ogrinfo_lib.cpp @@ -55,7 +55,7 @@ struct GDALVectorInfoOptions GDALVectorInfoFormat eFormat = FORMAT_TEXT; std::string osWHERE{}; CPLStringList aosLayers{}; - std::unique_ptr poSpatialFilter; + std::unique_ptr poSpatialFilter{}; bool bAllLayers = false; std::string osSQLStatement{}; std::string osDialect{}; From 2545a2cf3ffc4c8988d68b75a956d45a553adde9 Mon Sep 17 00:00:00 2001 From: Daniel Baston Date: Fri, 20 Sep 2024 13:44:18 -0400 Subject: [PATCH 197/710] Doc: Add more detail to THRESHOLD config option --- doc/source/user/configoptions.rst | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst index aa2bcaa0209c..164bc6f1ceb7 100644 --- a/doc/source/user/configoptions.rst +++ b/doc/source/user/configoptions.rst @@ -967,36 +967,41 @@ PROJ options - .. config:: CHECK_WITH_INVERT_PROJ :since: 1.7.0 + :default: NO Used by :source_file:`ogr/ogrct.cpp` and :source_file:`apps/gdalwarp_lib.cpp`. - This option can be used to control the behavior of gdalwarp when warping global + This option can be used to control the behavior of :program:`gdalwarp` when warping global datasets or when transforming from/to polar projections, which causes coordinate discontinuities. See http://trac.osgeo.org/gdal/ticket/2305. - The background is that PROJ does not guarantee that converting from src_srs to - dst_srs and then from dst_srs to src_srs will yield to the initial coordinates. + The background is that PROJ does not guarantee that converting from ``src_srs`` to + ``dst_srs`` and then from ``dst_srs`` to ``src_srs`` will yield the initial coordinates. This can lead to errors in the computation of the target bounding box of - gdalwarp, or to visual artifacts. + :program:`gdalwarp`, or to visual artifacts. - If CHECK_WITH_INVERT_PROJ option is not set, gdalwarp will check that the the + If :config:`CHECK_WITH_INVERT_PROJ` option is not set, :program:`gdalwarp` will check that the computed coordinates of the edges of the target image are in the validity area of the target projection. If they are not, it will retry computing them by - setting :config:`CHECK_WITH_INVERT_PROJ=TRUE` that forces ogrct.cpp to check the - consistency of each requested projection result with the invert projection. + setting :config:`CHECK_WITH_INVERT_PROJ=TRUE` that forces + :source_file:`ogr/ogrct.cpp` to check the consistency of each requested + projection result with the inverse projection. - If set to NO, gdalwarp will not attempt to use the invert projection. + If set to ``NO``, :program:`gdalwarp` will not attempt to use the inverse projection. - .. config:: THRESHOLD :since: 1.7.0 + :default: 0.1 for geographic SRS, 10000 otherwise Used by :source_file:`ogr/ogrct.cpp`. - Used in combination with :config:`CHECK_WITH_INVERT_PROJ=TRUE`. Define - the acceptable threshold used to check if the roundtrip from src_srs to - dst_srs and from dst_srs to srs_srs yield to the initial coordinates. The - value must be expressed in the units of the source SRS (typically degrees - for a geographic SRS, meters for a projected SRS) + Used in combination with :config:`CHECK_WITH_INVERT_PROJ=TRUE`. Defines + the acceptable threshold used to check if the round-trip from ``src_srs`` to + ``dst_srs`` and from ``dst_srs`` to ``srs_srs`` yields the initial coordinates. + The round-trip transformation will be considered successful if the ``x`` and ``y`` + values are both within :config:`THRESHOLD` of the original values. + The value must be expressed in the units of the source SRS (typically degrees + for a geographic SRS, meters for a projected SRS). - .. config:: OGR_ENABLE_PARTIAL_REPROJECTION :since: 1.8.0 From fe5137fdd24d99f2527ac4d55f8c120978ca768a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 04:34:32 +0200 Subject: [PATCH 198/710] ogr_wkb.h: add missing include --- ogr/ogr_wkb.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ogr/ogr_wkb.h b/ogr/ogr_wkb.h index ef4bd409702f..1932bf68920d 100644 --- a/ogr/ogr_wkb.h +++ b/ogr/ogr_wkb.h @@ -29,6 +29,8 @@ #ifndef OGR_WKB_H_INCLUDED #define OGR_WKB_H_INCLUDED +#include + #include "cpl_port.h" #include "ogr_core.h" From 5808d8dfbcde990ac2a31e0aa3a2b30f9810c99c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 05:34:26 +0200 Subject: [PATCH 199/710] Fix build issue with libgeotiff < 1.6 and VICAR driver as a plugin --- frmts/gtiff/gt_wkt_srs_priv.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frmts/gtiff/gt_wkt_srs_priv.h b/frmts/gtiff/gt_wkt_srs_priv.h index 21fc6c4bdc58..e3f83d4c3e38 100644 --- a/frmts/gtiff/gt_wkt_srs_priv.h +++ b/frmts/gtiff/gt_wkt_srs_priv.h @@ -40,14 +40,14 @@ #else -int GDALGTIFKeyGetASCII(GTIF *hGTIF, geokey_t key, char *szStr, - int szStrMaxLen); +int CPL_DLL GDALGTIFKeyGetASCII(GTIF *hGTIF, geokey_t key, char *szStr, + int szStrMaxLen); -int GDALGTIFKeyGetSHORT(GTIF *hGTIF, geokey_t key, unsigned short *pnVal, - int nIndex, int nCount); +int CPL_DLL GDALGTIFKeyGetSHORT(GTIF *hGTIF, geokey_t key, + unsigned short *pnVal, int nIndex, int nCount); -int GDALGTIFKeyGetDOUBLE(GTIF *hGTIF, geokey_t key, double *pdfVal, int nIndex, - int nCount); +int CPL_DLL GDALGTIFKeyGetDOUBLE(GTIF *hGTIF, geokey_t key, double *pdfVal, + int nIndex, int nCount); #endif From 123930a4fff693be460450c9a190458ea085aed1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 21 Sep 2024 23:10:08 +0200 Subject: [PATCH 200/710] Make OGR_VRT driver optional and buildable as plugin --- autotest/ogr/ogr_sql_test.py | 1 + autotest/pyscripts/test_ogrmerge.py | 11 ++- autotest/utilities/test_ogrinfo.py | 1 + frmts/pds/pds4vector.cpp | 2 +- gcore/gdal_priv.h | 3 +- ogr/CMakeLists.txt | 4 +- ogr/ogrsf_frmts/CMakeLists.txt | 2 +- ogr/ogrsf_frmts/generic/ogrlayerpool.h | 2 +- ogr/ogrsf_frmts/generic/ogrwarpedlayer.h | 2 +- ogr/ogrsf_frmts/vrt/CMakeLists.txt | 4 +- ogr/ogrsf_frmts/vrt/ogr_vrt.h | 3 - ogr/ogrsf_frmts/vrt/ogrvrtdatasource.cpp | 96 +---------------------- ogr/ogrsf_frmts/vrt/ogrvrtlayer.cpp | 4 +- ogr/ogrvrtgeometrytypes.cpp | 99 ++++++++++++++++++++++++ ogr/ogrvrtgeometrytypes.h | 16 ++++ 15 files changed, 138 insertions(+), 112 deletions(-) create mode 100644 ogr/ogrvrtgeometrytypes.cpp create mode 100644 ogr/ogrvrtgeometrytypes.h diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index e9c187219bed..c58f1e8bc12f 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -1285,6 +1285,7 @@ def test_ogr_sql_hstore_get_value_valid(data_ds, sql, expected): # Test 64 bit GetFeatureCount() +@pytest.mark.require_driver("OGR_VRT") def test_ogr_sql_45(): ds = ogr.Open( diff --git a/autotest/pyscripts/test_ogrmerge.py b/autotest/pyscripts/test_ogrmerge.py index b8f30feabbbf..7f585e7eccf9 100755 --- a/autotest/pyscripts/test_ogrmerge.py +++ b/autotest/pyscripts/test_ogrmerge.py @@ -36,10 +36,13 @@ from osgeo import gdal, ogr -pytestmark = pytest.mark.skipif( - test_py_scripts.get_py_script("ogrmerge") is None, - reason="ogrmerge.py not available", -) +pytestmark = [ + pytest.mark.require_driver("OGR_VRT"), + pytest.mark.skipif( + test_py_scripts.get_py_script("ogrmerge") is None, + reason="ogrmerge.py not available", + ), +] @pytest.fixture() diff --git a/autotest/utilities/test_ogrinfo.py b/autotest/utilities/test_ogrinfo.py index 1d15129dd13e..791651518bc7 100755 --- a/autotest/utilities/test_ogrinfo.py +++ b/autotest/utilities/test_ogrinfo.py @@ -521,6 +521,7 @@ def test_ogrinfo_23(ogrinfo_path, tmp_path): # Test metadata +@pytest.mark.require_driver("OGR_VRT") def test_ogrinfo_24(ogrinfo_path, tmp_path): vrt_fname = str(tmp_path / "test_ogrinfo_24.vrt") diff --git a/frmts/pds/pds4vector.cpp b/frmts/pds/pds4vector.cpp index 307de7f9b28c..8cdec4bfb7de 100644 --- a/frmts/pds/pds4vector.cpp +++ b/frmts/pds/pds4vector.cpp @@ -27,7 +27,7 @@ ****************************************************************************/ #include "pds4dataset.h" -#include "ogr_vrt.h" +#include "ogrvrtgeometrytypes.h" #include "ogr_p.h" diff --git a/gcore/gdal_priv.h b/gcore/gdal_priv.h index 9ad11d1bfb89..6db451a20d4f 100644 --- a/gcore/gdal_priv.h +++ b/gcore/gdal_priv.h @@ -4555,7 +4555,8 @@ void GDALDeserializeGCPListFromXML(const CPLXMLNode *psGCPList, void GDALSerializeOpenOptionsToXML(CPLXMLNode *psParentNode, CSLConstList papszOpenOptions); -char **GDALDeserializeOpenOptionsFromXML(const CPLXMLNode *psParentNode); +char CPL_DLL ** +GDALDeserializeOpenOptionsFromXML(const CPLXMLNode *psParentNode); int GDALCanFileAcceptSidecarFile(const char *pszFilename); diff --git a/ogr/CMakeLists.txt b/ogr/CMakeLists.txt index cc59f2a665a8..b7ddeae3e052 100644 --- a/ogr/CMakeLists.txt +++ b/ogr/CMakeLists.txt @@ -61,7 +61,9 @@ add_library( ogr_xerces.cpp ogr_geo_utils.cpp ogr_proj_p.cpp - ogr_wkb.cpp) + ogr_wkb.cpp + ogrvrtgeometrytypes.cpp +) add_dependencies(ogr generate_gdal_version_h) set(GDAL_DATA_FILES diff --git a/ogr/ogrsf_frmts/CMakeLists.txt b/ogr/ogrsf_frmts/CMakeLists.txt index c3753e8963ae..501dda1ba9e8 100644 --- a/ogr/ogrsf_frmts/CMakeLists.txt +++ b/ogr/ogrsf_frmts/CMakeLists.txt @@ -18,7 +18,7 @@ ogr_default_driver(shape "ESRI shape-file") # ###################################################################################################################### ogr_default_driver(kml KML) # when not found EXPAT, it use CPL_MINIXML (ex. android) -ogr_default_driver(vrt "VRT - Virtual Format") +ogr_optional_driver(vrt "VRT - Virtual Format") # Caution: if modifying AVC declaration here, also modify it in gdal.cmake ogr_optional_driver(avc AVC) diff --git a/ogr/ogrsf_frmts/generic/ogrlayerpool.h b/ogr/ogrsf_frmts/generic/ogrlayerpool.h index 724180af2a28..ec5974018590 100644 --- a/ogr/ogrsf_frmts/generic/ogrlayerpool.h +++ b/ogr/ogrsf_frmts/generic/ogrlayerpool.h @@ -101,7 +101,7 @@ class CPL_DLL OGRLayerPool /* OGRProxiedLayer */ /************************************************************************/ -class OGRProxiedLayer : public OGRAbstractProxiedLayer +class CPL_DLL OGRProxiedLayer : public OGRAbstractProxiedLayer { CPL_DISALLOW_COPY_ASSIGN(OGRProxiedLayer) diff --git a/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h b/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h index abe0050d41e0..e98fd2b244a3 100644 --- a/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h +++ b/ogr/ogrsf_frmts/generic/ogrwarpedlayer.h @@ -39,7 +39,7 @@ /* OGRWarpedLayer */ /************************************************************************/ -class OGRWarpedLayer : public OGRLayerDecorator +class CPL_DLL OGRWarpedLayer : public OGRLayerDecorator { CPL_DISALLOW_COPY_ASSIGN(OGRWarpedLayer) diff --git a/ogr/ogrsf_frmts/vrt/CMakeLists.txt b/ogr/ogrsf_frmts/vrt/CMakeLists.txt index 81021790e27e..868139a2c4d3 100644 --- a/ogr/ogrsf_frmts/vrt/CMakeLists.txt +++ b/ogr/ogrsf_frmts/vrt/CMakeLists.txt @@ -1,8 +1,8 @@ -# Depend from ogr_gpkg add_gdal_driver( TARGET ogr_VRT SOURCES ogr_vrt.h ogrvrtdatasource.cpp ogrvrtdriver.cpp ogrvrtlayer.cpp - BUILTIN) + PLUGIN_CAPABLE + NO_DEPS) gdal_standard_includes(ogr_VRT) target_include_directories(ogr_VRT PRIVATE $) diff --git a/ogr/ogrsf_frmts/vrt/ogr_vrt.h b/ogr/ogrsf_frmts/vrt/ogr_vrt.h index 0884f421df44..9032b356fec3 100644 --- a/ogr/ogrsf_frmts/vrt/ogr_vrt.h +++ b/ogr/ogrsf_frmts/vrt/ogr_vrt.h @@ -310,7 +310,4 @@ class OGRVRTDataSource final : public OGRDataSource bool IsInForbiddenNames(const char *pszOtherDSName) const; }; -OGRwkbGeometryType OGRVRTGetGeometryType(const char *pszGType, int *pbError); -CPLString CPL_DLL OGRVRTGetSerializedGeometryType(OGRwkbGeometryType eGeomType); - #endif // ndef OGR_VRT_H_INCLUDED diff --git a/ogr/ogrsf_frmts/vrt/ogrvrtdatasource.cpp b/ogr/ogrsf_frmts/vrt/ogrvrtdatasource.cpp index c52d7e6ca204..c3fb8daf9d30 100644 --- a/ogr/ogrsf_frmts/vrt/ogrvrtdatasource.cpp +++ b/ogr/ogrsf_frmts/vrt/ogrvrtdatasource.cpp @@ -48,101 +48,7 @@ #include "ogrunionlayer.h" #include "ogrwarpedlayer.h" #include "ogrsf_frmts.h" - -/************************************************************************/ -/* OGRVRTGetGeometryType() */ -/************************************************************************/ - -#define STRINGIFY(x) x, #x - -static const struct -{ - OGRwkbGeometryType eType; - const char *pszName; - bool bIsoFlags; -} asGeomTypeNames[] = { - {STRINGIFY(wkbUnknown), false}, - - {STRINGIFY(wkbPoint), false}, - {STRINGIFY(wkbLineString), false}, - {STRINGIFY(wkbPolygon), false}, - {STRINGIFY(wkbMultiPoint), false}, - {STRINGIFY(wkbMultiLineString), false}, - {STRINGIFY(wkbMultiPolygon), false}, - {STRINGIFY(wkbGeometryCollection), false}, - - {STRINGIFY(wkbCircularString), true}, - {STRINGIFY(wkbCompoundCurve), true}, - {STRINGIFY(wkbCurvePolygon), true}, - {STRINGIFY(wkbMultiCurve), true}, - {STRINGIFY(wkbMultiSurface), true}, - {STRINGIFY(wkbCurve), true}, - {STRINGIFY(wkbSurface), true}, - {STRINGIFY(wkbPolyhedralSurface), true}, - {STRINGIFY(wkbTIN), true}, - {STRINGIFY(wkbTriangle), true}, - - {STRINGIFY(wkbNone), false}, - {STRINGIFY(wkbLinearRing), false}, -}; - -OGRwkbGeometryType OGRVRTGetGeometryType(const char *pszGType, int *pbError) -{ - if (pbError) - *pbError = FALSE; - - for (const auto &entry : asGeomTypeNames) - { - if (EQUALN(pszGType, entry.pszName, strlen(entry.pszName))) - { - OGRwkbGeometryType eGeomType = entry.eType; - - if (strstr(pszGType, "25D") != nullptr || - strstr(pszGType, "Z") != nullptr) - eGeomType = wkbSetZ(eGeomType); - if (pszGType[strlen(pszGType) - 1] == 'M' || - pszGType[strlen(pszGType) - 2] == 'M') - eGeomType = wkbSetM(eGeomType); - return eGeomType; - } - } - - if (pbError) - *pbError = TRUE; - return wkbUnknown; -} - -/************************************************************************/ -/* OGRVRTGetSerializedGeometryType() */ -/************************************************************************/ - -CPLString OGRVRTGetSerializedGeometryType(OGRwkbGeometryType eGeomType) -{ - for (const auto &entry : asGeomTypeNames) - { - if (entry.eType == wkbFlatten(eGeomType)) - { - CPLString osRet(entry.pszName); - if (entry.bIsoFlags || OGR_GT_HasM(eGeomType)) - { - if (OGR_GT_HasZ(eGeomType)) - { - osRet += "Z"; - } - if (OGR_GT_HasM(eGeomType)) - { - osRet += "M"; - } - } - else if (OGR_GT_HasZ(eGeomType)) - { - osRet += "25D"; - } - return osRet; - } - } - return CPLString(); -} +#include "ogrvrtgeometrytypes.h" /************************************************************************/ /* OGRVRTDataSource() */ diff --git a/ogr/ogrsf_frmts/vrt/ogrvrtlayer.cpp b/ogr/ogrsf_frmts/vrt/ogrvrtlayer.cpp index af22aa41c077..94f39b53875d 100644 --- a/ogr/ogrsf_frmts/vrt/ogrvrtlayer.cpp +++ b/ogr/ogrsf_frmts/vrt/ogrvrtlayer.cpp @@ -52,8 +52,8 @@ #include "ogr_geometry.h" #include "ogr_spatialref.h" #include "ogrpgeogeometry.h" -#include "ogrsf_frmts/ogrsf_frmts.h" -#include "ogrsf_frmts/vrt/ogr_vrt.h" +#include "ogrsf_frmts.h" +#include "ogrvrtgeometrytypes.h" #define UNSUPPORTED_OP_READ_ONLY \ "%s : unsupported operation on a read-only datasource." diff --git a/ogr/ogrvrtgeometrytypes.cpp b/ogr/ogrvrtgeometrytypes.cpp new file mode 100644 index 000000000000..86c15c91e09d --- /dev/null +++ b/ogr/ogrvrtgeometrytypes.cpp @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: MIT +// Copyright 2024, Even Rouault + +#include "ogrvrtgeometrytypes.h" + +/************************************************************************/ +/* OGRVRTGetGeometryType() */ +/************************************************************************/ + +#define STRINGIFY(x) x, #x + +static const struct +{ + OGRwkbGeometryType eType; + const char *pszName; + bool bIsoFlags; +} asGeomTypeNames[] = { + {STRINGIFY(wkbUnknown), false}, + + {STRINGIFY(wkbPoint), false}, + {STRINGIFY(wkbLineString), false}, + {STRINGIFY(wkbPolygon), false}, + {STRINGIFY(wkbMultiPoint), false}, + {STRINGIFY(wkbMultiLineString), false}, + {STRINGIFY(wkbMultiPolygon), false}, + {STRINGIFY(wkbGeometryCollection), false}, + + {STRINGIFY(wkbCircularString), true}, + {STRINGIFY(wkbCompoundCurve), true}, + {STRINGIFY(wkbCurvePolygon), true}, + {STRINGIFY(wkbMultiCurve), true}, + {STRINGIFY(wkbMultiSurface), true}, + {STRINGIFY(wkbCurve), true}, + {STRINGIFY(wkbSurface), true}, + {STRINGIFY(wkbPolyhedralSurface), true}, + {STRINGIFY(wkbTIN), true}, + {STRINGIFY(wkbTriangle), true}, + + {STRINGIFY(wkbNone), false}, + {STRINGIFY(wkbLinearRing), false}, +}; + +OGRwkbGeometryType OGRVRTGetGeometryType(const char *pszGType, int *pbError) +{ + if (pbError) + *pbError = FALSE; + + for (const auto &entry : asGeomTypeNames) + { + if (EQUALN(pszGType, entry.pszName, strlen(entry.pszName))) + { + OGRwkbGeometryType eGeomType = entry.eType; + + if (strstr(pszGType, "25D") != nullptr || + strstr(pszGType, "Z") != nullptr) + eGeomType = wkbSetZ(eGeomType); + if (pszGType[strlen(pszGType) - 1] == 'M' || + pszGType[strlen(pszGType) - 2] == 'M') + eGeomType = wkbSetM(eGeomType); + return eGeomType; + } + } + + if (pbError) + *pbError = TRUE; + return wkbUnknown; +} + +/************************************************************************/ +/* OGRVRTGetSerializedGeometryType() */ +/************************************************************************/ + +std::string OGRVRTGetSerializedGeometryType(OGRwkbGeometryType eGeomType) +{ + for (const auto &entry : asGeomTypeNames) + { + if (entry.eType == wkbFlatten(eGeomType)) + { + std::string osRet(entry.pszName); + if (entry.bIsoFlags || OGR_GT_HasM(eGeomType)) + { + if (OGR_GT_HasZ(eGeomType)) + { + osRet += "Z"; + } + if (OGR_GT_HasM(eGeomType)) + { + osRet += "M"; + } + } + else if (OGR_GT_HasZ(eGeomType)) + { + osRet += "25D"; + } + return osRet; + } + } + return std::string(); +} diff --git a/ogr/ogrvrtgeometrytypes.h b/ogr/ogrvrtgeometrytypes.h new file mode 100644 index 000000000000..a5f055b580a5 --- /dev/null +++ b/ogr/ogrvrtgeometrytypes.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT +// Copyright 2024, Even Rouault + +#ifndef OGR_VRT_GEOMETRY_TYPES_H +#define OGR_VRT_GEOMETRY_TYPES_H + +#include "ogr_api.h" + +#include + +OGRwkbGeometryType CPL_DLL OGRVRTGetGeometryType(const char *pszGType, + int *pbError); +std::string CPL_DLL +OGRVRTGetSerializedGeometryType(OGRwkbGeometryType eGeomType); + +#endif From feced73d39066cf652af49ec996d03e454535a5f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 21 Sep 2024 23:27:50 +0200 Subject: [PATCH 201/710] Make KML driver optional and buildable as plugin --- Doxyfile | 1 - autotest/utilities/test_ogrlineref.py | 1 + ogr/CMakeLists.txt | 1 + ogr/{ogrsf_frmts/kml => }/ogr2kmlgeometry.cpp | 0 ogr/ogrsf_frmts/CMakeLists.txt | 2 +- ogr/ogrsf_frmts/kml/CMakeLists.txt | 7 ++++--- 6 files changed, 7 insertions(+), 5 deletions(-) rename ogr/{ogrsf_frmts/kml => }/ogr2kmlgeometry.cpp (100%) diff --git a/Doxyfile b/Doxyfile index 4bf006b3802a..c444e3aec4f8 100644 --- a/Doxyfile +++ b/Doxyfile @@ -389,7 +389,6 @@ INPUT = port \ ogr/ogrsf_frmts \ ogr/ogrsf_frmts/generic \ ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.cpp \ - ogr/ogrsf_frmts/kml/ogr2kmlgeometry.cpp \ swig/python/gdal-utils/scripts \ gnm \ doxygen_index.md diff --git a/autotest/utilities/test_ogrlineref.py b/autotest/utilities/test_ogrlineref.py index 2ed5eeee0a24..2a251870d828 100755 --- a/autotest/utilities/test_ogrlineref.py +++ b/autotest/utilities/test_ogrlineref.py @@ -123,6 +123,7 @@ def test_ogrlineref_4(ogrlineref_path, parts_shp, tmp_path): # test kml +@pytest.mark.require_driver("KML") def test_ogrlineref_5(ogrlineref_path, tmp_path): parts_kml = str(tmp_path / "parts.kml") diff --git a/ogr/CMakeLists.txt b/ogr/CMakeLists.txt index b7ddeae3e052..c515a8eccfed 100644 --- a/ogr/CMakeLists.txt +++ b/ogr/CMakeLists.txt @@ -63,6 +63,7 @@ add_library( ogr_proj_p.cpp ogr_wkb.cpp ogrvrtgeometrytypes.cpp + ogr2kmlgeometry.cpp ) add_dependencies(ogr generate_gdal_version_h) diff --git a/ogr/ogrsf_frmts/kml/ogr2kmlgeometry.cpp b/ogr/ogr2kmlgeometry.cpp similarity index 100% rename from ogr/ogrsf_frmts/kml/ogr2kmlgeometry.cpp rename to ogr/ogr2kmlgeometry.cpp diff --git a/ogr/ogrsf_frmts/CMakeLists.txt b/ogr/ogrsf_frmts/CMakeLists.txt index 501dda1ba9e8..194a5c0e5ad6 100644 --- a/ogr/ogrsf_frmts/CMakeLists.txt +++ b/ogr/ogrsf_frmts/CMakeLists.txt @@ -17,7 +17,7 @@ ogr_default_driver2(mitab TAB "MapInfo TAB and MIF/MID") ogr_default_driver(shape "ESRI shape-file") # ###################################################################################################################### -ogr_default_driver(kml KML) # when not found EXPAT, it use CPL_MINIXML (ex. android) +ogr_optional_driver(kml KML) # when not found EXPAT, it use CPL_MINIXML (ex. android) ogr_optional_driver(vrt "VRT - Virtual Format") # Caution: if modifying AVC declaration here, also modify it in gdal.cmake diff --git a/ogr/ogrsf_frmts/kml/CMakeLists.txt b/ogr/ogrsf_frmts/kml/CMakeLists.txt index 76b98e13c705..d1effa36f2df 100644 --- a/ogr/ogrsf_frmts/kml/CMakeLists.txt +++ b/ogr/ogrsf_frmts/kml/CMakeLists.txt @@ -1,8 +1,9 @@ -# gdal core depends on kml driver, so it should not be a plugin. add_gdal_driver( TARGET ogr_KML - SOURCES kmlutility.h ogr_kml.h ogrkmldriver.cpp ogr2kmlgeometry.cpp ogrkmldatasource.cpp ogrkmllayer.cpp - BUILTIN) + SOURCES kmlutility.h ogr_kml.h ogrkmldriver.cpp ogrkmldatasource.cpp ogrkmllayer.cpp + PLUGIN_CAPABLE + NO_DEPS +) gdal_standard_includes(ogr_KML) if (GDAL_USE_EXPAT) From b4ca4d36dd4ce05fe4337ae315dacc0e8eb18dcd Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 21 Sep 2024 23:59:36 +0200 Subject: [PATCH 202/710] autotest/cpp: make them succeed without Shapefile driver --- autotest/cpp/test_ogr.cpp | 15 ++++++--------- autotest/cpp/test_ogr_shape.cpp | 4 +++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/autotest/cpp/test_ogr.cpp b/autotest/cpp/test_ogr.cpp index 8fe4e7469977..a97275eb8361 100644 --- a/autotest/cpp/test_ogr.cpp +++ b/autotest/cpp/test_ogr.cpp @@ -51,7 +51,6 @@ namespace // Common fixture with test data struct test_ogr : public ::testing::Test { - std::string drv_shape_{"ESRI Shapefile"}; std::string data_{tut::common::data_basedir}; std::string data_tmp_{tut::common::tmp_basedir}; }; @@ -62,14 +61,6 @@ TEST_F(test_ogr, GetGDALDriverManager) ASSERT_TRUE(nullptr != GetGDALDriverManager()); } -// Test if Shapefile driver is registered -TEST_F(test_ogr, Shapefile_driver) -{ - GDALDriver *drv = - GetGDALDriverManager()->GetDriverByName(drv_shape_.c_str()); - ASSERT_TRUE(nullptr != drv); -} - template void testSpatialReferenceLeakOnCopy(OGRSpatialReference *poSRS) { @@ -1333,6 +1324,12 @@ TEST_F(test_ogr, OGRToOGCGeomType) // Test layer, dataset-feature and layer-feature iterators TEST_F(test_ogr, DatasetFeature_and_LayerFeature_iterators) { + if (!GDALGetDriverByName("ESRI Shapefile")) + { + GTEST_SKIP() << "ESRI Shapefile driver missing"; + return; + } + std::string file(data_ + SEP + "poly.shp"); GDALDatasetUniquePtr poDS(GDALDataset::Open(file.c_str(), GDAL_OF_VECTOR)); ASSERT_TRUE(poDS != nullptr); diff --git a/autotest/cpp/test_ogr_shape.cpp b/autotest/cpp/test_ogr_shape.cpp index f70c05c0cf64..697ee741a819 100644 --- a/autotest/cpp/test_ogr_shape.cpp +++ b/autotest/cpp/test_ogr_shape.cpp @@ -63,6 +63,8 @@ struct test_ogr_shape : public ::testing::Test void TearDown() override { + if (!drv_) + return; OGRDataSourceH ds = OGR_Dr_CreateDataSource(drv_, data_tmp_.c_str(), nullptr); if (ds == nullptr) @@ -545,7 +547,7 @@ TEST_F(test_ogr_shape, spatial_filtering) OGR_DS_Destroy(ds); } -TEST(test_ogr_shape_gdal, create) +TEST_F(test_ogr_shape, create_gdal) { GDALDriver *shpDriver = GetGDALDriverManager()->GetDriverByName("ESRI Shapefile"); From 966b01c3b94a0716146a762b331d0cdf5dfd6ba9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 00:01:17 +0200 Subject: [PATCH 203/710] Make Shapefile driver optional and buildable as plugin --- .github/workflows/cmake_builds.yml | 8 +++ ogr/ogrsf_frmts/CMakeLists.txt | 2 +- ogr/ogrsf_frmts/avc/CMakeLists.txt | 14 +++-- ogr/ogrsf_frmts/avc/avc.h | 9 +--- ogr/ogrsf_frmts/avc/avc_bin.cpp | 74 +++++++++++++++++++++++++-- ogr/ogrsf_frmts/shape/CMakeLists.txt | 10 ++-- ogr/ogrsf_frmts/shape/dbfopen.c | 2 + ogr/ogrsf_frmts/shape/shapefil.h | 4 +- ogr/ogrsf_frmts/shape/shp_vsi.c | 76 +++++++++++++++------------- 9 files changed, 142 insertions(+), 57 deletions(-) diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index 2eee98263c9d..e91031a969c2 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -546,6 +546,14 @@ jobs: shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2 + - name: Re-enable shapefile driver (otherwise lots of python tests would fail) + shell: bash -l {0} + run: | + cmake -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DOGR_ENABLE_DRIVER_SHAPE=ON -DOGR_ENABLE_DRIVER_SHAPE_PLUGIN=ON + - name: Build + shell: bash -l {0} + run: cmake --build $GITHUB_WORKSPACE/build --config RelWithDebInfo -j 2 + # Works around https://github.com/actions/runner-images/issues/10055 - name: Remove conflicting libraries shell: bash -l {0} diff --git a/ogr/ogrsf_frmts/CMakeLists.txt b/ogr/ogrsf_frmts/CMakeLists.txt index 194a5c0e5ad6..2f0a35385595 100644 --- a/ogr/ogrsf_frmts/CMakeLists.txt +++ b/ogr/ogrsf_frmts/CMakeLists.txt @@ -14,7 +14,7 @@ add_dependencies(ogrsf_frmts generate_gdal_version_h) ogr_default_driver(mem "Read/write driver for MEMORY virtual files") ogr_default_driver(geojson "GeoJSON/ESRIJSON/TopoJSON driver") ogr_default_driver2(mitab TAB "MapInfo TAB and MIF/MID") -ogr_default_driver(shape "ESRI shape-file") +ogr_optional_driver(shape "ESRI shape-file") # ###################################################################################################################### ogr_optional_driver(kml KML) # when not found EXPAT, it use CPL_MINIXML (ex. android) diff --git a/ogr/ogrsf_frmts/avc/CMakeLists.txt b/ogr/ogrsf_frmts/avc/CMakeLists.txt index 555bfbee44dd..f49a478568f2 100644 --- a/ogr/ogrsf_frmts/avc/CMakeLists.txt +++ b/ogr/ogrsf_frmts/avc/CMakeLists.txt @@ -21,8 +21,12 @@ add_gdal_driver( BUILTIN) gdal_standard_includes(ogr_AVC) -if (GDAL_USE_SHAPELIB_INTERNAL) - gdal_add_vendored_lib(ogr_AVC shapelib) -else () - gdal_target_link_libraries(ogr_AVC PRIVATE SHAPELIB::shp) -endif () +if (NOT OGR_ENABLE_DRIVER_SHAPE OR OGR_ENABLE_DRIVER_SHAPE_PLUGIN) +target_compile_definitions(ogr_AVC PRIVATE -DWITHOUT_SHAPEFILE) +else() + if (GDAL_USE_SHAPELIB_INTERNAL) + gdal_add_vendored_lib(ogr_AVC shapelib) + else () + gdal_target_link_libraries(ogr_AVC PRIVATE SHAPELIB::shp) + endif () +endif() diff --git a/ogr/ogrsf_frmts/avc/avc.h b/ogr/ogrsf_frmts/avc/avc.h index 907920d7be1f..1ff47030c4e7 100644 --- a/ogr/ogrsf_frmts/avc/avc.h +++ b/ogr/ogrsf_frmts/avc/avc.h @@ -118,14 +118,7 @@ #include "cpl_conv.h" #include "cpl_string.h" -#ifdef GDAL_COMPILATION -#ifdef RENAME_INTERNAL_SHAPELIB_SYMBOLS -#include "gdal_shapelib_symbol_rename.h" -#endif -#include "shapefil.h" -#else -#include "dbfopen.h" -#endif +typedef struct DBFInfo *DBFHandle; #include "avc_mbyte.h" diff --git a/ogr/ogrsf_frmts/avc/avc_bin.cpp b/ogr/ogrsf_frmts/avc/avc_bin.cpp index 4da41c06a4e4..2280f62cf80f 100644 --- a/ogr/ogrsf_frmts/avc/avc_bin.cpp +++ b/ogr/ogrsf_frmts/avc/avc_bin.cpp @@ -133,6 +133,75 @@ #include /* for isspace() */ +#ifdef WITHOUT_SHAPEFILE + +#define SHP_VSI_ONLY_SETUP_HOOKS +#define SAOffset vsi_l_offset +#define SHPAPI_CAL + +#define DBFAddField OGRAVC_DBFAddField +#define DBFAddNativeFieldType OGRAVC_DBFAddNativeFieldType +#define DBFAlterFieldDefn OGRAVC_DBFAlterFieldDefn +#define DBFCloneEmpty OGRAVC_DBFCloneEmpty +#define DBFClose OGRAVC_DBFClose +#define DBFCreateEx OGRAVC_DBFCreateEx +#define DBFCreate OGRAVC_DBFCreate +#define DBFCreateLL OGRAVC_DBFCreateLL +#define DBFDeleteField OGRAVC_DBFDeleteField +#define DBFFlushRecord OGRAVC_DBFFlushRecord +#define DBFGetCodePage OGRAVC_DBFGetCodePage +#define DBFGetFieldCount OGRAVC_DBFGetFieldCount +#define DBFGetFieldIndex OGRAVC_DBFGetFieldIndex +#define DBFGetFieldInfo OGRAVC_DBFGetFieldInfo +#define DBFGetLenWithoutExtension OGRAVC_DBFGetLenWithoutExtension +#define DBFGetNativeFieldType OGRAVC_DBFGetNativeFieldType +#define DBFGetNullCharacter OGRAVC_DBFGetNullCharacter +#define DBFGetRecordCount OGRAVC_DBFGetRecordCount +#define DBFIsAttributeNULL OGRAVC_DBFIsAttributeNULL +#define DBFIsRecordDeleted OGRAVC_DBFIsRecordDeleted +#define DBFIsValueNULL OGRAVC_DBFIsValueNULL +#define DBFLoadRecord OGRAVC_DBFLoadRecord +#define DBFMarkRecordDeleted OGRAVC_DBFMarkRecordDeleted +#define DBFOpen OGRAVC_DBFOpen +#define DBFOpenLL OGRAVC_DBFOpenLL +#define DBFReadAttribute OGRAVC_DBFReadAttribute +#define DBFReadDoubleAttribute OGRAVC_DBFReadDoubleAttribute +#define DBFReadIntegerAttribute OGRAVC_DBFReadIntegerAttribute +#define DBFReadLogicalAttribute OGRAVC_DBFReadLogicalAttribute +#define DBFReadStringAttribute OGRAVC_DBFReadStringAttribute +#define DBFReadDateAttribute OGRAVC_DBFReadDateAttribute +#define DBFReadTuple OGRAVC_DBFReadTuple +#define DBFReorderFields OGRAVC_DBFReorderFields +#define DBFSetLastModifiedDate OGRAVC_DBFSetLastModifiedDate +#define DBFSetWriteEndOfFileChar OGRAVC_DBFSetWriteEndOfFileChar +#define DBFUpdateHeader OGRAVC_DBFUpdateHeader +#define DBFWriteAttributeDirectly OGRAVC_DBFWriteAttributeDirectly +#define DBFWriteAttribute OGRAVC_DBFWriteAttribute +#define DBFWriteDoubleAttribute OGRAVC_DBFWriteDoubleAttribute +#define DBFWriteHeader OGRAVC_DBFWriteHeader +#define DBFWriteIntegerAttribute OGRAVC_DBFWriteIntegerAttribute +#define DBFWriteLogicalAttribute OGRAVC_DBFWriteLogicalAttribute +#define DBFWriteNULLAttribute OGRAVC_DBFWriteNULLAttribute +#define DBFWriteStringAttribute OGRAVC_DBFWriteStringAttribute +#define DBFWriteDateAttribute OGRAVC_DBFWriteDateAttribute +#define DBFWriteTuple OGRAVC_DBFWriteTuple + +#define VSI_SHP_WriteMoreDataOK OGRAVC_VSI_SHP_WriteMoreDataOK +#define SASetupDefaultHooks OGRAVC_SASetupDefaultHooks + +#include "../shape/shapefil.h" +#include "../shape/dbfopen.c" +#include "../shape/shp_vsi.c" + +#else + +#ifdef RENAME_INTERNAL_SHAPELIB_SYMBOLS +#include "gdal_shapelib_symbol_rename.h" +#endif +#include "shapefil.h" + +#endif // WITHOUT_SHAPEFILE + /* Used by avc_binwr.c */ extern int _AVCBinReadNextArcDir(AVCRawBinFile *psFile, AVCTableDef *psArcDir); @@ -2454,9 +2523,8 @@ AVCBinFile *_AVCBinReadOpenDBFTable(const char *pszDBFFilename, for (iField = 0; iField < psTableDef->numFields; iField++) { - int nWidth, nDecimals; + int nWidth = 0, nDecimals = 0; /* DBFFieldType eDBFType; */ - char cNativeType; /*------------------------------------------------------------- * Fetch DBF Field info and convert to Arc/Info type... @@ -2466,7 +2534,7 @@ AVCBinFile *_AVCBinReadOpenDBFTable(const char *pszDBFFilename, /* eDBFType = */ DBFGetFieldInfo(hDBFFile, iField, pasFieldDef[iField].szName, &nWidth, &nDecimals); - cNativeType = DBFGetNativeFieldType(hDBFFile, iField); + const char cNativeType = DBFGetNativeFieldType(hDBFFile, iField); pasFieldDef[iField].nFmtWidth = (GInt16)nWidth; pasFieldDef[iField].nFmtPrec = (GInt16)nDecimals; diff --git a/ogr/ogrsf_frmts/shape/CMakeLists.txt b/ogr/ogrsf_frmts/shape/CMakeLists.txt index 52f74314eeb6..706f8834ad97 100644 --- a/ogr/ogrsf_frmts/shape/CMakeLists.txt +++ b/ogr/ogrsf_frmts/shape/CMakeLists.txt @@ -1,8 +1,8 @@ -# depends from frmts/aigrid, avc add_gdal_driver( TARGET ogr_Shape SOURCES shape2ogr.cpp shp_vsi.c ogrshapedatasource.cpp ogrshapedriver.cpp ogrshapelayer.cpp - BUILTIN + PLUGIN_CAPABLE + NO_DEPS STRONG_CXX_WFLAGS ) gdal_standard_includes(ogr_Shape) @@ -20,7 +20,11 @@ if (GDAL_USE_SHAPELIB_INTERNAL) endif () target_compile_definitions(shapelib PUBLIC -DUSE_CPL) set_property(TARGET shapelib PROPERTY POSITION_INDEPENDENT_CODE ${GDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) - target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) + if (OGR_ENABLE_DRIVER_SHAPE_PLUGIN) + target_sources(ogr_Shape PRIVATE $) + else() + target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) + endif() gdal_add_vendored_lib(ogr_Shape shapelib) elseif (HAVE_SHAPELIB) gdal_target_link_libraries(ogr_Shape PRIVATE SHAPELIB::shp) diff --git a/ogr/ogrsf_frmts/shape/dbfopen.c b/ogr/ogrsf_frmts/shape/dbfopen.c index 90ed7ea8e9c8..8bac07247992 100644 --- a/ogr/ogrsf_frmts/shape/dbfopen.c +++ b/ogr/ogrsf_frmts/shape/dbfopen.c @@ -24,6 +24,7 @@ #include "cpl_string.h" #else +#ifndef STRCASECMP #if defined(_MSC_VER) #define STRCASECMP(a, b) (_stricmp(a, b)) #elif defined(_WIN32) @@ -32,6 +33,7 @@ #include #define STRCASECMP(a, b) (strcasecmp(a, b)) #endif +#endif #if defined(_MSC_VER) #if _MSC_VER < 1900 diff --git a/ogr/ogrsf_frmts/shape/shapefil.h b/ogr/ogrsf_frmts/shape/shapefil.h index 308dc6416cc1..061980e8391c 100644 --- a/ogr/ogrsf_frmts/shape/shapefil.h +++ b/ogr/ogrsf_frmts/shape/shapefil.h @@ -419,7 +419,7 @@ extern "C" /************************************************************************/ /* DBF Support. */ /************************************************************************/ - typedef struct + typedef struct DBFInfo { SAHooks sHooks; @@ -467,7 +467,7 @@ extern "C" int bRequireNextWriteSeek; } DBFInfo; - typedef DBFInfo *DBFHandle; + typedef struct DBFInfo *DBFHandle; typedef enum { diff --git a/ogr/ogrsf_frmts/shape/shp_vsi.c b/ogr/ogrsf_frmts/shape/shp_vsi.c index 1185145417e6..76a719a2197d 100644 --- a/ogr/ogrsf_frmts/shape/shp_vsi.c +++ b/ogr/ogrsf_frmts/shape/shp_vsi.c @@ -34,6 +34,8 @@ #include "cpl_vsi_error.h" #include +#include "shapefil_private.h" + typedef struct { VSILFILE *fp; @@ -43,26 +45,6 @@ typedef struct SAOffset nCurOffset; } OGRSHPDBFFile; -/************************************************************************/ -/* VSI_SHP_GetVSIL() */ -/************************************************************************/ - -VSILFILE *VSI_SHP_GetVSIL(SAFile file) -{ - OGRSHPDBFFile *pFile = (OGRSHPDBFFile *)file; - return pFile->fp; -} - -/************************************************************************/ -/* VSI_SHP_GetFilename() */ -/************************************************************************/ - -const char *VSI_SHP_GetFilename(SAFile file) -{ - OGRSHPDBFFile *pFile = (OGRSHPDBFFile *)file; - return pFile->pszFilename; -} - /************************************************************************/ /* VSI_SHP_OpenInternal() */ /************************************************************************/ @@ -73,8 +55,8 @@ static SAFile VSI_SHP_OpenInternal(const char *pszFilename, { OGRSHPDBFFile *pFile; VSILFILE *fp = VSIFOpenExL(pszFilename, pszAccess, TRUE); - if (fp == NULL) - return NULL; + if (fp == SHPLIB_NULLPTR) + return SHPLIB_NULLPTR; pFile = (OGRSHPDBFFile *)CPLCalloc(1, sizeof(OGRSHPDBFFile)); pFile->fp = fp; pFile->pszFilename = CPLStrdup(pszFilename); @@ -95,18 +77,6 @@ static SAFile VSI_SHP_Open(const char *pszFilename, const char *pszAccess, return VSI_SHP_OpenInternal(pszFilename, pszAccess, FALSE); } -/************************************************************************/ -/* VSI_SHP_Open2GBLimit() */ -/************************************************************************/ - -static SAFile VSI_SHP_Open2GBLimit(const char *pszFilename, - const char *pszAccess, void *userData) - -{ - (void)userData; - return VSI_SHP_OpenInternal(pszFilename, pszAccess, TRUE); -} - /************************************************************************/ /* VSI_SHP_Read() */ /************************************************************************/ @@ -259,7 +229,41 @@ void SASetupDefaultHooks(SAHooks *psHooks) psHooks->Error = VSI_SHP_Error; psHooks->Atof = CPLAtof; - psHooks->pvUserData = NULL; + psHooks->pvUserData = SHPLIB_NULLPTR; +} + +#ifndef SHP_VSI_ONLY_SETUP_HOOKS + +/************************************************************************/ +/* VSI_SHP_GetVSIL() */ +/************************************************************************/ + +VSILFILE *VSI_SHP_GetVSIL(SAFile file) +{ + OGRSHPDBFFile *pFile = (OGRSHPDBFFile *)file; + return pFile->fp; +} + +/************************************************************************/ +/* VSI_SHP_GetFilename() */ +/************************************************************************/ + +const char *VSI_SHP_GetFilename(SAFile file) +{ + OGRSHPDBFFile *pFile = (OGRSHPDBFFile *)file; + return pFile->pszFilename; +} + +/************************************************************************/ +/* VSI_SHP_Open2GBLimit() */ +/************************************************************************/ + +static SAFile VSI_SHP_Open2GBLimit(const char *pszFilename, + const char *pszAccess, void *userData) + +{ + (void)userData; + return VSI_SHP_OpenInternal(pszFilename, pszAccess, TRUE); } /************************************************************************/ @@ -280,3 +284,5 @@ const SAHooks *VSI_SHP_GetHook(int b2GBLimit) { return (b2GBLimit) ? &sOGRHook2GBLimit : &sOGRHook; } + +#endif From b2ae27f9cca74839ff5a303b283151b4eba74046 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 01:25:02 +0200 Subject: [PATCH 204/710] Fixes for CMAKE_UNITY_BUILD --- ogr/ogr2kmlgeometry.cpp | 4 ++-- ogr/ogr_wkb.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ogr/ogr2kmlgeometry.cpp b/ogr/ogr2kmlgeometry.cpp index 329ef74ed2f1..e6a2f77931d1 100644 --- a/ogr/ogr2kmlgeometry.cpp +++ b/ogr/ogr2kmlgeometry.cpp @@ -43,8 +43,6 @@ #include "ogr_geometry.h" #include "ogr_p.h" -constexpr double EPSILON = 1e-8; - /************************************************************************/ /* MakeKMLCoordinate() */ /************************************************************************/ @@ -53,6 +51,8 @@ static void MakeKMLCoordinate(char *pszTarget, size_t nTargetLen, double x, double y, double z, bool b3D) { + constexpr double EPSILON = 1e-8; + if (y < -90 || y > 90) { if (y > 90 && y < 90 + EPSILON) diff --git a/ogr/ogr_wkb.cpp b/ogr/ogr_wkb.cpp index c2b39d03b3ce..d128c16794bb 100644 --- a/ogr/ogr_wkb.cpp +++ b/ogr/ogr_wkb.cpp @@ -828,8 +828,6 @@ bool OGRWKBIntersectsPessimistic(const GByte *pabyWkb, size_t nWKBSize, /* epsilonEqual() */ /************************************************************************/ -constexpr double EPSILON = 1.0E-5; - static inline bool epsilonEqual(double a, double b, double eps) { return ::fabs(a - b) < eps; @@ -864,6 +862,8 @@ static inline double GetY(const GByte *data, uint32_t i, int nDim, static bool OGRWKBIsClockwiseRing(const GByte *data, const uint32_t nPoints, const int nDim, const bool bNeedSwap) { + constexpr double EPSILON = 1.0E-5; + // WARNING: keep in sync OGRLineString::isClockwise(), // OGRCurve::isClockwise() and OGRWKBIsClockwiseRing() From 996230db4117960a9f57d11cabade3256cd59b78 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 01:12:12 +0200 Subject: [PATCH 205/710] Preparatory work to make GeoJSON driver optional --- Doxyfile | 1 - apps/CMakeLists.txt | 6 +- apps/gdalinfo_lib.cpp | 3 +- apps/ogrinfo_lib.cpp | 2 +- frmts/eeda/CMakeLists.txt | 1 - frmts/eeda/eedacommon.cpp | 2 +- frmts/eeda/eedadataset.cpp | 4 +- frmts/eeda/eedaidataset.cpp | 3 +- frmts/mbtiles/CMakeLists.txt | 2 +- frmts/mbtiles/mbtilesdataset.cpp | 2 +- frmts/plmosaic/CMakeLists.txt | 2 +- frmts/plmosaic/plmosaicdataset.cpp | 3 +- gcore/CMakeLists.txt | 2 +- gcore/gdal_rat.cpp | 2 +- gcore/gdaljp2metadata.cpp | 2 +- gcore/nasakeywordhandler.cpp | 2 +- ogr/CMakeLists.txt | 6 +- ogr/ogresrijsongeometry.cpp | 649 +++++++++++++ ogr/ogresrijsongeometry.h | 26 + ogr/ogrfeature.cpp | 2 +- ogr/ogrgeojsongeometry.cpp | 861 +++++++++++++++++ ogr/ogrgeojsongeometry.h | 65 ++ .../geojson => }/ogrgeojsonwriter.cpp | 322 +++--- .../geojson => }/ogrgeojsonwriter.h | 73 +- ogr/ogrgeometryfactory.cpp | 3 +- ogr/ogrlibjsonutils.cpp | 426 ++++++++ ogr/ogrlibjsonutils.h | 53 + ogr/ogrsf_frmts/amigocloud/CMakeLists.txt | 3 +- .../amigocloud/ogramigoclouddatasource.cpp | 2 +- .../amigocloud/ogramigocloudlayer.cpp | 2 +- .../amigocloud/ogramigocloudtablelayer.cpp | 2 +- ogr/ogrsf_frmts/carto/CMakeLists.txt | 3 +- ogr/ogrsf_frmts/carto/ogrcartodatasource.cpp | 2 +- ogr/ogrsf_frmts/carto/ogrcartolayer.cpp | 2 +- ogr/ogrsf_frmts/carto/ogrcartotablelayer.cpp | 2 +- ogr/ogrsf_frmts/elastic/CMakeLists.txt | 1 - .../elastic/ogrelasticaggregationlayer.cpp | 2 +- .../elastic/ogrelasticdatasource.cpp | 2 +- ogr/ogrsf_frmts/elastic/ogrelasticlayer.cpp | 6 +- ogr/ogrsf_frmts/geojson/CMakeLists.txt | 1 - ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp | 634 +----------- .../geojson/ogrgeojsondatasource.cpp | 1 + ogr/ogrsf_frmts/geojson/ogrgeojsonreader.cpp | 913 +----------------- ogr/ogrsf_frmts/geojson/ogrgeojsonreader.h | 72 -- .../geojson/ogrgeojsonseqdriver.cpp | 2 + ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp | 185 ---- ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h | 14 - .../ogrjsoncollectionstreamingparser.cpp | 4 +- ogr/ogrsf_frmts/geojson/ogrtopojsonreader.cpp | 1 + ogr/ogrsf_frmts/gmlas/CMakeLists.txt | 2 +- ogr/ogrsf_frmts/gmlas/ogrgmlaswriter.cpp | 2 +- ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp | 2 + .../jsonfg/ogrjsonfgwritelayer.cpp | 2 +- ogr/ogrsf_frmts/plscenes/CMakeLists.txt | 1 - ogr/ogrsf_frmts/plscenes/ogr_plscenes.h | 3 +- .../plscenes/ogrplscenesdatav1dataset.cpp | 2 +- .../plscenes/ogrplscenesdatav1layer.cpp | 4 +- ogr/ogrsf_frmts/xlsx/CMakeLists.txt | 2 +- 58 files changed, 2266 insertions(+), 2135 deletions(-) create mode 100644 ogr/ogresrijsongeometry.cpp create mode 100644 ogr/ogresrijsongeometry.h create mode 100644 ogr/ogrgeojsongeometry.cpp create mode 100644 ogr/ogrgeojsongeometry.h rename ogr/{ogrsf_frmts/geojson => }/ogrgeojsonwriter.cpp (91%) rename ogr/{ogrsf_frmts/geojson => }/ogrgeojsonwriter.h (55%) create mode 100644 ogr/ogrlibjsonutils.cpp create mode 100644 ogr/ogrlibjsonutils.h diff --git a/Doxyfile b/Doxyfile index c444e3aec4f8..1fd627a3489d 100644 --- a/Doxyfile +++ b/Doxyfile @@ -388,7 +388,6 @@ INPUT = port \ ogr \ ogr/ogrsf_frmts \ ogr/ogrsf_frmts/generic \ - ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.cpp \ swig/python/gdal-utils/scripts \ gnm \ doxygen_index.md diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 4ce116ae1b01..667cb229374c 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -28,7 +28,7 @@ gdal_standard_includes(appslib) target_compile_options(appslib PRIVATE ${GDAL_CXX_WARNING_FLAGS} ${WFLAG_OLD_STYLE_CAST}) target_include_directories( appslib PRIVATE $ $ - $ $) + $) gdal_target_link_libraries(appslib PRIVATE PROJ::proj) @@ -132,7 +132,7 @@ if (BUILD_APPS) add_dependencies(${UTILCMD} generate_gdal_version_h) target_include_directories( ${UTILCMD} PRIVATE $ $ - $ $) + $) if (MSVC) set_target_properties(${UTILCMD} PROPERTIES LINK_FLAGS "wsetargv.obj") endif () @@ -200,7 +200,7 @@ if (BUILD_APPS) endif () target_include_directories( ${UTILCMD} PRIVATE $ $ - $ $) + $) target_link_libraries(${UTILCMD} PRIVATE $) endforeach () endif () diff --git a/apps/gdalinfo_lib.cpp b/apps/gdalinfo_lib.cpp index e61ef39ff8e5..32dfb6c3b191 100644 --- a/apps/gdalinfo_lib.cpp +++ b/apps/gdalinfo_lib.cpp @@ -58,7 +58,8 @@ #include "ogr_api.h" #include "ogr_srs_api.h" #include "ogr_spatialref.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" +#include "ogrgeojsongeometry.h" #include "ogrgeojsonwriter.h" using std::vector; diff --git a/apps/ogrinfo_lib.cpp b/apps/ogrinfo_lib.cpp index b227370ddbb9..6d2497f6d15b 100644 --- a/apps/ogrinfo_lib.cpp +++ b/apps/ogrinfo_lib.cpp @@ -29,7 +29,7 @@ #include "cpl_port.h" #include "cpl_json.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include "cpl_string.h" #include "gdal_utils.h" #include "gdal_utils_priv.h" diff --git a/frmts/eeda/CMakeLists.txt b/frmts/eeda/CMakeLists.txt index 716ae8385688..d531229bb5b6 100644 --- a/frmts/eeda/CMakeLists.txt +++ b/frmts/eeda/CMakeLists.txt @@ -8,7 +8,6 @@ add_gdal_driver( "NOT GDAL_USE_JSONC_INTERNAL" NO_DEPS) gdal_standard_includes(gdal_EEDA) -target_include_directories(gdal_EEDA PRIVATE $) set(GDAL_DATA_FILES LICENSE.TXT diff --git a/frmts/eeda/eedacommon.cpp b/frmts/eeda/eedacommon.cpp index 0ba2c6a82492..3200a55da979 100644 --- a/frmts/eeda/eedacommon.cpp +++ b/frmts/eeda/eedacommon.cpp @@ -28,7 +28,7 @@ #include "cpl_http.h" #include "eeda.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include #include diff --git a/frmts/eeda/eedadataset.cpp b/frmts/eeda/eedadataset.cpp index f41c4b4781d3..7d71b0582370 100644 --- a/frmts/eeda/eedadataset.cpp +++ b/frmts/eeda/eedadataset.cpp @@ -27,10 +27,10 @@ ****************************************************************************/ #include "gdal_priv.h" +#include "ogrsf_frmts.h" #include "cpl_http.h" #include "cpl_conv.h" -#include "ogrgeojsonreader.h" -#include "ogrgeojsonwriter.h" +#include "ogrlibjsonutils.h" #include "ogr_swq.h" #include "eeda.h" diff --git a/frmts/eeda/eedaidataset.cpp b/frmts/eeda/eedaidataset.cpp index e6df9d0d2a8a..60c466158ffc 100644 --- a/frmts/eeda/eedaidataset.cpp +++ b/frmts/eeda/eedaidataset.cpp @@ -29,8 +29,7 @@ #include "gdal_priv.h" #include "cpl_http.h" #include "cpl_conv.h" -#include "ogrgeojsonreader.h" -#include "ogrgeojsonwriter.h" +#include "ogrlibjsonutils.h" #include "eeda.h" #include diff --git a/frmts/mbtiles/CMakeLists.txt b/frmts/mbtiles/CMakeLists.txt index 856a030fd6e2..f1850f94c2b7 100644 --- a/frmts/mbtiles/CMakeLists.txt +++ b/frmts/mbtiles/CMakeLists.txt @@ -26,5 +26,5 @@ if (NOT GDAL_USE_JSONC_INTERNAL) gdal_target_link_libraries(gdal_MBTiles PRIVATE ${JSONC_TARGET}) endif () target_include_directories( - gdal_MBTiles PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/geojson ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/gpkg + gdal_MBTiles PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/gpkg ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/sqlite) diff --git a/frmts/mbtiles/mbtilesdataset.cpp b/frmts/mbtiles/mbtilesdataset.cpp index 6ef43a125b20..0f1e04659fd3 100644 --- a/frmts/mbtiles/mbtilesdataset.cpp +++ b/frmts/mbtiles/mbtilesdataset.cpp @@ -44,7 +44,7 @@ #include "ogrsqlitebase.h" #include "zlib.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include #include diff --git a/frmts/plmosaic/CMakeLists.txt b/frmts/plmosaic/CMakeLists.txt index d2ac81f9a4e0..ff7ca47643d1 100644 --- a/frmts/plmosaic/CMakeLists.txt +++ b/frmts/plmosaic/CMakeLists.txt @@ -1,7 +1,7 @@ add_gdal_driver(TARGET gdal_PLMOSAIC SOURCES plmosaicdataset.cpp PLUGIN_CAPABLE_IF "NOT GDAL_USE_JSONC_INTERNAL" NO_DEPS) gdal_standard_includes(gdal_PLMOSAIC) -target_include_directories(gdal_PLMOSAIC PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/geojson) + if (GDAL_USE_JSONC_INTERNAL) gdal_add_vendored_lib(gdal_PLMOSAIC libjson) else () diff --git a/frmts/plmosaic/plmosaicdataset.cpp b/frmts/plmosaic/plmosaicdataset.cpp index fd759dd1a60c..f00353119a00 100644 --- a/frmts/plmosaic/plmosaicdataset.cpp +++ b/frmts/plmosaic/plmosaicdataset.cpp @@ -34,8 +34,7 @@ #include "ogr_spatialref.h" #include "ogrsf_frmts.h" #include "../vrt/gdal_vrt.h" - -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include diff --git a/gcore/CMakeLists.txt b/gcore/CMakeLists.txt index a629c7f163b2..d91dc6bb4d60 100644 --- a/gcore/CMakeLists.txt +++ b/gcore/CMakeLists.txt @@ -69,7 +69,7 @@ add_dependencies(gcore generate_gdal_version_h) target_compile_options(gcore PRIVATE ${GDAL_CXX_WARNING_FLAGS} ${WFLAG_OLD_STYLE_CAST} ${WFLAG_EFFCXX}) target_include_directories( gcore - PRIVATE $ $ + PRIVATE $ $ $ $) set_property(TARGET gcore PROPERTY POSITION_INDEPENDENT_CODE ${GDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) diff --git a/gcore/gdal_rat.cpp b/gcore/gdal_rat.cpp index a9e8a5cdaaa5..a881b2c76d11 100644 --- a/gcore/gdal_rat.cpp +++ b/gcore/gdal_rat.cpp @@ -54,7 +54,7 @@ #ifdef __clang__ #pragma clang diagnostic pop #endif -#include "ogrgeojsonwriter.h" +#include "ogrlibjsonutils.h" /** * \class GDALRasterAttributeTable diff --git a/gcore/gdaljp2metadata.cpp b/gcore/gdaljp2metadata.cpp index 3852ecd93fd1..b7f91169ec94 100644 --- a/gcore/gdaljp2metadata.cpp +++ b/gcore/gdaljp2metadata.cpp @@ -57,7 +57,7 @@ #include "ogr_core.h" #include "ogr_geometry.h" #include "ogr_spatialref.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" /*! @cond Doxygen_Suppress */ diff --git a/gcore/nasakeywordhandler.cpp b/gcore/nasakeywordhandler.cpp index 5656f26f6618..6b8c6b385345 100644 --- a/gcore/nasakeywordhandler.cpp +++ b/gcore/nasakeywordhandler.cpp @@ -53,7 +53,7 @@ ****************************************************************************/ #include "nasakeywordhandler.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include //! @cond Doxygen_Suppress diff --git a/ogr/CMakeLists.txt b/ogr/CMakeLists.txt index c515a8eccfed..30b10c9a4581 100644 --- a/ogr/CMakeLists.txt +++ b/ogr/CMakeLists.txt @@ -64,6 +64,10 @@ add_library( ogr_wkb.cpp ogrvrtgeometrytypes.cpp ogr2kmlgeometry.cpp + ogrlibjsonutils.cpp + ogrgeojsongeometry.cpp + ogrgeojsonwriter.cpp + ogresrijsongeometry.cpp ) add_dependencies(ogr generate_gdal_version_h) @@ -87,7 +91,7 @@ set_property( include(GdalStandardIncludes) gdal_standard_includes(ogr) target_compile_options(ogr PRIVATE ${GDAL_CXX_WARNING_FLAGS} ${WFLAG_OLD_STYLE_CAST} ${WFLAG_EFFCXX}) -target_include_directories(ogr PRIVATE $ $) +target_include_directories(ogr PRIVATE $) set_property(TARGET ogr PROPERTY POSITION_INDEPENDENT_CODE ${GDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) diff --git a/ogr/ogresrijsongeometry.cpp b/ogr/ogresrijsongeometry.cpp new file mode 100644 index 000000000000..099acdce841d --- /dev/null +++ b/ogr/ogresrijsongeometry.cpp @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: MIT +// Copyright 2024, Even Rouault + +/*! @cond Doxygen_Suppress */ + +#include "ogresrijsongeometry.h" + +#include "ogrlibjsonutils.h" + +#include "ogr_geometry.h" +#include "ogr_spatialref.h" + +static OGRPoint *OGRESRIJSONReadPoint(json_object *poObj); +static OGRGeometry *OGRESRIJSONReadLineString(json_object *poObj); +static OGRGeometry *OGRESRIJSONReadPolygon(json_object *poObj); +static OGRMultiPoint *OGRESRIJSONReadMultiPoint(json_object *poObj); + +/************************************************************************/ +/* OGRESRIJSONReadGeometry() */ +/************************************************************************/ + +OGRGeometry *OGRESRIJSONReadGeometry(json_object *poObj) +{ + OGRGeometry *poGeometry = nullptr; + + if (OGRGeoJSONFindMemberByName(poObj, "x")) + poGeometry = OGRESRIJSONReadPoint(poObj); + else if (OGRGeoJSONFindMemberByName(poObj, "paths")) + poGeometry = OGRESRIJSONReadLineString(poObj); + else if (OGRGeoJSONFindMemberByName(poObj, "rings")) + poGeometry = OGRESRIJSONReadPolygon(poObj); + else if (OGRGeoJSONFindMemberByName(poObj, "points")) + poGeometry = OGRESRIJSONReadMultiPoint(poObj); + + return poGeometry; +} + +/************************************************************************/ +/* OGR_G_CreateGeometryFromEsriJson() */ +/************************************************************************/ + +/** Create a OGR geometry from a ESRIJson geometry object */ +OGRGeometryH OGR_G_CreateGeometryFromEsriJson(const char *pszJson) +{ + if (nullptr == pszJson) + { + // Translation failed. + return nullptr; + } + + json_object *poObj = nullptr; + if (!OGRJSonParse(pszJson, &poObj)) + return nullptr; + + OGRGeometry *poGeometry = OGRESRIJSONReadGeometry(poObj); + + // Release JSON tree. + json_object_put(poObj); + + return OGRGeometry::ToHandle(poGeometry); +} + +/************************************************************************/ +/* OGRESRIJSONGetType() */ +/************************************************************************/ + +OGRwkbGeometryType OGRESRIJSONGetGeometryType(json_object *poObj) +{ + if (nullptr == poObj) + return wkbUnknown; + + json_object *poObjType = OGRGeoJSONFindMemberByName(poObj, "geometryType"); + if (nullptr == poObjType) + { + return wkbNone; + } + + const char *name = json_object_get_string(poObjType); + if (EQUAL(name, "esriGeometryPoint")) + return wkbPoint; + else if (EQUAL(name, "esriGeometryPolyline")) + return wkbLineString; + else if (EQUAL(name, "esriGeometryPolygon")) + return wkbPolygon; + else if (EQUAL(name, "esriGeometryMultiPoint")) + return wkbMultiPoint; + else + return wkbUnknown; +} + +/************************************************************************/ +/* OGRESRIJSONGetCoordinateToDouble() */ +/************************************************************************/ + +static double OGRESRIJSONGetCoordinateToDouble(json_object *poObjCoord, + const char *pszCoordName, + bool &bValid) +{ + const int iType = json_object_get_type(poObjCoord); + if (json_type_double != iType && json_type_int != iType) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid '%s' coordinate. " + "Type is not double or integer for \'%s\'.", + pszCoordName, json_object_to_json_string(poObjCoord)); + bValid = false; + return 0.0; + } + + return json_object_get_double(poObjCoord); +} + +/************************************************************************/ +/* OGRESRIJSONGetCoordinate() */ +/************************************************************************/ + +static double OGRESRIJSONGetCoordinate(json_object *poObj, + const char *pszCoordName, bool &bValid) +{ + json_object *poObjCoord = OGRGeoJSONFindMemberByName(poObj, pszCoordName); + if (nullptr == poObjCoord) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid Point object. " + "Missing '%s' member.", + pszCoordName); + bValid = false; + return 0.0; + } + + return OGRESRIJSONGetCoordinateToDouble(poObjCoord, pszCoordName, bValid); +} + +/************************************************************************/ +/* OGRESRIJSONReadPoint() */ +/************************************************************************/ + +OGRPoint *OGRESRIJSONReadPoint(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + bool bValid = true; + const double dfX = OGRESRIJSONGetCoordinate(poObj, "x", bValid); + const double dfY = OGRESRIJSONGetCoordinate(poObj, "y", bValid); + if (!bValid) + return nullptr; + + json_object *poObjZ = OGRGeoJSONFindMemberByName(poObj, "z"); + if (nullptr == poObjZ) + return new OGRPoint(dfX, dfY); + + const double dfZ = OGRESRIJSONGetCoordinateToDouble(poObjZ, "z", bValid); + if (!bValid) + return nullptr; + return new OGRPoint(dfX, dfY, dfZ); +} + +/************************************************************************/ +/* OGRESRIJSONReaderParseZM() */ +/************************************************************************/ + +static void OGRESRIJSONReaderParseZM(json_object *poObj, bool *bHasZ, + bool *bHasM) +{ + CPLAssert(nullptr != poObj); + // The ESRI geojson spec states that geometries other than point can + // have the attributes hasZ and hasM. A geometry that has a z value + // implies the 3rd number in the tuple is z. if hasM is true, but hasZ + // is not, it is the M value. + bool bZ = false; + json_object *poObjHasZ = OGRGeoJSONFindMemberByName(poObj, "hasZ"); + if (poObjHasZ != nullptr) + { + if (json_object_get_type(poObjHasZ) == json_type_boolean) + { + bZ = CPL_TO_BOOL(json_object_get_boolean(poObjHasZ)); + } + } + + bool bM = false; + json_object *poObjHasM = OGRGeoJSONFindMemberByName(poObj, "hasM"); + if (poObjHasM != nullptr) + { + if (json_object_get_type(poObjHasM) == json_type_boolean) + { + bM = CPL_TO_BOOL(json_object_get_boolean(poObjHasM)); + } + } + if (bHasZ != nullptr) + *bHasZ = bZ; + if (bHasM != nullptr) + *bHasM = bM; +} + +/************************************************************************/ +/* OGRESRIJSONReaderParseXYZMArray() */ +/************************************************************************/ + +static bool OGRESRIJSONReaderParseXYZMArray(json_object *poObjCoords, + bool /*bHasZ*/, bool bHasM, + double *pdfX, double *pdfY, + double *pdfZ, double *pdfM, + int *pnNumCoords) +{ + if (poObjCoords == nullptr) + { + CPLDebug("ESRIJSON", + "OGRESRIJSONReaderParseXYZMArray: got null object."); + return false; + } + + if (json_type_array != json_object_get_type(poObjCoords)) + { + CPLDebug("ESRIJSON", + "OGRESRIJSONReaderParseXYZMArray: got non-array object."); + return false; + } + + const auto coordDimension = json_object_array_length(poObjCoords); + + // Allow 4 coordinates if M is present, but it is eventually ignored. + if (coordDimension < 2 || coordDimension > 4) + { + CPLDebug("ESRIJSON", + "OGRESRIJSONReaderParseXYZMArray: got an unexpected " + "array object."); + return false; + } + + // Read X coordinate. + json_object *poObjCoord = json_object_array_get_idx(poObjCoords, 0); + if (poObjCoord == nullptr) + { + CPLDebug("ESRIJSON", + "OGRESRIJSONReaderParseXYZMArray: got null object."); + return false; + } + + bool bValid = true; + const double dfX = + OGRESRIJSONGetCoordinateToDouble(poObjCoord, "x", bValid); + + // Read Y coordinate. + poObjCoord = json_object_array_get_idx(poObjCoords, 1); + if (poObjCoord == nullptr) + { + CPLDebug("ESRIJSON", + "OGRESRIJSONReaderParseXYZMArray: got null object."); + return false; + } + + const double dfY = + OGRESRIJSONGetCoordinateToDouble(poObjCoord, "y", bValid); + if (!bValid) + return false; + + // Read Z or M or Z and M coordinates. + if (coordDimension > 2) + { + poObjCoord = json_object_array_get_idx(poObjCoords, 2); + if (poObjCoord == nullptr) + { + CPLDebug("ESRIJSON", + "OGRESRIJSONReaderParseXYZMArray: got null object."); + return false; + } + + const double dfZorM = OGRESRIJSONGetCoordinateToDouble( + poObjCoord, (coordDimension > 3 || !bHasM) ? "z" : "m", bValid); + if (!bValid) + return false; + if (pdfZ != nullptr) + { + if (coordDimension > 3 || !bHasM) + *pdfZ = dfZorM; + else + *pdfZ = 0.0; + } + if (pdfM != nullptr && coordDimension == 3) + { + if (bHasM) + *pdfM = dfZorM; + else + *pdfM = 0.0; + } + if (coordDimension == 4) + { + poObjCoord = json_object_array_get_idx(poObjCoords, 3); + if (poObjCoord == nullptr) + { + CPLDebug("ESRIJSON", + "OGRESRIJSONReaderParseXYZMArray: got null object."); + return false; + } + + const double dfM = + OGRESRIJSONGetCoordinateToDouble(poObjCoord, "m", bValid); + if (!bValid) + return false; + if (pdfM != nullptr) + *pdfM = dfM; + } + } + else + { + if (pdfZ != nullptr) + *pdfZ = 0.0; + if (pdfM != nullptr) + *pdfM = 0.0; + } + + if (pnNumCoords != nullptr) + *pnNumCoords = static_cast(coordDimension); + if (pdfX != nullptr) + *pdfX = dfX; + if (pdfY != nullptr) + *pdfY = dfY; + + return true; +} + +/************************************************************************/ +/* OGRESRIJSONReadLineString() */ +/************************************************************************/ + +OGRGeometry *OGRESRIJSONReadLineString(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + bool bHasZ = false; + bool bHasM = false; + + OGRESRIJSONReaderParseZM(poObj, &bHasZ, &bHasM); + + json_object *poObjPaths = OGRGeoJSONFindMemberByName(poObj, "paths"); + if (nullptr == poObjPaths) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid LineString object. " + "Missing \'paths\' member."); + return nullptr; + } + + if (json_type_array != json_object_get_type(poObjPaths)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid LineString object. " + "Invalid \'paths\' member."); + return nullptr; + } + + OGRMultiLineString *poMLS = nullptr; + OGRGeometry *poRet = nullptr; + const auto nPaths = json_object_array_length(poObjPaths); + for (auto iPath = decltype(nPaths){0}; iPath < nPaths; iPath++) + { + json_object *poObjPath = json_object_array_get_idx(poObjPaths, iPath); + if (poObjPath == nullptr || + json_type_array != json_object_get_type(poObjPath)) + { + delete poRet; + CPLDebug("ESRIJSON", "LineString: got non-array object."); + return nullptr; + } + + OGRLineString *poLine = new OGRLineString(); + if (nPaths > 1) + { + if (iPath == 0) + { + poMLS = new OGRMultiLineString(); + poRet = poMLS; + } + poMLS->addGeometryDirectly(poLine); + } + else + { + poRet = poLine; + } + const auto nPoints = json_object_array_length(poObjPath); + for (auto i = decltype(nPoints){0}; i < nPoints; i++) + { + int nNumCoords = 2; + json_object *poObjCoords = json_object_array_get_idx(poObjPath, i); + double dfX = 0.0; + double dfY = 0.0; + double dfZ = 0.0; + double dfM = 0.0; + if (!OGRESRIJSONReaderParseXYZMArray(poObjCoords, bHasZ, bHasM, + &dfX, &dfY, &dfZ, &dfM, + &nNumCoords)) + { + delete poRet; + return nullptr; + } + + if (nNumCoords == 3 && !bHasM) + { + poLine->addPoint(dfX, dfY, dfZ); + } + else if (nNumCoords == 3) + { + poLine->addPointM(dfX, dfY, dfM); + } + else if (nNumCoords == 4) + { + poLine->addPoint(dfX, dfY, dfZ, dfM); + } + else + { + poLine->addPoint(dfX, dfY); + } + } + } + + if (poRet == nullptr) + poRet = new OGRLineString(); + + return poRet; +} + +/************************************************************************/ +/* OGRESRIJSONReadPolygon() */ +/************************************************************************/ + +OGRGeometry *OGRESRIJSONReadPolygon(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + bool bHasZ = false; + bool bHasM = false; + + OGRESRIJSONReaderParseZM(poObj, &bHasZ, &bHasM); + + json_object *poObjRings = OGRGeoJSONFindMemberByName(poObj, "rings"); + if (nullptr == poObjRings) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid Polygon object. " + "Missing \'rings\' member."); + return nullptr; + } + + if (json_type_array != json_object_get_type(poObjRings)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid Polygon object. " + "Invalid \'rings\' member."); + return nullptr; + } + + const auto nRings = json_object_array_length(poObjRings); + OGRGeometry **papoGeoms = new OGRGeometry *[nRings]; + for (auto iRing = decltype(nRings){0}; iRing < nRings; iRing++) + { + json_object *poObjRing = json_object_array_get_idx(poObjRings, iRing); + if (poObjRing == nullptr || + json_type_array != json_object_get_type(poObjRing)) + { + for (auto j = decltype(iRing){0}; j < iRing; j++) + delete papoGeoms[j]; + delete[] papoGeoms; + CPLDebug("ESRIJSON", "Polygon: got non-array object."); + return nullptr; + } + + OGRPolygon *poPoly = new OGRPolygon(); + auto poLine = std::make_unique(); + papoGeoms[iRing] = poPoly; + + const auto nPoints = json_object_array_length(poObjRing); + for (auto i = decltype(nPoints){0}; i < nPoints; i++) + { + int nNumCoords = 2; + json_object *poObjCoords = json_object_array_get_idx(poObjRing, i); + double dfX = 0.0; + double dfY = 0.0; + double dfZ = 0.0; + double dfM = 0.0; + if (!OGRESRIJSONReaderParseXYZMArray(poObjCoords, bHasZ, bHasM, + &dfX, &dfY, &dfZ, &dfM, + &nNumCoords)) + { + for (auto j = decltype(iRing){0}; j <= iRing; j++) + delete papoGeoms[j]; + delete[] papoGeoms; + return nullptr; + } + + if (nNumCoords == 3 && !bHasM) + { + poLine->addPoint(dfX, dfY, dfZ); + } + else if (nNumCoords == 3) + { + poLine->addPointM(dfX, dfY, dfM); + } + else if (nNumCoords == 4) + { + poLine->addPoint(dfX, dfY, dfZ, dfM); + } + else + { + poLine->addPoint(dfX, dfY); + } + } + poPoly->addRingDirectly(poLine.release()); + } + + OGRGeometry *poRet = OGRGeometryFactory::organizePolygons( + papoGeoms, static_cast(nRings), nullptr, nullptr); + delete[] papoGeoms; + + return poRet; +} + +/************************************************************************/ +/* OGRESRIJSONReadMultiPoint() */ +/************************************************************************/ + +OGRMultiPoint *OGRESRIJSONReadMultiPoint(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + bool bHasZ = false; + bool bHasM = false; + + OGRESRIJSONReaderParseZM(poObj, &bHasZ, &bHasM); + + json_object *poObjPoints = OGRGeoJSONFindMemberByName(poObj, "points"); + if (nullptr == poObjPoints) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid MultiPoint object. " + "Missing \'points\' member."); + return nullptr; + } + + if (json_type_array != json_object_get_type(poObjPoints)) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid MultiPoint object. " + "Invalid \'points\' member."); + return nullptr; + } + + OGRMultiPoint *poMulti = new OGRMultiPoint(); + + const auto nPoints = json_object_array_length(poObjPoints); + for (auto i = decltype(nPoints){0}; i < nPoints; i++) + { + int nNumCoords = 2; + json_object *poObjCoords = json_object_array_get_idx(poObjPoints, i); + double dfX = 0.0; + double dfY = 0.0; + double dfZ = 0.0; + double dfM = 0.0; + if (!OGRESRIJSONReaderParseXYZMArray(poObjCoords, bHasZ, bHasM, &dfX, + &dfY, &dfZ, &dfM, &nNumCoords)) + { + delete poMulti; + return nullptr; + } + + if (nNumCoords == 3 && !bHasM) + { + poMulti->addGeometryDirectly(new OGRPoint(dfX, dfY, dfZ)); + } + else if (nNumCoords == 3) + { + OGRPoint *poPoint = new OGRPoint(dfX, dfY); + poPoint->setM(dfM); + poMulti->addGeometryDirectly(poPoint); + } + else if (nNumCoords == 4) + { + poMulti->addGeometryDirectly(new OGRPoint(dfX, dfY, dfZ, dfM)); + } + else + { + poMulti->addGeometryDirectly(new OGRPoint(dfX, dfY)); + } + } + + return poMulti; +} + +/************************************************************************/ +/* OGRESRIJSONReadSpatialReference() */ +/************************************************************************/ + +OGRSpatialReference *OGRESRIJSONReadSpatialReference(json_object *poObj) +{ + /* -------------------------------------------------------------------- */ + /* Read spatial reference definition. */ + /* -------------------------------------------------------------------- */ + OGRSpatialReference *poSRS = nullptr; + + json_object *poObjSrs = + OGRGeoJSONFindMemberByName(poObj, "spatialReference"); + if (nullptr != poObjSrs) + { + json_object *poObjWkid = + OGRGeoJSONFindMemberByName(poObjSrs, "latestWkid"); + if (poObjWkid == nullptr) + poObjWkid = OGRGeoJSONFindMemberByName(poObjSrs, "wkid"); + if (poObjWkid == nullptr) + { + json_object *poObjWkt = OGRGeoJSONFindMemberByName(poObjSrs, "wkt"); + if (poObjWkt == nullptr) + return nullptr; + + const char *pszWKT = json_object_get_string(poObjWkt); + poSRS = new OGRSpatialReference(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (OGRERR_NONE != poSRS->importFromWkt(pszWKT)) + { + delete poSRS; + poSRS = nullptr; + } + else + { + auto poSRSMatch = poSRS->FindBestMatch(70); + if (poSRSMatch) + { + poSRS->Release(); + poSRS = poSRSMatch; + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + } + } + + return poSRS; + } + + const int nEPSG = json_object_get_int(poObjWkid); + + poSRS = new OGRSpatialReference(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (OGRERR_NONE != poSRS->importFromEPSG(nEPSG)) + { + delete poSRS; + poSRS = nullptr; + } + } + + return poSRS; +} + +/*! @endcond */ diff --git a/ogr/ogresrijsongeometry.h b/ogr/ogresrijsongeometry.h new file mode 100644 index 000000000000..0468f9db4aa5 --- /dev/null +++ b/ogr/ogresrijsongeometry.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MIT +// Copyright 2024, Even Rouault + +#ifndef OGRESRIJSONGEOMETRY_H_INCLUDED +#define OGRESRIJSONGEOMETRY_H_INCLUDED + +/*! @cond Doxygen_Suppress */ + +#include "cpl_port.h" +#include "cpl_json_header.h" + +#include "ogr_api.h" + +class OGRGeometry; +class OGRPoint; +class OGRMultiPoint; +class OGRSpatialReference; + +OGRGeometry CPL_DLL *OGRESRIJSONReadGeometry(json_object *poObj); +OGRSpatialReference CPL_DLL * +OGRESRIJSONReadSpatialReference(json_object *poObj); +OGRwkbGeometryType CPL_DLL OGRESRIJSONGetGeometryType(json_object *poObj); + +/*! @endcond */ + +#endif diff --git a/ogr/ogrfeature.cpp b/ogr/ogrfeature.cpp index b1afa3fd3473..684f22f53fac 100644 --- a/ogr/ogrfeature.cpp +++ b/ogr/ogrfeature.cpp @@ -54,7 +54,7 @@ #include "ogr_featurestyle.h" #include "ogr_geometry.h" #include "ogr_p.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include "cpl_json_header.h" diff --git a/ogr/ogrgeojsongeometry.cpp b/ogr/ogrgeojsongeometry.cpp new file mode 100644 index 000000000000..cde200e83692 --- /dev/null +++ b/ogr/ogrgeojsongeometry.cpp @@ -0,0 +1,861 @@ +// SPDX-License-Identifier: MIT +// Copyright 2007, Mateusz Loskot +// Copyright 2008-2024, Even Rouault + +/*! @cond Doxygen_Suppress */ + +#include "ogrgeojsongeometry.h" +#include "ogrlibjsonutils.h" + +#include "ogr_geometry.h" +#include "ogr_spatialref.h" + +static OGRPoint *OGRGeoJSONReadPoint(json_object *poObj); +static OGRMultiPoint *OGRGeoJSONReadMultiPoint(json_object *poObj); +static OGRLineString *OGRGeoJSONReadLineString(json_object *poObj, + bool bRaw = false); +static OGRMultiLineString *OGRGeoJSONReadMultiLineString(json_object *poObj); +static OGRLinearRing *OGRGeoJSONReadLinearRing(json_object *poObj); +static OGRMultiPolygon *OGRGeoJSONReadMultiPolygon(json_object *poObj); +static OGRGeometryCollection * +OGRGeoJSONReadGeometryCollection(json_object *poObj, + OGRSpatialReference *poSRS = nullptr); + +/************************************************************************/ +/* OGRGeoJSONGetType */ +/************************************************************************/ + +GeoJSONObject::Type OGRGeoJSONGetType(json_object *poObj) +{ + if (nullptr == poObj) + return GeoJSONObject::eUnknown; + + json_object *poObjType = OGRGeoJSONFindMemberByName(poObj, "type"); + if (nullptr == poObjType) + return GeoJSONObject::eUnknown; + + const char *name = json_object_get_string(poObjType); + if (EQUAL(name, "Point")) + return GeoJSONObject::ePoint; + else if (EQUAL(name, "LineString")) + return GeoJSONObject::eLineString; + else if (EQUAL(name, "Polygon")) + return GeoJSONObject::ePolygon; + else if (EQUAL(name, "MultiPoint")) + return GeoJSONObject::eMultiPoint; + else if (EQUAL(name, "MultiLineString")) + return GeoJSONObject::eMultiLineString; + else if (EQUAL(name, "MultiPolygon")) + return GeoJSONObject::eMultiPolygon; + else if (EQUAL(name, "GeometryCollection")) + return GeoJSONObject::eGeometryCollection; + else if (EQUAL(name, "Feature")) + return GeoJSONObject::eFeature; + else if (EQUAL(name, "FeatureCollection")) + return GeoJSONObject::eFeatureCollection; + else + return GeoJSONObject::eUnknown; +} + +/************************************************************************/ +/* OGRGeoJSONGetOGRGeometryType() */ +/************************************************************************/ + +OGRwkbGeometryType OGRGeoJSONGetOGRGeometryType(json_object *poObj) +{ + if (nullptr == poObj) + return wkbUnknown; + + json_object *poObjType = CPL_json_object_object_get(poObj, "type"); + if (nullptr == poObjType) + return wkbUnknown; + + OGRwkbGeometryType eType = wkbUnknown; + const char *name = json_object_get_string(poObjType); + if (EQUAL(name, "Point")) + eType = wkbPoint; + else if (EQUAL(name, "LineString")) + eType = wkbLineString; + else if (EQUAL(name, "Polygon")) + eType = wkbPolygon; + else if (EQUAL(name, "MultiPoint")) + eType = wkbMultiPoint; + else if (EQUAL(name, "MultiLineString")) + eType = wkbMultiLineString; + else if (EQUAL(name, "MultiPolygon")) + eType = wkbMultiPolygon; + else if (EQUAL(name, "GeometryCollection")) + eType = wkbGeometryCollection; + else + return wkbUnknown; + + json_object *poCoordinates; + if (eType == wkbGeometryCollection) + { + json_object *poGeometries = + CPL_json_object_object_get(poObj, "geometries"); + if (poGeometries && + json_object_get_type(poGeometries) == json_type_array && + json_object_array_length(poGeometries) > 0) + { + if (OGR_GT_HasZ(OGRGeoJSONGetOGRGeometryType( + json_object_array_get_idx(poGeometries, 0)))) + eType = OGR_GT_SetZ(eType); + } + } + else + { + poCoordinates = CPL_json_object_object_get(poObj, "coordinates"); + if (poCoordinates && + json_object_get_type(poCoordinates) == json_type_array && + json_object_array_length(poCoordinates) > 0) + { + while (true) + { + auto poChild = json_object_array_get_idx(poCoordinates, 0); + if (!(poChild && + json_object_get_type(poChild) == json_type_array && + json_object_array_length(poChild) > 0)) + { + if (json_object_array_length(poCoordinates) == 3) + eType = OGR_GT_SetZ(eType); + break; + } + poCoordinates = poChild; + } + } + } + + return eType; +} + +/************************************************************************/ +/* OGRGeoJSONReadGeometry */ +/************************************************************************/ + +OGRGeometry *OGRGeoJSONReadGeometry(json_object *poObj, + OGRSpatialReference *poParentSRS) +{ + + OGRGeometry *poGeometry = nullptr; + OGRSpatialReference *poSRS = nullptr; + lh_entry *entry = OGRGeoJSONFindMemberEntryByName(poObj, "crs"); + if (entry != nullptr) + { + json_object *poObjSrs = + static_cast(const_cast(entry->v)); + if (poObjSrs != nullptr) + { + poSRS = OGRGeoJSONReadSpatialReference(poObj); + } + } + + OGRSpatialReference *poSRSToAssign = nullptr; + if (entry != nullptr) + { + poSRSToAssign = poSRS; + } + else if (poParentSRS) + { + poSRSToAssign = poParentSRS; + } + else + { + // Assign WGS84 if no CRS defined on geometry. + poSRSToAssign = OGRSpatialReference::GetWGS84SRS(); + } + + GeoJSONObject::Type objType = OGRGeoJSONGetType(poObj); + if (GeoJSONObject::ePoint == objType) + poGeometry = OGRGeoJSONReadPoint(poObj); + else if (GeoJSONObject::eMultiPoint == objType) + poGeometry = OGRGeoJSONReadMultiPoint(poObj); + else if (GeoJSONObject::eLineString == objType) + poGeometry = OGRGeoJSONReadLineString(poObj); + else if (GeoJSONObject::eMultiLineString == objType) + poGeometry = OGRGeoJSONReadMultiLineString(poObj); + else if (GeoJSONObject::ePolygon == objType) + poGeometry = OGRGeoJSONReadPolygon(poObj); + else if (GeoJSONObject::eMultiPolygon == objType) + poGeometry = OGRGeoJSONReadMultiPolygon(poObj); + else if (GeoJSONObject::eGeometryCollection == objType) + poGeometry = OGRGeoJSONReadGeometryCollection(poObj, poSRSToAssign); + else + { + CPLDebug("GeoJSON", "Unsupported geometry type detected. " + "Feature gets NULL geometry assigned."); + } + + if (poGeometry && GeoJSONObject::eGeometryCollection != objType) + poGeometry->assignSpatialReference(poSRSToAssign); + + if (poSRS) + poSRS->Release(); + + return poGeometry; +} + +/************************************************************************/ +/* OGRGeoJSONGetCoordinate() */ +/************************************************************************/ + +static double OGRGeoJSONGetCoordinate(json_object *poObj, + const char *pszCoordName, int nIndex, + bool &bValid) +{ + json_object *poObjCoord = json_object_array_get_idx(poObj, nIndex); + if (nullptr == poObjCoord) + { + CPLDebug("GeoJSON", "Point: got null object for %s.", pszCoordName); + bValid = false; + return 0.0; + } + + const int iType = json_object_get_type(poObjCoord); + if (json_type_double != iType && json_type_int != iType) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid '%s' coordinate. " + "Type is not double or integer for \'%s\'.", + pszCoordName, json_object_to_json_string(poObjCoord)); + bValid = false; + return 0.0; + } + + return json_object_get_double(poObjCoord); +} + +/************************************************************************/ +/* OGRGeoJSONReadRawPoint */ +/************************************************************************/ + +static bool OGRGeoJSONReadRawPoint(json_object *poObj, OGRPoint &point) +{ + CPLAssert(nullptr != poObj); + + if (json_type_array == json_object_get_type(poObj)) + { + const auto nSize = json_object_array_length(poObj); + + if (nSize < GeoJSONObject::eMinCoordinateDimension) + { + CPLDebug("GeoJSON", "Invalid coord dimension. " + "At least 2 dimensions must be present."); + return false; + } + + bool bValid = true; + const double dfX = OGRGeoJSONGetCoordinate(poObj, "x", 0, bValid); + const double dfY = OGRGeoJSONGetCoordinate(poObj, "y", 1, bValid); + point.setX(dfX); + point.setY(dfY); + + // Read Z coordinate. + if (nSize >= GeoJSONObject::eMaxCoordinateDimension) + { + // Don't *expect* mixed-dimension geometries, although the + // spec doesn't explicitly forbid this. + const double dfZ = OGRGeoJSONGetCoordinate(poObj, "z", 2, bValid); + point.setZ(dfZ); + } + else + { + point.flattenTo2D(); + } + return bValid; + } + + return false; +} + +/************************************************************************/ +/* OGRGeoJSONReadPoint */ +/************************************************************************/ + +OGRPoint *OGRGeoJSONReadPoint(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + json_object *poObjCoords = OGRGeoJSONFindMemberByName(poObj, "coordinates"); + if (nullptr == poObjCoords) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid Point object. Missing \'coordinates\' member."); + return nullptr; + } + + OGRPoint *poPoint = new OGRPoint(); + if (!OGRGeoJSONReadRawPoint(poObjCoords, *poPoint)) + { + CPLDebug("GeoJSON", "Point: raw point parsing failure."); + delete poPoint; + return nullptr; + } + + return poPoint; +} + +/************************************************************************/ +/* OGRGeoJSONReadMultiPoint */ +/************************************************************************/ + +OGRMultiPoint *OGRGeoJSONReadMultiPoint(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + json_object *poObjPoints = OGRGeoJSONFindMemberByName(poObj, "coordinates"); + if (nullptr == poObjPoints) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid MultiPoint object. " + "Missing \'coordinates\' member."); + return nullptr; + } + + OGRMultiPoint *poMultiPoint = nullptr; + if (json_type_array == json_object_get_type(poObjPoints)) + { + const auto nPoints = json_object_array_length(poObjPoints); + + poMultiPoint = new OGRMultiPoint(); + + for (auto i = decltype(nPoints){0}; i < nPoints; ++i) + { + json_object *poObjCoords = + json_object_array_get_idx(poObjPoints, i); + + OGRPoint pt; + if (poObjCoords != nullptr && + !OGRGeoJSONReadRawPoint(poObjCoords, pt)) + { + delete poMultiPoint; + CPLDebug("GeoJSON", "LineString: raw point parsing failure."); + return nullptr; + } + poMultiPoint->addGeometry(&pt); + } + } + + return poMultiPoint; +} + +/************************************************************************/ +/* OGRGeoJSONReadLineString */ +/************************************************************************/ + +OGRLineString *OGRGeoJSONReadLineString(json_object *poObj, bool bRaw) +{ + CPLAssert(nullptr != poObj); + + json_object *poObjPoints = nullptr; + + if (!bRaw) + { + poObjPoints = OGRGeoJSONFindMemberByName(poObj, "coordinates"); + if (nullptr == poObjPoints) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid LineString object. " + "Missing \'coordinates\' member."); + return nullptr; + } + } + else + { + poObjPoints = poObj; + } + + OGRLineString *poLine = nullptr; + + if (json_type_array == json_object_get_type(poObjPoints)) + { + const auto nPoints = json_object_array_length(poObjPoints); + + poLine = new OGRLineString(); + poLine->setNumPoints(static_cast(nPoints)); + + for (auto i = decltype(nPoints){0}; i < nPoints; ++i) + { + json_object *poObjCoords = + json_object_array_get_idx(poObjPoints, i); + if (poObjCoords == nullptr) + { + delete poLine; + CPLDebug("GeoJSON", "LineString: got null object."); + return nullptr; + } + + OGRPoint pt; + if (!OGRGeoJSONReadRawPoint(poObjCoords, pt)) + { + delete poLine; + CPLDebug("GeoJSON", "LineString: raw point parsing failure."); + return nullptr; + } + if (pt.getCoordinateDimension() == 2) + { + poLine->setPoint(static_cast(i), pt.getX(), pt.getY()); + } + else + { + poLine->setPoint(static_cast(i), pt.getX(), pt.getY(), + pt.getZ()); + } + } + } + + return poLine; +} + +/************************************************************************/ +/* OGRGeoJSONReadMultiLineString */ +/************************************************************************/ + +OGRMultiLineString *OGRGeoJSONReadMultiLineString(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + json_object *poObjLines = OGRGeoJSONFindMemberByName(poObj, "coordinates"); + if (nullptr == poObjLines) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid MultiLineString object. " + "Missing \'coordinates\' member."); + return nullptr; + } + + OGRMultiLineString *poMultiLine = nullptr; + + if (json_type_array == json_object_get_type(poObjLines)) + { + const auto nLines = json_object_array_length(poObjLines); + + poMultiLine = new OGRMultiLineString(); + + for (auto i = decltype(nLines){0}; i < nLines; ++i) + { + json_object *poObjLine = json_object_array_get_idx(poObjLines, i); + + OGRLineString *poLine; + if (poObjLine != nullptr) + poLine = OGRGeoJSONReadLineString(poObjLine, true); + else + poLine = new OGRLineString(); + + if (nullptr != poLine) + { + poMultiLine->addGeometryDirectly(poLine); + } + } + } + + return poMultiLine; +} + +/************************************************************************/ +/* OGRGeoJSONReadLinearRing */ +/************************************************************************/ + +OGRLinearRing *OGRGeoJSONReadLinearRing(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + OGRLinearRing *poRing = nullptr; + + if (json_type_array == json_object_get_type(poObj)) + { + const auto nPoints = json_object_array_length(poObj); + + poRing = new OGRLinearRing(); + poRing->setNumPoints(static_cast(nPoints)); + + for (auto i = decltype(nPoints){0}; i < nPoints; ++i) + { + json_object *poObjCoords = json_object_array_get_idx(poObj, i); + if (poObjCoords == nullptr) + { + delete poRing; + CPLDebug("GeoJSON", "LinearRing: got null object."); + return nullptr; + } + + OGRPoint pt; + if (!OGRGeoJSONReadRawPoint(poObjCoords, pt)) + { + delete poRing; + CPLDebug("GeoJSON", "LinearRing: raw point parsing failure."); + return nullptr; + } + + if (2 == pt.getCoordinateDimension()) + poRing->setPoint(static_cast(i), pt.getX(), pt.getY()); + else + poRing->setPoint(static_cast(i), pt.getX(), pt.getY(), + pt.getZ()); + } + } + + return poRing; +} + +/************************************************************************/ +/* OGRGeoJSONReadPolygon */ +/************************************************************************/ + +OGRPolygon *OGRGeoJSONReadPolygon(json_object *poObj, bool bRaw) +{ + CPLAssert(nullptr != poObj); + + json_object *poObjRings = nullptr; + + if (!bRaw) + { + poObjRings = OGRGeoJSONFindMemberByName(poObj, "coordinates"); + if (nullptr == poObjRings) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid Polygon object. " + "Missing \'coordinates\' member."); + return nullptr; + } + } + else + { + poObjRings = poObj; + } + + OGRPolygon *poPolygon = nullptr; + + if (json_type_array == json_object_get_type(poObjRings)) + { + const auto nRings = json_object_array_length(poObjRings); + if (nRings > 0) + { + json_object *poObjPoints = json_object_array_get_idx(poObjRings, 0); + if (poObjPoints == nullptr) + { + poPolygon = new OGRPolygon(); + } + else + { + OGRLinearRing *poRing = OGRGeoJSONReadLinearRing(poObjPoints); + if (nullptr != poRing) + { + poPolygon = new OGRPolygon(); + poPolygon->addRingDirectly(poRing); + } + } + + for (auto i = decltype(nRings){1}; + i < nRings && nullptr != poPolygon; ++i) + { + poObjPoints = json_object_array_get_idx(poObjRings, i); + if (poObjPoints != nullptr) + { + OGRLinearRing *poRing = + OGRGeoJSONReadLinearRing(poObjPoints); + if (nullptr != poRing) + { + poPolygon->addRingDirectly(poRing); + } + } + } + } + else + { + poPolygon = new OGRPolygon(); + } + } + + return poPolygon; +} + +/************************************************************************/ +/* OGRGeoJSONReadMultiPolygon */ +/************************************************************************/ + +OGRMultiPolygon *OGRGeoJSONReadMultiPolygon(json_object *poObj) +{ + CPLAssert(nullptr != poObj); + + json_object *poObjPolys = OGRGeoJSONFindMemberByName(poObj, "coordinates"); + if (nullptr == poObjPolys) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid MultiPolygon object. " + "Missing \'coordinates\' member."); + return nullptr; + } + + OGRMultiPolygon *poMultiPoly = nullptr; + + if (json_type_array == json_object_get_type(poObjPolys)) + { + const auto nPolys = json_object_array_length(poObjPolys); + + poMultiPoly = new OGRMultiPolygon(); + + for (auto i = decltype(nPolys){0}; i < nPolys; ++i) + { + json_object *poObjPoly = json_object_array_get_idx(poObjPolys, i); + if (poObjPoly == nullptr) + { + poMultiPoly->addGeometryDirectly(new OGRPolygon()); + } + else + { + OGRPolygon *poPoly = OGRGeoJSONReadPolygon(poObjPoly, true); + if (nullptr != poPoly) + { + poMultiPoly->addGeometryDirectly(poPoly); + } + } + } + } + + return poMultiPoly; +} + +/************************************************************************/ +/* OGRGeoJSONReadGeometryCollection */ +/************************************************************************/ + +OGRGeometryCollection * +OGRGeoJSONReadGeometryCollection(json_object *poObj, OGRSpatialReference *poSRS) +{ + CPLAssert(nullptr != poObj); + + json_object *poObjGeoms = OGRGeoJSONFindMemberByName(poObj, "geometries"); + if (nullptr == poObjGeoms) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Invalid GeometryCollection object. " + "Missing \'geometries\' member."); + return nullptr; + } + + OGRGeometryCollection *poCollection = nullptr; + + if (json_type_array == json_object_get_type(poObjGeoms)) + { + poCollection = new OGRGeometryCollection(); + poCollection->assignSpatialReference(poSRS); + + const auto nGeoms = json_object_array_length(poObjGeoms); + for (auto i = decltype(nGeoms){0}; i < nGeoms; ++i) + { + json_object *poObjGeom = json_object_array_get_idx(poObjGeoms, i); + if (poObjGeom == nullptr) + { + CPLDebug("GeoJSON", "Skipping null sub-geometry"); + continue; + } + + OGRGeometry *poGeometry = OGRGeoJSONReadGeometry(poObjGeom, poSRS); + if (nullptr != poGeometry) + { + poCollection->addGeometryDirectly(poGeometry); + } + } + } + + return poCollection; +} + +/************************************************************************/ +/* OGRGeoJSONGetGeometryName() */ +/************************************************************************/ + +const char *OGRGeoJSONGetGeometryName(OGRGeometry const *poGeometry) +{ + CPLAssert(nullptr != poGeometry); + + const OGRwkbGeometryType eType = wkbFlatten(poGeometry->getGeometryType()); + + if (wkbPoint == eType) + return "Point"; + else if (wkbLineString == eType) + return "LineString"; + else if (wkbPolygon == eType) + return "Polygon"; + else if (wkbMultiPoint == eType) + return "MultiPoint"; + else if (wkbMultiLineString == eType) + return "MultiLineString"; + else if (wkbMultiPolygon == eType) + return "MultiPolygon"; + else if (wkbGeometryCollection == eType) + return "GeometryCollection"; + + return "Unknown"; +} + +/************************************************************************/ +/* OGRGeoJSONReadSpatialReference */ +/************************************************************************/ + +OGRSpatialReference *OGRGeoJSONReadSpatialReference(json_object *poObj) +{ + + /* -------------------------------------------------------------------- */ + /* Read spatial reference definition. */ + /* -------------------------------------------------------------------- */ + OGRSpatialReference *poSRS = nullptr; + + json_object *poObjSrs = OGRGeoJSONFindMemberByName(poObj, "crs"); + if (nullptr != poObjSrs) + { + json_object *poObjSrsType = + OGRGeoJSONFindMemberByName(poObjSrs, "type"); + if (poObjSrsType == nullptr) + return nullptr; + + const char *pszSrsType = json_object_get_string(poObjSrsType); + + // TODO: Add URL and URN types support. + if (STARTS_WITH_CI(pszSrsType, "NAME")) + { + json_object *poObjSrsProps = + OGRGeoJSONFindMemberByName(poObjSrs, "properties"); + if (poObjSrsProps == nullptr) + return nullptr; + + json_object *poNameURL = + OGRGeoJSONFindMemberByName(poObjSrsProps, "name"); + if (poNameURL == nullptr) + return nullptr; + + const char *pszName = json_object_get_string(poNameURL); + + // Mostly to emulate GDAL 2.x behavior + // See https://github.com/OSGeo/gdal/issues/2035 + if (EQUAL(pszName, "urn:ogc:def:crs:OGC:1.3:CRS84")) + pszName = "EPSG:4326"; + + poSRS = new OGRSpatialReference(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (OGRERR_NONE != + poSRS->SetFromUserInput( + pszName, + OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get())) + { + delete poSRS; + poSRS = nullptr; + } + } + + else if (STARTS_WITH_CI(pszSrsType, "EPSG")) + { + json_object *poObjSrsProps = + OGRGeoJSONFindMemberByName(poObjSrs, "properties"); + if (poObjSrsProps == nullptr) + return nullptr; + + json_object *poObjCode = + OGRGeoJSONFindMemberByName(poObjSrsProps, "code"); + if (poObjCode == nullptr) + return nullptr; + + int nEPSG = json_object_get_int(poObjCode); + + poSRS = new OGRSpatialReference(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (OGRERR_NONE != poSRS->importFromEPSG(nEPSG)) + { + delete poSRS; + poSRS = nullptr; + } + } + + else if (STARTS_WITH_CI(pszSrsType, "URL") || + STARTS_WITH_CI(pszSrsType, "LINK")) + { + json_object *poObjSrsProps = + OGRGeoJSONFindMemberByName(poObjSrs, "properties"); + if (poObjSrsProps == nullptr) + return nullptr; + + json_object *poObjURL = + OGRGeoJSONFindMemberByName(poObjSrsProps, "url"); + + if (nullptr == poObjURL) + { + poObjURL = OGRGeoJSONFindMemberByName(poObjSrsProps, "href"); + } + if (poObjURL == nullptr) + return nullptr; + + const char *pszURL = json_object_get_string(poObjURL); + + poSRS = new OGRSpatialReference(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (OGRERR_NONE != poSRS->importFromUrl(pszURL)) + { + delete poSRS; + poSRS = nullptr; + } + } + + else if (EQUAL(pszSrsType, "OGC")) + { + json_object *poObjSrsProps = + OGRGeoJSONFindMemberByName(poObjSrs, "properties"); + if (poObjSrsProps == nullptr) + return nullptr; + + json_object *poObjURN = + OGRGeoJSONFindMemberByName(poObjSrsProps, "urn"); + if (poObjURN == nullptr) + return nullptr; + + poSRS = new OGRSpatialReference(); + poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); + if (OGRERR_NONE != + poSRS->importFromURN(json_object_get_string(poObjURN))) + { + delete poSRS; + poSRS = nullptr; + } + } + } + + // Strip AXIS, since geojson has (easting, northing) / (longitude, latitude) + // order. According to http://www.geojson.org/geojson-spec.html#id2 : + // "Point coordinates are in x, y order (easting, northing for projected + // coordinates, longitude, latitude for geographic coordinates)". + if (poSRS != nullptr) + { + OGR_SRSNode *poGEOGCS = poSRS->GetAttrNode("GEOGCS"); + if (poGEOGCS != nullptr) + poGEOGCS->StripNodes("AXIS"); + } + + return poSRS; +} + +/************************************************************************/ +/* OGR_G_CreateGeometryFromJson */ +/************************************************************************/ + +/** Create a OGR geometry from a GeoJSON geometry object */ +OGRGeometryH OGR_G_CreateGeometryFromJson(const char *pszJson) +{ + if (nullptr == pszJson) + { + // Translation failed. + return nullptr; + } + + json_object *poObj = nullptr; + if (!OGRJSonParse(pszJson, &poObj)) + return nullptr; + + OGRGeometry *poGeometry = OGRGeoJSONReadGeometry(poObj); + + // Release JSON tree. + json_object_put(poObj); + + return OGRGeometry::ToHandle(poGeometry); +} + +/*! @endcond */ diff --git a/ogr/ogrgeojsongeometry.h b/ogr/ogrgeojsongeometry.h new file mode 100644 index 000000000000..07d6af036641 --- /dev/null +++ b/ogr/ogrgeojsongeometry.h @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MIT +// Copyright 2007, Mateusz Loskot +// Copyright 2008-2024, Even Rouault + +#ifndef OGRGEOJSONGEOMETRY_H_INCLUDED +#define OGRGEOJSONGEOMETRY_H_INCLUDED + +/*! @cond Doxygen_Suppress */ + +#include "cpl_port.h" +#include "cpl_json_header.h" + +#include "ogr_api.h" + +class OGRGeometry; +class OGRPolygon; +class OGRSpatialReference; + +/************************************************************************/ +/* GeoJSONObject */ +/************************************************************************/ + +struct GeoJSONObject +{ + enum Type + { + eUnknown = wkbUnknown, // non-GeoJSON properties + ePoint = wkbPoint, + eLineString = wkbLineString, + ePolygon = wkbPolygon, + eMultiPoint = wkbMultiPoint, + eMultiLineString = wkbMultiLineString, + eMultiPolygon = wkbMultiPolygon, + eGeometryCollection = wkbGeometryCollection, + eFeature, + eFeatureCollection + }; + + enum CoordinateDimension + { + eMinCoordinateDimension = 2, + eMaxCoordinateDimension = 3 + }; +}; + +/************************************************************************/ +/* GeoJSON Geometry Translators */ +/************************************************************************/ + +GeoJSONObject::Type CPL_DLL OGRGeoJSONGetType(json_object *poObj); + +OGRwkbGeometryType CPL_DLL OGRGeoJSONGetOGRGeometryType(json_object *poObj); + +OGRGeometry CPL_DLL * +OGRGeoJSONReadGeometry(json_object *poObj, + OGRSpatialReference *poParentSRS = nullptr); +OGRSpatialReference CPL_DLL *OGRGeoJSONReadSpatialReference(json_object *poObj); + +OGRPolygon *OGRGeoJSONReadPolygon(json_object *poObj, bool bRaw = false); + +const char *OGRGeoJSONGetGeometryName(OGRGeometry const *poGeometry); + +/*! @endcond */ + +#endif diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.cpp b/ogr/ogrgeojsonwriter.cpp similarity index 91% rename from ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.cpp rename to ogr/ogrgeojsonwriter.cpp index cfac52354625..bcf059ed7411 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.cpp +++ b/ogr/ogrgeojsonwriter.cpp @@ -27,12 +27,16 @@ * DEALINGS IN THE SOFTWARE. ****************************************************************************/ +/*! @cond Doxygen_Suppress */ + #define JSON_C_VER_013 (13 << 8) #include "ogrgeojsonwriter.h" -#include "ogrgeojsonutils.h" -#include "ogr_geojson.h" -#include "ogrgeojsonreader.h" +#include "ogr_geometry.h" +#include "ogrgeojsongeometry.h" +#include "ogrlibjsonutils.h" +#include "ogr_feature.h" +#include "ogr_p.h" #include // JSON-C #if (!defined(JSON_C_VERSION_NUM)) || (JSON_C_VERSION_NUM < JSON_C_VER_013) @@ -41,7 +45,6 @@ #include #include "ogr_api.h" -#include "ogr_p.h" #include #include @@ -52,6 +55,46 @@ static json_object * json_object_new_float_with_significant_figures(float fVal, int nSignificantFigures); +static json_object * +OGRGeoJSONWritePoint(const OGRPoint *poPoint, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteLineString(const OGRLineString *poLine, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteMultiPoint(const OGRMultiPoint *poGeometry, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteMultiLineString(const OGRMultiLineString *poGeometry, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteMultiPolygon(const OGRMultiPolygon *poGeometry, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteGeometryCollection(const OGRGeometryCollection *poGeometry, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteCoords(double const &fX, double const &fY, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteCoords(double const &fX, double const &fY, double const &fZ, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteLineCoords(const OGRLineString *poLine, + const OGRGeoJSONWriteOptions &oOptions); + +static json_object * +OGRGeoJSONWriteRingCoords(const OGRLinearRing *poLine, bool bIsExteriorRing, + const OGRGeoJSONWriteOptions &oOptions); + /************************************************************************/ /* SetRFC7946Settings() */ /************************************************************************/ @@ -928,8 +971,8 @@ json_object *OGRGeoJSONWriteAttributes(OGRFeature *poFeature, else if (OFTInteger64 == eType) { if (eSubType == OFSTBoolean) - poObjProp = json_object_new_boolean( - (json_bool)poFeature->GetFieldAsInteger64(nField)); + poObjProp = json_object_new_boolean(static_cast( + poFeature->GetFieldAsInteger64(nField))); else poObjProp = json_object_new_int64( poFeature->GetFieldAsInteger64(nField)); @@ -1501,6 +1544,67 @@ json_object *OGRGeoJSONWriteRingCoords(const OGRLinearRing *poLine, return poObjCoords; } +/************************************************************************/ +/* OGR_json_float_with_significant_figures_to_string() */ +/************************************************************************/ + +static int OGR_json_float_with_significant_figures_to_string( + struct json_object *jso, struct printbuf *pb, int /* level */, + int /* flags */) +{ + char szBuffer[75] = {}; + int nSize = 0; + const float fVal = static_cast(json_object_get_double(jso)); + if (std::isnan(fVal)) + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "NaN"); + else if (std::isinf(fVal)) + { + if (fVal > 0) + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "Infinity"); + else + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "-Infinity"); + } + else + { + const void *userData = +#if (!defined(JSON_C_VERSION_NUM)) || (JSON_C_VERSION_NUM < JSON_C_VER_013) + jso->_userdata; +#else + json_object_get_userdata(jso); +#endif + const uintptr_t nSignificantFigures = + reinterpret_cast(userData); + const bool bSignificantFiguresIsNegative = + (nSignificantFigures >> (8 * sizeof(nSignificantFigures) - 1)) != 0; + const int nInitialSignificantFigures = + bSignificantFiguresIsNegative + ? 8 + : static_cast(nSignificantFigures); + nSize = OGRFormatFloat(szBuffer, sizeof(szBuffer), fVal, + nInitialSignificantFigures, 'g'); + } + + return printbuf_memappend(pb, szBuffer, nSize); +} + +/************************************************************************/ +/* json_object_new_float_with_significant_figures() */ +/************************************************************************/ + +json_object * +json_object_new_float_with_significant_figures(float fVal, + int nSignificantFigures) +{ + json_object *jso = json_object_new_double(fVal); + json_object_set_serializer( + jso, OGR_json_float_with_significant_figures_to_string, + reinterpret_cast(static_cast(nSignificantFigures)), + nullptr); + return jso; +} + +/*! @endcond */ + /************************************************************************/ /* OGR_G_ExportToJson */ /************************************************************************/ @@ -1609,209 +1713,3 @@ char *OGR_G_ExportToJsonEx(OGRGeometryH hGeometry, char **papszOptions) // Translation failed. return nullptr; } - -/************************************************************************/ -/* OGR_json_double_with_precision_to_string() */ -/************************************************************************/ - -static int OGR_json_double_with_precision_to_string(struct json_object *jso, - struct printbuf *pb, - int /* level */, - int /* flags */) -{ - const void *userData = -#if (!defined(JSON_C_VERSION_NUM)) || (JSON_C_VERSION_NUM < JSON_C_VER_013) - jso->_userdata; -#else - json_object_get_userdata(jso); -#endif - // Precision is stored as a uintptr_t content casted to void* - const uintptr_t nPrecision = reinterpret_cast(userData); - char szBuffer[75] = {}; - const double dfVal = json_object_get_double(jso); - if (fabs(dfVal) > 1e50 && !std::isinf(dfVal)) - { - CPLsnprintf(szBuffer, sizeof(szBuffer), "%.17g", dfVal); - } - else - { - const bool bPrecisionIsNegative = - (nPrecision >> (8 * sizeof(nPrecision) - 1)) != 0; - OGRFormatDouble(szBuffer, sizeof(szBuffer), dfVal, '.', - bPrecisionIsNegative ? 15 - : static_cast(nPrecision)); - } - return printbuf_memappend(pb, szBuffer, static_cast(strlen(szBuffer))); -} - -/************************************************************************/ -/* json_object_new_double_with_precision() */ -/************************************************************************/ - -json_object *json_object_new_double_with_precision(double dfVal, - int nCoordPrecision) -{ - json_object *jso = json_object_new_double(dfVal); - json_object_set_serializer(jso, OGR_json_double_with_precision_to_string, - (void *)(uintptr_t)nCoordPrecision, nullptr); - return jso; -} - -/************************************************************************/ -/* OGR_json_double_with_significant_figures_to_string() */ -/************************************************************************/ - -static int OGR_json_double_with_significant_figures_to_string( - struct json_object *jso, struct printbuf *pb, int /* level */, - int /* flags */) -{ - char szBuffer[75] = {}; - int nSize = 0; - const double dfVal = json_object_get_double(jso); - if (std::isnan(dfVal)) - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "NaN"); - else if (std::isinf(dfVal)) - { - if (dfVal > 0) - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "Infinity"); - else - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "-Infinity"); - } - else - { - char szFormatting[32] = {}; - const void *userData = -#if (!defined(JSON_C_VERSION_NUM)) || (JSON_C_VERSION_NUM < JSON_C_VER_013) - jso->_userdata; -#else - json_object_get_userdata(jso); -#endif - const uintptr_t nSignificantFigures = - reinterpret_cast(userData); - const bool bSignificantFiguresIsNegative = - (nSignificantFigures >> (8 * sizeof(nSignificantFigures) - 1)) != 0; - const int nInitialSignificantFigures = - bSignificantFiguresIsNegative - ? 17 - : static_cast(nSignificantFigures); - CPLsnprintf(szFormatting, sizeof(szFormatting), "%%.%dg", - nInitialSignificantFigures); - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), szFormatting, dfVal); - const char *pszDot = strchr(szBuffer, '.'); - - // Try to avoid .xxxx999999y or .xxxx000000y rounding issues by - // decreasing a bit precision. - if (nInitialSignificantFigures > 10 && pszDot != nullptr && - (strstr(pszDot, "999999") != nullptr || - strstr(pszDot, "000000") != nullptr)) - { - bool bOK = false; - for (int i = 1; i <= 3; i++) - { - CPLsnprintf(szFormatting, sizeof(szFormatting), "%%.%dg", - nInitialSignificantFigures - i); - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), szFormatting, - dfVal); - pszDot = strchr(szBuffer, '.'); - if (pszDot != nullptr && strstr(pszDot, "999999") == nullptr && - strstr(pszDot, "000000") == nullptr) - { - bOK = true; - break; - } - } - if (!bOK) - { - CPLsnprintf(szFormatting, sizeof(szFormatting), "%%.%dg", - nInitialSignificantFigures); - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), szFormatting, - dfVal); - } - } - - if (nSize + 2 < static_cast(sizeof(szBuffer)) && - strchr(szBuffer, '.') == nullptr && - strchr(szBuffer, 'e') == nullptr) - { - nSize += - CPLsnprintf(szBuffer + nSize, sizeof(szBuffer) - nSize, ".0"); - } - } - - return printbuf_memappend(pb, szBuffer, nSize); -} - -/************************************************************************/ -/* json_object_new_double_with_significant_figures() */ -/************************************************************************/ - -json_object * -json_object_new_double_with_significant_figures(double dfVal, - int nSignificantFigures) -{ - json_object *jso = json_object_new_double(dfVal); - json_object_set_serializer( - jso, OGR_json_double_with_significant_figures_to_string, - reinterpret_cast(static_cast(nSignificantFigures)), - nullptr); - return jso; -} - -/************************************************************************/ -/* OGR_json_float_with_significant_figures_to_string() */ -/************************************************************************/ - -static int OGR_json_float_with_significant_figures_to_string( - struct json_object *jso, struct printbuf *pb, int /* level */, - int /* flags */) -{ - char szBuffer[75] = {}; - int nSize = 0; - const float fVal = static_cast(json_object_get_double(jso)); - if (std::isnan(fVal)) - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "NaN"); - else if (std::isinf(fVal)) - { - if (fVal > 0) - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "Infinity"); - else - nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "-Infinity"); - } - else - { - const void *userData = -#if (!defined(JSON_C_VERSION_NUM)) || (JSON_C_VERSION_NUM < JSON_C_VER_013) - jso->_userdata; -#else - json_object_get_userdata(jso); -#endif - const uintptr_t nSignificantFigures = - reinterpret_cast(userData); - const bool bSignificantFiguresIsNegative = - (nSignificantFigures >> (8 * sizeof(nSignificantFigures) - 1)) != 0; - const int nInitialSignificantFigures = - bSignificantFiguresIsNegative - ? 8 - : static_cast(nSignificantFigures); - nSize = OGRFormatFloat(szBuffer, sizeof(szBuffer), fVal, - nInitialSignificantFigures, 'g'); - } - - return printbuf_memappend(pb, szBuffer, nSize); -} - -/************************************************************************/ -/* json_object_new_float_with_significant_figures() */ -/************************************************************************/ - -json_object * -json_object_new_float_with_significant_figures(float fVal, - int nSignificantFigures) -{ - json_object *jso = json_object_new_double(fVal); - json_object_set_serializer( - jso, OGR_json_float_with_significant_figures_to_string, - reinterpret_cast(static_cast(nSignificantFigures)), - nullptr); - return jso; -} diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.h b/ogr/ogrgeojsonwriter.h similarity index 55% rename from ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.h rename to ogr/ogrgeojsonwriter.h index 9f28db52c30a..250f8449ac11 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonwriter.h +++ b/ogr/ogrgeojsonwriter.h @@ -31,46 +31,22 @@ #ifndef OGR_GEOJSONWRITER_H_INCLUDED #define OGR_GEOJSONWRITER_H_INCLUDED +/*! @cond Doxygen_Suppress */ + #include "ogr_core.h" #include "cpl_json_header.h" #include "cpl_string.h" -/************************************************************************/ -/* FORWARD DECLARATIONS */ -/************************************************************************/ -#ifdef __cplusplus class OGRFeature; class OGRGeometry; -class OGRPoint; -class OGRMultiPoint; -class OGRLineString; -class OGRMultiLineString; -class OGRLinearRing; class OGRPolygon; -class OGRMultiPolygon; -class OGRGeometryCollection; -#endif - -CPL_C_START -/* %.XXXf formatting */ -json_object CPL_DLL *json_object_new_double_with_precision(double dfVal, - int nCoordPrecision); - -/* %.XXXg formatting */ -json_object CPL_DLL * -json_object_new_double_with_significant_figures(double dfVal, - int nSignificantFigures); -CPL_C_END /************************************************************************/ /* GeoJSON Geometry Translators */ /************************************************************************/ -#ifdef __cplusplus -class OGRCoordinateTransformation; -/*! @cond Doxygen_Suppress */ -class OGRGeoJSONWriteOptions +class CPL_DLL OGRGeoJSONWriteOptions { public: bool bWriteBBOX = false; @@ -92,49 +68,28 @@ class OGRGeoJSONWriteOptions void SetIDOptions(CSLConstList papszOptions); }; -/*! @endcond */ +OGREnvelope3D CPL_DLL OGRGeoJSONGetBBox(const OGRGeometry *poGeometry, + const OGRGeoJSONWriteOptions &oOptions); + +json_object CPL_DLL * +OGRGeoJSONWriteFeature(OGRFeature *poFeature, + const OGRGeoJSONWriteOptions &oOptions); -OGREnvelope3D OGRGeoJSONGetBBox(const OGRGeometry *poGeometry, - const OGRGeoJSONWriteOptions &oOptions); -json_object *OGRGeoJSONWriteFeature(OGRFeature *poFeature, - const OGRGeoJSONWriteOptions &oOptions); void OGRGeoJSONWriteId(const OGRFeature *poFeature, json_object *poObj, bool bIdAlreadyWritten, const OGRGeoJSONWriteOptions &oOptions); + json_object *OGRGeoJSONWriteAttributes( OGRFeature *poFeature, bool bWriteIdIfFoundInAttributes = true, const OGRGeoJSONWriteOptions &oOptions = OGRGeoJSONWriteOptions()); + json_object CPL_DLL * OGRGeoJSONWriteGeometry(const OGRGeometry *poGeometry, const OGRGeoJSONWriteOptions &oOptions); -json_object *OGRGeoJSONWritePoint(const OGRPoint *poPoint, - const OGRGeoJSONWriteOptions &oOptions); -json_object *OGRGeoJSONWriteLineString(const OGRLineString *poLine, - const OGRGeoJSONWriteOptions &oOptions); + json_object *OGRGeoJSONWritePolygon(const OGRPolygon *poPolygon, const OGRGeoJSONWriteOptions &oOptions); -json_object *OGRGeoJSONWriteMultiPoint(const OGRMultiPoint *poGeometry, - const OGRGeoJSONWriteOptions &oOptions); -json_object * -OGRGeoJSONWriteMultiLineString(const OGRMultiLineString *poGeometry, - const OGRGeoJSONWriteOptions &oOptions); -json_object * -OGRGeoJSONWriteMultiPolygon(const OGRMultiPolygon *poGeometry, - const OGRGeoJSONWriteOptions &oOptions); -json_object * -OGRGeoJSONWriteGeometryCollection(const OGRGeometryCollection *poGeometry, - const OGRGeoJSONWriteOptions &oOptions); - -json_object *OGRGeoJSONWriteCoords(double const &fX, double const &fY, - const OGRGeoJSONWriteOptions &oOptions); -json_object *OGRGeoJSONWriteCoords(double const &fX, double const &fY, - double const &fZ, - const OGRGeoJSONWriteOptions &oOptions); -json_object *OGRGeoJSONWriteLineCoords(const OGRLineString *poLine, - const OGRGeoJSONWriteOptions &oOptions); -json_object *OGRGeoJSONWriteRingCoords(const OGRLinearRing *poLine, - bool bIsExteriorRing, - const OGRGeoJSONWriteOptions &oOptions); -#endif + +/*! @endcond */ #endif /* OGR_GEOJSONWRITER_H_INCLUDED */ diff --git a/ogr/ogrgeometryfactory.cpp b/ogr/ogrgeometryfactory.cpp index ff9bad541cb8..27e76c6c2df1 100644 --- a/ogr/ogrgeometryfactory.cpp +++ b/ogr/ogrgeometryfactory.cpp @@ -44,7 +44,8 @@ #ifdef HAVE_GEOS #include "geos_c.h" #endif -#include "ogrgeojsonreader.h" + +#include "ogrgeojsongeometry.h" #include #include diff --git a/ogr/ogrlibjsonutils.cpp b/ogr/ogrlibjsonutils.cpp new file mode 100644 index 000000000000..610d48cf04cb --- /dev/null +++ b/ogr/ogrlibjsonutils.cpp @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: MIT +// Copyright 2007, Mateusz Loskot +// Copyright 2008-2024, Even Rouault + +/*! @cond Doxygen_Suppress */ + +#include "ogrlibjsonutils.h" + +#include "cpl_string.h" +#include "ogr_p.h" + +#include + +/************************************************************************/ +/* OGRJSonParse() */ +/************************************************************************/ + +bool OGRJSonParse(const char *pszText, json_object **ppoObj, bool bVerboseError) +{ + if (ppoObj == nullptr) + return false; + json_tokener *jstok = json_tokener_new(); + const int nLen = pszText == nullptr ? 0 : static_cast(strlen(pszText)); + *ppoObj = json_tokener_parse_ex(jstok, pszText, nLen); + if (jstok->err != json_tokener_success) + { + if (bVerboseError) + { + CPLError(CE_Failure, CPLE_AppDefined, + "JSON parsing error: %s (at offset %d)", + json_tokener_error_desc(jstok->err), jstok->char_offset); + } + + json_tokener_free(jstok); + *ppoObj = nullptr; + return false; + } + json_tokener_free(jstok); + return true; +} + +/************************************************************************/ +/* CPL_json_object_object_get() */ +/************************************************************************/ + +// This is the same as json_object_object_get() except it will not raise +// deprecation warning. + +json_object *CPL_json_object_object_get(struct json_object *obj, + const char *key) +{ + json_object *poRet = nullptr; + json_object_object_get_ex(obj, key, &poRet); + return poRet; +} + +/************************************************************************/ +/* json_ex_get_object_by_path() */ +/************************************************************************/ + +json_object *json_ex_get_object_by_path(json_object *poObj, const char *pszPath) +{ + if (poObj == nullptr || json_object_get_type(poObj) != json_type_object || + pszPath == nullptr || *pszPath == '\0') + { + return nullptr; + } + char **papszTokens = CSLTokenizeString2(pszPath, ".", 0); + for (int i = 0; papszTokens[i] != nullptr; i++) + { + poObj = CPL_json_object_object_get(poObj, papszTokens[i]); + if (poObj == nullptr) + break; + if (papszTokens[i + 1] != nullptr) + { + if (json_object_get_type(poObj) != json_type_object) + { + poObj = nullptr; + break; + } + } + } + CSLDestroy(papszTokens); + return poObj; +} + +/************************************************************************/ +/* OGRGeoJSONFindMemberByName */ +/************************************************************************/ + +lh_entry *OGRGeoJSONFindMemberEntryByName(json_object *poObj, + const char *pszName) +{ + if (nullptr == pszName || nullptr == poObj) + return nullptr; + + if (nullptr != json_object_get_object(poObj)) + { + lh_entry *entry = json_object_get_object(poObj)->head; + while (entry != nullptr) + { + if (EQUAL(static_cast(entry->k), pszName)) + return entry; + entry = entry->next; + } + } + + return nullptr; +} + +json_object *OGRGeoJSONFindMemberByName(json_object *poObj, const char *pszName) +{ + lh_entry *entry = OGRGeoJSONFindMemberEntryByName(poObj, pszName); + if (nullptr == entry) + return nullptr; + return static_cast(const_cast(entry->v)); +} + +/************************************************************************/ +/* OGR_json_double_with_precision_to_string() */ +/************************************************************************/ + +static int OGR_json_double_with_precision_to_string(struct json_object *jso, + struct printbuf *pb, + int /* level */, + int /* flags */) +{ + const void *userData = +#if (!defined(JSON_C_VERSION_NUM)) || (JSON_C_VERSION_NUM < JSON_C_VER_013) + jso->_userdata; +#else + json_object_get_userdata(jso); +#endif + // Precision is stored as a uintptr_t content casted to void* + const uintptr_t nPrecision = reinterpret_cast(userData); + char szBuffer[75] = {}; + const double dfVal = json_object_get_double(jso); + if (fabs(dfVal) > 1e50 && !std::isinf(dfVal)) + { + CPLsnprintf(szBuffer, sizeof(szBuffer), "%.17g", dfVal); + } + else + { + const bool bPrecisionIsNegative = + (nPrecision >> (8 * sizeof(nPrecision) - 1)) != 0; + OGRFormatDouble(szBuffer, sizeof(szBuffer), dfVal, '.', + bPrecisionIsNegative ? 15 + : static_cast(nPrecision)); + } + return printbuf_memappend(pb, szBuffer, static_cast(strlen(szBuffer))); +} + +/************************************************************************/ +/* json_object_new_double_with_precision() */ +/************************************************************************/ + +json_object *json_object_new_double_with_precision(double dfVal, + int nCoordPrecision) +{ + json_object *jso = json_object_new_double(dfVal); + json_object_set_serializer( + jso, OGR_json_double_with_precision_to_string, + reinterpret_cast(static_cast(nCoordPrecision)), + nullptr); + return jso; +} + +/************************************************************************/ +/* OGR_json_double_with_significant_figures_to_string() */ +/************************************************************************/ + +static int OGR_json_double_with_significant_figures_to_string( + struct json_object *jso, struct printbuf *pb, int /* level */, + int /* flags */) +{ + char szBuffer[75] = {}; + int nSize = 0; + const double dfVal = json_object_get_double(jso); + if (std::isnan(dfVal)) + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "NaN"); + else if (std::isinf(dfVal)) + { + if (dfVal > 0) + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "Infinity"); + else + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), "-Infinity"); + } + else + { + char szFormatting[32] = {}; + const void *userData = +#if (!defined(JSON_C_VERSION_NUM)) || (JSON_C_VERSION_NUM < JSON_C_VER_013) + jso->_userdata; +#else + json_object_get_userdata(jso); +#endif + const uintptr_t nSignificantFigures = + reinterpret_cast(userData); + const bool bSignificantFiguresIsNegative = + (nSignificantFigures >> (8 * sizeof(nSignificantFigures) - 1)) != 0; + const int nInitialSignificantFigures = + bSignificantFiguresIsNegative + ? 17 + : static_cast(nSignificantFigures); + CPLsnprintf(szFormatting, sizeof(szFormatting), "%%.%dg", + nInitialSignificantFigures); + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), szFormatting, dfVal); + const char *pszDot = strchr(szBuffer, '.'); + + // Try to avoid .xxxx999999y or .xxxx000000y rounding issues by + // decreasing a bit precision. + if (nInitialSignificantFigures > 10 && pszDot != nullptr && + (strstr(pszDot, "999999") != nullptr || + strstr(pszDot, "000000") != nullptr)) + { + bool bOK = false; + for (int i = 1; i <= 3; i++) + { + CPLsnprintf(szFormatting, sizeof(szFormatting), "%%.%dg", + nInitialSignificantFigures - i); + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), szFormatting, + dfVal); + pszDot = strchr(szBuffer, '.'); + if (pszDot != nullptr && strstr(pszDot, "999999") == nullptr && + strstr(pszDot, "000000") == nullptr) + { + bOK = true; + break; + } + } + if (!bOK) + { + CPLsnprintf(szFormatting, sizeof(szFormatting), "%%.%dg", + nInitialSignificantFigures); + nSize = CPLsnprintf(szBuffer, sizeof(szBuffer), szFormatting, + dfVal); + } + } + + if (nSize + 2 < static_cast(sizeof(szBuffer)) && + strchr(szBuffer, '.') == nullptr && + strchr(szBuffer, 'e') == nullptr) + { + nSize += + CPLsnprintf(szBuffer + nSize, sizeof(szBuffer) - nSize, ".0"); + } + } + + return printbuf_memappend(pb, szBuffer, nSize); +} + +/************************************************************************/ +/* json_object_new_double_with_significant_figures() */ +/************************************************************************/ + +json_object * +json_object_new_double_with_significant_figures(double dfVal, + int nSignificantFigures) +{ + json_object *jso = json_object_new_double(dfVal); + json_object_set_serializer( + jso, OGR_json_double_with_significant_figures_to_string, + reinterpret_cast(static_cast(nSignificantFigures)), + nullptr); + return jso; +} + +/************************************************************************/ +/* GeoJSONPropertyToFieldType() */ +/************************************************************************/ + +constexpr GIntBig MY_INT64_MAX = + (static_cast(0x7FFFFFFF) << 32) | 0xFFFFFFFF; +constexpr GIntBig MY_INT64_MIN = static_cast(0x80000000) << 32; + +OGRFieldType GeoJSONPropertyToFieldType(json_object *poObject, + OGRFieldSubType &eSubType, + bool bArrayAsString) +{ + eSubType = OFSTNone; + + if (poObject == nullptr) + { + return OFTString; + } + + json_type type = json_object_get_type(poObject); + + if (json_type_boolean == type) + { + eSubType = OFSTBoolean; + return OFTInteger; + } + else if (json_type_double == type) + return OFTReal; + else if (json_type_int == type) + { + GIntBig nVal = json_object_get_int64(poObject); + if (!CPL_INT64_FITS_ON_INT32(nVal)) + { + if (nVal == MY_INT64_MIN || nVal == MY_INT64_MAX) + { + static bool bWarned = false; + if (!bWarned) + { + bWarned = true; + CPLError( + CE_Warning, CPLE_AppDefined, + "Integer values probably ranging out of 64bit integer " + "range have been found. Will be clamped to " + "INT64_MIN/INT64_MAX"); + } + } + return OFTInteger64; + } + else + { + return OFTInteger; + } + } + else if (json_type_string == type) + return OFTString; + else if (json_type_array == type) + { + if (bArrayAsString) + { + eSubType = OFSTJSON; + return OFTString; + } + const auto nSize = json_object_array_length(poObject); + if (nSize == 0) + { + eSubType = OFSTJSON; + return OFTString; + } + OGRFieldType eType = OFTIntegerList; + for (auto i = decltype(nSize){0}; i < nSize; i++) + { + json_object *poRow = json_object_array_get_idx(poObject, i); + if (poRow != nullptr) + { + type = json_object_get_type(poRow); + if (type == json_type_string) + { + if (i == 0 || eType == OFTStringList) + { + eType = OFTStringList; + } + else + { + eSubType = OFSTJSON; + return OFTString; + } + } + else if (type == json_type_double) + { + if (eSubType == OFSTNone && + (i == 0 || eType == OFTRealList || + eType == OFTIntegerList || eType == OFTInteger64List)) + { + eType = OFTRealList; + } + else + { + eSubType = OFSTJSON; + return OFTString; + } + } + else if (type == json_type_int) + { + if (eSubType == OFSTNone && eType == OFTIntegerList) + { + GIntBig nVal = json_object_get_int64(poRow); + if (!CPL_INT64_FITS_ON_INT32(nVal)) + eType = OFTInteger64List; + } + else if (eSubType == OFSTNone && + (eType == OFTInteger64List || + eType == OFTRealList)) + { + // ok + } + else + { + eSubType = OFSTJSON; + return OFTString; + } + } + else if (type == json_type_boolean) + { + if (i == 0 || + (eType == OFTIntegerList && eSubType == OFSTBoolean)) + { + eSubType = OFSTBoolean; + } + else + { + eSubType = OFSTJSON; + return OFTString; + } + } + else + { + eSubType = OFSTJSON; + return OFTString; + } + } + else + { + eSubType = OFSTJSON; + return OFTString; + } + } + + return eType; + } + else if (json_type_object == type) + { + eSubType = OFSTJSON; + return OFTString; + } + + return OFTString; // null +} + +/*! @endcond */ diff --git a/ogr/ogrlibjsonutils.h b/ogr/ogrlibjsonutils.h new file mode 100644 index 000000000000..732eaa59d751 --- /dev/null +++ b/ogr/ogrlibjsonutils.h @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT +// Copyright 2007, Mateusz Loskot +// Copyright 2008-2024, Even Rouault + +#ifndef OGRLIBJSONUTILS_H_INCLUDED +#define OGRLIBJSONUTILS_H_INCLUDED + +/*! @cond Doxygen_Suppress */ + +#include "cpl_port.h" +#include "cpl_json_header.h" + +#include "ogr_api.h" + +bool CPL_DLL OGRJSonParse(const char *pszText, json_object **ppoObj, + bool bVerboseError = true); + +json_object CPL_DLL *CPL_json_object_object_get(struct json_object *obj, + const char *key); +json_object CPL_DLL *json_ex_get_object_by_path(json_object *poObj, + const char *pszPath); + +/************************************************************************/ +/* GeoJSON Parsing Utilities */ +/************************************************************************/ + +lh_entry CPL_DLL *OGRGeoJSONFindMemberEntryByName(json_object *poObj, + const char *pszName); +json_object CPL_DLL *OGRGeoJSONFindMemberByName(json_object *poObj, + const char *pszName); + +/************************************************************************/ +/* GeoJSONPropertyToFieldType */ +/************************************************************************/ + +OGRFieldType CPL_DLL GeoJSONPropertyToFieldType(json_object *poObject, + OGRFieldSubType &eSubType, + bool bArrayAsString = false); + +CPL_C_START +/* %.XXXf formatting */ +json_object CPL_DLL *json_object_new_double_with_precision(double dfVal, + int nCoordPrecision); + +/* %.XXXg formatting */ +json_object CPL_DLL * +json_object_new_double_with_significant_figures(double dfVal, + int nSignificantFigures); +CPL_C_END + +/*! @endcond */ + +#endif diff --git a/ogr/ogrsf_frmts/amigocloud/CMakeLists.txt b/ogr/ogrsf_frmts/amigocloud/CMakeLists.txt index 31ea887cec98..70c866dbb589 100644 --- a/ogr/ogrsf_frmts/amigocloud/CMakeLists.txt +++ b/ogr/ogrsf_frmts/amigocloud/CMakeLists.txt @@ -10,8 +10,7 @@ add_gdal_driver( "NOT GDAL_USE_JSONC_INTERNAL" NO_DEPS) gdal_standard_includes(ogr_AmigoCloud) -target_include_directories(ogr_AmigoCloud PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/pgdump - ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/geojson) +target_include_directories(ogr_AmigoCloud PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/pgdump) if (GDAL_USE_JSONC_INTERNAL) gdal_add_vendored_lib(ogr_AmigoCloud libjson) else () diff --git a/ogr/ogrsf_frmts/amigocloud/ogramigoclouddatasource.cpp b/ogr/ogrsf_frmts/amigocloud/ogramigoclouddatasource.cpp index 3cd0ed5cfe01..675c6d23c70d 100644 --- a/ogr/ogrsf_frmts/amigocloud/ogramigoclouddatasource.cpp +++ b/ogr/ogrsf_frmts/amigocloud/ogramigoclouddatasource.cpp @@ -28,7 +28,7 @@ #include "ogr_amigocloud.h" #include "ogr_pgdump.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include CPLString OGRAMIGOCLOUDGetOptionValue(const char *pszFilename, diff --git a/ogr/ogrsf_frmts/amigocloud/ogramigocloudlayer.cpp b/ogr/ogrsf_frmts/amigocloud/ogramigocloudlayer.cpp index be5e991987b4..3aa6771d887c 100644 --- a/ogr/ogrsf_frmts/amigocloud/ogramigocloudlayer.cpp +++ b/ogr/ogrsf_frmts/amigocloud/ogramigocloudlayer.cpp @@ -28,7 +28,7 @@ #include "ogr_amigocloud.h" #include "ogr_p.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" /************************************************************************/ /* OGRAmigoCloudLayer() */ diff --git a/ogr/ogrsf_frmts/amigocloud/ogramigocloudtablelayer.cpp b/ogr/ogrsf_frmts/amigocloud/ogramigocloudtablelayer.cpp index 5dc99fe45604..43e559e7519e 100644 --- a/ogr/ogrsf_frmts/amigocloud/ogramigocloudtablelayer.cpp +++ b/ogr/ogrsf_frmts/amigocloud/ogramigocloudtablelayer.cpp @@ -29,7 +29,7 @@ #include "ogr_amigocloud.h" #include "ogr_p.h" #include "ogr_pgdump.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include #include diff --git a/ogr/ogrsf_frmts/carto/CMakeLists.txt b/ogr/ogrsf_frmts/carto/CMakeLists.txt index 01a5c295c076..0d3478c7b77b 100644 --- a/ogr/ogrsf_frmts/carto/CMakeLists.txt +++ b/ogr/ogrsf_frmts/carto/CMakeLists.txt @@ -17,8 +17,7 @@ if(NOT TARGET ogr_Carto) endif() gdal_standard_includes(ogr_Carto) -target_include_directories(ogr_Carto PRIVATE $ - $) +target_include_directories(ogr_Carto PRIVATE $) if (GDAL_USE_JSONC_INTERNAL) gdal_add_vendored_lib(ogr_Carto libjson) else () diff --git a/ogr/ogrsf_frmts/carto/ogrcartodatasource.cpp b/ogr/ogrsf_frmts/carto/ogrcartodatasource.cpp index a50ec0d778ac..b65d21944b74 100644 --- a/ogr/ogrsf_frmts/carto/ogrcartodatasource.cpp +++ b/ogr/ogrsf_frmts/carto/ogrcartodatasource.cpp @@ -28,7 +28,7 @@ #include "ogr_carto.h" #include "ogr_pgdump.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" /************************************************************************/ /* OGRCARTODataSource() */ diff --git a/ogr/ogrsf_frmts/carto/ogrcartolayer.cpp b/ogr/ogrsf_frmts/carto/ogrcartolayer.cpp index 11661b08ae0b..d5d7c85f7f8b 100644 --- a/ogr/ogrsf_frmts/carto/ogrcartolayer.cpp +++ b/ogr/ogrsf_frmts/carto/ogrcartolayer.cpp @@ -28,7 +28,7 @@ #include "ogr_carto.h" #include "ogr_p.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" /************************************************************************/ /* OGRCARTOLayer() */ diff --git a/ogr/ogrsf_frmts/carto/ogrcartotablelayer.cpp b/ogr/ogrsf_frmts/carto/ogrcartotablelayer.cpp index 460073385666..24c953db5ab7 100644 --- a/ogr/ogrsf_frmts/carto/ogrcartotablelayer.cpp +++ b/ogr/ogrsf_frmts/carto/ogrcartotablelayer.cpp @@ -29,7 +29,7 @@ #include "ogr_carto.h" #include "ogr_p.h" #include "ogr_pgdump.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" /************************************************************************/ /* OGRCARTOEscapeIdentifier( ) */ diff --git a/ogr/ogrsf_frmts/elastic/CMakeLists.txt b/ogr/ogrsf_frmts/elastic/CMakeLists.txt index 0f8544eb801c..f34e98f5f7fa 100644 --- a/ogr/ogrsf_frmts/elastic/CMakeLists.txt +++ b/ogr/ogrsf_frmts/elastic/CMakeLists.txt @@ -16,7 +16,6 @@ if(NOT TARGET ogr_Elastic) endif() gdal_standard_includes(ogr_Elastic) -target_include_directories(ogr_Elastic PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/geojson) if (GDAL_USE_JSONC_INTERNAL) gdal_add_vendored_lib(ogr_Elastic libjson) else () diff --git a/ogr/ogrsf_frmts/elastic/ogrelasticaggregationlayer.cpp b/ogr/ogrsf_frmts/elastic/ogrelasticaggregationlayer.cpp index cbda028cd36d..7020d3db7852 100644 --- a/ogr/ogrsf_frmts/elastic/ogrelasticaggregationlayer.cpp +++ b/ogr/ogrsf_frmts/elastic/ogrelasticaggregationlayer.cpp @@ -27,7 +27,7 @@ ****************************************************************************/ #include "ogr_elastic.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include "cpl_json.h" #include diff --git a/ogr/ogrsf_frmts/elastic/ogrelasticdatasource.cpp b/ogr/ogrsf_frmts/elastic/ogrelasticdatasource.cpp index 02a9b464c3e5..dcbec753091a 100644 --- a/ogr/ogrsf_frmts/elastic/ogrelasticdatasource.cpp +++ b/ogr/ogrsf_frmts/elastic/ogrelasticdatasource.cpp @@ -32,7 +32,7 @@ #include "cpl_string.h" #include "cpl_csv.h" #include "cpl_http.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include "ogr_swq.h" /************************************************************************/ diff --git a/ogr/ogrsf_frmts/elastic/ogrelasticlayer.cpp b/ogr/ogrsf_frmts/elastic/ogrelasticlayer.cpp index 5b153981025d..1d8d8810735f 100644 --- a/ogr/ogrsf_frmts/elastic/ogrelasticlayer.cpp +++ b/ogr/ogrsf_frmts/elastic/ogrelasticlayer.cpp @@ -34,9 +34,9 @@ #include "ogr_api.h" #include "ogr_p.h" #include "ogr_swq.h" -#include "../geojson/ogrgeojsonwriter.h" -#include "../geojson/ogrgeojsonreader.h" -#include "../geojson/ogrgeojsonutils.h" +#include "ogrgeojsonwriter.h" +#include "ogrlibjsonutils.h" +#include "ogrgeojsongeometry.h" #include "ogr_geo_utils.h" #include diff --git a/ogr/ogrsf_frmts/geojson/CMakeLists.txt b/ogr/ogrsf_frmts/geojson/CMakeLists.txt index 90760d556600..6e9bea85b064 100644 --- a/ogr/ogrsf_frmts/geojson/CMakeLists.txt +++ b/ogr/ogrsf_frmts/geojson/CMakeLists.txt @@ -6,7 +6,6 @@ add_gdal_driver( ogrgeojsonreader.cpp ogrgeojsonutils.cpp ogrgeojsonwritelayer.cpp - ogrgeojsonwriter.cpp ogrgeojsondriver.cpp ogrgeojsonseqdriver.cpp ogresrijsonreader.cpp diff --git a/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp b/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp index 1a3a13402b97..6095775c8e2a 100644 --- a/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp +++ b/ogr/ogrsf_frmts/geojson/ogresrijsonreader.cpp @@ -30,7 +30,7 @@ ****************************************************************************/ #include "cpl_port.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include #include @@ -49,6 +49,7 @@ #include "ogr_geojson.h" #include "ogrgeojsonreader.h" #include "ogrgeojsonutils.h" +#include "ogresrijsongeometry.h" // #include "symbol_renames.h" @@ -411,51 +412,6 @@ bool OGRESRIJSONReader::AddFeature(OGRFeature *poFeature) return true; } -/************************************************************************/ -/* OGRESRIJSONReadGeometry() */ -/************************************************************************/ - -OGRGeometry *OGRESRIJSONReadGeometry(json_object *poObj) -{ - OGRGeometry *poGeometry = nullptr; - - if (OGRGeoJSONFindMemberByName(poObj, "x")) - poGeometry = OGRESRIJSONReadPoint(poObj); - else if (OGRGeoJSONFindMemberByName(poObj, "paths")) - poGeometry = OGRESRIJSONReadLineString(poObj); - else if (OGRGeoJSONFindMemberByName(poObj, "rings")) - poGeometry = OGRESRIJSONReadPolygon(poObj); - else if (OGRGeoJSONFindMemberByName(poObj, "points")) - poGeometry = OGRESRIJSONReadMultiPoint(poObj); - - return poGeometry; -} - -/************************************************************************/ -/* OGR_G_CreateGeometryFromEsriJson() */ -/************************************************************************/ - -/** Create a OGR geometry from a ESRIJson geometry object */ -OGRGeometryH OGR_G_CreateGeometryFromEsriJson(const char *pszJson) -{ - if (nullptr == pszJson) - { - // Translation failed. - return nullptr; - } - - json_object *poObj = nullptr; - if (!OGRJSonParse(pszJson, &poObj)) - return nullptr; - - OGRGeometry *poGeometry = OGRESRIJSONReadGeometry(poObj); - - // Release JSON tree. - json_object_put(poObj); - - return OGRGeometry::ToHandle(poGeometry); -} - /************************************************************************/ /* EsriDateToOGRDate() */ /************************************************************************/ @@ -625,589 +581,3 @@ OGRGeoJSONLayer *OGRESRIJSONReader::ReadFeatureCollection(json_object *poObj) CPLAssert(nullptr != poLayer_); return poLayer_; } - -/************************************************************************/ -/* OGRESRIJSONGetType() */ -/************************************************************************/ - -OGRwkbGeometryType OGRESRIJSONGetGeometryType(json_object *poObj) -{ - if (nullptr == poObj) - return wkbUnknown; - - json_object *poObjType = OGRGeoJSONFindMemberByName(poObj, "geometryType"); - if (nullptr == poObjType) - { - return wkbNone; - } - - const char *name = json_object_get_string(poObjType); - if (EQUAL(name, "esriGeometryPoint")) - return wkbPoint; - else if (EQUAL(name, "esriGeometryPolyline")) - return wkbLineString; - else if (EQUAL(name, "esriGeometryPolygon")) - return wkbPolygon; - else if (EQUAL(name, "esriGeometryMultiPoint")) - return wkbMultiPoint; - else - return wkbUnknown; -} - -/************************************************************************/ -/* OGRESRIJSONGetCoordinateToDouble() */ -/************************************************************************/ - -static double OGRESRIJSONGetCoordinateToDouble(json_object *poObjCoord, - const char *pszCoordName, - bool &bValid) -{ - const int iType = json_object_get_type(poObjCoord); - if (json_type_double != iType && json_type_int != iType) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid '%s' coordinate. " - "Type is not double or integer for \'%s\'.", - pszCoordName, json_object_to_json_string(poObjCoord)); - bValid = false; - return 0.0; - } - - return json_object_get_double(poObjCoord); -} - -/************************************************************************/ -/* OGRESRIJSONGetCoordinate() */ -/************************************************************************/ - -static double OGRESRIJSONGetCoordinate(json_object *poObj, - const char *pszCoordName, bool &bValid) -{ - json_object *poObjCoord = OGRGeoJSONFindMemberByName(poObj, pszCoordName); - if (nullptr == poObjCoord) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid Point object. " - "Missing '%s' member.", - pszCoordName); - bValid = false; - return 0.0; - } - - return OGRESRIJSONGetCoordinateToDouble(poObjCoord, pszCoordName, bValid); -} - -/************************************************************************/ -/* OGRESRIJSONReadPoint() */ -/************************************************************************/ - -OGRPoint *OGRESRIJSONReadPoint(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - bool bValid = true; - const double dfX = OGRESRIJSONGetCoordinate(poObj, "x", bValid); - const double dfY = OGRESRIJSONGetCoordinate(poObj, "y", bValid); - if (!bValid) - return nullptr; - - json_object *poObjZ = OGRGeoJSONFindMemberByName(poObj, "z"); - if (nullptr == poObjZ) - return new OGRPoint(dfX, dfY); - - const double dfZ = OGRESRIJSONGetCoordinateToDouble(poObjZ, "z", bValid); - if (!bValid) - return nullptr; - return new OGRPoint(dfX, dfY, dfZ); -} - -/************************************************************************/ -/* OGRESRIJSONReaderParseZM() */ -/************************************************************************/ - -static void OGRESRIJSONReaderParseZM(json_object *poObj, bool *bHasZ, - bool *bHasM) -{ - CPLAssert(nullptr != poObj); - // The ESRI geojson spec states that geometries other than point can - // have the attributes hasZ and hasM. A geometry that has a z value - // implies the 3rd number in the tuple is z. if hasM is true, but hasZ - // is not, it is the M value. - bool bZ = false; - json_object *poObjHasZ = OGRGeoJSONFindMemberByName(poObj, "hasZ"); - if (poObjHasZ != nullptr) - { - if (json_object_get_type(poObjHasZ) == json_type_boolean) - { - bZ = CPL_TO_BOOL(json_object_get_boolean(poObjHasZ)); - } - } - - bool bM = false; - json_object *poObjHasM = OGRGeoJSONFindMemberByName(poObj, "hasM"); - if (poObjHasM != nullptr) - { - if (json_object_get_type(poObjHasM) == json_type_boolean) - { - bM = CPL_TO_BOOL(json_object_get_boolean(poObjHasM)); - } - } - if (bHasZ != nullptr) - *bHasZ = bZ; - if (bHasM != nullptr) - *bHasM = bM; -} - -/************************************************************************/ -/* OGRESRIJSONReaderParseXYZMArray() */ -/************************************************************************/ - -static bool OGRESRIJSONReaderParseXYZMArray(json_object *poObjCoords, - bool /*bHasZ*/, bool bHasM, - double *pdfX, double *pdfY, - double *pdfZ, double *pdfM, - int *pnNumCoords) -{ - if (poObjCoords == nullptr) - { - CPLDebug("ESRIJSON", - "OGRESRIJSONReaderParseXYZMArray: got null object."); - return false; - } - - if (json_type_array != json_object_get_type(poObjCoords)) - { - CPLDebug("ESRIJSON", - "OGRESRIJSONReaderParseXYZMArray: got non-array object."); - return false; - } - - const auto coordDimension = json_object_array_length(poObjCoords); - - // Allow 4 coordinates if M is present, but it is eventually ignored. - if (coordDimension < 2 || coordDimension > 4) - { - CPLDebug("ESRIJSON", - "OGRESRIJSONReaderParseXYZMArray: got an unexpected " - "array object."); - return false; - } - - // Read X coordinate. - json_object *poObjCoord = json_object_array_get_idx(poObjCoords, 0); - if (poObjCoord == nullptr) - { - CPLDebug("ESRIJSON", - "OGRESRIJSONReaderParseXYZMArray: got null object."); - return false; - } - - bool bValid = true; - const double dfX = - OGRESRIJSONGetCoordinateToDouble(poObjCoord, "x", bValid); - - // Read Y coordinate. - poObjCoord = json_object_array_get_idx(poObjCoords, 1); - if (poObjCoord == nullptr) - { - CPLDebug("ESRIJSON", - "OGRESRIJSONReaderParseXYZMArray: got null object."); - return false; - } - - const double dfY = - OGRESRIJSONGetCoordinateToDouble(poObjCoord, "y", bValid); - if (!bValid) - return false; - - // Read Z or M or Z and M coordinates. - if (coordDimension > 2) - { - poObjCoord = json_object_array_get_idx(poObjCoords, 2); - if (poObjCoord == nullptr) - { - CPLDebug("ESRIJSON", - "OGRESRIJSONReaderParseXYZMArray: got null object."); - return false; - } - - const double dfZorM = OGRESRIJSONGetCoordinateToDouble( - poObjCoord, (coordDimension > 3 || !bHasM) ? "z" : "m", bValid); - if (!bValid) - return false; - if (pdfZ != nullptr) - { - if (coordDimension > 3 || !bHasM) - *pdfZ = dfZorM; - else - *pdfZ = 0.0; - } - if (pdfM != nullptr && coordDimension == 3) - { - if (bHasM) - *pdfM = dfZorM; - else - *pdfM = 0.0; - } - if (coordDimension == 4) - { - poObjCoord = json_object_array_get_idx(poObjCoords, 3); - if (poObjCoord == nullptr) - { - CPLDebug("ESRIJSON", - "OGRESRIJSONReaderParseXYZMArray: got null object."); - return false; - } - - const double dfM = - OGRESRIJSONGetCoordinateToDouble(poObjCoord, "m", bValid); - if (!bValid) - return false; - if (pdfM != nullptr) - *pdfM = dfM; - } - } - else - { - if (pdfZ != nullptr) - *pdfZ = 0.0; - if (pdfM != nullptr) - *pdfM = 0.0; - } - - if (pnNumCoords != nullptr) - *pnNumCoords = static_cast(coordDimension); - if (pdfX != nullptr) - *pdfX = dfX; - if (pdfY != nullptr) - *pdfY = dfY; - - return true; -} - -/************************************************************************/ -/* OGRESRIJSONReadLineString() */ -/************************************************************************/ - -OGRGeometry *OGRESRIJSONReadLineString(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - bool bHasZ = false; - bool bHasM = false; - - OGRESRIJSONReaderParseZM(poObj, &bHasZ, &bHasM); - - json_object *poObjPaths = OGRGeoJSONFindMemberByName(poObj, "paths"); - if (nullptr == poObjPaths) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid LineString object. " - "Missing \'paths\' member."); - return nullptr; - } - - if (json_type_array != json_object_get_type(poObjPaths)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid LineString object. " - "Invalid \'paths\' member."); - return nullptr; - } - - OGRMultiLineString *poMLS = nullptr; - OGRGeometry *poRet = nullptr; - const auto nPaths = json_object_array_length(poObjPaths); - for (auto iPath = decltype(nPaths){0}; iPath < nPaths; iPath++) - { - json_object *poObjPath = json_object_array_get_idx(poObjPaths, iPath); - if (poObjPath == nullptr || - json_type_array != json_object_get_type(poObjPath)) - { - delete poRet; - CPLDebug("ESRIJSON", "LineString: got non-array object."); - return nullptr; - } - - OGRLineString *poLine = new OGRLineString(); - if (nPaths > 1) - { - if (iPath == 0) - { - poMLS = new OGRMultiLineString(); - poRet = poMLS; - } - poMLS->addGeometryDirectly(poLine); - } - else - { - poRet = poLine; - } - const auto nPoints = json_object_array_length(poObjPath); - for (auto i = decltype(nPoints){0}; i < nPoints; i++) - { - int nNumCoords = 2; - json_object *poObjCoords = json_object_array_get_idx(poObjPath, i); - double dfX = 0.0; - double dfY = 0.0; - double dfZ = 0.0; - double dfM = 0.0; - if (!OGRESRIJSONReaderParseXYZMArray(poObjCoords, bHasZ, bHasM, - &dfX, &dfY, &dfZ, &dfM, - &nNumCoords)) - { - delete poRet; - return nullptr; - } - - if (nNumCoords == 3 && !bHasM) - { - poLine->addPoint(dfX, dfY, dfZ); - } - else if (nNumCoords == 3) - { - poLine->addPointM(dfX, dfY, dfM); - } - else if (nNumCoords == 4) - { - poLine->addPoint(dfX, dfY, dfZ, dfM); - } - else - { - poLine->addPoint(dfX, dfY); - } - } - } - - if (poRet == nullptr) - poRet = new OGRLineString(); - - return poRet; -} - -/************************************************************************/ -/* OGRESRIJSONReadPolygon() */ -/************************************************************************/ - -OGRGeometry *OGRESRIJSONReadPolygon(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - bool bHasZ = false; - bool bHasM = false; - - OGRESRIJSONReaderParseZM(poObj, &bHasZ, &bHasM); - - json_object *poObjRings = OGRGeoJSONFindMemberByName(poObj, "rings"); - if (nullptr == poObjRings) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid Polygon object. " - "Missing \'rings\' member."); - return nullptr; - } - - if (json_type_array != json_object_get_type(poObjRings)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid Polygon object. " - "Invalid \'rings\' member."); - return nullptr; - } - - const auto nRings = json_object_array_length(poObjRings); - OGRGeometry **papoGeoms = new OGRGeometry *[nRings]; - for (auto iRing = decltype(nRings){0}; iRing < nRings; iRing++) - { - json_object *poObjRing = json_object_array_get_idx(poObjRings, iRing); - if (poObjRing == nullptr || - json_type_array != json_object_get_type(poObjRing)) - { - for (auto j = decltype(iRing){0}; j < iRing; j++) - delete papoGeoms[j]; - delete[] papoGeoms; - CPLDebug("ESRIJSON", "Polygon: got non-array object."); - return nullptr; - } - - OGRPolygon *poPoly = new OGRPolygon(); - auto poLine = std::make_unique(); - papoGeoms[iRing] = poPoly; - - const auto nPoints = json_object_array_length(poObjRing); - for (auto i = decltype(nPoints){0}; i < nPoints; i++) - { - int nNumCoords = 2; - json_object *poObjCoords = json_object_array_get_idx(poObjRing, i); - double dfX = 0.0; - double dfY = 0.0; - double dfZ = 0.0; - double dfM = 0.0; - if (!OGRESRIJSONReaderParseXYZMArray(poObjCoords, bHasZ, bHasM, - &dfX, &dfY, &dfZ, &dfM, - &nNumCoords)) - { - for (auto j = decltype(iRing){0}; j <= iRing; j++) - delete papoGeoms[j]; - delete[] papoGeoms; - return nullptr; - } - - if (nNumCoords == 3 && !bHasM) - { - poLine->addPoint(dfX, dfY, dfZ); - } - else if (nNumCoords == 3) - { - poLine->addPointM(dfX, dfY, dfM); - } - else if (nNumCoords == 4) - { - poLine->addPoint(dfX, dfY, dfZ, dfM); - } - else - { - poLine->addPoint(dfX, dfY); - } - } - poPoly->addRingDirectly(poLine.release()); - } - - OGRGeometry *poRet = OGRGeometryFactory::organizePolygons( - papoGeoms, static_cast(nRings), nullptr, nullptr); - delete[] papoGeoms; - - return poRet; -} - -/************************************************************************/ -/* OGRESRIJSONReadMultiPoint() */ -/************************************************************************/ - -OGRMultiPoint *OGRESRIJSONReadMultiPoint(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - bool bHasZ = false; - bool bHasM = false; - - OGRESRIJSONReaderParseZM(poObj, &bHasZ, &bHasM); - - json_object *poObjPoints = OGRGeoJSONFindMemberByName(poObj, "points"); - if (nullptr == poObjPoints) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid MultiPoint object. " - "Missing \'points\' member."); - return nullptr; - } - - if (json_type_array != json_object_get_type(poObjPoints)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid MultiPoint object. " - "Invalid \'points\' member."); - return nullptr; - } - - OGRMultiPoint *poMulti = new OGRMultiPoint(); - - const auto nPoints = json_object_array_length(poObjPoints); - for (auto i = decltype(nPoints){0}; i < nPoints; i++) - { - int nNumCoords = 2; - json_object *poObjCoords = json_object_array_get_idx(poObjPoints, i); - double dfX = 0.0; - double dfY = 0.0; - double dfZ = 0.0; - double dfM = 0.0; - if (!OGRESRIJSONReaderParseXYZMArray(poObjCoords, bHasZ, bHasM, &dfX, - &dfY, &dfZ, &dfM, &nNumCoords)) - { - delete poMulti; - return nullptr; - } - - if (nNumCoords == 3 && !bHasM) - { - poMulti->addGeometryDirectly(new OGRPoint(dfX, dfY, dfZ)); - } - else if (nNumCoords == 3) - { - OGRPoint *poPoint = new OGRPoint(dfX, dfY); - poPoint->setM(dfM); - poMulti->addGeometryDirectly(poPoint); - } - else if (nNumCoords == 4) - { - poMulti->addGeometryDirectly(new OGRPoint(dfX, dfY, dfZ, dfM)); - } - else - { - poMulti->addGeometryDirectly(new OGRPoint(dfX, dfY)); - } - } - - return poMulti; -} - -/************************************************************************/ -/* OGRESRIJSONReadSpatialReference() */ -/************************************************************************/ - -OGRSpatialReference *OGRESRIJSONReadSpatialReference(json_object *poObj) -{ - /* -------------------------------------------------------------------- */ - /* Read spatial reference definition. */ - /* -------------------------------------------------------------------- */ - OGRSpatialReference *poSRS = nullptr; - - json_object *poObjSrs = - OGRGeoJSONFindMemberByName(poObj, "spatialReference"); - if (nullptr != poObjSrs) - { - json_object *poObjWkid = - OGRGeoJSONFindMemberByName(poObjSrs, "latestWkid"); - if (poObjWkid == nullptr) - poObjWkid = OGRGeoJSONFindMemberByName(poObjSrs, "wkid"); - if (poObjWkid == nullptr) - { - json_object *poObjWkt = OGRGeoJSONFindMemberByName(poObjSrs, "wkt"); - if (poObjWkt == nullptr) - return nullptr; - - const char *pszWKT = json_object_get_string(poObjWkt); - poSRS = new OGRSpatialReference(); - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (OGRERR_NONE != poSRS->importFromWkt(pszWKT)) - { - delete poSRS; - poSRS = nullptr; - } - else - { - auto poSRSMatch = poSRS->FindBestMatch(70); - if (poSRSMatch) - { - poSRS->Release(); - poSRS = poSRSMatch; - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - } - } - - return poSRS; - } - - const int nEPSG = json_object_get_int(poObjWkid); - - poSRS = new OGRSpatialReference(); - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (OGRERR_NONE != poSRS->importFromEPSG(nEPSG)) - { - delete poSRS; - poSRS = nullptr; - } - } - - return poSRS; -} diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp index 426f07ef90ef..3079d4ff19b5 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsondatasource.cpp @@ -51,6 +51,7 @@ #include "ogr_feature.h" #include "ogr_geometry.h" #include "ogr_spatialref.h" +#include "ogrlibjsonutils.h" #include "ogrgeojsonreader.h" #include "ogrgeojsonutils.h" #include "ogrgeojsonwriter.h" diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.cpp index 2cbe962b301c..995135f3749c 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.cpp @@ -29,7 +29,9 @@ #include "ogrgeojsonreader.h" #include "ogrgeojsonutils.h" +#include "ogrgeojsongeometry.h" #include "ogr_geojson.h" +#include "ogrlibjsonutils.h" #include "ogrjsoncollectionstreamingparser.h" #include "ogr_api.h" @@ -1043,149 +1045,6 @@ void OGRGeoJSONReader::ReadLayer(OGRGeoJSONDataSource *poDS, poDS->AddLayer(poLayer); } -/************************************************************************/ -/* OGRGeoJSONReadSpatialReference */ -/************************************************************************/ - -OGRSpatialReference *OGRGeoJSONReadSpatialReference(json_object *poObj) -{ - - /* -------------------------------------------------------------------- */ - /* Read spatial reference definition. */ - /* -------------------------------------------------------------------- */ - OGRSpatialReference *poSRS = nullptr; - - json_object *poObjSrs = OGRGeoJSONFindMemberByName(poObj, "crs"); - if (nullptr != poObjSrs) - { - json_object *poObjSrsType = - OGRGeoJSONFindMemberByName(poObjSrs, "type"); - if (poObjSrsType == nullptr) - return nullptr; - - const char *pszSrsType = json_object_get_string(poObjSrsType); - - // TODO: Add URL and URN types support. - if (STARTS_WITH_CI(pszSrsType, "NAME")) - { - json_object *poObjSrsProps = - OGRGeoJSONFindMemberByName(poObjSrs, "properties"); - if (poObjSrsProps == nullptr) - return nullptr; - - json_object *poNameURL = - OGRGeoJSONFindMemberByName(poObjSrsProps, "name"); - if (poNameURL == nullptr) - return nullptr; - - const char *pszName = json_object_get_string(poNameURL); - - // Mostly to emulate GDAL 2.x behavior - // See https://github.com/OSGeo/gdal/issues/2035 - if (EQUAL(pszName, "urn:ogc:def:crs:OGC:1.3:CRS84")) - pszName = "EPSG:4326"; - - poSRS = new OGRSpatialReference(); - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (OGRERR_NONE != - poSRS->SetFromUserInput( - pszName, - OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get())) - { - delete poSRS; - poSRS = nullptr; - } - } - - else if (STARTS_WITH_CI(pszSrsType, "EPSG")) - { - json_object *poObjSrsProps = - OGRGeoJSONFindMemberByName(poObjSrs, "properties"); - if (poObjSrsProps == nullptr) - return nullptr; - - json_object *poObjCode = - OGRGeoJSONFindMemberByName(poObjSrsProps, "code"); - if (poObjCode == nullptr) - return nullptr; - - int nEPSG = json_object_get_int(poObjCode); - - poSRS = new OGRSpatialReference(); - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (OGRERR_NONE != poSRS->importFromEPSG(nEPSG)) - { - delete poSRS; - poSRS = nullptr; - } - } - - else if (STARTS_WITH_CI(pszSrsType, "URL") || - STARTS_WITH_CI(pszSrsType, "LINK")) - { - json_object *poObjSrsProps = - OGRGeoJSONFindMemberByName(poObjSrs, "properties"); - if (poObjSrsProps == nullptr) - return nullptr; - - json_object *poObjURL = - OGRGeoJSONFindMemberByName(poObjSrsProps, "url"); - - if (nullptr == poObjURL) - { - poObjURL = OGRGeoJSONFindMemberByName(poObjSrsProps, "href"); - } - if (poObjURL == nullptr) - return nullptr; - - const char *pszURL = json_object_get_string(poObjURL); - - poSRS = new OGRSpatialReference(); - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (OGRERR_NONE != poSRS->importFromUrl(pszURL)) - { - delete poSRS; - poSRS = nullptr; - } - } - - else if (EQUAL(pszSrsType, "OGC")) - { - json_object *poObjSrsProps = - OGRGeoJSONFindMemberByName(poObjSrs, "properties"); - if (poObjSrsProps == nullptr) - return nullptr; - - json_object *poObjURN = - OGRGeoJSONFindMemberByName(poObjSrsProps, "urn"); - if (poObjURN == nullptr) - return nullptr; - - poSRS = new OGRSpatialReference(); - poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER); - if (OGRERR_NONE != - poSRS->importFromURN(json_object_get_string(poObjURN))) - { - delete poSRS; - poSRS = nullptr; - } - } - } - - // Strip AXIS, since geojson has (easting, northing) / (longitude, latitude) - // order. According to http://www.geojson.org/geojson-spec.html#id2 : - // "Point coordinates are in x, y order (easting, northing for projected - // coordinates, longitude, latitude for geographic coordinates)". - if (poSRS != nullptr) - { - OGR_SRSNode *poGEOGCS = poSRS->GetAttrNode("GEOGCS"); - if (poGEOGCS != nullptr) - poGEOGCS->StripNodes("AXIS"); - } - - return poSRS; -} - /************************************************************************/ /* GenerateLayerDefn() */ /************************************************************************/ @@ -2459,775 +2318,9 @@ void OGRGeoJSONReader::ReadFeatureCollection(OGRGeoJSONLayer *poLayer, } /************************************************************************/ -/* OGRGeoJSONFindMemberByName */ -/************************************************************************/ - -lh_entry *OGRGeoJSONFindMemberEntryByName(json_object *poObj, - const char *pszName) -{ - if (nullptr == pszName || nullptr == poObj) - return nullptr; - - if (nullptr != json_object_get_object(poObj)) - { - lh_entry *entry = json_object_get_object(poObj)->head; - while (entry != nullptr) - { - if (EQUAL(static_cast(entry->k), pszName)) - return entry; - entry = entry->next; - } - } - - return nullptr; -} - -json_object *OGRGeoJSONFindMemberByName(json_object *poObj, const char *pszName) -{ - lh_entry *entry = OGRGeoJSONFindMemberEntryByName(poObj, pszName); - if (nullptr == entry) - return nullptr; - return (json_object *)entry->v; -} - -/************************************************************************/ -/* OGRGeoJSONGetType */ -/************************************************************************/ - -GeoJSONObject::Type OGRGeoJSONGetType(json_object *poObj) -{ - if (nullptr == poObj) - return GeoJSONObject::eUnknown; - - json_object *poObjType = OGRGeoJSONFindMemberByName(poObj, "type"); - if (nullptr == poObjType) - return GeoJSONObject::eUnknown; - - const char *name = json_object_get_string(poObjType); - if (EQUAL(name, "Point")) - return GeoJSONObject::ePoint; - else if (EQUAL(name, "LineString")) - return GeoJSONObject::eLineString; - else if (EQUAL(name, "Polygon")) - return GeoJSONObject::ePolygon; - else if (EQUAL(name, "MultiPoint")) - return GeoJSONObject::eMultiPoint; - else if (EQUAL(name, "MultiLineString")) - return GeoJSONObject::eMultiLineString; - else if (EQUAL(name, "MultiPolygon")) - return GeoJSONObject::eMultiPolygon; - else if (EQUAL(name, "GeometryCollection")) - return GeoJSONObject::eGeometryCollection; - else if (EQUAL(name, "Feature")) - return GeoJSONObject::eFeature; - else if (EQUAL(name, "FeatureCollection")) - return GeoJSONObject::eFeatureCollection; - else - return GeoJSONObject::eUnknown; -} - -/************************************************************************/ -/* OGRGeoJSONGetOGRGeometryType() */ -/************************************************************************/ - -OGRwkbGeometryType OGRGeoJSONGetOGRGeometryType(json_object *poObj) -{ - if (nullptr == poObj) - return wkbUnknown; - - json_object *poObjType = CPL_json_object_object_get(poObj, "type"); - if (nullptr == poObjType) - return wkbUnknown; - - OGRwkbGeometryType eType = wkbUnknown; - const char *name = json_object_get_string(poObjType); - if (EQUAL(name, "Point")) - eType = wkbPoint; - else if (EQUAL(name, "LineString")) - eType = wkbLineString; - else if (EQUAL(name, "Polygon")) - eType = wkbPolygon; - else if (EQUAL(name, "MultiPoint")) - eType = wkbMultiPoint; - else if (EQUAL(name, "MultiLineString")) - eType = wkbMultiLineString; - else if (EQUAL(name, "MultiPolygon")) - eType = wkbMultiPolygon; - else if (EQUAL(name, "GeometryCollection")) - eType = wkbGeometryCollection; - else - return wkbUnknown; - - json_object *poCoordinates; - if (eType == wkbGeometryCollection) - { - json_object *poGeometries = - CPL_json_object_object_get(poObj, "geometries"); - if (poGeometries && - json_object_get_type(poGeometries) == json_type_array && - json_object_array_length(poGeometries) > 0) - { - if (OGR_GT_HasZ(OGRGeoJSONGetOGRGeometryType( - json_object_array_get_idx(poGeometries, 0)))) - eType = OGR_GT_SetZ(eType); - } - } - else - { - poCoordinates = CPL_json_object_object_get(poObj, "coordinates"); - if (poCoordinates && - json_object_get_type(poCoordinates) == json_type_array && - json_object_array_length(poCoordinates) > 0) - { - while (true) - { - auto poChild = json_object_array_get_idx(poCoordinates, 0); - if (!(poChild && - json_object_get_type(poChild) == json_type_array && - json_object_array_length(poChild) > 0)) - { - if (json_object_array_length(poCoordinates) == 3) - eType = OGR_GT_SetZ(eType); - break; - } - poCoordinates = poChild; - } - } - } - - return eType; -} - -/************************************************************************/ -/* OGRGeoJSONReadGeometry */ -/************************************************************************/ - -OGRGeometry *OGRGeoJSONReadGeometry(json_object *poObj, - OGRSpatialReference *poParentSRS) -{ - - OGRGeometry *poGeometry = nullptr; - OGRSpatialReference *poSRS = nullptr; - lh_entry *entry = OGRGeoJSONFindMemberEntryByName(poObj, "crs"); - if (entry != nullptr) - { - json_object *poObjSrs = (json_object *)entry->v; - if (poObjSrs != nullptr) - { - poSRS = OGRGeoJSONReadSpatialReference(poObj); - } - } - - OGRSpatialReference *poSRSToAssign = nullptr; - if (entry != nullptr) - { - poSRSToAssign = poSRS; - } - else if (poParentSRS) - { - poSRSToAssign = poParentSRS; - } - else - { - // Assign WGS84 if no CRS defined on geometry. - poSRSToAssign = OGRSpatialReference::GetWGS84SRS(); - } - - GeoJSONObject::Type objType = OGRGeoJSONGetType(poObj); - if (GeoJSONObject::ePoint == objType) - poGeometry = OGRGeoJSONReadPoint(poObj); - else if (GeoJSONObject::eMultiPoint == objType) - poGeometry = OGRGeoJSONReadMultiPoint(poObj); - else if (GeoJSONObject::eLineString == objType) - poGeometry = OGRGeoJSONReadLineString(poObj); - else if (GeoJSONObject::eMultiLineString == objType) - poGeometry = OGRGeoJSONReadMultiLineString(poObj); - else if (GeoJSONObject::ePolygon == objType) - poGeometry = OGRGeoJSONReadPolygon(poObj); - else if (GeoJSONObject::eMultiPolygon == objType) - poGeometry = OGRGeoJSONReadMultiPolygon(poObj); - else if (GeoJSONObject::eGeometryCollection == objType) - poGeometry = OGRGeoJSONReadGeometryCollection(poObj, poSRSToAssign); - else - { - CPLDebug("GeoJSON", "Unsupported geometry type detected. " - "Feature gets NULL geometry assigned."); - } - - if (poGeometry && GeoJSONObject::eGeometryCollection != objType) - poGeometry->assignSpatialReference(poSRSToAssign); - - if (poSRS) - poSRS->Release(); - - return poGeometry; -} - -/************************************************************************/ -/* OGRGeoJSONGetCoordinate() */ -/************************************************************************/ - -static double OGRGeoJSONGetCoordinate(json_object *poObj, - const char *pszCoordName, int nIndex, - bool &bValid) -{ - json_object *poObjCoord = json_object_array_get_idx(poObj, nIndex); - if (nullptr == poObjCoord) - { - CPLDebug("GeoJSON", "Point: got null object for %s.", pszCoordName); - bValid = false; - return 0.0; - } - - const int iType = json_object_get_type(poObjCoord); - if (json_type_double != iType && json_type_int != iType) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid '%s' coordinate. " - "Type is not double or integer for \'%s\'.", - pszCoordName, json_object_to_json_string(poObjCoord)); - bValid = false; - return 0.0; - } - - return json_object_get_double(poObjCoord); -} - -/************************************************************************/ -/* OGRGeoJSONReadRawPoint */ -/************************************************************************/ - -bool OGRGeoJSONReadRawPoint(json_object *poObj, OGRPoint &point) -{ - CPLAssert(nullptr != poObj); - - if (json_type_array == json_object_get_type(poObj)) - { - const auto nSize = json_object_array_length(poObj); - - if (nSize < GeoJSONObject::eMinCoordinateDimension) - { - CPLDebug("GeoJSON", "Invalid coord dimension. " - "At least 2 dimensions must be present."); - return false; - } - - bool bValid = true; - const double dfX = OGRGeoJSONGetCoordinate(poObj, "x", 0, bValid); - const double dfY = OGRGeoJSONGetCoordinate(poObj, "y", 1, bValid); - point.setX(dfX); - point.setY(dfY); - - // Read Z coordinate. - if (nSize >= GeoJSONObject::eMaxCoordinateDimension) - { - // Don't *expect* mixed-dimension geometries, although the - // spec doesn't explicitly forbid this. - const double dfZ = OGRGeoJSONGetCoordinate(poObj, "z", 2, bValid); - point.setZ(dfZ); - } - else - { - point.flattenTo2D(); - } - return bValid; - } - - return false; -} - -/************************************************************************/ -/* OGRGeoJSONReadPoint */ -/************************************************************************/ - -OGRPoint *OGRGeoJSONReadPoint(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - json_object *poObjCoords = OGRGeoJSONFindMemberByName(poObj, "coordinates"); - if (nullptr == poObjCoords) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid Point object. Missing \'coordinates\' member."); - return nullptr; - } - - OGRPoint *poPoint = new OGRPoint(); - if (!OGRGeoJSONReadRawPoint(poObjCoords, *poPoint)) - { - CPLDebug("GeoJSON", "Point: raw point parsing failure."); - delete poPoint; - return nullptr; - } - - return poPoint; -} - -/************************************************************************/ -/* OGRGeoJSONReadMultiPoint */ -/************************************************************************/ - -OGRMultiPoint *OGRGeoJSONReadMultiPoint(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - json_object *poObjPoints = OGRGeoJSONFindMemberByName(poObj, "coordinates"); - if (nullptr == poObjPoints) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid MultiPoint object. " - "Missing \'coordinates\' member."); - return nullptr; - } - - OGRMultiPoint *poMultiPoint = nullptr; - if (json_type_array == json_object_get_type(poObjPoints)) - { - const auto nPoints = json_object_array_length(poObjPoints); - - poMultiPoint = new OGRMultiPoint(); - - for (auto i = decltype(nPoints){0}; i < nPoints; ++i) - { - json_object *poObjCoords = - json_object_array_get_idx(poObjPoints, i); - - OGRPoint pt; - if (poObjCoords != nullptr && - !OGRGeoJSONReadRawPoint(poObjCoords, pt)) - { - delete poMultiPoint; - CPLDebug("GeoJSON", "LineString: raw point parsing failure."); - return nullptr; - } - poMultiPoint->addGeometry(&pt); - } - } - - return poMultiPoint; -} - -/************************************************************************/ -/* OGRGeoJSONReadLineString */ -/************************************************************************/ - -OGRLineString *OGRGeoJSONReadLineString(json_object *poObj, bool bRaw) -{ - CPLAssert(nullptr != poObj); - - json_object *poObjPoints = nullptr; - - if (!bRaw) - { - poObjPoints = OGRGeoJSONFindMemberByName(poObj, "coordinates"); - if (nullptr == poObjPoints) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid LineString object. " - "Missing \'coordinates\' member."); - return nullptr; - } - } - else - { - poObjPoints = poObj; - } - - OGRLineString *poLine = nullptr; - - if (json_type_array == json_object_get_type(poObjPoints)) - { - const auto nPoints = json_object_array_length(poObjPoints); - - poLine = new OGRLineString(); - poLine->setNumPoints(static_cast(nPoints)); - - for (auto i = decltype(nPoints){0}; i < nPoints; ++i) - { - json_object *poObjCoords = - json_object_array_get_idx(poObjPoints, i); - if (poObjCoords == nullptr) - { - delete poLine; - CPLDebug("GeoJSON", "LineString: got null object."); - return nullptr; - } - - OGRPoint pt; - if (!OGRGeoJSONReadRawPoint(poObjCoords, pt)) - { - delete poLine; - CPLDebug("GeoJSON", "LineString: raw point parsing failure."); - return nullptr; - } - if (pt.getCoordinateDimension() == 2) - { - poLine->setPoint(static_cast(i), pt.getX(), pt.getY()); - } - else - { - poLine->setPoint(static_cast(i), pt.getX(), pt.getY(), - pt.getZ()); - } - } - } - - return poLine; -} - -/************************************************************************/ -/* OGRGeoJSONReadMultiLineString */ -/************************************************************************/ - -OGRMultiLineString *OGRGeoJSONReadMultiLineString(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - json_object *poObjLines = OGRGeoJSONFindMemberByName(poObj, "coordinates"); - if (nullptr == poObjLines) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid MultiLineString object. " - "Missing \'coordinates\' member."); - return nullptr; - } - - OGRMultiLineString *poMultiLine = nullptr; - - if (json_type_array == json_object_get_type(poObjLines)) - { - const auto nLines = json_object_array_length(poObjLines); - - poMultiLine = new OGRMultiLineString(); - - for (auto i = decltype(nLines){0}; i < nLines; ++i) - { - json_object *poObjLine = json_object_array_get_idx(poObjLines, i); - - OGRLineString *poLine; - if (poObjLine != nullptr) - poLine = OGRGeoJSONReadLineString(poObjLine, true); - else - poLine = new OGRLineString(); - - if (nullptr != poLine) - { - poMultiLine->addGeometryDirectly(poLine); - } - } - } - - return poMultiLine; -} - -/************************************************************************/ -/* OGRGeoJSONReadLinearRing */ -/************************************************************************/ - -OGRLinearRing *OGRGeoJSONReadLinearRing(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - OGRLinearRing *poRing = nullptr; - - if (json_type_array == json_object_get_type(poObj)) - { - const auto nPoints = json_object_array_length(poObj); - - poRing = new OGRLinearRing(); - poRing->setNumPoints(static_cast(nPoints)); - - for (auto i = decltype(nPoints){0}; i < nPoints; ++i) - { - json_object *poObjCoords = json_object_array_get_idx(poObj, i); - if (poObjCoords == nullptr) - { - delete poRing; - CPLDebug("GeoJSON", "LinearRing: got null object."); - return nullptr; - } - - OGRPoint pt; - if (!OGRGeoJSONReadRawPoint(poObjCoords, pt)) - { - delete poRing; - CPLDebug("GeoJSON", "LinearRing: raw point parsing failure."); - return nullptr; - } - - if (2 == pt.getCoordinateDimension()) - poRing->setPoint(static_cast(i), pt.getX(), pt.getY()); - else - poRing->setPoint(static_cast(i), pt.getX(), pt.getY(), - pt.getZ()); - } - } - - return poRing; -} - -/************************************************************************/ -/* OGRGeoJSONReadPolygon */ -/************************************************************************/ - -OGRPolygon *OGRGeoJSONReadPolygon(json_object *poObj, bool bRaw) -{ - CPLAssert(nullptr != poObj); - - json_object *poObjRings = nullptr; - - if (!bRaw) - { - poObjRings = OGRGeoJSONFindMemberByName(poObj, "coordinates"); - if (nullptr == poObjRings) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid Polygon object. " - "Missing \'coordinates\' member."); - return nullptr; - } - } - else - { - poObjRings = poObj; - } - - OGRPolygon *poPolygon = nullptr; - - if (json_type_array == json_object_get_type(poObjRings)) - { - const auto nRings = json_object_array_length(poObjRings); - if (nRings > 0) - { - json_object *poObjPoints = json_object_array_get_idx(poObjRings, 0); - if (poObjPoints == nullptr) - { - poPolygon = new OGRPolygon(); - } - else - { - OGRLinearRing *poRing = OGRGeoJSONReadLinearRing(poObjPoints); - if (nullptr != poRing) - { - poPolygon = new OGRPolygon(); - poPolygon->addRingDirectly(poRing); - } - } - - for (auto i = decltype(nRings){1}; - i < nRings && nullptr != poPolygon; ++i) - { - poObjPoints = json_object_array_get_idx(poObjRings, i); - if (poObjPoints != nullptr) - { - OGRLinearRing *poRing = - OGRGeoJSONReadLinearRing(poObjPoints); - if (nullptr != poRing) - { - poPolygon->addRingDirectly(poRing); - } - } - } - } - else - { - poPolygon = new OGRPolygon(); - } - } - - return poPolygon; -} - -/************************************************************************/ -/* OGRGeoJSONReadMultiPolygon */ -/************************************************************************/ - -OGRMultiPolygon *OGRGeoJSONReadMultiPolygon(json_object *poObj) -{ - CPLAssert(nullptr != poObj); - - json_object *poObjPolys = OGRGeoJSONFindMemberByName(poObj, "coordinates"); - if (nullptr == poObjPolys) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid MultiPolygon object. " - "Missing \'coordinates\' member."); - return nullptr; - } - - OGRMultiPolygon *poMultiPoly = nullptr; - - if (json_type_array == json_object_get_type(poObjPolys)) - { - const auto nPolys = json_object_array_length(poObjPolys); - - poMultiPoly = new OGRMultiPolygon(); - - for (auto i = decltype(nPolys){0}; i < nPolys; ++i) - { - json_object *poObjPoly = json_object_array_get_idx(poObjPolys, i); - if (poObjPoly == nullptr) - { - poMultiPoly->addGeometryDirectly(new OGRPolygon()); - } - else - { - OGRPolygon *poPoly = OGRGeoJSONReadPolygon(poObjPoly, true); - if (nullptr != poPoly) - { - poMultiPoly->addGeometryDirectly(poPoly); - } - } - } - } - - return poMultiPoly; -} - -/************************************************************************/ -/* OGRGeoJSONReadGeometryCollection */ +/* OGRGeoJSONGetExtent3D() */ /************************************************************************/ -OGRGeometryCollection * -OGRGeoJSONReadGeometryCollection(json_object *poObj, OGRSpatialReference *poSRS) -{ - CPLAssert(nullptr != poObj); - - json_object *poObjGeoms = OGRGeoJSONFindMemberByName(poObj, "geometries"); - if (nullptr == poObjGeoms) - { - CPLError(CE_Failure, CPLE_AppDefined, - "Invalid GeometryCollection object. " - "Missing \'geometries\' member."); - return nullptr; - } - - OGRGeometryCollection *poCollection = nullptr; - - if (json_type_array == json_object_get_type(poObjGeoms)) - { - poCollection = new OGRGeometryCollection(); - poCollection->assignSpatialReference(poSRS); - - const auto nGeoms = json_object_array_length(poObjGeoms); - for (auto i = decltype(nGeoms){0}; i < nGeoms; ++i) - { - json_object *poObjGeom = json_object_array_get_idx(poObjGeoms, i); - if (poObjGeom == nullptr) - { - CPLDebug("GeoJSON", "Skipping null sub-geometry"); - continue; - } - - OGRGeometry *poGeometry = OGRGeoJSONReadGeometry(poObjGeom, poSRS); - if (nullptr != poGeometry) - { - poCollection->addGeometryDirectly(poGeometry); - } - } - } - - return poCollection; -} - -/************************************************************************/ -/* OGR_G_CreateGeometryFromJson */ -/************************************************************************/ - -/** Create a OGR geometry from a GeoJSON geometry object */ -OGRGeometryH OGR_G_CreateGeometryFromJson(const char *pszJson) -{ - if (nullptr == pszJson) - { - // Translation failed. - return nullptr; - } - - json_object *poObj = nullptr; - if (!OGRJSonParse(pszJson, &poObj)) - return nullptr; - - OGRGeometry *poGeometry = OGRGeoJSONReadGeometry(poObj); - - // Release JSON tree. - json_object_put(poObj); - - return (OGRGeometryH)poGeometry; -} - -/************************************************************************/ -/* json_ex_get_object_by_path() */ -/************************************************************************/ - -json_object *json_ex_get_object_by_path(json_object *poObj, const char *pszPath) -{ - if (poObj == nullptr || json_object_get_type(poObj) != json_type_object || - pszPath == nullptr || *pszPath == '\0') - { - return nullptr; - } - char **papszTokens = CSLTokenizeString2(pszPath, ".", 0); - for (int i = 0; papszTokens[i] != nullptr; i++) - { - poObj = CPL_json_object_object_get(poObj, papszTokens[i]); - if (poObj == nullptr) - break; - if (papszTokens[i + 1] != nullptr) - { - if (json_object_get_type(poObj) != json_type_object) - { - poObj = nullptr; - break; - } - } - } - CSLDestroy(papszTokens); - return poObj; -} - -/************************************************************************/ -/* OGRJSonParse() */ -/************************************************************************/ - -bool OGRJSonParse(const char *pszText, json_object **ppoObj, bool bVerboseError) -{ - if (ppoObj == nullptr) - return false; - json_tokener *jstok = json_tokener_new(); - const int nLen = pszText == nullptr ? 0 : static_cast(strlen(pszText)); - *ppoObj = json_tokener_parse_ex(jstok, pszText, nLen); - if (jstok->err != json_tokener_success) - { - if (bVerboseError) - { - CPLError(CE_Failure, CPLE_AppDefined, - "JSON parsing error: %s (at offset %d)", - json_tokener_error_desc(jstok->err), jstok->char_offset); - } - - json_tokener_free(jstok); - *ppoObj = nullptr; - return false; - } - json_tokener_free(jstok); - return true; -} - -/************************************************************************/ -/* CPL_json_object_object_get() */ -/************************************************************************/ - -// This is the same as json_object_object_get() except it will not raise -// deprecation warning. - -json_object *CPL_json_object_object_get(struct json_object *obj, - const char *key) -{ - json_object *poRet = nullptr; - json_object_object_get_ex(obj, key, &poRet); - return poRet; -} - bool OGRGeoJSONGetExtent3D(json_object *poObj, OGREnvelope3D *poEnvelope) { if (!poEnvelope || !poObj) diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.h b/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.h index d06457ca3bcb..8aa54a568ab1 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.h +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonreader.h @@ -61,33 +61,6 @@ class OGRFeature; class OGRGeoJSONLayer; class OGRSpatialReference; -/************************************************************************/ -/* GeoJSONObject */ -/************************************************************************/ - -struct GeoJSONObject -{ - enum Type - { - eUnknown = wkbUnknown, // non-GeoJSON properties - ePoint = wkbPoint, - eLineString = wkbLineString, - ePolygon = wkbPolygon, - eMultiPoint = wkbMultiPoint, - eMultiLineString = wkbMultiLineString, - eMultiPolygon = wkbMultiPolygon, - eGeometryCollection = wkbGeometryCollection, - eFeature, - eFeatureCollection - }; - - enum CoordinateDimension - { - eMinCoordinateDimension = 2, - eMaxCoordinateDimension = 3 - }; -}; - /************************************************************************/ /* OGRGeoJSONBaseReader */ /************************************************************************/ @@ -258,21 +231,6 @@ void OGRGeoJSONReaderAddOrUpdateField( /* GeoJSON Parsing Utilities */ /************************************************************************/ -lh_entry *OGRGeoJSONFindMemberEntryByName(json_object *poObj, - const char *pszName); -json_object *OGRGeoJSONFindMemberByName(json_object *poObj, - const char *pszName); -GeoJSONObject::Type OGRGeoJSONGetType(json_object *poObj); - -json_object CPL_DLL *json_ex_get_object_by_path(json_object *poObj, - const char *pszPath); - -json_object CPL_DLL *CPL_json_object_object_get(struct json_object *obj, - const char *key); - -bool CPL_DLL OGRJSonParse(const char *pszText, json_object **ppoObj, - bool bVerboseError = true); - bool OGRGeoJSONUpdateLayerGeomType(bool &bFirstGeom, OGRwkbGeometryType eGeomType, OGRwkbGeometryType &eLayerGeomType); @@ -280,28 +238,6 @@ bool OGRGeoJSONUpdateLayerGeomType(bool &bFirstGeom, // Get the 3D extent from the geometry coordinates of a feature bool OGRGeoJSONGetExtent3D(json_object *poObj, OGREnvelope3D *poEnvelope); -/************************************************************************/ -/* GeoJSON Geometry Translators */ -/************************************************************************/ - -OGRwkbGeometryType OGRGeoJSONGetOGRGeometryType(json_object *poObj); - -bool OGRGeoJSONReadRawPoint(json_object *poObj, OGRPoint &point); -OGRGeometry CPL_DLL * -OGRGeoJSONReadGeometry(json_object *poObj, - OGRSpatialReference *poParentSRS = nullptr); -OGRPoint *OGRGeoJSONReadPoint(json_object *poObj); -OGRMultiPoint *OGRGeoJSONReadMultiPoint(json_object *poObj); -OGRLineString *OGRGeoJSONReadLineString(json_object *poObj, bool bRaw = false); -OGRMultiLineString *OGRGeoJSONReadMultiLineString(json_object *poObj); -OGRLinearRing *OGRGeoJSONReadLinearRing(json_object *poObj); -OGRPolygon *OGRGeoJSONReadPolygon(json_object *poObj, bool bRaw = false); -OGRMultiPolygon *OGRGeoJSONReadMultiPolygon(json_object *poObj); -OGRGeometryCollection * -OGRGeoJSONReadGeometryCollection(json_object *poObj, - OGRSpatialReference *poSRS = nullptr); -OGRSpatialReference *OGRGeoJSONReadSpatialReference(json_object *poObj); - /************************************************************************/ /* OGRESRIJSONReader */ /************************************************************************/ @@ -341,14 +277,6 @@ class OGRESRIJSONReader OGRGeoJSONLayer *ReadFeatureCollection(json_object *poObj); }; -OGRGeometry *OGRESRIJSONReadGeometry(json_object *poObj); -OGRSpatialReference *OGRESRIJSONReadSpatialReference(json_object *poObj); -OGRwkbGeometryType OGRESRIJSONGetGeometryType(json_object *poObj); -OGRPoint *OGRESRIJSONReadPoint(json_object *poObj); -OGRGeometry *OGRESRIJSONReadLineString(json_object *poObj); -OGRGeometry *OGRESRIJSONReadPolygon(json_object *poObj); -OGRMultiPoint *OGRESRIJSONReadMultiPoint(json_object *poObj); - /************************************************************************/ /* OGRTopoJSONReader */ /************************************************************************/ diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp index b1fa548ec798..f2ab83c28650 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonseqdriver.cpp @@ -32,8 +32,10 @@ #include "cpl_vsi_error.h" #include "ogr_geojson.h" +#include "ogrlibjsonutils.h" #include "ogrgeojsonreader.h" #include "ogrgeojsonwriter.h" +#include "ogrgeojsongeometry.h" #include #include diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp index 6a0655b3fae0..5f8c783c89df 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.cpp @@ -1029,163 +1029,6 @@ GeoJSONSourceType JSONFGDriverGetSourceType(GDALOpenInfo *poOpenInfo) return srcType; } -/************************************************************************/ -/* GeoJSONPropertyToFieldType() */ -/************************************************************************/ - -constexpr GIntBig MY_INT64_MAX = (((GIntBig)0x7FFFFFFF) << 32) | 0xFFFFFFFF; -constexpr GIntBig MY_INT64_MIN = ((GIntBig)0x80000000) << 32; - -OGRFieldType GeoJSONPropertyToFieldType(json_object *poObject, - OGRFieldSubType &eSubType, - bool bArrayAsString) -{ - eSubType = OFSTNone; - - if (poObject == nullptr) - { - return OFTString; - } - - json_type type = json_object_get_type(poObject); - - if (json_type_boolean == type) - { - eSubType = OFSTBoolean; - return OFTInteger; - } - else if (json_type_double == type) - return OFTReal; - else if (json_type_int == type) - { - GIntBig nVal = json_object_get_int64(poObject); - if (!CPL_INT64_FITS_ON_INT32(nVal)) - { - if (nVal == MY_INT64_MIN || nVal == MY_INT64_MAX) - { - static bool bWarned = false; - if (!bWarned) - { - bWarned = true; - CPLError( - CE_Warning, CPLE_AppDefined, - "Integer values probably ranging out of 64bit integer " - "range have been found. Will be clamped to " - "INT64_MIN/INT64_MAX"); - } - } - return OFTInteger64; - } - else - { - return OFTInteger; - } - } - else if (json_type_string == type) - return OFTString; - else if (json_type_array == type) - { - if (bArrayAsString) - { - eSubType = OFSTJSON; - return OFTString; - } - const auto nSize = json_object_array_length(poObject); - if (nSize == 0) - { - eSubType = OFSTJSON; - return OFTString; - } - OGRFieldType eType = OFTIntegerList; - for (auto i = decltype(nSize){0}; i < nSize; i++) - { - json_object *poRow = json_object_array_get_idx(poObject, i); - if (poRow != nullptr) - { - type = json_object_get_type(poRow); - if (type == json_type_string) - { - if (i == 0 || eType == OFTStringList) - { - eType = OFTStringList; - } - else - { - eSubType = OFSTJSON; - return OFTString; - } - } - else if (type == json_type_double) - { - if (eSubType == OFSTNone && - (i == 0 || eType == OFTRealList || - eType == OFTIntegerList || eType == OFTInteger64List)) - { - eType = OFTRealList; - } - else - { - eSubType = OFSTJSON; - return OFTString; - } - } - else if (type == json_type_int) - { - if (eSubType == OFSTNone && eType == OFTIntegerList) - { - GIntBig nVal = json_object_get_int64(poRow); - if (!CPL_INT64_FITS_ON_INT32(nVal)) - eType = OFTInteger64List; - } - else if (eSubType == OFSTNone && - (eType == OFTInteger64List || - eType == OFTRealList)) - { - // ok - } - else - { - eSubType = OFSTJSON; - return OFTString; - } - } - else if (type == json_type_boolean) - { - if (i == 0 || - (eType == OFTIntegerList && eSubType == OFSTBoolean)) - { - eSubType = OFSTBoolean; - } - else - { - eSubType = OFSTJSON; - return OFTString; - } - } - else - { - eSubType = OFSTJSON; - return OFTString; - } - } - else - { - eSubType = OFSTJSON; - return OFTString; - } - } - - return eType; - } - else if (json_type_object == type) - { - eSubType = OFSTJSON; - return OFTString; - } - - return OFTString; // null -} - /************************************************************************/ /* GeoJSONStringPropertyToFieldType() */ /************************************************************************/ @@ -1221,31 +1064,3 @@ OGRFieldType GeoJSONStringPropertyToFieldType(json_object *poObject, } return OFTString; } - -/************************************************************************/ -/* OGRGeoJSONGetGeometryName() */ -/************************************************************************/ - -const char *OGRGeoJSONGetGeometryName(OGRGeometry const *poGeometry) -{ - CPLAssert(nullptr != poGeometry); - - const OGRwkbGeometryType eType = wkbFlatten(poGeometry->getGeometryType()); - - if (wkbPoint == eType) - return "Point"; - else if (wkbLineString == eType) - return "LineString"; - else if (wkbPolygon == eType) - return "Polygon"; - else if (wkbMultiPoint == eType) - return "MultiPoint"; - else if (wkbMultiLineString == eType) - return "MultiLineString"; - else if (wkbMultiPolygon == eType) - return "MultiPolygon"; - else if (wkbGeometryCollection == eType) - return "GeometryCollection"; - - return "Unknown"; -} diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h index b2c9ebb98eeb..e250321baa21 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsonutils.h @@ -65,14 +65,6 @@ bool ESRIJSONIsObject(const char *pszText, GDALOpenInfo *poOpenInfo); bool TopoJSONIsObject(const char *pszText, GDALOpenInfo *poOpenInfo); bool JSONFGIsObject(const char *pszText, GDALOpenInfo *poOpenInfo); -/************************************************************************/ -/* GeoJSONPropertyToFieldType */ -/************************************************************************/ - -OGRFieldType CPL_DLL GeoJSONPropertyToFieldType(json_object *poObject, - OGRFieldSubType &eSubType, - bool bArrayAsString = false); - /************************************************************************/ /* GeoJSONStringPropertyToFieldType */ /************************************************************************/ @@ -80,10 +72,4 @@ OGRFieldType CPL_DLL GeoJSONPropertyToFieldType(json_object *poObject, OGRFieldType GeoJSONStringPropertyToFieldType(json_object *poObject, int &nTZFlag); -/************************************************************************/ -/* OGRGeoJSONGetGeometryName */ -/************************************************************************/ - -const char *OGRGeoJSONGetGeometryName(OGRGeometry const *poGeometry); - #endif // OGR_GEOJSONUTILS_H_INCLUDED diff --git a/ogr/ogrsf_frmts/geojson/ogrjsoncollectionstreamingparser.cpp b/ogr/ogrsf_frmts/geojson/ogrjsoncollectionstreamingparser.cpp index 5d4fcd9655ca..8dc56dcd069b 100644 --- a/ogr/ogrsf_frmts/geojson/ogrjsoncollectionstreamingparser.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrjsoncollectionstreamingparser.cpp @@ -29,7 +29,9 @@ #include "ogrjsoncollectionstreamingparser.h" #include "cpl_string.h" -#include "ogrgeojsonreader.h" // CPL_json_object_object_get +#include "ogrlibjsonutils.h" // CPL_json_object_object_get + +#include "ogr_feature.h" #define JSON_C_VER_013 (13 << 8) diff --git a/ogr/ogrsf_frmts/geojson/ogrtopojsonreader.cpp b/ogr/ogrsf_frmts/geojson/ogrtopojsonreader.cpp index 71a44a3438d3..ed8113cf8dc1 100644 --- a/ogr/ogrsf_frmts/geojson/ogrtopojsonreader.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrtopojsonreader.cpp @@ -28,6 +28,7 @@ #include "ogrgeojsonreader.h" #include "ogrgeojsonutils.h" +#include "ogrlibjsonutils.h" #include "ogr_geojson.h" #include // JSON-C #include "ogr_api.h" diff --git a/ogr/ogrsf_frmts/gmlas/CMakeLists.txt b/ogr/ogrsf_frmts/gmlas/CMakeLists.txt index 5d9c1d76af4b..2c8d69fc7c43 100644 --- a/ogr/ogrsf_frmts/gmlas/CMakeLists.txt +++ b/ogr/ogrsf_frmts/gmlas/CMakeLists.txt @@ -37,7 +37,7 @@ set_property( PROPERTY RESOURCE "${GDAL_DATA_FILES}") target_include_directories( - ogr_GMLAS PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/geojson ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/mem + ogr_GMLAS PRIVATE ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/mem ${GDAL_VECTOR_FORMAT_SOURCE_DIR}/pgdump) # Internal libs first diff --git a/ogr/ogrsf_frmts/gmlas/ogrgmlaswriter.cpp b/ogr/ogrsf_frmts/gmlas/ogrgmlaswriter.cpp index 9b3c8a467061..912978386d56 100644 --- a/ogr/ogrsf_frmts/gmlas/ogrgmlaswriter.cpp +++ b/ogr/ogrsf_frmts/gmlas/ogrgmlaswriter.cpp @@ -30,7 +30,7 @@ #include "ogr_gmlas.h" #include "ogr_p.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include "cpl_time.h" #include diff --git a/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp b/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp index b6d95ae424cf..a49624f51390 100644 --- a/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp +++ b/ogr/ogrsf_frmts/jsonfg/ogrjsonfgreader.cpp @@ -30,6 +30,8 @@ #include "ogrgeojsonreader.h" #include "ogrgeojsonutils.h" +#include "ogrlibjsonutils.h" +#include "ogrgeojsongeometry.h" #include "ogr_geojson.h" #include "cpl_vsi_virtual.h" diff --git a/ogr/ogrsf_frmts/jsonfg/ogrjsonfgwritelayer.cpp b/ogr/ogrsf_frmts/jsonfg/ogrjsonfgwritelayer.cpp index 20e7d39ce9a8..5ed5c38dd074 100644 --- a/ogr/ogrsf_frmts/jsonfg/ogrjsonfgwritelayer.cpp +++ b/ogr/ogrsf_frmts/jsonfg/ogrjsonfgwritelayer.cpp @@ -28,7 +28,7 @@ #include "ogr_jsonfg.h" #include "cpl_time.h" -#include "ogrgeojsonreader.h" // OGRJSonParse() +#include "ogrlibjsonutils.h" // OGRJSonParse() #include diff --git a/ogr/ogrsf_frmts/plscenes/CMakeLists.txt b/ogr/ogrsf_frmts/plscenes/CMakeLists.txt index bce3876d78cb..7f378eae5747 100644 --- a/ogr/ogrsf_frmts/plscenes/CMakeLists.txt +++ b/ogr/ogrsf_frmts/plscenes/CMakeLists.txt @@ -22,7 +22,6 @@ set_property( PROPERTY RESOURCE "${GDAL_DATA_FILES}") gdal_standard_includes(ogr_PLSCENES) -target_include_directories(ogr_PLSCENES PRIVATE $) if (GDAL_USE_JSONC_INTERNAL) gdal_add_vendored_lib(ogr_PLSCENES libjson) diff --git a/ogr/ogrsf_frmts/plscenes/ogr_plscenes.h b/ogr/ogrsf_frmts/plscenes/ogr_plscenes.h index 3cc0c0b176ae..bc486c36854e 100644 --- a/ogr/ogrsf_frmts/plscenes/ogr_plscenes.h +++ b/ogr/ogrsf_frmts/plscenes/ogr_plscenes.h @@ -34,8 +34,7 @@ #include "ogrsf_frmts.h" #include "ogr_srs_api.h" #include "cpl_http.h" -#include "ogr_geojson.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include "ogr_swq.h" #include #include diff --git a/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp b/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp index 7ad724680d37..67080a4ff056 100644 --- a/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp +++ b/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1dataset.cpp @@ -27,7 +27,7 @@ ****************************************************************************/ #include "ogr_plscenes.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" #include /************************************************************************/ diff --git a/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1layer.cpp b/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1layer.cpp index e2a5e2b9951c..906bea0075ab 100644 --- a/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1layer.cpp +++ b/ogr/ogrsf_frmts/plscenes/ogrplscenesdatav1layer.cpp @@ -27,7 +27,9 @@ ****************************************************************************/ #include "ogr_plscenes.h" -#include "ogrgeojsonreader.h" +#include "ogrlibjsonutils.h" +#include "ogrgeojsongeometry.h" +#include "ogrgeojsonwriter.h" #include /************************************************************************/ diff --git a/ogr/ogrsf_frmts/xlsx/CMakeLists.txt b/ogr/ogrsf_frmts/xlsx/CMakeLists.txt index e902499c1f0b..4870ac9030e7 100644 --- a/ogr/ogrsf_frmts/xlsx/CMakeLists.txt +++ b/ogr/ogrsf_frmts/xlsx/CMakeLists.txt @@ -1,7 +1,7 @@ add_gdal_driver(TARGET ogr_XLSX SOURCES ogr_xlsx.h ogrxlsxdatasource.cpp ogrxlsxdriver.cpp PLUGIN_CAPABLE NO_DEPS) gdal_standard_includes(ogr_XLSX) target_include_directories( - ogr_XLSX PRIVATE $ $ + ogr_XLSX PRIVATE $ $) if (GDAL_USE_EXPAT) target_compile_definitions(ogr_XLSX PRIVATE -DHAVE_EXPAT) From 6ebc888a8ec43f38dc9b0d9faf9fc686fbf07c84 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 01:11:51 +0200 Subject: [PATCH 206/710] Make GeoJSON (and companion TopoJSON, ESRIJSON, GeoJSONSeq) drivers optional or buildable as a plugin (if not builtin, the JSONFG driver is not available) --- autotest/alg/rasterize.py | 3 ++ autotest/ogr/ogr_geojsonseq.py | 2 ++ autotest/ogr/ogr_topojson.py | 2 ++ autotest/pyscripts/test_gdal_polygonize.py | 1 + autotest/utilities/test_gdal_footprint.py | 1 + autotest/utilities/test_gdal_footprint_lib.py | 2 ++ autotest/utilities/test_gdal_grid_lib.py | 36 +++++++++++++++++++ autotest/utilities/test_gdal_rasterize_lib.py | 1 + autotest/utilities/test_gdalwarp_lib.py | 9 +++++ autotest/utilities/test_ogr2ogr.py | 2 ++ autotest/utilities/test_ogr2ogr_lib.py | 1 + autotest/utilities/test_ogrinfo_lib.py | 3 ++ ogr/ogrsf_frmts/CMakeLists.txt | 7 ++-- ogr/ogrsf_frmts/geojson/CMakeLists.txt | 17 +++++---- ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp | 6 ++++ ogr/ogrsf_frmts/jsonfg/CMakeLists.txt | 2 +- 16 files changed, 85 insertions(+), 10 deletions(-) diff --git a/autotest/alg/rasterize.py b/autotest/alg/rasterize.py index 9fe6370de860..2edced6a6501 100755 --- a/autotest/alg/rasterize.py +++ b/autotest/alg/rasterize.py @@ -853,6 +853,9 @@ def test_rasterize_merge_alg_add_polygon(wkt): ############################################################################### + + +@pytest.mark.require_driver("GeoJSON") def test_rasterize_bugfix_gh6981(): bad_geometry = { diff --git a/autotest/ogr/ogr_geojsonseq.py b/autotest/ogr/ogr_geojsonseq.py index 89acd3a61ca9..f4e1b329d76f 100755 --- a/autotest/ogr/ogr_geojsonseq.py +++ b/autotest/ogr/ogr_geojsonseq.py @@ -34,6 +34,8 @@ from osgeo import gdal, ogr, osr +pytestmark = pytest.mark.require_driver("GeoJSONSeq") + def _ogr_geojsonseq_create(filename, lco, expect_rs): diff --git a/autotest/ogr/ogr_topojson.py b/autotest/ogr/ogr_topojson.py index 1580ca7d7c9f..dec5f2802693 100755 --- a/autotest/ogr/ogr_topojson.py +++ b/autotest/ogr/ogr_topojson.py @@ -35,6 +35,8 @@ from osgeo import gdal, ogr +pytestmark = pytest.mark.require_driver("TopoJSON") + ############################################################################### # Test TopoJSON diff --git a/autotest/pyscripts/test_gdal_polygonize.py b/autotest/pyscripts/test_gdal_polygonize.py index 90eec0de5b96..f36c145528cf 100755 --- a/autotest/pyscripts/test_gdal_polygonize.py +++ b/autotest/pyscripts/test_gdal_polygonize.py @@ -263,6 +263,7 @@ def test_gdal_polygonize_4bis(script_path, tmp_path): # Test -8 +@pytest.mark.require_driver("GeoJSON") def test_gdal_polygonize_minus_8(script_path, tmp_path): outfilename = str(tmp_path / "out.geojson") diff --git a/autotest/utilities/test_gdal_footprint.py b/autotest/utilities/test_gdal_footprint.py index 75661c891056..39d169c33c11 100755 --- a/autotest/utilities/test_gdal_footprint.py +++ b/autotest/utilities/test_gdal_footprint.py @@ -53,6 +53,7 @@ def gdal_footprint_path(): ############################################################################### +@pytest.mark.require_driver("GeoJSON") def test_gdal_footprint_basic(gdal_footprint_path, tmp_path): footprint_json = str(tmp_path / "out_footprint.json") diff --git a/autotest/utilities/test_gdal_footprint_lib.py b/autotest/utilities/test_gdal_footprint_lib.py index d8a969b887e5..3ed6df840d63 100755 --- a/autotest/utilities/test_gdal_footprint_lib.py +++ b/autotest/utilities/test_gdal_footprint_lib.py @@ -123,6 +123,7 @@ def test_gdal_footprint_lib_destSRS(): # +@pytest.mark.require_driver("GeoJSON") def test_gdal_footprint_lib_inline_geojson(): ret = gdal.Footprint("", "../gcore/data/byte.tif", format="GeoJSON") @@ -134,6 +135,7 @@ def test_gdal_footprint_lib_inline_geojson(): # +@pytest.mark.require_driver("GeoJSON") def test_gdal_footprint_lib_inline_wkt(): ret = gdal.Footprint("", "../gcore/data/byte.tif", format="WKT") diff --git a/autotest/utilities/test_gdal_grid_lib.py b/autotest/utilities/test_gdal_grid_lib.py index 6ad09bd42d45..e7c546bdf3df 100755 --- a/autotest/utilities/test_gdal_grid_lib.py +++ b/autotest/utilities/test_gdal_grid_lib.py @@ -175,6 +175,7 @@ def test_gdal_grid_lib_2(tmp_vsimem, env): # May fail on minimum builds without qhull @gdaltest.disable_exceptions() +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_3(): wkt = "POLYGON ((37.3495241627097 55.6901648563184 187.680953979492,37.349543273449 55.6901565410051 187.714370727539,37.3495794832707 55.6901531392856 187.67333984375,37.3496210575104 55.6901595647556 187.6396484375,37.3496398329735 55.6901716597552 187.596603393555,37.3496726900339 55.6901780852222 187.681350708008,37.3496793955565 55.6901829988139 187.933898925781,37.3496921360493 55.6901860225623 187.934280395508,37.3497162759304 55.6902037870796 187.435394287109,37.3497484624386 55.6902094566047 187.515319824219,37.3497618734837 55.6902241973661 190.329940795898,37.3497511446476 55.690238560154 190.345748901367,37.3497404158115 55.6902567026153 190.439697265625,37.3497142642736 55.6902650179072 189.086044311523,37.349688783288 55.6902608602615 187.763305664062,37.3496626317501 55.6902468754498 187.53678894043,37.3496378213167 55.6902412059301 187.598648071289,37.3496103286743 55.6902400720261 187.806274414062,37.3495902121067 55.6902313787607 187.759521484375,37.3495734483004 55.6902177719067 187.578125,37.349532879889 55.6902035980954 187.56965637207,37.3495161160827 55.6901939599008 187.541793823242,37.3495187982917 55.6901754394418 187.610427856445,37.3495241627097 55.6901648563184 187.680953979492))" @@ -227,6 +228,7 @@ def _shift_by(geom, dx, dy): @pytest.mark.parametrize("alg", ["invdist", "invdistnn"]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_invdistnn_quadrant_all_params(alg): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10,1 0 100000000)" @@ -253,6 +255,7 @@ def test_gdal_grid_lib_invdistnn_quadrant_all_params(alg): @pytest.mark.parametrize("alg", ["invdist", "invdistnn"]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_invdistnn_quadrant_insufficient_radius(alg): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -272,6 +275,7 @@ def test_gdal_grid_lib_invdistnn_quadrant_insufficient_radius(alg): _compare_arrays(ds, [[0.0]]) # insufficient radius. should be > sqrt(2) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_invdistnn_quadrant_min_points_not_reached(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -291,6 +295,7 @@ def test_gdal_grid_lib_invdistnn_quadrant_min_points_not_reached(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_invdistnn_quadrant_missing_point_in_one_quadrant(): # Missing point in 0.5 -0.5 quadrant @@ -311,6 +316,7 @@ def test_gdal_grid_lib_invdistnn_quadrant_missing_point_in_one_quadrant(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_invdistnn_quadrant_ignore_extra_points(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10,1 0 100000000)" @@ -341,6 +347,7 @@ def test_gdal_grid_lib_invdistnn_quadrant_ignore_extra_points(): _compare_arrays(ds, [[10.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_quadrant_all_params(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10,1 0 100)" @@ -362,6 +369,7 @@ def test_gdal_grid_lib_average_quadrant_all_params(): _compare_arrays(ds, [[expected_val]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_quadrant_insufficient_radius(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -381,6 +389,7 @@ def test_gdal_grid_lib_average_quadrant_insufficient_radius(): _compare_arrays(ds, [[0.0]]) # insufficient radius. should be > sqrt(2) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_quadrant_min_points_not_reached(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -400,6 +409,7 @@ def test_gdal_grid_lib_average_quadrant_min_points_not_reached(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_quadrant_missing_point_in_one_quadrant(): # Missing point in 0.5 -0.5 quadrant @@ -420,6 +430,7 @@ def test_gdal_grid_lib_average_quadrant_missing_point_in_one_quadrant(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_quadrant_ignore_extra_points(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10,1 0 100000000)" @@ -450,6 +461,7 @@ def test_gdal_grid_lib_average_quadrant_ignore_extra_points(): _compare_arrays(ds, [[10.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_minimum_quadrant_all_params(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 9,1 0 5)" @@ -469,6 +481,7 @@ def test_gdal_grid_lib_minimum_quadrant_all_params(): _compare_arrays(ds, [[5.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_minimum_quadrant_insufficient_radius(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -488,6 +501,7 @@ def test_gdal_grid_lib_minimum_quadrant_insufficient_radius(): _compare_arrays(ds, [[0.0]]) # insufficient radius. should be > sqrt(2) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_minimum_quadrant_min_points_not_reached(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -507,6 +521,7 @@ def test_gdal_grid_lib_minimum_quadrant_min_points_not_reached(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_minimum_quadrant_missing_point_in_one_quadrant(): # Missing point in 0.5 -0.5 quadrant @@ -527,6 +542,7 @@ def test_gdal_grid_lib_minimum_quadrant_missing_point_in_one_quadrant(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_minimum_quadrant_ignore_extra_points(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10,1 0 1)" @@ -546,6 +562,7 @@ def test_gdal_grid_lib_minimum_quadrant_ignore_extra_points(): _compare_arrays(ds, [[10.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_maximum_quadrant_all_params(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 11,1 0 50)" @@ -565,6 +582,7 @@ def test_gdal_grid_lib_maximum_quadrant_all_params(): _compare_arrays(ds, [[50.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_maximum_quadrant_insufficient_radius(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -584,6 +602,7 @@ def test_gdal_grid_lib_maximum_quadrant_insufficient_radius(): _compare_arrays(ds, [[0.0]]) # insufficient radius. should be > sqrt(2) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_maximum_quadrant_min_points_not_reached(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10)" @@ -603,6 +622,7 @@ def test_gdal_grid_lib_maximum_quadrant_min_points_not_reached(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_maximum_quadrant_missing_point_in_one_quadrant(): # Missing point in 0.5 -0.5 quadrant @@ -623,6 +643,7 @@ def test_gdal_grid_lib_maximum_quadrant_missing_point_in_one_quadrant(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_maximum_quadrant_ignore_extra_points(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 10,1 0 100)" @@ -642,6 +663,7 @@ def test_gdal_grid_lib_maximum_quadrant_ignore_extra_points(): _compare_arrays(ds, [[10.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_range_quadrant_all_params(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 1,1 0 50)" @@ -661,6 +683,7 @@ def test_gdal_grid_lib_range_quadrant_all_params(): _compare_arrays(ds, [[50.0 - 1.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_range_quadrant_insufficient_radius(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 0)" @@ -680,6 +703,7 @@ def test_gdal_grid_lib_range_quadrant_insufficient_radius(): _compare_arrays(ds, [[0.0]]) # insufficient radius. should be > sqrt(2) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_range_quadrant_min_points_not_reached(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 0)" @@ -699,6 +723,7 @@ def test_gdal_grid_lib_range_quadrant_min_points_not_reached(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_range_quadrant_missing_point_in_one_quadrant(): # Missing point in 0.5 -0.5 quadrant @@ -719,6 +744,7 @@ def test_gdal_grid_lib_range_quadrant_missing_point_in_one_quadrant(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_range_quadrant_ignore_extra_points(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 1,1 0 100)" @@ -738,6 +764,7 @@ def test_gdal_grid_lib_range_quadrant_ignore_extra_points(): _compare_arrays(ds, [[9.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_count_quadrant_all_params(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 1,1 0 50)" @@ -757,6 +784,7 @@ def test_gdal_grid_lib_count_quadrant_all_params(): _compare_arrays(ds, [[5]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_count_quadrant_insufficient_radius(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 0)" @@ -776,6 +804,7 @@ def test_gdal_grid_lib_count_quadrant_insufficient_radius(): _compare_arrays(ds, [[0.0]]) # insufficient radius. should be > sqrt(2) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_count_quadrant_min_points_not_reached(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 0)" @@ -795,6 +824,7 @@ def test_gdal_grid_lib_count_quadrant_min_points_not_reached(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_count_quadrant_missing_point_in_one_quadrant(): # Missing point in 0.5 -0.5 quadrant @@ -815,6 +845,7 @@ def test_gdal_grid_lib_count_quadrant_missing_point_in_one_quadrant(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_count_quadrant_ignore_extra_points(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 1,1 0 100)" @@ -834,6 +865,7 @@ def test_gdal_grid_lib_count_quadrant_ignore_extra_points(): _compare_arrays(ds, [[4.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_distance_quadrant_all_params(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 1,1 0 50)" @@ -855,6 +887,7 @@ def test_gdal_grid_lib_average_distance_quadrant_all_params(): _compare_arrays(ds, [[expected_val]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_distance_quadrant_insufficient_radius(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 0)" @@ -874,6 +907,7 @@ def test_gdal_grid_lib_average_distance_quadrant_insufficient_radius(): _compare_arrays(ds, [[0.0]]) # insufficient radius. should be > sqrt(2) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_distance_quadrant_min_points_not_reached(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 0)" @@ -893,6 +927,7 @@ def test_gdal_grid_lib_average_distance_quadrant_min_points_not_reached(): _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_distance_quadrant_missing_point_in_one_quadrant(): # Missing point in 0.5 -0.5 quadrant @@ -913,6 +948,7 @@ def test_gdal_grid_lib_average_distance_quadrant_missing_point_in_one_quadrant() _compare_arrays(ds, [[0.0]]) +@pytest.mark.require_driver("GeoJSON") def test_gdal_grid_lib_average_distance_quadrant_ignore_extra_points(): wkt = "MULTIPOINT(0.5 0.5 10,-0.5 0.5 10,-0.5 -0.5 10,0.5 -0.5 1,1 0 100)" diff --git a/autotest/utilities/test_gdal_rasterize_lib.py b/autotest/utilities/test_gdal_rasterize_lib.py index fa8103c7adf2..b9b1caeef949 100755 --- a/autotest/utilities/test_gdal_rasterize_lib.py +++ b/autotest/utilities/test_gdal_rasterize_lib.py @@ -798,6 +798,7 @@ def test_gdal_rasterize_lib_dict_arguments(): # Test doesn't crash without options +@pytest.mark.require_driver("GeoJSON") def test_gdal_rasterize_no_options(tmp_vsimem): """Test doesn't crash without options""" diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index 2d55e07c212a..cbaafbdd0b61 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -795,6 +795,7 @@ def test_gdalwarp_lib_45(): @pytest.mark.require_driver("CSV") +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_46(tmp_vsimem): ds = gdal.Warp( @@ -891,6 +892,7 @@ def test_gdalwarp_lib_46(tmp_vsimem): # Test -crop_to_cutline -tr X Y -wo CUTLINE_ALL_TOUCHED=YES (fixes for #1360) +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_cutline_all_touched_single_pixel(tmp_vsimem): cutlineDSName = ( @@ -941,6 +943,7 @@ def test_gdalwarp_lib_cutline_all_touched_single_pixel(tmp_vsimem): @pytest.mark.require_driver("CSV") +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_crop_to_cutline_slightly_shifted_wrt_pixel_boundaries(tmp_vsimem): cutlineDSName = ( @@ -1505,6 +1508,7 @@ def test_gdalwarp_lib_dstnodata(dstNodata): # Test automatic densification of cutline (#6375) +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_128(tmp_vsimem): mem_ds = gdal.GetDriverByName("MEM").Create("", 1177, 4719) @@ -1600,6 +1604,7 @@ def test_gdalwarp_lib_128(tmp_vsimem): @pytest.mark.require_geos +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_129(tmp_vsimem): mem_ds = gdal.GetDriverByName("MEM").Create("", 1000, 2000) @@ -3058,6 +3063,7 @@ def test_gdalwarp_lib_scale_offset(): # Test cutline with zero-width sliver +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_cutline_zero_width_sliver(tmp_vsimem): # Geometry valid in EPSG:4326, but that has a zero-width sliver @@ -3079,6 +3085,7 @@ def test_gdalwarp_lib_cutline_zero_width_sliver(tmp_vsimem): # Test cutline with zero-width sliver +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_cutline_zero_width_sliver_remove_empty_polygon(tmp_vsimem): geojson = { @@ -3120,6 +3127,7 @@ def test_gdalwarp_lib_cutline_zero_width_sliver_remove_empty_polygon(tmp_vsimem) # Test cutline with zero-width sliver +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_cutline_zero_width_sliver_remove_empty_inner_ring(tmp_vsimem): geojson = { @@ -3913,6 +3921,7 @@ def test_gdalwarp_lib_working_data_type_with_source_dataset_of_different_types() @pytest.mark.require_geos +@pytest.mark.require_driver("GeoJSON") def test_gdalwarp_lib_cutline_crossing_antimeridian_in_EPSG_32601_and_raster_in_EPSG_4326( tmp_vsimem, ): diff --git a/autotest/utilities/test_ogr2ogr.py b/autotest/utilities/test_ogr2ogr.py index bc218fb45855..55a210cd3cc7 100755 --- a/autotest/utilities/test_ogr2ogr.py +++ b/autotest/utilities/test_ogr2ogr.py @@ -2082,6 +2082,7 @@ def ogr2ogr_62_json(tmp_path): return fname +@pytest.mark.require_driver("GeoJSON") def test_ogr2ogr_62(ogr2ogr_path, ogr2ogr_62_json, tmp_path): dst_json = str(tmp_path / "test_ogr2ogr_62.json") @@ -2097,6 +2098,7 @@ def test_ogr2ogr_62(ogr2ogr_path, ogr2ogr_62_json, tmp_path): assert "bar" in data and "baz" in data +@pytest.mark.require_driver("GeoJSON") def test_ogr2ogr_62bis(ogr2ogr_path, ogr2ogr_62_json, tmp_path): dst_json = str(tmp_path / "test_ogr2ogr_62bis.json") diff --git a/autotest/utilities/test_ogr2ogr_lib.py b/autotest/utilities/test_ogr2ogr_lib.py index f51f7cd1ace3..eeaa8a64b149 100755 --- a/autotest/utilities/test_ogr2ogr_lib.py +++ b/autotest/utilities/test_ogr2ogr_lib.py @@ -2182,6 +2182,7 @@ def test_ogr2ogr_lib_reprojection_curve_geometries_forced_geom_type(geometryType @pytest.mark.require_driver("CSV") +@pytest.mark.require_driver("GeoJSON") def test_ogr2ogr_lib_reprojection_curve_geometries_output_does_not_support_curve( tmp_vsimem, ): diff --git a/autotest/utilities/test_ogrinfo_lib.py b/autotest/utilities/test_ogrinfo_lib.py index 876f75e935ff..b3606cc66906 100755 --- a/autotest/utilities/test_ogrinfo_lib.py +++ b/autotest/utilities/test_ogrinfo_lib.py @@ -317,6 +317,7 @@ def test_ogrinfo_lib_json_relationships(): # Test json output with OFSTJSON field +@pytest.mark.require_driver("GeoJSON") def test_ogrinfo_lib_json_OFSTJSON(): ds = gdal.OpenEx( @@ -375,6 +376,7 @@ def test_ogrinfo_lib_json_OFSTJSON(): # Test json output with -fields=NO +@pytest.mark.require_driver("GeoJSON") def test_ogrinfo_lib_json_fields_NO(): ds = gdal.OpenEx( @@ -398,6 +400,7 @@ def test_ogrinfo_lib_json_fields_NO(): # Test json output with -geom=NO +@pytest.mark.require_driver("GeoJSON") def test_ogrinfo_lib_json_geom_NO(): ds = gdal.OpenEx( diff --git a/ogr/ogrsf_frmts/CMakeLists.txt b/ogr/ogrsf_frmts/CMakeLists.txt index 2f0a35385595..7e8c9bf14420 100644 --- a/ogr/ogrsf_frmts/CMakeLists.txt +++ b/ogr/ogrsf_frmts/CMakeLists.txt @@ -12,7 +12,7 @@ add_dependencies(ogrsf_frmts generate_gdal_version_h) # Obligatory for building GDAL; always compile in. ogr_default_driver(mem "Read/write driver for MEMORY virtual files") -ogr_default_driver(geojson "GeoJSON/ESRIJSON/TopoJSON driver") +ogr_optional_driver(geojson "GeoJSON/ESRIJSON/TopoJSON driver") ogr_default_driver2(mitab TAB "MapInfo TAB and MIF/MID") ogr_optional_driver(shape "ESRI shape-file") @@ -46,13 +46,16 @@ ogr_optional_driver(jml JML) ogr_optional_driver(vdv "VDV-451/VDV-452/INTREST Data Format") ogr_optional_driver(flatgeobuf FlatGeobuf) ogr_optional_driver(mapml MapML) -ogr_optional_driver(jsonfg JSONFG) + if( NOT WORDS_BIGENDIAN ) ogr_optional_driver(miramon "MiraMonVector") endif() # ###################################################################################################################### # +if (NOT OGR_ENABLE_DRIVER_GEOJSON_PLUGIN) + ogr_dependent_driver(jsonfg JSONFG "OGR_ENABLE_DRIVER_GEOJSON") +endif() ogr_dependent_driver(sdts SDTS "GDAL_ENABLE_DRIVER_SDTS") diff --git a/ogr/ogrsf_frmts/geojson/CMakeLists.txt b/ogr/ogrsf_frmts/geojson/CMakeLists.txt index 6e9bea85b064..6d0a3d7628a5 100644 --- a/ogr/ogrsf_frmts/geojson/CMakeLists.txt +++ b/ogr/ogrsf_frmts/geojson/CMakeLists.txt @@ -1,6 +1,5 @@ -# should not be a plugin - dependency from elastic add_gdal_driver( - TARGET ogr_geojson + TARGET ogr_GeoJSON SOURCES ogrgeojsondatasource.cpp ogrgeojsonlayer.cpp ogrgeojsonreader.cpp @@ -13,11 +12,15 @@ add_gdal_driver( ogrtopojsonreader.cpp ogrtopojsondriver.cpp ogrjsoncollectionstreamingparser.cpp - BUILTIN) -gdal_standard_includes(ogr_geojson) -target_include_directories(ogr_geojson PRIVATE $) + PLUGIN_CAPABLE_IF "NOT GDAL_USE_JSONC_INTERNAL" + NO_DEPS) +gdal_standard_includes(ogr_GeoJSON) +target_include_directories(ogr_GeoJSON PRIVATE $) if (GDAL_USE_JSONC_INTERNAL) - gdal_add_vendored_lib(ogr_geojson libjson) + gdal_add_vendored_lib(ogr_GeoJSON libjson) else () - gdal_target_link_libraries(ogr_geojson PRIVATE ${JSONC_TARGET}) + gdal_target_link_libraries(ogr_GeoJSON PRIVATE ${JSONC_TARGET}) endif () +if (OGR_ENABLE_DRIVER_GEOJSON_PLUGIN) + target_compile_definitions(ogr_GeoJSON PRIVATE BUILT_AS_PLUGIN) +endif() diff --git a/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp b/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp index aefcaa07a52b..ba51fd7fb1cf 100644 --- a/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp +++ b/ogr/ogrsf_frmts/geojson/ogrgeojsondriver.cpp @@ -799,4 +799,10 @@ void RegisterOGRGeoJSON() poDriver->pfnUnloadDriver = OGRGeoJSONDriverUnload; GetGDALDriverManager()->RegisterDriver(poDriver); + +#ifdef BUILT_AS_PLUGIN + RegisterOGRTopoJSON(); + RegisterOGRESRIJSON(); + RegisterOGRGeoJSONSeq(); +#endif } diff --git a/ogr/ogrsf_frmts/jsonfg/CMakeLists.txt b/ogr/ogrsf_frmts/jsonfg/CMakeLists.txt index 1c37903aece5..95c809176337 100644 --- a/ogr/ogrsf_frmts/jsonfg/CMakeLists.txt +++ b/ogr/ogrsf_frmts/jsonfg/CMakeLists.txt @@ -13,7 +13,7 @@ add_gdal_driver( BUILTIN) gdal_standard_includes(ogr_JSONFG) target_include_directories(ogr_JSONFG PRIVATE $ - $) + $) if (GDAL_USE_JSONC_INTERNAL) gdal_add_vendored_lib(ogr_JSONFG libjson) else () From 28a00b0825828874618f60f71234ec854e688786 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 04:33:38 +0200 Subject: [PATCH 207/710] Make MapInfo driver optional or buildable as a plugin (if not builtin, Northwoord driver is not available, and Shapefile attribute indices too) --- .github/workflows/cmake_builds.yml | 2 +- autotest/gcore/tiff_read.py | 3 +++ autotest/ogr/ogr_basic_test.py | 2 ++ autotest/ogr/ogr_feature.py | 1 + autotest/ogr/ogr_index_test.py | 2 ++ autotest/ogr/ogr_sql_test.py | 3 +++ autotest/osr/osr_micoordsys.py | 2 ++ autotest/utilities/test_gdal_rasterize.py | 1 + autotest/utilities/test_gdalsrsinfo.py | 1 + autotest/utilities/test_gdaltindex_lib.py | 1 + cmake/helpers/GdalDriverHelper.cmake | 6 ------ frmts/CMakeLists.txt | 4 +++- gdal.cmake | 10 ++++++++++ ogr/CMakeLists.txt | 6 ++++-- ogr/ogrsf_frmts/CMakeLists.txt | 7 ++++++- ogr/ogrsf_frmts/generic/CMakeLists.txt | 4 ++++ ogr/ogrsf_frmts/generic/ogr_miattrind.cpp | 17 +++++++++++++++++ ogr/ogrsf_frmts/generic/ogrlayer.cpp | 7 ++++++- ogr/ogrsf_frmts/mitab/CMakeLists.txt | 3 ++- ogr/ogrspatialreference.cpp | 6 ++++-- 20 files changed, 73 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cmake_builds.yml b/.github/workflows/cmake_builds.yml index e91031a969c2..30e4f87dec0b 100644 --- a/.github/workflows/cmake_builds.yml +++ b/.github/workflows/cmake_builds.yml @@ -456,7 +456,7 @@ jobs: # Build PDF driver as plugin due to the PDFium build including libopenjp2 symbols which would conflict with external libopenjp2 run: | mkdir -p $GITHUB_WORKSPACE/build - cmake -G "${generator}" -Werror=dev "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal" "-DUSE_CCACHE=ON" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DGDAL_ENABLE_PLUGINS:BOOL=ON -DGDAL_ENABLE_PLUGINS_NO_DEPS:BOOL=ON -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT -DBUILD_JAVA_BINDINGS=OFF -DBUILD_CSHARP_BINDINGS=ON -DGDAL_USE_MYSQL:BOOL=OFF -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DWERROR_DEV_FLAG="-Werror=dev" -DCMAKE_BUILD_TYPE=Release -DPDFIUM_ROOT=$GITHUB_WORKSPACE/install-pdfium -DGDAL_ENABLE_DRIVER_PDF_PLUGIN:BOOL=ON -DCMAKE_UNITY_BUILD=ON + cmake -G "${generator}" -Werror=dev "-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install-gdal" "-DUSE_CCACHE=ON" "-DCMAKE_PREFIX_PATH=${CONDA}/envs/gdalenv" -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} -S "$GITHUB_WORKSPACE" -B "$GITHUB_WORKSPACE/build" -DGDAL_ENABLE_PLUGINS:BOOL=ON -DGDAL_ENABLE_PLUGINS_NO_DEPS:BOOL=ON -DGDAL_USE_PUBLICDECOMPWT:BOOL=ON -DPUBLICDECOMPWT_URL=https://github.com/rouault/PublicDecompWT -DBUILD_JAVA_BINDINGS=OFF -DBUILD_CSHARP_BINDINGS=ON -DGDAL_USE_MYSQL:BOOL=OFF -DCMAKE_C_FLAGS=" /WX" -DCMAKE_CXX_FLAGS=" /WX" -DWERROR_DEV_FLAG="-Werror=dev" -DCMAKE_BUILD_TYPE=Release -DPDFIUM_ROOT=$GITHUB_WORKSPACE/install-pdfium -DGDAL_ENABLE_DRIVER_PDF_PLUGIN:BOOL=ON -DCMAKE_UNITY_BUILD=ON -DOGR_ENABLE_DRIVER_TAB_PLUGIN=OFF - name: Build shell: bash -l {0} run: cmake --build $GITHUB_WORKSPACE/build --config Release -j 2 diff --git a/autotest/gcore/tiff_read.py b/autotest/gcore/tiff_read.py index 3a18f98579a5..ba2b0d4677a7 100755 --- a/autotest/gcore/tiff_read.py +++ b/autotest/gcore/tiff_read.py @@ -774,6 +774,7 @@ def test_tiff_read_stats_from_pam(tmp_path): # Test extracting georeferencing from a .TAB file +@pytest.mark.require_driver("MapInfo File") def test_tiff_read_from_tab(tmp_path): ds = gdal.GetDriverByName("GTiff").Create(tmp_path / "tiff_read_from_tab.tif", 1, 1) @@ -2822,6 +2823,7 @@ def test_tiff_read_one_strip_no_bytecount(): # Test GDAL_GEOREF_SOURCES +@pytest.mark.require_driver("MapInfo File") @pytest.mark.parametrize( "config_option_value,copy_pam,copy_worldfile,copy_tabfile,expected_srs,expected_gt", [ @@ -3027,6 +3029,7 @@ def test_tiff_read_nogeoref( # Test GDAL_GEOREF_SOURCES +@pytest.mark.require_driver("MapInfo File") @pytest.mark.parametrize( "config_option_value,copy_pam,copy_worldfile,copy_tabfile,expected_srs,expected_gt", [ diff --git a/autotest/ogr/ogr_basic_test.py b/autotest/ogr/ogr_basic_test.py index ee33cc6d5b8a..09703b7deaf5 100755 --- a/autotest/ogr/ogr_basic_test.py +++ b/autotest/ogr/ogr_basic_test.py @@ -1228,6 +1228,8 @@ def test_driver_open_throw_2(): with gdaltest.enable_exceptions(): drv = ogr.GetDriverByName("MapInfo File") + if not drv: + pytest.skip("MapInfo driver not available") assert isinstance(drv, ogr.Driver) diff --git a/autotest/ogr/ogr_feature.py b/autotest/ogr/ogr_feature.py index 51a9713feaa4..55ec34f9e2e0 100755 --- a/autotest/ogr/ogr_feature.py +++ b/autotest/ogr/ogr_feature.py @@ -923,6 +923,7 @@ def test_ogr_feature_GetFieldAsISO8601DateTime(): assert feature.GetFieldAsISO8601DateTime("field_datetime") == "" +@pytest.mark.require_driver("MapInfo File") def test_ogr_feature_dump_readable(): ds = ogr.Open("data/mitab/single_point_mapinfo.tab") diff --git a/autotest/ogr/ogr_index_test.py b/autotest/ogr/ogr_index_test.py index 0044b44ccbe0..6b8fa6f5b10b 100755 --- a/autotest/ogr/ogr_index_test.py +++ b/autotest/ogr/ogr_index_test.py @@ -34,6 +34,8 @@ from osgeo import ogr +pytestmark = pytest.mark.require_driver("MapInfo File") + ############################################################################### diff --git a/autotest/ogr/ogr_sql_test.py b/autotest/ogr/ogr_sql_test.py index c58f1e8bc12f..4dba5ef4d41f 100755 --- a/autotest/ogr/ogr_sql_test.py +++ b/autotest/ogr/ogr_sql_test.py @@ -419,6 +419,7 @@ def test_ogr_sql_13(data_ds): # Verify selection of, and on ogr_style and ogr_geom_wkt. +@pytest.mark.require_driver("MapInfo File") def test_ogr_sql_14(): expect = [ @@ -452,6 +453,7 @@ def test_ogr_sql_15(data_ds): ############################################################################### +@pytest.mark.require_driver("MapInfo File") def test_ogr_sql_16(): expect = [2] @@ -465,6 +467,7 @@ def test_ogr_sql_16(): ############################################################################### # Test the RFC 21 CAST operator. # +@pytest.mark.require_driver("MapInfo File") def test_ogr_sql_17(): expect = ["1", "2"] diff --git a/autotest/osr/osr_micoordsys.py b/autotest/osr/osr_micoordsys.py index 9e18c2922b43..6352b3baf19e 100755 --- a/autotest/osr/osr_micoordsys.py +++ b/autotest/osr/osr_micoordsys.py @@ -33,6 +33,8 @@ from osgeo import osr +pytestmark = pytest.mark.require_driver("MapInfo File") + ############################################################################### # Test the osr.SpatialReference.ImportFromMICoordSys() function. # diff --git a/autotest/utilities/test_gdal_rasterize.py b/autotest/utilities/test_gdal_rasterize.py index f6049bd5becb..ebd55da69f3e 100755 --- a/autotest/utilities/test_gdal_rasterize.py +++ b/autotest/utilities/test_gdal_rasterize.py @@ -56,6 +56,7 @@ def gdal_rasterize_path(): # Simple polygon rasterization (adapted from alg/rasterize.py). +@pytest.mark.require_driver("MapInfo File") def test_gdal_rasterize_1(gdal_rasterize_path, tmp_path): output_tif = str(tmp_path / "rast1.tif") diff --git a/autotest/utilities/test_gdalsrsinfo.py b/autotest/utilities/test_gdalsrsinfo.py index 02e14de5e7ae..0012eb3ca2f2 100755 --- a/autotest/utilities/test_gdalsrsinfo.py +++ b/autotest/utilities/test_gdalsrsinfo.py @@ -144,6 +144,7 @@ def test_gdalsrsinfo_6(gdalsrsinfo_path): # Test -o mapinfo option +@pytest.mark.require_driver("MapInfo File") def test_gdalsrsinfo_7(gdalsrsinfo_path): ret = gdaltest.runexternal(gdalsrsinfo_path + " -o mapinfo ../gcore/data/byte.tif") diff --git a/autotest/utilities/test_gdaltindex_lib.py b/autotest/utilities/test_gdaltindex_lib.py index 1ca76128ea5f..1d0296b7f11a 100644 --- a/autotest/utilities/test_gdaltindex_lib.py +++ b/autotest/utilities/test_gdaltindex_lib.py @@ -234,6 +234,7 @@ def test_gdaltindex_lib_outputSRS_writeAbsoluePath(tmp_path, four_tile_index): # Test -f, -lyr_name +@pytest.mark.require_driver("MapInfo File") def test_gdaltindex_lib_format_layerName(tmp_path, four_tiles): index_mif = str(tmp_path / "test_gdaltindex6.mif") diff --git a/cmake/helpers/GdalDriverHelper.cmake b/cmake/helpers/GdalDriverHelper.cmake index ad07cb3d407c..d8588adbf1f4 100644 --- a/cmake/helpers/GdalDriverHelper.cmake +++ b/cmake/helpers/GdalDriverHelper.cmake @@ -559,9 +559,3 @@ macro(ogr_default_driver name desc) add_feature_info(ogr_${key} OGR_ENABLE_DRIVER_${key} "${desc}") add_subdirectory(${name}) endmacro() -macro(ogr_default_driver2 name key desc) - set(OGR_ENABLE_DRIVER_${key} ON CACHE BOOL "${desc}" FORCE) - add_feature_info(ogr_${key} OGR_ENABLE_DRIVER_${key} "${desc}") - add_subdirectory(${name}) -endmacro() - diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index 875eaade5385..e824bf1c783b 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -85,7 +85,9 @@ gdal_optional_format(blx "Magellan BLX Topo File Format") gdal_optional_format(msgn "Meteosat Second Generation (MSG) Native Archive Format (.nat)") gdal_optional_format(til "EarthWatch .TIL Driver") gdal_optional_format(r "R Object Data Store") -gdal_optional_format(northwood "NWT_GRD/NWT_GRC -- Northwood/Vertical Mapper File Format") +if (NOT OGR_ENABLE_DRIVER_TAB_PLUGIN) + gdal_dependent_format(northwood "NWT_GRD/NWT_GRC -- Northwood/Vertical Mapper File Format" "OGR_ENABLE_DRIVER_TAB") +endif() gdal_optional_format(saga "SAGA GIS Binary Driver") gdal_optional_format(xyz "ASCII Gridded XYZ") include(avif/driver_declaration.cmake) diff --git a/gdal.cmake b/gdal.cmake index f806dad4a74d..383014a35a5f 100644 --- a/gdal.cmake +++ b/gdal.cmake @@ -314,6 +314,16 @@ if (GDAL_USE_SHAPELIB_INTERNAL) mark_as_advanced(RENAME_INTERNAL_SHAPELIB_SYMBOLS) endif () +# Must be set before including ogr +option(OGR_ENABLE_DRIVER_TAB + "Set ON to build MapInfo TAB and MIF/MID driver (required by Northwoord driver, and Shapefile attribute indexing)" + ${OGR_BUILD_OPTIONAL_DRIVERS}) +if(OGR_ENABLE_DRIVER_TAB AND + NOT DEFINED OGR_ENABLE_DRIVER_TAB_PLUGIN AND + GDAL_ENABLE_PLUGINS_NO_DEPS) + option(OGR_ENABLE_DRIVER_TAB_PLUGIN "Set ON to build OGR MapInfo TAB and MIF/MID driver as plugin" ON) +endif() + # Core components add_subdirectory(alg) add_subdirectory(ogr) diff --git a/ogr/CMakeLists.txt b/ogr/CMakeLists.txt index 30b10c9a4581..18ce7f1b9ed3 100644 --- a/ogr/CMakeLists.txt +++ b/ogr/CMakeLists.txt @@ -95,6 +95,10 @@ target_include_directories(ogr PRIVATE $) set_property(TARGET ogr PROPERTY POSITION_INDEPENDENT_CODE ${GDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) +if (OGR_ENABLE_DRIVER_TAB AND NOT OGR_ENABLE_DRIVER_TAB_PLUGIN) + target_compile_definitions(ogr PRIVATE -DHAVE_MITAB) +endif() + if (GDAL_USE_ZLIB_INTERNAL) gdal_add_vendored_lib(ogr libz) endif () @@ -103,8 +107,6 @@ if (GDAL_USE_JSONC_INTERNAL) gdal_add_vendored_lib(ogr libjson) endif () -target_compile_definitions(ogr PRIVATE HAVE_MITAB) - gdal_target_link_libraries(ogr PRIVATE PROJ::proj) # External libs then diff --git a/ogr/ogrsf_frmts/CMakeLists.txt b/ogr/ogrsf_frmts/CMakeLists.txt index 7e8c9bf14420..f93f82db90c9 100644 --- a/ogr/ogrsf_frmts/CMakeLists.txt +++ b/ogr/ogrsf_frmts/CMakeLists.txt @@ -13,7 +13,12 @@ add_dependencies(ogrsf_frmts generate_gdal_version_h) # Obligatory for building GDAL; always compile in. ogr_default_driver(mem "Read/write driver for MEMORY virtual files") ogr_optional_driver(geojson "GeoJSON/ESRIJSON/TopoJSON driver") -ogr_default_driver2(mitab TAB "MapInfo TAB and MIF/MID") + +add_feature_info(ogr_TAB OGR_ENABLE_DRIVER_TAB "MapInfo TAB and MIF/MID") +if (OGR_ENABLE_DRIVER_TAB) + add_subdirectory(mitab) +endif () + ogr_optional_driver(shape "ESRI shape-file") # ###################################################################################################################### diff --git a/ogr/ogrsf_frmts/generic/CMakeLists.txt b/ogr/ogrsf_frmts/generic/CMakeLists.txt index 2b30b07f0f7e..d91cca719e6b 100644 --- a/ogr/ogrsf_frmts/generic/CMakeLists.txt +++ b/ogr/ogrsf_frmts/generic/CMakeLists.txt @@ -47,3 +47,7 @@ add_dependencies(ogrsf_generic generate_gdal_version_h) target_compile_options(ogrsf_generic PRIVATE ${GDAL_CXX_WARNING_FLAGS} ${WFLAG_OLD_STYLE_CAST} ${WFLAG_EFFCXX}) target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) set_property(TARGET ogrsf_generic PROPERTY POSITION_INDEPENDENT_CODE ${GDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) + +if (OGR_ENABLE_DRIVER_TAB AND NOT OGR_ENABLE_DRIVER_TAB_PLUGIN) +target_compile_definitions(ogrsf_generic PRIVATE -DHAVE_MITAB) +endif() diff --git a/ogr/ogrsf_frmts/generic/ogr_miattrind.cpp b/ogr/ogrsf_frmts/generic/ogr_miattrind.cpp index 50e38ac69af4..80e6ed515570 100644 --- a/ogr/ogrsf_frmts/generic/ogr_miattrind.cpp +++ b/ogr/ogrsf_frmts/generic/ogr_miattrind.cpp @@ -29,6 +29,9 @@ ****************************************************************************/ #include "ogr_attrind.h" + +#ifdef HAVE_MITAB + #include "mitab/mitab_priv.h" #include "cpl_minixml.h" @@ -829,3 +832,17 @@ OGRErr OGRMIAttrIndex::Clear() { return OGRERR_UNSUPPORTED_OPERATION; } + +#else + +/************************************************************************/ +/* OGRCreateDefaultLayerIndex() */ +/************************************************************************/ + +OGRLayerAttrIndex *OGRCreateDefaultLayerIndex() + +{ + return nullptr; +} + +#endif diff --git a/ogr/ogrsf_frmts/generic/ogrlayer.cpp b/ogr/ogrsf_frmts/generic/ogrlayer.cpp index 82ac8464b073..c4e5fa06dd02 100644 --- a/ogr/ogrsf_frmts/generic/ogrlayer.cpp +++ b/ogr/ogrsf_frmts/generic/ogrlayer.cpp @@ -1943,9 +1943,11 @@ void OGR_L_ResetReading(OGRLayerH hLayer) /************************************************************************/ //! @cond Doxygen_Suppress -OGRErr OGRLayer::InitializeIndexSupport(const char *pszFilename) +OGRErr +OGRLayer::InitializeIndexSupport([[maybe_unused]] const char *pszFilename) { +#ifdef HAVE_MITAB OGRErr eErr; if (m_poAttrIndex != nullptr) @@ -1961,6 +1963,9 @@ OGRErr OGRLayer::InitializeIndexSupport(const char *pszFilename) } return eErr; +#else + return OGRERR_FAILURE; +#endif } //! @endcond diff --git a/ogr/ogrsf_frmts/mitab/CMakeLists.txt b/ogr/ogrsf_frmts/mitab/CMakeLists.txt index 6bec0cb2224c..49c10553efd1 100644 --- a/ogr/ogrsf_frmts/mitab/CMakeLists.txt +++ b/ogr/ogrsf_frmts/mitab/CMakeLists.txt @@ -2,7 +2,8 @@ add_gdal_driver( TARGET ogr_TAB DEF MITAB_ENABLED - BUILTIN + PLUGIN_CAPABLE + NO_DEPS SOURCES mitab_rawbinblock.cpp mitab_mapheaderblock.cpp mitab_mapindexblock.cpp diff --git a/ogr/ogrspatialreference.cpp b/ogr/ogrspatialreference.cpp index fc9bec8b49de..25db55399e4f 100644 --- a/ogr/ogrspatialreference.cpp +++ b/ogr/ogrspatialreference.cpp @@ -11251,7 +11251,8 @@ OGRErr OSRExportToMICoordSys(OGRSpatialReferenceH hSRS, char **ppszReturn) * OGRERR_UNSUPPORTED_OPERATION if MITAB library was not linked in. */ -OGRErr OGRSpatialReference::exportToMICoordSys(char **ppszResult) const +OGRErr OGRSpatialReference::exportToMICoordSys( + [[maybe_unused]] char **ppszResult) const { #ifdef HAVE_MITAB @@ -11305,7 +11306,8 @@ OGRErr OSRImportFromMICoordSys(OGRSpatialReferenceH hSRS, * OGRERR_UNSUPPORTED_OPERATION if MITAB library was not linked in. */ -OGRErr OGRSpatialReference::importFromMICoordSys(const char *pszCoordSys) +OGRErr OGRSpatialReference::importFromMICoordSys( + [[maybe_unused]] const char *pszCoordSys) { #ifdef HAVE_MITAB From 58cc577359c81d91ff417976698efc260424d21c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 14:03:55 +0200 Subject: [PATCH 208/710] /vsicurl/: no longer forward Authorization header when doing redirection to other hosts. Can be configured with the CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT=YES/NO/IF_SAME_HOST config option --- autotest/gcore/vsicurl.py | 195 ++++++++++++++++++++++- doc/source/spelling_wordlist.txt | 1 + doc/source/user/configoptions.rst | 14 ++ doc/source/user/virtual_file_systems.rst | 5 + port/cpl_http.cpp | 44 +++-- port/cpl_vsil_curl.cpp | 86 ++++++++-- port/cpl_vsil_curl_class.h | 6 +- 7 files changed, 323 insertions(+), 28 deletions(-) diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index 45746a20f57b..69f6a54d2f92 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -233,11 +233,204 @@ def server(): webserver.server_stop(process, port) +############################################################################### +# Test regular redirection + + +@pytest.mark.parametrize( + "authorization_header_allowed", [None, "YES", "NO", "IF_SAME_HOST"] +) +def test_vsicurl_test_redirect(server, authorization_header_allowed): + + gdal.VSICurlClearCache() + + expected_headers = None + unexpected_headers = [] + if authorization_header_allowed != "NO": + expected_headers = {"Authorization": "Bearer xxx"} + else: + unexpected_headers = ["Authorization"] + + handler = webserver.SequentialHandler() + handler.add("GET", "/test_redirect/", 404) + handler.add( + "HEAD", + "/test_redirect/test.bin", + 301, + {"Location": "http://localhost:%d/redirected/test.bin" % server.port}, + expected_headers={"Authorization": "Bearer xxx"}, + ) + + # Curl always forward Authorization if same server when handling itself + # the redirect, so this means that CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT=NO + # is not honored for that particular request. To honour it, we would have + # to disable CURLOPT_FOLLOWLOCATION and implement it at hand + handler.add( + "HEAD", + "/redirected/test.bin", + 200, + {"Content-Length": "3"}, + expected_headers={"Authorization": "Bearer xxx"}, + ) + + handler.add( + "GET", + "/redirected/test.bin", + 200, + {"Content-Length": "3"}, + b"xyz", + expected_headers=expected_headers, + unexpected_headers=unexpected_headers, + ) + + options = {"GDAL_HTTP_HEADERS": "Authorization: Bearer xxx"} + if authorization_header_allowed: + options[ + "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT" + ] = authorization_header_allowed + with webserver.install_http_handler(handler), gdal.config_options(options): + f = gdal.VSIFOpenL( + "/vsicurl/http://localhost:%d/test_redirect/test.bin" % server.port, + "rb", + ) + assert f is not None + try: + assert gdal.VSIFReadL(1, 3, f) == b"xyz" + finally: + gdal.VSIFCloseL(f) + + +############################################################################### +# Test regular redirection + + +@pytest.mark.parametrize( + "authorization_header_allowed", [None, "YES", "NO", "IF_SAME_HOST"] +) +def test_vsicurl_test_redirect_different_server(server, authorization_header_allowed): + + gdal.VSICurlClearCache() + + expected_headers = None + unexpected_headers = [] + if authorization_header_allowed == "YES": + expected_headers = {"Authorization": "Bearer xxx"} + else: + unexpected_headers = ["Authorization"] + + handler = webserver.SequentialHandler() + handler.add("GET", "/test_redirect/", 404) + handler.add( + "HEAD", + "/test_redirect/test.bin", + 301, + {"Location": "http://127.0.0.1:%d/redirected/test.bin" % server.port}, + expected_headers={"Authorization": "Bearer xxx"}, + ) + handler.add( + "HEAD", + "/redirected/test.bin", + 200, + {"Content-Length": "3"}, + expected_headers=expected_headers, + unexpected_headers=unexpected_headers, + ) + handler.add( + "GET", + "/redirected/test.bin", + 200, + {"Content-Length": "3"}, + b"xyz", + expected_headers=expected_headers, + unexpected_headers=unexpected_headers, + ) + + options = {"GDAL_HTTP_HEADERS": "Authorization: Bearer xxx"} + if authorization_header_allowed: + options[ + "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT" + ] = authorization_header_allowed + with webserver.install_http_handler(handler), gdal.config_options(options): + f = gdal.VSIFOpenL( + "/vsicurl/http://localhost:%d/test_redirect/test.bin" % server.port, + "rb", + ) + try: + assert gdal.VSIFReadL(1, 3, f) == b"xyz" + finally: + gdal.VSIFCloseL(f) + + +############################################################################### +# Test regular redirection + + +@gdaltest.enable_exceptions() +@pytest.mark.require_curl(7, 61, 0) +@pytest.mark.parametrize( + "authorization_header_allowed", [None, "YES", "NO", "IF_SAME_HOST"] +) +def test_vsicurl_test_redirect_different_server_with_bearer( + server, authorization_header_allowed +): + + gdal.VSICurlClearCache() + + expected_headers = None + unexpected_headers = [] + if authorization_header_allowed == "YES": + expected_headers = {"Authorization": "Bearer xxx"} + else: + unexpected_headers = ["Authorization"] + + handler = webserver.SequentialHandler() + handler.add("GET", "/test_redirect/", 404) + handler.add( + "HEAD", + "/test_redirect/test.bin", + 301, + {"Location": "http://127.0.0.1:%d/redirected/test.bin" % server.port}, + expected_headers={"Authorization": "Bearer xxx"}, + ) + handler.add( + "HEAD", + "/redirected/test.bin", + 200, + {"Content-Length": "3"}, + expected_headers=expected_headers, + unexpected_headers=unexpected_headers, + ) + handler.add( + "GET", + "/redirected/test.bin", + 200, + {"Content-Length": "3"}, + b"xyz", + expected_headers=expected_headers, + unexpected_headers=unexpected_headers, + ) + + options = {"GDAL_HTTP_AUTH": "BEARER", "GDAL_HTTP_BEARER": "xxx"} + if authorization_header_allowed: + options[ + "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT" + ] = authorization_header_allowed + with webserver.install_http_handler(handler), gdal.config_options(options): + f = gdal.VSIFOpenL( + "/vsicurl/http://localhost:%d/test_redirect/test.bin" % server.port, + "rb", + ) + try: + assert gdal.VSIFReadL(1, 3, f) == b"xyz" + finally: + gdal.VSIFCloseL(f) + + ############################################################################### # Test redirection with Expires= type of signed URLs -def test_vsicurl_test_redirect(server): +def test_vsicurl_test_redirect_with_expires(server): gdal.VSICurlClearCache() diff --git a/doc/source/spelling_wordlist.txt b/doc/source/spelling_wordlist.txt index bba8115394f7..474f9db809cb 100644 --- a/doc/source/spelling_wordlist.txt +++ b/doc/source/spelling_wordlist.txt @@ -2732,6 +2732,7 @@ RecordBatch RecordBatchAsNumpy recurse recursionLevel +redirections Redistributable reentrancy reentrant diff --git a/doc/source/user/configoptions.rst b/doc/source/user/configoptions.rst index aa2bcaa0209c..f9cea46e84bc 100644 --- a/doc/source/user/configoptions.rst +++ b/doc/source/user/configoptions.rst @@ -626,6 +626,20 @@ Networking options Try to query quietly redirected URLs to Amazon S3 signed URLs during their validity period, so as to minimize round-trips. +- .. config:: CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT + :choices: YES, NO, IF_SAME_HOST + :default: IF_SAME_HOST + :since: 3.10 + + Determines if the HTTP ``Authorization`` header must be forwarded when + redirections are followed: + + - ``NO`` to always disable forwarding of Authorization header + - ``YES`` to always enable forwarding of Authorization header (was the + default value prior to GDAL 3.10) + - ``IF_SAME_HOST`` to enable forwarding of Authorization header only if + the redirection is to the same host. + - .. config:: CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE :choices: YES, NO diff --git a/doc/source/user/virtual_file_systems.rst b/doc/source/user/virtual_file_systems.rst index 860025e30452..6f6a29698933 100644 --- a/doc/source/user/virtual_file_systems.rst +++ b/doc/source/user/virtual_file_systems.rst @@ -413,6 +413,11 @@ As an alternative, starting with GDAL 3.6, the :config:`GDAL_HTTP_HEADERS` configuration option can also be used to specify headers. :config:`CPL_CURL_VERBOSE=YES` allows one to see them and more, when combined with ``--debug``. +Starting with GDAL 3.10, the ``Authorization`` header is no longer automatically +forwarded when redirections are followed. +That behavior can be configured by setting the +:config:`CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT` configuration option. + Starting with GDAL 2.3, the :config:`GDAL_HTTP_MAX_RETRY` (number of attempts) and :config:`GDAL_HTTP_RETRY_DELAY` (in seconds) configuration option can be set, so that request retries are done in case of HTTP errors 429, 502, 503 or 504. Starting with GDAL 3.6, the following configuration options control the TCP keep-alive functionality (cf https://daniel.haxx.se/blog/2020/02/10/curl-ootw-keepalive-time/ for a detailed explanation): diff --git a/port/cpl_http.cpp b/port/cpl_http.cpp index 471adadbc20b..7d8751036744 100644 --- a/port/cpl_http.cpp +++ b/port/cpl_http.cpp @@ -2237,14 +2237,22 @@ void *CPLHTTPSetOptions(void *pcurl, const char *pszURL, CURLAUTH_ANYSAFE); else if (EQUAL(pszHttpAuth, "BEARER")) { - const char *pszBearer = CSLFetchNameValue(papszOptions, "HTTP_BEARER"); - if (pszBearer == nullptr) - pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr); - if (pszBearer != nullptr) - unchecked_curl_easy_setopt(http_handle, CURLOPT_XOAUTH2_BEARER, - pszBearer); - unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, - CURLAUTH_BEARER); + const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef( + papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES"); + const bool bAuthorizationHeaderAllowed = + CPLTestBool(pszAuthorizationHeaderAllowed); + if (bAuthorizationHeaderAllowed) + { + const char *pszBearer = + CSLFetchNameValue(papszOptions, "HTTP_BEARER"); + if (pszBearer == nullptr) + pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr); + if (pszBearer != nullptr) + unchecked_curl_easy_setopt(http_handle, CURLOPT_XOAUTH2_BEARER, + pszBearer); + unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, + CURLAUTH_BEARER); + } } else if (EQUAL(pszHttpAuth, "NEGOTIATE")) unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, @@ -2365,6 +2373,15 @@ void *CPLHTTPSetOptions(void *pcurl, const char *pszURL, 1L); unchecked_curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1); + const char *pszUnrestrictedAuth = CPLGetConfigOption( + "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT", + "IF_SAME_HOST"); + if (!EQUAL(pszUnrestrictedAuth, "IF_SAME_HOST") && + CPLTestBool(pszUnrestrictedAuth)) + { + unchecked_curl_easy_setopt(http_handle, CURLOPT_UNRESTRICTED_AUTH, 1); + } + unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXREDIRS, 10); unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL); @@ -2664,6 +2681,11 @@ void *CPLHTTPSetOptions(void *pcurl, const char *pszURL, } if (!bHeadersDone) { + const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef( + papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES"); + const bool bAuthorizationHeaderAllowed = + CPLTestBool(pszAuthorizationHeaderAllowed); + // We accept both raw headers with \r\n as a separator, or as // a comma separated list of foo: bar values. const CPLStringList aosTokens( @@ -2672,7 +2694,11 @@ void *CPLHTTPSetOptions(void *pcurl, const char *pszURL, : CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS)); for (int i = 0; i < aosTokens.size(); ++i) { - headers = curl_slist_append(headers, aosTokens[i]); + if (bAuthorizationHeaderAllowed || + !STARTS_WITH_CI(aosTokens[i], "Authorization:")) + { + headers = curl_slist_append(headers, aosTokens[i]); + } } } } diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp index 48c76ff69bdf..1efe8884105c 100644 --- a/port/cpl_vsil_curl.cpp +++ b/port/cpl_vsil_curl.cpp @@ -634,6 +634,7 @@ void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct, VSILFILE *fp, psStruct->nStartOffset = 0; psStruct->nEndOffset = 0; psStruct->nHTTPCode = 0; + psStruct->nFirstHTTPCode = 0; psStruct->nContentLength = 0; psStruct->bFoundContentRange = false; psStruct->bError = false; @@ -676,7 +677,10 @@ size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb, char *pszSpace = strchr(pszLine, ' '); if (pszSpace) { - psStruct->nHTTPCode = atoi(pszSpace + 1); + const int nHTTPCode = atoi(pszSpace + 1); + if (psStruct->nFirstHTTPCode == 0) + psStruct->nFirstHTTPCode = nHTTPCode; + psStruct->nHTTPCode = nHTTPCode; } } else if (STARTS_WITH_CI(pszLine, "Content-Length: ")) @@ -1236,15 +1240,23 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, if (!osEffectiveURL.empty() && strstr(osEffectiveURL.c_str(), osURL.c_str()) == nullptr) { - CPLDebug(poFS->GetDebugKey(), "Effective URL: %s", - osEffectiveURL.c_str()); - - if (m_bUseRedirectURLIfNoQueryStringParams && - osEffectiveURL.find('?') == std::string::npos) + // Moved permanently ? + if (sWriteFuncHeaderData.nFirstHTTPCode == 301 || + (m_bUseRedirectURLIfNoQueryStringParams && + osEffectiveURL.find('?') == std::string::npos)) { + CPLDebug(poFS->GetDebugKey(), + "Using effective URL %s permanently", + osEffectiveURL.c_str()); oFileProp.osRedirectURL = osEffectiveURL; poFS->SetCachedFileProp(m_pszURL, oFileProp); } + else + { + CPLDebug(poFS->GetDebugKey(), + "Using effective URL %s temporarily", + osEffectiveURL.c_str()); + } // Is this is a redirect to a S3 URL? if (VSICurlIsS3LikeSignedURL(osEffectiveURL.c_str()) && @@ -1603,7 +1615,9 @@ vsi_l_offset VSICurlHandle::Tell() /* GetRedirectURLIfValid() */ /************************************************************************/ -std::string VSICurlHandle::GetRedirectURLIfValid(bool &bHasExpired) const +std::string +VSICurlHandle::GetRedirectURLIfValid(bool &bHasExpired, + CPLStringList &aosHTTPOptions) const { bHasExpired = false; poFS->GetCachedFileProp(m_pszURL, oFileProp); @@ -1635,6 +1649,39 @@ std::string VSICurlHandle::GetRedirectURLIfValid(bool &bHasExpired) const bHasExpired = false; } + if (m_pszURL != osURL) + { + const char *pszAuthorizationHeaderAllowed = CPLGetConfigOption( + "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT", + "IF_SAME_HOST"); + if (EQUAL(pszAuthorizationHeaderAllowed, "IF_SAME_HOST")) + { + const auto ExtractServer = [](const std::string &s) + { + size_t afterHTTPPos = 0; + if (STARTS_WITH(s.c_str(), "http://")) + afterHTTPPos = strlen("http://"); + else if (STARTS_WITH(s.c_str(), "https://")) + afterHTTPPos = strlen("https://"); + const auto posSlash = s.find('/', afterHTTPPos); + if (posSlash != std::string::npos) + return s.substr(afterHTTPPos, posSlash - afterHTTPPos); + else + return s.substr(afterHTTPPos); + }; + + if (ExtractServer(osURL) != ExtractServer(m_pszURL)) + { + aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", + "NO"); + } + } + else if (!CPLTestBool(pszAuthorizationHeaderAllowed)) + { + aosHTTPOptions.SetNameValue("AUTHORIZATION_HEADER_ALLOWED", "NO"); + } + } + return osURL; } @@ -1809,7 +1856,9 @@ std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, ManagePlanetaryComputerSigning(); bool bHasExpired = false; - std::string osURL(GetRedirectURLIfValid(bHasExpired)); + + CPLStringList aosHTTPOptions(m_aosHTTPOptions); + std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); bool bUsedRedirect = osURL != m_pszURL; WriteFuncStruct sWriteFuncData; @@ -1819,7 +1868,7 @@ std::string VSICurlHandle::DownloadRegion(const vsi_l_offset startOffset, retry: CURL *hCurlHandle = curl_easy_init(); struct curl_slist *headers = - VSICurlSetOptions(hCurlHandle, osURL.c_str(), m_aosHTTPOptions.List()); + VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); if (!AllowAutomaticRedirection()) unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0); @@ -2382,7 +2431,9 @@ int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData, ManagePlanetaryComputerSigning(); bool bHasExpired = false; - std::string osURL(GetRedirectURLIfValid(bHasExpired)); + + CPLStringList aosHTTPOptions(m_aosHTTPOptions); + std::string osURL(GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); if (bHasExpired) { return VSIVirtualHandle::ReadMultiRange(nRanges, ppData, panOffsets, @@ -2447,7 +2498,7 @@ int VSICurlHandle::ReadMultiRange(int const nRanges, void **const ppData, // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); struct curl_slist *headers = VSICurlSetOptions( - hCurlHandle, osURL.c_str(), m_aosHTTPOptions.List()); + hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); VSICURLInitWriteFuncStruct(&asWriteFuncData[iRequest], this, pfnReadCbk, pReadCbkUserData); @@ -3039,18 +3090,19 @@ size_t VSICurlHandle::PRead(void *pBuffer, size_t nSize, NetworkStatisticsFile oContextFile(m_osFilename.c_str()); NetworkStatisticsAction oContextAction("PRead"); + CPLStringList aosHTTPOptions(m_aosHTTPOptions); std::string osURL; { std::lock_guard oLock(m_oMutex); ManagePlanetaryComputerSigning(); bool bHasExpired; - osURL = GetRedirectURLIfValid(bHasExpired); + osURL = GetRedirectURLIfValid(bHasExpired, aosHTTPOptions); } CURL *hCurlHandle = curl_easy_init(); struct curl_slist *headers = - VSICurlSetOptions(hCurlHandle, osURL.c_str(), m_aosHTTPOptions.List()); + VSICurlSetOptions(hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); WriteFuncStruct sWriteFuncData; VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr); @@ -3210,7 +3262,9 @@ void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, ManagePlanetaryComputerSigning(); bool bHasExpired = false; - const std::string l_osURL(GetRedirectURLIfValid(bHasExpired)); + CPLStringList aosHTTPOptions(m_aosHTTPOptions); + const std::string l_osURL( + GetRedirectURLIfValid(bHasExpired, aosHTTPOptions)); if (bHasExpired) { return; @@ -3277,7 +3331,7 @@ void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, static_cast(m_aoAdviseReadRanges.size())); #endif - const auto task = [this](const std::string &osURL) + const auto task = [this, aosHTTPOptions](const std::string &osURL) { CURLM *hMultiHandle = curl_multi_init(); @@ -3325,7 +3379,7 @@ void VSICurlHandle::AdviseRead(int nRanges, const vsi_l_offset *panOffsets, // unchecked_curl_easy_setopt(hCurlHandle, CURLOPT_PIPEWAIT, 1); struct curl_slist *headers = VSICurlSetOptions( - hCurlHandle, osURL.c_str(), m_aosHTTPOptions.List()); + hCurlHandle, osURL.c_str(), aosHTTPOptions.List()); VSICURLInitWriteFuncStruct(&asWriteFuncData[i], this, pfnReadCbk, pReadCbkUserData); diff --git a/port/cpl_vsil_curl_class.h b/port/cpl_vsil_curl_class.h index e371d497d4a0..92786b34c5d5 100644 --- a/port/cpl_vsil_curl_class.h +++ b/port/cpl_vsil_curl_class.h @@ -116,7 +116,8 @@ struct WriteFuncStruct bool bMultiRange = false; vsi_l_offset nStartOffset = 0; vsi_l_offset nEndOffset = 0; - int nHTTPCode = 0; + int nHTTPCode = 0; // potentially after redirect + int nFirstHTTPCode = 0; // the one of the redirect vsi_l_offset nContentLength = 0; bool bFoundContentRange = false; bool bError = false; @@ -423,7 +424,8 @@ class VSICurlHandle : public VSIVirtualHandle int ReadMultiRangeSingleGet(int nRanges, void **ppData, const vsi_l_offset *panOffsets, const size_t *panSizes); - std::string GetRedirectURLIfValid(bool &bHasExpired) const; + std::string GetRedirectURLIfValid(bool &bHasExpired, + CPLStringList &aosHTTPOptions) const; void UpdateRedirectInfo(CURL *hCurlHandle, const WriteFuncStruct &sWriteFuncHeaderData); From 1bd433312e51c4a47d004bfea08116e458a3fa87 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 15:39:30 +0200 Subject: [PATCH 209/710] Backup potential useful links to old Trac wiki [ci skip] --- doc/old_trac_pointers.md | 8 ++++++++ frmts/wcs/README.md | 1 + 2 files changed, 9 insertions(+) create mode 100644 doc/old_trac_pointers.md create mode 100644 frmts/wcs/README.md diff --git a/doc/old_trac_pointers.md b/doc/old_trac_pointers.md new file mode 100644 index 000000000000..b40174b7562b --- /dev/null +++ b/doc/old_trac_pointers.md @@ -0,0 +1,8 @@ +Google Summer of Code ideas: https://web.archive.org/web/20240812224510/https://trac.osgeo.org/gdal/wiki/SummerOfCode +Google Summer of Code selected projects: https://web.archive.org/web/20240812232901/https://trac.osgeo.org/gdal/wiki/SoCProjects +Old Sponsorship program: http://web.archive.org/web/20230921154549/https://trac.osgeo.org/gdal/wiki/Sponsorship +Old Build hints: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/BuildHints +Additional notes on utilities: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/UserDocs +CodeSnippets: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/CodeSnippets +LayerAlgebra: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/LayerAlgebra +Old FAQ: http://web.archive.org/https://trac.osgeo.org/gdal/wiki/FAQ diff --git a/frmts/wcs/README.md b/frmts/wcs/README.md new file mode 100644 index 000000000000..2dd8345c6bc1 --- /dev/null +++ b/frmts/wcs/README.md @@ -0,0 +1 @@ +WCS Basics and GDAL: http://web.archive.org/web/20240413133419/https://trac.osgeo.org/gdal/wiki/WCS%2Binteroperability From 3b37f5a2bdee3b0553d331358df125edc536bdf8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 16:04:59 +0200 Subject: [PATCH 210/710] Doc: remove links to Trac wiki (or point to web.archive.org) --- .../api/csharp/csharp_compile_legacy.rst | 12 ++-- doc/source/api/csharp/csharp_raster.rst | 70 +++++++++---------- doc/source/api/index.rst | 5 -- .../development/building_from_source.rst | 2 +- .../rfc48_geographical_networks_support.rst | 2 +- doc/source/drivers/raster/cog.rst | 6 +- doc/source/drivers/raster/ecw.rst | 1 - doc/source/drivers/raster/gtiff.rst | 3 - doc/source/drivers/raster/jp2ecw.rst | 1 - doc/source/drivers/raster/msg.rst | 29 -------- doc/source/drivers/raster/netcdf.rst | 3 - doc/source/drivers/raster/pds4.rst | 11 +-- doc/source/drivers/raster/postgisraster.rst | 2 +- doc/source/drivers/vector/pgeo.rst | 3 +- doc/source/drivers/vector/shapefile.rst | 6 +- doc/source/faq.rst | 2 +- 16 files changed, 46 insertions(+), 112 deletions(-) diff --git a/doc/source/api/csharp/csharp_compile_legacy.rst b/doc/source/api/csharp/csharp_compile_legacy.rst index da817c98207a..e9579c045e3f 100644 --- a/doc/source/api/csharp/csharp_compile_legacy.rst +++ b/doc/source/api/csharp/csharp_compile_legacy.rst @@ -84,7 +84,7 @@ To test the compiled binaries, you can use: nmake /f makefile.vc test` -This command will invoke some of the sample applications. +This command will invoke some of the sample applications. .. note:: For the tests to work the location of the proj and gdal DLLs should be available in the PATH. @@ -93,7 +93,7 @@ Using MONO on Windows If you have the Windows version of the MONO package installed you can compile the C# code using the MONO compiler. In this case uncomment the following entry in csharp.opt: -:program:`MONO = YES` +:program:`MONO = YES` .. note:: mcs.exe must be in the PATH. @@ -165,7 +165,7 @@ To test the compiled binaries, you can use: nmake test -This command will invoke some of the sample applications. +This command will invoke some of the sample applications. .. note:: For the tests to work the location of the proj and gdal libraries should be available in the PATH. @@ -179,8 +179,4 @@ To run one of the prebuilt executables - you can run them with Mono as follows : :program:`mono GDALInfo.exe` Both the managed libraries (i.e. the DLLs) and the unmanaged libraries must be available to Mono. -This is in more detail in `the Mono documentation `__ - -.. note:: This document was amended from the previous version at `https://trac.osgeo.org/gdal/wiki/GdalOgrCsharpCompile `__ - - +This is in more detail in `the Mono documentation `__ diff --git a/doc/source/api/csharp/csharp_raster.rst b/doc/source/api/csharp/csharp_raster.rst index ac0e23ea40b8..fde6af99db4b 100644 --- a/doc/source/api/csharp/csharp_raster.rst +++ b/doc/source/api/csharp/csharp_raster.rst @@ -15,40 +15,40 @@ The :file:`Band` class contains the following :file:`ReadRaster`/:file:`WriteRas .. code-block:: C# - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, byte[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, short[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, int[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, float[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, + + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, double[] buffer, int buf_xSize, int buf_ySize, int pixelSpace, int lineSpace){} - + public CPLErr ReadRaster(int xOff, int yOff, int xSize, int ySize, IntPtr buffer, i nt buf_xSize, int buf_ySize, DataType buf_type, int pixelSpace, int lineSpace){} - - public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, IntPtr buffer, + + public CPLErr WriteRaster(int xOff, int yOff, int xSize, int ySize, IntPtr buffer, int buf_xSize, int buf_ySize, DataType buf_type, int pixelSpace, int lineSpace){} The only difference between these functions is the actual type of the buffer parameter. @@ -76,7 +76,7 @@ When reading the image this way the C# API will copy the image data between the band.ReadRaster(0, 0, width, height, r, width, height, 0, 0); // Copying the pixels into the C# bitmap int i, j; - for (i = 0; i< width; i++) + for (i = 0; i< width; i++) { for (j=0; j`__ diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst index 127aea2f66a1..cb4d070b787d 100644 --- a/doc/source/api/index.rst +++ b/doc/source/api/index.rst @@ -117,7 +117,6 @@ API Go Julia - Lua Original Node.js bindings Node.js fork with full Promise-based async and TypeScript support Perl @@ -126,10 +125,6 @@ API Ruby Rust - .. warning:: - For Perl, since GDAL 3.5 the link `Perl `__ is deprecated, use above link instead. - - There are also more Pythonic ways of using the vector/OGR functions with diff --git a/doc/source/development/building_from_source.rst b/doc/source/development/building_from_source.rst index 6a067888836e..adaa100d5bb8 100644 --- a/doc/source/development/building_from_source.rst +++ b/doc/source/development/building_from_source.rst @@ -2546,5 +2546,5 @@ crashes will occur at runtime (often at process termination with a Autoconf/nmake (GDAL versions < 3.5.0) -------------------------------------------------------------------------------- -See https://trac.osgeo.org/gdal/wiki/BuildHints for hints for GDAL < 3.5 +See http://web.archive.org/https://trac.osgeo.org/gdal/wiki/BuildHints for hints for GDAL < 3.5 autoconf and nmake build systems. diff --git a/doc/source/development/rfc/rfc48_geographical_networks_support.rst b/doc/source/development/rfc/rfc48_geographical_networks_support.rst index 0e6bcd2d40fb..32a453bf77cd 100644 --- a/doc/source/development/rfc/rfc48_geographical_networks_support.rst +++ b/doc/source/development/rfc/rfc48_geographical_networks_support.rst @@ -19,7 +19,7 @@ project “GDAL/OGR Geography Network support” into GDAL library. GNM create, manage and analyse networks built over spatial data in GDAL. GSoC project description: -`http://trac.osgeo.org/gdal/wiki/geography_network_support `__ +`http://web.archive.org/web/20240812232429/https://trac.osgeo.org/gdal/wiki/geography_network_support `__ GDAL fork with all changes in trunk: `https://github.com/MikhanGusev/gdal `__ diff --git a/doc/source/drivers/raster/cog.rst b/doc/source/drivers/raster/cog.rst index 381cecdc9e7a..c7a50573c960 100644 --- a/doc/source/drivers/raster/cog.rst +++ b/doc/source/drivers/raster/cog.rst @@ -53,9 +53,7 @@ General creation options * ``JPEG`` should generally only be used with Byte data (8 bit per channel). But if GDAL is built with internal libtiff and libjpeg, it is possible to read and write TIFF files with 12bit JPEG compressed TIFF - files (seen as UInt16 bands with NBITS=12). See the `"8 and 12 bit - JPEG in TIFF" `__ wiki - page for more details. + files (seen as UInt16 bands with NBITS=12). For the COG driver, JPEG compression for 3 or 4-band images automatically selects the PHOTOMETRIC=YCBCR colorspace with a 4:2:2 subsampling of the Y,Cb,Cr components. @@ -612,8 +610,6 @@ See Also -------- - :ref:`raster.gtiff` driver -- `How to generate and read cloud optimized GeoTIFF - files `__ (before GDAL 3.1) - If your source dataset is an internally tiled geotiff with the desired georeferencing and compression, using `cogger `__ (possibly along with gdaladdo to create overviews) will be much faster than the COG driver. diff --git a/doc/source/drivers/raster/ecw.rst b/doc/source/drivers/raster/ecw.rst index f30d728f1ea4..add3aa839c5a 100644 --- a/doc/source/drivers/raster/ecw.rst +++ b/doc/source/drivers/raster/ecw.rst @@ -327,4 +327,3 @@ See Also `Hexagon Geospatial public forum `__ - Community contributed `patches `__ to apply to ECW SDK 3.3 sources -- `GDAL ECW Build Hints `__ diff --git a/doc/source/drivers/raster/gtiff.rst b/doc/source/drivers/raster/gtiff.rst index 0381a5b8f0e0..0ba9c0de89bc 100644 --- a/doc/source/drivers/raster/gtiff.rst +++ b/doc/source/drivers/raster/gtiff.rst @@ -384,9 +384,6 @@ The TIFF format only supports R,G,B components for palettes / color tables. Thus on writing the alpha information will be silently discarded. -You may want to read hints to `generate and read cloud optimized GeoTIFF -files `__ - Creation Options ~~~~~~~~~~~~~~~~ diff --git a/doc/source/drivers/raster/jp2ecw.rst b/doc/source/drivers/raster/jp2ecw.rst index fe1df5c8d7d1..3a4c59172e88 100644 --- a/doc/source/drivers/raster/jp2ecw.rst +++ b/doc/source/drivers/raster/jp2ecw.rst @@ -379,4 +379,3 @@ See Also - Support for non-GDAL specific issues should be directed to the `Hexagon Geospatial public forum `__ -- `GDAL ECW Build Hints `__ diff --git a/doc/source/drivers/raster/msg.rst b/doc/source/drivers/raster/msg.rst index 2ad9c9d03740..357c14a4c4e1 100644 --- a/doc/source/drivers/raster/msg.rst +++ b/doc/source/drivers/raster/msg.rst @@ -35,37 +35,8 @@ Driver capabilities Build Instructions ------------------ -CMake builds -++++++++++++ - See the ``GDAL_USE_PUBLICDECOMPWT`` option of :ref:`building_from_source`. -Other build systems -+++++++++++++++++++ - -Clone the EUMETSAT library for wavelet decompression into ``frmts/msg``. - -If you are building with Visual Studio 6.0, extract the .vc makefiles -for the PublicDecompWT from the file `PublicDecompWTMakefiles.zip` -stored in that directory. - -If you build using the GNUMakefile, use *--with-msg* option to enable -MSG driver: - -:: - - ./configure --with-msg - -If you find that some adjustments are needed in the makefile and/or the msg -source files, please "commit" them. The EUMETSAT library promises to be -"platform independent", but as we are working with Microsoft Windows and -Visual Studio 6.0, we did not have the facilities to check if the rest -of the msg driver is. Furthermore, apply steps 4 to 7 from the :ref:`raster_driver_tut`, section "Adding -Driver to GDAL Tree". - -MSG Wiki page is available at http://trac.osgeo.org/gdal/wiki/MSG. It's -dedicated to document building and usage hints - Specification of Source Dataset ------------------------------- diff --git a/doc/source/drivers/raster/netcdf.rst b/doc/source/drivers/raster/netcdf.rst index 78e95193eb70..030f82381d60 100644 --- a/doc/source/drivers/raster/netcdf.rst +++ b/doc/source/drivers/raster/netcdf.rst @@ -750,9 +750,6 @@ This driver is compiled with the UNIDATA NetCDF library. You need to download or compile the NetCDF library before configuring GDAL with NetCDF support. -See `NetCDF GDAL wiki `__ for -build instructions and information regarding HDF4, NetCDF-4 and HDF5. - See Also: --------- diff --git a/doc/source/drivers/raster/pds4.rst b/doc/source/drivers/raster/pds4.rst index d584853f04a8..0dd5add475e9 100644 --- a/doc/source/drivers/raster/pds4.rst +++ b/doc/source/drivers/raster/pds4.rst @@ -194,10 +194,7 @@ The following dataset creation options are available: and not creating from an existing PDS4 file, the data/pds4_template.xml file will be used. For GDAL utilities to find this default PDS4 template, GDAL's data directory should be - defined in your environment (typically on Windows builds). Consult - the - `wiki `__ - for more information. + defined in your environment (typically on Windows builds). - .. co:: LATITUDE_TYPE :choices: Planetocentric, Planetographic @@ -537,12 +534,6 @@ Converting a shapefile to a PDS4 dataset with a CSV-delimited table $ ogr2ogr my_out_pds4.xml in.shp -Limitations ------------ - -As a new driver and new format, please report any issues to the bug -tracker, as explained on the `wiki `__ - See Also: --------- diff --git a/doc/source/drivers/raster/postgisraster.rst b/doc/source/drivers/raster/postgisraster.rst index 3927e23e5738..55266b389f32 100644 --- a/doc/source/drivers/raster/postgisraster.rst +++ b/doc/source/drivers/raster/postgisraster.rst @@ -130,6 +130,6 @@ See Also -------- - `GDAL PostGISRaster driver - Wiki `__ + Wiki `__ - `PostGIS Raster documentation `__ diff --git a/doc/source/drivers/vector/pgeo.rst b/doc/source/drivers/vector/pgeo.rst index 2176296c7f42..f363b2feaead 100644 --- a/doc/source/drivers/vector/pgeo.rst +++ b/doc/source/drivers/vector/pgeo.rst @@ -104,8 +104,7 @@ How to use PGeo driver with unixODBC and MDB Tools (on Unix and Linux) This article gives step-by-step explanation of how to use OGR with unixODBC package and how to access Personal Geodatabase with PGeo -driver. See also `GDAL wiki for other -details `__ +driver. Prerequisites ~~~~~~~~~~~~~ diff --git a/doc/source/drivers/vector/shapefile.rst b/doc/source/drivers/vector/shapefile.rst index 67cd3abee67f..c978c611612b 100644 --- a/doc/source/drivers/vector/shapefile.rst +++ b/doc/source/drivers/vector/shapefile.rst @@ -166,9 +166,7 @@ terminated with an error. Note that this can make it very difficult to translate a mixed geometry layer from another format into Shapefile format using ogr2ogr, since ogr2ogr has no support for separating out geometries from a source -layer. See the -`FAQ `__ -for a solution. +layer. Shapefile feature attributes are stored in an associated .dbf file, and so attributes suffer a number of limitations: @@ -484,5 +482,3 @@ See Also -------- - `Shapelib Page `__ -- `User Notes on OGR Shapefile - Driver `__ diff --git a/doc/source/faq.rst b/doc/source/faq.rst index d49c2f00bf77..d3025583cd26 100644 --- a/doc/source/faq.rst +++ b/doc/source/faq.rst @@ -4,7 +4,7 @@ FAQ ================================================================================ -.. TODO maybe migrate the chapters 2 and following of https://trac.osgeo.org/gdal/wiki/FAQ +.. TODO maybe migrate the chapters 2 and following of http://web.archive.org/web/https://trac.osgeo.org/gdal/wiki/FAQ .. only:: not latex From be866a575fc667f77eb0e7b6dc267ffa21ffce33 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 18:14:17 +0200 Subject: [PATCH 211/710] autotest: skip nwt_grd/nwt_grc when drivers not there --- autotest/gdrivers/nwt_grc.py | 3 +++ autotest/gdrivers/nwt_grd.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/autotest/gdrivers/nwt_grc.py b/autotest/gdrivers/nwt_grc.py index 36d2e1013274..fcfb7c87e1c4 100755 --- a/autotest/gdrivers/nwt_grc.py +++ b/autotest/gdrivers/nwt_grc.py @@ -30,6 +30,9 @@ import gdaltest +import pytest + +pytestmark = pytest.mark.require_driver("NWT_GRC") ############################################################################### # Test a GRC dataset diff --git a/autotest/gdrivers/nwt_grd.py b/autotest/gdrivers/nwt_grd.py index 04d8ca300946..abf91af2e220 100755 --- a/autotest/gdrivers/nwt_grd.py +++ b/autotest/gdrivers/nwt_grd.py @@ -34,6 +34,8 @@ from osgeo import gdal +pytestmark = pytest.mark.require_driver("NWT_GRD") + ############################################################################### # Test a GRD dataset with three bands + Z From b7830458dbabf9d3ba483781c3e4705c049f9e43 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 16:40:46 +0200 Subject: [PATCH 212/710] Allow Northwoord drivers to be built if TAB driver is disabled (but without write support for NWT_GRD) --- autotest/gdrivers/nwt_grd.py | 8 + autotest/osr/osr_micoordsys.py | 2 - frmts/CMakeLists.txt | 4 +- frmts/northwood/CMakeLists.txt | 4 + frmts/northwood/grcdataset.cpp | 7 +- frmts/northwood/grddataset.cpp | 28 +- ogr/CMakeLists.txt | 6 +- ogr/{ogrsf_frmts/mitab => }/mitab_bounds.cpp | 13 +- ..._spatialref.cpp => ogrmitabspatialref.cpp} | 586 +++++++++++++----- ogr/ogrmitabspatialref.h | 75 +++ ogr/ogrsf_frmts/mitab/CMakeLists.txt | 3 - ogr/ogrsf_frmts/mitab/mitab.h | 56 -- ogr/ogrsf_frmts/mitab/mitab_coordsys.cpp | 389 ------------ ogr/ogrsf_frmts/mitab/mitab_priv.h | 29 +- ogr/ogrsf_frmts/mitab/mitab_tabfile.cpp | 107 ++++ ogr/ogrsf_frmts/mitab/mitab_utils.cpp | 62 -- ogr/ogrsf_frmts/mitab/mitab_utils.h | 3 - ogr/ogrspatialreference.cpp | 26 +- 18 files changed, 672 insertions(+), 736 deletions(-) rename ogr/{ogrsf_frmts/mitab => }/mitab_bounds.cpp (99%) rename ogr/{ogrsf_frmts/mitab/mitab_spatialref.cpp => ogrmitabspatialref.cpp} (88%) create mode 100644 ogr/ogrmitabspatialref.h delete mode 100644 ogr/ogrsf_frmts/mitab/mitab_coordsys.cpp diff --git a/autotest/gdrivers/nwt_grd.py b/autotest/gdrivers/nwt_grd.py index abf91af2e220..21ec5df5fe5d 100755 --- a/autotest/gdrivers/nwt_grd.py +++ b/autotest/gdrivers/nwt_grd.py @@ -31,6 +31,7 @@ import shutil import gdaltest +import pytest from osgeo import gdal @@ -53,6 +54,13 @@ def test_nwt_grd_1(): def test_nwt_grd_2(): + + if ( + gdal.GetDriverByName("NWT_GRD").GetMetadataItem(gdal.DMD_CREATIONDATATYPES) + is None + ): + pytest.skip("NWT_GRD driver has no write support due to missing MITAB driver") + """ Test writing a GRD via CreateCopy """ diff --git a/autotest/osr/osr_micoordsys.py b/autotest/osr/osr_micoordsys.py index 6352b3baf19e..9e18c2922b43 100755 --- a/autotest/osr/osr_micoordsys.py +++ b/autotest/osr/osr_micoordsys.py @@ -33,8 +33,6 @@ from osgeo import osr -pytestmark = pytest.mark.require_driver("MapInfo File") - ############################################################################### # Test the osr.SpatialReference.ImportFromMICoordSys() function. # diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index e824bf1c783b..875eaade5385 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -85,9 +85,7 @@ gdal_optional_format(blx "Magellan BLX Topo File Format") gdal_optional_format(msgn "Meteosat Second Generation (MSG) Native Archive Format (.nat)") gdal_optional_format(til "EarthWatch .TIL Driver") gdal_optional_format(r "R Object Data Store") -if (NOT OGR_ENABLE_DRIVER_TAB_PLUGIN) - gdal_dependent_format(northwood "NWT_GRD/NWT_GRC -- Northwood/Vertical Mapper File Format" "OGR_ENABLE_DRIVER_TAB") -endif() +gdal_optional_format(northwood "NWT_GRD/NWT_GRC -- Northwood/Vertical Mapper File Format") gdal_optional_format(saga "SAGA GIS Binary Driver") gdal_optional_format(xyz "ASCII Gridded XYZ") include(avif/driver_declaration.cmake) diff --git a/frmts/northwood/CMakeLists.txt b/frmts/northwood/CMakeLists.txt index 5d89d7d9d6c7..1e5e23fc6d3c 100644 --- a/frmts/northwood/CMakeLists.txt +++ b/frmts/northwood/CMakeLists.txt @@ -5,3 +5,7 @@ add_gdal_driver( SOURCES grddataset.cpp grcdataset.cpp northwood.cpp northwood.h BUILTIN STRONG_CXX_WFLAGS) gdal_standard_includes(gdal_northwood) + +if (NOT OGR_ENABLE_DRIVER_TAB OR OGR_ENABLE_DRIVER_TAB_PLUGIN) + target_compile_definitions(gdal_northwood PRIVATE -DNO_MITAB_SUPPORT) +endif() diff --git a/frmts/northwood/grcdataset.cpp b/frmts/northwood/grcdataset.cpp index 27b0008da439..149fb1c235bc 100644 --- a/frmts/northwood/grcdataset.cpp +++ b/frmts/northwood/grcdataset.cpp @@ -30,12 +30,7 @@ #include "gdal_frmts.h" #include "gdal_pam.h" #include "northwood.h" - -#ifdef MSVC -#include "..\..\ogr\ogrsf_frmts\mitab\mitab.h" -#else -#include "../../ogr/ogrsf_frmts/mitab/mitab.h" -#endif +#include "ogrmitabspatialref.h" /************************************************************************/ /* ==================================================================== */ diff --git a/frmts/northwood/grddataset.cpp b/frmts/northwood/grddataset.cpp index 44249cb174f1..8a3645205210 100644 --- a/frmts/northwood/grddataset.cpp +++ b/frmts/northwood/grddataset.cpp @@ -34,12 +34,15 @@ #include "gdal_frmts.h" #include "gdal_pam.h" #include "northwood.h" +#include "ogrmitabspatialref.h" +#ifndef NO_MITAB_SUPPORT #ifdef MSVC #include "..\..\ogr\ogrsf_frmts\mitab\mitab.h" #else #include "../../ogr/ogrsf_frmts/mitab/mitab.h" #endif +#endif constexpr float NODATA = -1.e37f; constexpr double SCALE16BIT = 65534.0; @@ -79,9 +82,11 @@ class NWT_GRDDataset final : public GDALPamDataset bool bUpdateHeader; mutable OGRSpatialReference *m_poSRS = nullptr; +#ifndef NO_MITAB_SUPPORT // Update the header data with latest changes int UpdateHeader(); int WriteTab(); +#endif NWT_GRDDataset(const NWT_GRDDataset &) = delete; NWT_GRDDataset &operator=(const NWT_GRDDataset &) = delete; @@ -92,6 +97,8 @@ class NWT_GRDDataset final : public GDALPamDataset static GDALDataset *Open(GDALOpenInfo *); static int Identify(GDALOpenInfo *); + +#ifndef NO_MITAB_SUPPORT static GDALDataset *Create(const char *pszFilename, int nXSize, int nYSize, int nBandsIn, GDALDataType eType, char **papszParamList); @@ -100,13 +107,17 @@ class NWT_GRDDataset final : public GDALPamDataset char **papszOptions, GDALProgressFunc pfnProgress, void *pProgressData); +#endif CPLErr GetGeoTransform(double *padfTransform) override; CPLErr SetGeoTransform(double *padfTransform) override; CPLErr FlushCache(bool bAtClosing) override; const OGRSpatialReference *GetSpatialRef() const override; + +#ifndef NO_MITAB_SUPPORT CPLErr SetSpatialRef(const OGRSpatialReference *poSRS) override; +#endif }; /************************************************************************/ @@ -468,7 +479,9 @@ CPLErr NWT_GRDDataset::FlushCache(bool bAtClosing) // Ensure the header and TAB file are up to date if (bUpdateHeader) { +#ifndef NO_MITAB_SUPPORT UpdateHeader(); +#endif } // Call the parent method @@ -540,9 +553,11 @@ const OGRSpatialReference *NWT_GRDDataset::GetSpatialRef() const OGRSpatialReference *poSpatialRef = MITABCoordSys2SpatialRef(pGrd->cMICoordSys); m_poSRS = poSpatialRef; + return m_poSRS; } +#ifndef NO_MITAB_SUPPORT /************************************************************************/ /* SetSpatialRef() */ /************************************************************************/ @@ -563,6 +578,7 @@ CPLErr NWT_GRDDataset::SetSpatialRef(const OGRSpatialReference *poSRS) return CE_None; } +#endif /************************************************************************/ /* Identify() */ @@ -671,6 +687,7 @@ GDALDataset *NWT_GRDDataset::Open(GDALOpenInfo *poOpenInfo) return poDS; } +#ifndef NO_MITAB_SUPPORT /************************************************************************/ /* UpdateHeader() */ /************************************************************************/ @@ -1124,6 +1141,7 @@ GDALDataset *NWT_GRDDataset::CreateCopy(const char *pszFilename, return poDstDS; } +#endif // NO_MITAB_SUPPORT /************************************************************************/ /* GDALRegister_GRD() */ @@ -1142,7 +1160,6 @@ void GDALRegister_NWT_GRD() poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC, "drivers/raster/nwtgrd.html"); poDriver->SetMetadataItem(GDAL_DMD_EXTENSION, "grd"); poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES"); - poDriver->SetMetadataItem(GDAL_DMD_CREATIONDATATYPES, "Float32"); poDriver->SetMetadataItem( GDAL_DMD_OPENOPTIONLIST, @@ -1151,6 +1168,12 @@ void GDALRegister_NWT_GRD() "(RGBZ). Only used in read-only mode' default='4'/>" ""); + poDriver->pfnOpen = NWT_GRDDataset::Open; + poDriver->pfnIdentify = NWT_GRDDataset::Identify; + +#ifndef NO_MITAB_SUPPORT + + poDriver->SetMetadataItem(GDAL_DMD_CREATIONDATATYPES, "Float32"); poDriver->SetMetadataItem( GDAL_DMD_CREATIONOPTIONLIST, "" @@ -1172,10 +1195,9 @@ void GDALRegister_NWT_GRD() "MapInfo' default='0'/>" ""); - poDriver->pfnOpen = NWT_GRDDataset::Open; - poDriver->pfnIdentify = NWT_GRDDataset::Identify; poDriver->pfnCreate = NWT_GRDDataset::Create; poDriver->pfnCreateCopy = NWT_GRDDataset::CreateCopy; +#endif GetGDALDriverManager()->RegisterDriver(poDriver); } diff --git a/ogr/CMakeLists.txt b/ogr/CMakeLists.txt index 18ce7f1b9ed3..6768428fb569 100644 --- a/ogr/CMakeLists.txt +++ b/ogr/CMakeLists.txt @@ -68,6 +68,8 @@ add_library( ogrgeojsongeometry.cpp ogrgeojsonwriter.cpp ogresrijsongeometry.cpp + ogrmitabspatialref.cpp + mitab_bounds.cpp ) add_dependencies(ogr generate_gdal_version_h) @@ -95,10 +97,6 @@ target_include_directories(ogr PRIVATE $) set_property(TARGET ogr PROPERTY POSITION_INDEPENDENT_CODE ${GDAL_OBJECT_LIBRARIES_POSITION_INDEPENDENT_CODE}) target_sources(${GDAL_LIB_TARGET_NAME} PRIVATE $) -if (OGR_ENABLE_DRIVER_TAB AND NOT OGR_ENABLE_DRIVER_TAB_PLUGIN) - target_compile_definitions(ogr PRIVATE -DHAVE_MITAB) -endif() - if (GDAL_USE_ZLIB_INTERNAL) gdal_add_vendored_lib(ogr libz) endif () diff --git a/ogr/ogrsf_frmts/mitab/mitab_bounds.cpp b/ogr/mitab_bounds.cpp similarity index 99% rename from ogr/ogrsf_frmts/mitab/mitab_bounds.cpp rename to ogr/mitab_bounds.cpp index 5101f04fc6da..2b13ff2003fb 100644 --- a/ogr/ogrsf_frmts/mitab/mitab_bounds.cpp +++ b/ogr/mitab_bounds.cpp @@ -29,8 +29,7 @@ **********************************************************************/ #include "cpl_port.h" -#include "mitab.h" -#include "mitab_priv.h" +#include "ogrmitabspatialref.h" #include #include @@ -20553,13 +20552,3 @@ void MITABFreeCoordSysTable() gpasExtBoundsList = nullptr; nExtBoundsListCount = -1; } - -/********************************************************************** - * MITABCoordSysTableLoaded() - * - * Returns TRUE if a coordsys table was loaded, FALSE otherwise. - **********************************************************************/ -bool MITABCoordSysTableLoaded() -{ - return nExtBoundsListCount >= 0; -} diff --git a/ogr/ogrsf_frmts/mitab/mitab_spatialref.cpp b/ogr/ogrmitabspatialref.cpp similarity index 88% rename from ogr/ogrsf_frmts/mitab/mitab_spatialref.cpp rename to ogr/ogrmitabspatialref.cpp index 2109ebd73705..2c544a173cd2 100644 --- a/ogr/ogrsf_frmts/mitab/mitab_spatialref.cpp +++ b/ogr/ogrmitabspatialref.cpp @@ -1,36 +1,14 @@ -/********************************************************************** - * - * Name: mitab_spatialref.cpp - * Project: MapInfo TAB Read/Write library - * Language: C++ - * Purpose: Implementation of the SpatialRef stuff in the TABFile class. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ********************************************************************** - * Copyright (c) 1999-2001, Frank Warmerdam - * Copyright (c) 2014, Even Rouault - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - **********************************************************************/ +// SPDX-License-Identifier: MIT +// Copyright 1999-2003, Daniel Morissette +// Copyright (c) 1999-2001, Frank Warmerdam +// Implementation translation between MIF CoordSys format, and +// and OGRSpatialRef format. + +/*! @cond Doxygen_Suppress */ + +#include "ogrmitabspatialref.h" #include "cpl_port.h" -#include "mitab.h" #include #include @@ -41,10 +19,95 @@ #include "cpl_conv.h" #include "cpl_error.h" #include "cpl_string.h" -#include "mitab_priv.h" #include "ogr_spatialref.h" #include "ogr_srs_api.h" +typedef struct +{ + int nDatumEPSGCode; + int nMapInfoDatumID; + const char *pszOGCDatumName; + int nEllipsoid; + double dfShiftX; + double dfShiftY; + double dfShiftZ; + double dfDatumParm0; /* RotX */ + double dfDatumParm1; /* RotY */ + double dfDatumParm2; /* RotZ */ + double dfDatumParm3; /* Scale Factor */ + double dfDatumParm4; /* Prime Meridian */ +} MapInfoDatumInfo; + +typedef struct +{ + int nMapInfoId; + const char *pszMapinfoName; + double dfA; /* semi major axis in meters */ + double dfInvFlattening; /* Inverse flattening */ +} MapInfoSpheroidInfo; + +/********************************************************************** + * MapInfo Units string to numeric ID conversion + **********************************************************************/ +typedef struct +{ + int nUnitId; + const char *pszAbbrev; +} MapInfoUnitsInfo; + +static const MapInfoUnitsInfo gasUnitsList[] = { + {0, "mi"}, {1, "km"}, {2, "in"}, {3, "ft"}, + {4, "yd"}, {5, "mm"}, {6, "cm"}, {7, "m"}, + {8, "survey ft"}, {8, "survey foot"}, // alternate + {13, nullptr}, {9, "nmi"}, {30, "li"}, {31, "ch"}, + {32, "rd"}, {-1, nullptr}}; + +/********************************************************************** + * TABUnitIdToString() + * + * Return the MIF units name for specified units id. + * Return "" if no match found. + * + * The returned string should not be freed by the caller. + **********************************************************************/ +static const char *TABUnitIdToString(int nId) +{ + const MapInfoUnitsInfo *psList = gasUnitsList; + + while (psList->nUnitId != -1) + { + if (psList->nUnitId == nId) + return psList->pszAbbrev; + psList++; + } + + return ""; +} + +/********************************************************************** + * TABUnitIdFromString() + * + * Return the units ID for specified MIF units name + * + * Returns -1 if no match found. + **********************************************************************/ +static int TABUnitIdFromString(const char *pszName) +{ + if (pszName == nullptr) + return 13; + + const MapInfoUnitsInfo *psList = gasUnitsList; + + while (psList->nUnitId != -1) + { + if (psList->pszAbbrev != nullptr && EQUAL(psList->pszAbbrev, pszName)) + return psList->nUnitId; + psList++; + } + + return -1; +} + /* -------------------------------------------------------------------- */ /* This table was automatically generated by doing translations */ /* between mif and tab for each datum, and extracting the */ @@ -691,54 +754,6 @@ static const MapInfoLCCSRS asMapInfoLCCSRSList[] = { {0, 0, 74, -98.5, 36.6666666667, 38.5666666667, 37.2666666667}, }; -/********************************************************************** - * TABFile::GetSpatialRef() - * - * Returns a reference to an OGRSpatialReference for this dataset. - * If the projection parameters have not been parsed yet, then we will - * parse them before returning. - * - * The returned object is owned and maintained by this TABFile and - * should not be modified or freed by the caller. - * - * Returns NULL if the SpatialRef cannot be accessed. - **********************************************************************/ -OGRSpatialReference *TABFile::GetSpatialRef() -{ - if (m_poMAPFile == nullptr) - { - CPLError(CE_Failure, CPLE_AssertionFailed, - "GetSpatialRef() failed: file has not been opened yet."); - return nullptr; - } - - if (GetGeomType() == wkbNone) - return nullptr; - - /*----------------------------------------------------------------- - * If projection params have already been processed, just use them. - *----------------------------------------------------------------*/ - if (m_poSpatialRef != nullptr) - return m_poSpatialRef; - - /*----------------------------------------------------------------- - * Fetch the parameters from the header. - *----------------------------------------------------------------*/ - TABProjInfo sTABProj; - - TABMAPHeaderBlock *poHeader = nullptr; - if ((poHeader = m_poMAPFile->GetHeaderBlock()) == nullptr || - poHeader->GetProjInfo(&sTABProj) != 0) - { - CPLError(CE_Failure, CPLE_FileIO, - "GetSpatialRef() failed reading projection parameters."); - return nullptr; - } - - m_poSpatialRef = GetSpatialRefFromTABProj(sTABProj); - return m_poSpatialRef; -} - /********************************************************************** * TABFile::GetSpatialRefFromTABProj() **********************************************************************/ @@ -750,7 +765,7 @@ static bool TAB_EQUAL(double a, double b) } OGRSpatialReference * -TABFile::GetSpatialRefFromTABProj(const TABProjInfo &sTABProj) +TABFileGetSpatialRefFromTABProj(const TABProjInfo &sTABProj) { /*----------------------------------------------------------------- * Get the units name, and translation factor. @@ -1432,65 +1447,6 @@ TABFile::GetSpatialRefFromTABProj(const TABProjInfo &sTABProj) return poSpatialRef; } -/********************************************************************** - * TABFile::SetSpatialRef() - * - * Set the OGRSpatialReference for this dataset. - * A reference to the OGRSpatialReference will be kept, and it will also - * be converted into a TABProjInfo to be stored in the .MAP header. - * - * Returns 0 on success, and -1 on error. - **********************************************************************/ -int TABFile::SetSpatialRef(OGRSpatialReference *poSpatialRef) -{ - if (m_eAccessMode != TABWrite) - { - CPLError(CE_Failure, CPLE_NotSupported, - "SetSpatialRef() can be used only with Write access."); - return -1; - } - - if (m_poMAPFile == nullptr) - { - CPLError(CE_Failure, CPLE_AssertionFailed, - "SetSpatialRef() failed: file has not been opened yet."); - return -1; - } - - if (poSpatialRef == nullptr) - { - CPLError(CE_Failure, CPLE_AssertionFailed, - "SetSpatialRef() failed: Called with NULL poSpatialRef."); - return -1; - } - - /*----------------------------------------------------------------- - * Keep a copy of the OGRSpatialReference... - * Note: we have to take the reference count into account... - *----------------------------------------------------------------*/ - if (m_poSpatialRef && m_poSpatialRef->Dereference() == 0) - delete m_poSpatialRef; - - m_poSpatialRef = poSpatialRef->Clone(); - - TABProjInfo sTABProj; - int nParamCount = 0; - GetTABProjFromSpatialRef(poSpatialRef, sTABProj, nParamCount); - - /*----------------------------------------------------------------- - * Set the new parameters in the .MAP header. - * This will also trigger lookup of default bounds for the projection. - *----------------------------------------------------------------*/ - if (SetProjInfo(&sTABProj) != 0) - { - CPLError(CE_Failure, CPLE_FileIO, - "SetSpatialRef() failed setting projection parameters."); - return -1; - } - - return 0; -} - static int MITABGetCustomDatum(const OGRSpatialReference *poSpatialRef, TABProjInfo &sTABProj) { @@ -1549,8 +1505,8 @@ static int MITABGetCustomDatum(const OGRSpatialReference *poSpatialRef, return TRUE; } -int TABFile::GetTABProjFromSpatialRef(const OGRSpatialReference *poSpatialRef, - TABProjInfo &sTABProj, int &nParamCount) +int TABFileGetTABProjFromSpatialRef(const OGRSpatialReference *poSpatialRef, + TABProjInfo &sTABProj, int &nParamCount) { /*----------------------------------------------------------------- * Initialize TABProjInfo @@ -2318,3 +2274,349 @@ int TABFile::GetTABProjFromSpatialRef(const OGRSpatialReference *poSpatialRef, return 0; } + +extern const MapInfoDatumInfo asDatumInfoList[]; +extern const MapInfoSpheroidInfo asSpheroidInfoList[]; + +/************************************************************************/ +/* MITABCoordSys2SpatialRef() */ +/* */ +/* Convert a MIF COORDSYS string into a new OGRSpatialReference */ +/* object. */ +/************************************************************************/ + +OGRSpatialReference *MITABCoordSys2SpatialRef(const char *pszCoordSys) + +{ + TABProjInfo sTABProj; + if (MITABCoordSys2TABProjInfo(pszCoordSys, &sTABProj) < 0) + return nullptr; + OGRSpatialReference *poSR = TABFileGetSpatialRefFromTABProj(sTABProj); + + // Report on translation. + char *pszWKT = nullptr; + + poSR->exportToWkt(&pszWKT); + if (pszWKT != nullptr) + { + CPLDebug("MITAB", "This CoordSys value:\n%s\nwas translated to:\n%s", + pszCoordSys, pszWKT); + CPLFree(pszWKT); + } + + return poSR; +} + +/************************************************************************/ +/* MITABSpatialRef2CoordSys() */ +/* */ +/* Converts a OGRSpatialReference object into a MIF COORDSYS */ +/* string. */ +/* */ +/* The function returns a newly allocated string that should be */ +/* CPLFree()'d by the caller. */ +/************************************************************************/ + +char *MITABSpatialRef2CoordSys(const OGRSpatialReference *poSR) + +{ + if (poSR == nullptr) + return nullptr; + + TABProjInfo sTABProj; + int nParamCount = 0; + TABFileGetTABProjFromSpatialRef(poSR, sTABProj, nParamCount); + + // Do coordsys lookup. + double dXMin = 0.0; + double dYMin = 0.0; + double dXMax = 0.0; + double dYMax = 0.0; + bool bHasBounds = false; + if (sTABProj.nProjId > 1 && + MITABLookupCoordSysBounds(&sTABProj, dXMin, dYMin, dXMax, dYMax, true)) + { + bHasBounds = true; + } + + // Translate the units. + const char *pszMIFUnits = TABUnitIdToString(sTABProj.nUnitsId); + + // Build coordinate system definition. + CPLString osCoordSys; + + if (sTABProj.nProjId != 0) + { + osCoordSys.Printf("Earth Projection %d", sTABProj.nProjId); + } + else + { + osCoordSys.Printf("NonEarth Units"); + } + + // Append Datum. + if (sTABProj.nProjId != 0) + { + osCoordSys += CPLSPrintf(", %d", sTABProj.nDatumId); + + if (sTABProj.nDatumId == 999 || sTABProj.nDatumId == 9999) + { + osCoordSys += + CPLSPrintf(", %d, %.15g, %.15g, %.15g", sTABProj.nEllipsoidId, + sTABProj.dDatumShiftX, sTABProj.dDatumShiftY, + sTABProj.dDatumShiftZ); + } + + if (sTABProj.nDatumId == 9999) + { + osCoordSys += + CPLSPrintf(", %.15g, %.15g, %.15g, %.15g, %.15g", + sTABProj.adDatumParams[0], sTABProj.adDatumParams[1], + sTABProj.adDatumParams[2], sTABProj.adDatumParams[3], + sTABProj.adDatumParams[4]); + } + } + + // Append units. + if (sTABProj.nProjId != 1 && pszMIFUnits != nullptr) + { + if (sTABProj.nProjId != 0) + osCoordSys += ","; + + osCoordSys += CPLSPrintf(" \"%s\"", pszMIFUnits); + } + + // Append Projection Params. + for (int iParam = 0; iParam < nParamCount; iParam++) + osCoordSys += CPLSPrintf(", %.15g", sTABProj.adProjParams[iParam]); + + // Append user bounds. + if (bHasBounds) + { + if (fabs(dXMin - floor(dXMin + 0.5)) < 1e-8 && + fabs(dYMin - floor(dYMin + 0.5)) < 1e-8 && + fabs(dXMax - floor(dXMax + 0.5)) < 1e-8 && + fabs(dYMax - floor(dYMax + 0.5)) < 1e-8) + { + osCoordSys += + CPLSPrintf(" Bounds (%d, %d) (%d, %d)", static_cast(dXMin), + static_cast(dYMin), static_cast(dXMax), + static_cast(dYMax)); + } + else + { + osCoordSys += CPLSPrintf(" Bounds (%f, %f) (%f, %f)", dXMin, dYMin, + dXMax, dYMax); + } + } + + // Report on translation. + char *pszWKT = nullptr; + + poSR->exportToWkt(&pszWKT); + if (pszWKT != nullptr) + { + CPLDebug("MITAB", "This WKT Projection:\n%s\n\ntranslates to:\n%s", + pszWKT, osCoordSys.c_str()); + CPLFree(pszWKT); + } + + return CPLStrdup(osCoordSys.c_str()); +} + +/************************************************************************/ +/* MITABExtractCoordSysBounds */ +/* */ +/* Return true if MIF coordsys string contains a BOUNDS parameter and */ +/* Set x/y min/max values. */ +/************************************************************************/ + +bool MITABExtractCoordSysBounds(const char *pszCoordSys, double &dXMin, + double &dYMin, double &dXMax, double &dYMax) + +{ + if (pszCoordSys == nullptr) + return false; + + char **papszFields = + CSLTokenizeStringComplex(pszCoordSys, " ,()", TRUE, FALSE); + + int iBounds = CSLFindString(papszFields, "Bounds"); + + if (iBounds >= 0 && iBounds + 4 < CSLCount(papszFields)) + { + dXMin = CPLAtof(papszFields[++iBounds]); + dYMin = CPLAtof(papszFields[++iBounds]); + dXMax = CPLAtof(papszFields[++iBounds]); + dYMax = CPLAtof(papszFields[++iBounds]); + CSLDestroy(papszFields); + return true; + } + + CSLDestroy(papszFields); + return false; +} + +/********************************************************************** + * MITABCoordSys2TABProjInfo() + * + * Convert a MIF COORDSYS string into a TABProjInfo structure. + * + * Returns 0 on success, -1 on error. + **********************************************************************/ +int MITABCoordSys2TABProjInfo(const char *pszCoordSys, TABProjInfo *psProj) + +{ + // Set all fields to zero, equivalent of NonEarth Units "mi" + memset(psProj, 0, sizeof(TABProjInfo)); + + if (pszCoordSys == nullptr) + return -1; + + // Parse the passed string into words. + while (*pszCoordSys == ' ') + pszCoordSys++; // Eat leading spaces. + if (STARTS_WITH_CI(pszCoordSys, "CoordSys") && pszCoordSys[8] != '\0') + pszCoordSys += 9; + + char **papszFields = + CSLTokenizeStringComplex(pszCoordSys, " ,", TRUE, FALSE); + + // Clip off Bounds information. + int iBounds = CSLFindString(papszFields, "Bounds"); + + while (iBounds != -1 && papszFields[iBounds] != nullptr) + { + CPLFree(papszFields[iBounds]); + papszFields[iBounds] = nullptr; + iBounds++; + } + + // Fetch the projection. + char **papszNextField = nullptr; + + if (CSLCount(papszFields) >= 3 && EQUAL(papszFields[0], "Earth") && + EQUAL(papszFields[1], "Projection")) + { + int nProjId = atoi(papszFields[2]); + if (nProjId >= 3000) + nProjId -= 3000; + else if (nProjId >= 2000) + nProjId -= 2000; + else if (nProjId >= 1000) + nProjId -= 1000; + + psProj->nProjId = static_cast(nProjId); + papszNextField = papszFields + 3; + } + else if (CSLCount(papszFields) >= 2 && EQUAL(papszFields[0], "NonEarth")) + { + // NonEarth Units "..." Bounds (x, y) (x, y) + psProj->nProjId = 0; + papszNextField = papszFields + 2; + + if (papszNextField[0] != nullptr && EQUAL(papszNextField[0], "Units")) + papszNextField++; + } + else + { + // Invalid projection string ??? + if (CSLCount(papszFields) > 0) + CPLError(CE_Warning, CPLE_IllegalArg, + "Failed parsing CoordSys: '%s'", pszCoordSys); + CSLDestroy(papszFields); + return -1; + } + + // Fetch the datum information. + int nDatum = 0; + + if (psProj->nProjId != 0 && CSLCount(papszNextField) > 0) + { + nDatum = atoi(papszNextField[0]); + papszNextField++; + } + + if ((nDatum == 999 || nDatum == 9999) && CSLCount(papszNextField) >= 4) + { + psProj->nEllipsoidId = static_cast(atoi(papszNextField[0])); + psProj->dDatumShiftX = CPLAtof(papszNextField[1]); + psProj->dDatumShiftY = CPLAtof(papszNextField[2]); + psProj->dDatumShiftZ = CPLAtof(papszNextField[3]); + papszNextField += 4; + + if (nDatum == 9999 && CSLCount(papszNextField) >= 5) + { + psProj->adDatumParams[0] = CPLAtof(papszNextField[0]); + psProj->adDatumParams[1] = CPLAtof(papszNextField[1]); + psProj->adDatumParams[2] = CPLAtof(papszNextField[2]); + psProj->adDatumParams[3] = CPLAtof(papszNextField[3]); + psProj->adDatumParams[4] = CPLAtof(papszNextField[4]); + papszNextField += 5; + } + } + else if (nDatum != 999 && nDatum != 9999) + { + // Find the datum, and collect its parameters if possible. + const MapInfoDatumInfo *psDatumInfo = nullptr; + + int iDatum = 0; // Used after for. + for (; asDatumInfoList[iDatum].nMapInfoDatumID != -1; iDatum++) + { + if (asDatumInfoList[iDatum].nMapInfoDatumID == nDatum) + { + psDatumInfo = asDatumInfoList + iDatum; + break; + } + } + + if (asDatumInfoList[iDatum].nMapInfoDatumID == -1) + { + // Use WGS84. + psDatumInfo = asDatumInfoList + 0; + } + + if (psDatumInfo != nullptr) + { + psProj->nEllipsoidId = static_cast(psDatumInfo->nEllipsoid); + psProj->nDatumId = + static_cast(psDatumInfo->nMapInfoDatumID); + psProj->dDatumShiftX = psDatumInfo->dfShiftX; + psProj->dDatumShiftY = psDatumInfo->dfShiftY; + psProj->dDatumShiftZ = psDatumInfo->dfShiftZ; + psProj->adDatumParams[0] = psDatumInfo->dfDatumParm0; + psProj->adDatumParams[1] = psDatumInfo->dfDatumParm1; + psProj->adDatumParams[2] = psDatumInfo->dfDatumParm2; + psProj->adDatumParams[3] = psDatumInfo->dfDatumParm3; + psProj->adDatumParams[4] = psDatumInfo->dfDatumParm4; + } + } + + // Fetch the units string. + if (CSLCount(papszNextField) > 0) + { + if (isdigit(static_cast(papszNextField[0][0]))) + { + psProj->nUnitsId = static_cast(atoi(papszNextField[0])); + } + else + { + psProj->nUnitsId = + static_cast(TABUnitIdFromString(papszNextField[0])); + } + papszNextField++; + } + + // Finally the projection parameters. + for (int iParam = 0; iParam < 7 && CSLCount(papszNextField) > 0; iParam++) + { + psProj->adProjParams[iParam] = CPLAtof(papszNextField[0]); + papszNextField++; + } + + CSLDestroy(papszFields); + + return 0; +} + +/*! @endcond */ diff --git a/ogr/ogrmitabspatialref.h b/ogr/ogrmitabspatialref.h new file mode 100644 index 000000000000..dff205d46174 --- /dev/null +++ b/ogr/ogrmitabspatialref.h @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright 1999-2003, Daniel Morissette +// Copyright (c) 1999-2001, Frank Warmerdam +// Implementation translation between MIF CoordSys format, and +// and OGRSpatialRef format. + +#ifndef OGRMITABSPATIALREF_H_INCLUDED +#define OGRMITABSPATIALREF_H_INCLUDED + +/*! @cond Doxygen_Suppress */ + +#include "cpl_port.h" + +class OGRSpatialReference; + +/*--------------------------------------------------------------------- + * TABProjInfo + * struct used to store the projection parameters from the .MAP header + *--------------------------------------------------------------------*/ +typedef struct TABProjInfo_t +{ + GByte nProjId; // See MapInfo Ref. Manual, App. F and G + GByte nEllipsoidId; + GByte nUnitsId; + double adProjParams[7]; // params in same order as in .MIF COORDSYS + + GInt16 nDatumId; // Datum Id added in MapInfo 7.8+ (.map V500) + double dDatumShiftX; // Before that, we had to always lookup datum + double dDatumShiftY; // parameters to establish datum id + double dDatumShiftZ; + double adDatumParams[5]; + + // Affine parameters only in .map version 500 and up + GByte nAffineFlag; // 0=No affine param, 1=Affine params + GByte nAffineUnits; + double dAffineParamA; // Affine params + double dAffineParamB; + double dAffineParamC; + double dAffineParamD; + double dAffineParamE; + double dAffineParamF; +} TABProjInfo; + +OGRSpatialReference CPL_DLL * +TABFileGetSpatialRefFromTABProj(const TABProjInfo &sTABProj); + +int CPL_DLL +TABFileGetTABProjFromSpatialRef(const OGRSpatialReference *poSpatialRef, + TABProjInfo &sTABProj, int &nParamCount); + +OGRSpatialReference CPL_DLL *MITABCoordSys2SpatialRef(const char *pszCoordSys); + +char CPL_DLL *MITABSpatialRef2CoordSys(const OGRSpatialReference *poSR); + +bool CPL_DLL MITABExtractCoordSysBounds(const char *pszCoordSys, double &dXMin, + double &dYMin, double &dXMax, + double &dYMax); + +int CPL_DLL MITABCoordSys2TABProjInfo(const char *pszCoordSys, + TABProjInfo *psProj); + +/*--------------------------------------------------------------------- + * The following are used for coordsys bounds lookup + *--------------------------------------------------------------------*/ + +bool CPL_DLL MITABLookupCoordSysBounds(TABProjInfo *psCS, double &dXMin, + double &dYMin, double &dXMax, + double &dYMax, + bool bOnlyUserTable = false); +int CPL_DLL MITABLoadCoordSysTable(const char *pszFname); +void CPL_DLL MITABFreeCoordSysTable(); + +/*! @endcond */ + +#endif diff --git a/ogr/ogrsf_frmts/mitab/CMakeLists.txt b/ogr/ogrsf_frmts/mitab/CMakeLists.txt index 49c10553efd1..330c0c823ef0 100644 --- a/ogr/ogrsf_frmts/mitab/CMakeLists.txt +++ b/ogr/ogrsf_frmts/mitab/CMakeLists.txt @@ -19,11 +19,8 @@ add_gdal_driver( mitab_utils.cpp mitab_imapinfofile.cpp mitab_middatafile.cpp - mitab_bounds.cpp mitab_maptoolblock.cpp mitab_tooldef.cpp - mitab_coordsys.cpp - mitab_spatialref.cpp mitab_ogr_driver.cpp mitab_indfile.cpp mitab_tabview.cpp diff --git a/ogr/ogrsf_frmts/mitab/mitab.h b/ogr/ogrsf_frmts/mitab/mitab.h index 5a61a0173006..17007d0fb616 100644 --- a/ogr/ogrsf_frmts/mitab/mitab.h +++ b/ogr/ogrsf_frmts/mitab/mitab.h @@ -345,12 +345,6 @@ class TABFile final : public IMapInfoFile virtual OGRSpatialReference *GetSpatialRef() override; - static OGRSpatialReference * - GetSpatialRefFromTABProj(const TABProjInfo &sTABProj); - static int GetTABProjFromSpatialRef(const OGRSpatialReference *poSpatialRef, - TABProjInfo &sTABProj, - int &nParamCount); - virtual int GetFeatureCountByType(int &numPoints, int &numLines, int &numRegions, int &numTexts, GBool bForce = TRUE) override; @@ -2246,54 +2240,4 @@ class TABDebugFeature final : public TABFeature virtual void DumpMIF(FILE *fpOut = nullptr) override; }; -/* -------------------------------------------------------------------- */ -/* Some stuff related to spatial reference system handling. */ -/* */ -/* In GDAL we make use of the coordsys transformation from */ -/* other places (sometimes even from plugins), so we */ -/* deliberately export these two functions from the DLL. */ -/* -------------------------------------------------------------------- */ - -char CPL_DLL *MITABSpatialRef2CoordSys(const OGRSpatialReference *); -OGRSpatialReference CPL_DLL *MITABCoordSys2SpatialRef(const char *); - -bool MITABExtractCoordSysBounds(const char *pszCoordSys, double &dXMin, - double &dYMin, double &dXMax, double &dYMax); -int MITABCoordSys2TABProjInfo(const char *pszCoordSys, TABProjInfo *psProj); - -typedef struct -{ - int nDatumEPSGCode; - int nMapInfoDatumID; - const char *pszOGCDatumName; - int nEllipsoid; - double dfShiftX; - double dfShiftY; - double dfShiftZ; - double dfDatumParm0; /* RotX */ - double dfDatumParm1; /* RotY */ - double dfDatumParm2; /* RotZ */ - double dfDatumParm3; /* Scale Factor */ - double dfDatumParm4; /* Prime Meridian */ -} MapInfoDatumInfo; - -typedef struct -{ - int nMapInfoId; - const char *pszMapinfoName; - double dfA; /* semi major axis in meters */ - double dfInvFlattening; /* Inverse flattening */ -} MapInfoSpheroidInfo; - -/*--------------------------------------------------------------------- - * The following are used for coordsys bounds lookup - *--------------------------------------------------------------------*/ - -bool MITABLookupCoordSysBounds(TABProjInfo *psCS, double &dXMin, double &dYMin, - double &dXMax, double &dYMax, - bool bOnlyUserTable = false); -int MITABLoadCoordSysTable(const char *pszFname); -void MITABFreeCoordSysTable(); -bool MITABCoordSysTableLoaded(); // TODO(schwehr): Unused? - #endif /* MITAB_H_INCLUDED_ */ diff --git a/ogr/ogrsf_frmts/mitab/mitab_coordsys.cpp b/ogr/ogrsf_frmts/mitab/mitab_coordsys.cpp deleted file mode 100644 index 27d41b79a1ae..000000000000 --- a/ogr/ogrsf_frmts/mitab/mitab_coordsys.cpp +++ /dev/null @@ -1,389 +0,0 @@ -/********************************************************************** - * - * Name: mitab_coordsys.cpp - * Project: MapInfo TAB Read/Write library - * Language: C++ - * Purpose: Implementation translation between MIF CoordSys format, and - * and OGRSpatialRef format. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ********************************************************************** - * Copyright (c) 1999-2001, Frank Warmerdam - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - **********************************************************************/ - -#include "cpl_port.h" -#include "mitab.h" -#include "mitab_utils.h" - -#include -#include -#include -#include - -#include "cpl_conv.h" -#include "cpl_error.h" -#include "cpl_string.h" -#include "mitab_priv.h" -#include "ogr_spatialref.h" - -extern const MapInfoDatumInfo asDatumInfoList[]; -extern const MapInfoSpheroidInfo asSpheroidInfoList[]; - -/************************************************************************/ -/* MITABCoordSys2SpatialRef() */ -/* */ -/* Convert a MIF COORDSYS string into a new OGRSpatialReference */ -/* object. */ -/************************************************************************/ - -OGRSpatialReference *MITABCoordSys2SpatialRef(const char *pszCoordSys) - -{ - TABProjInfo sTABProj; - if (MITABCoordSys2TABProjInfo(pszCoordSys, &sTABProj) < 0) - return nullptr; - OGRSpatialReference *poSR = TABFile::GetSpatialRefFromTABProj(sTABProj); - - // Report on translation. - char *pszWKT = nullptr; - - poSR->exportToWkt(&pszWKT); - if (pszWKT != nullptr) - { - CPLDebug("MITAB", "This CoordSys value:\n%s\nwas translated to:\n%s", - pszCoordSys, pszWKT); - CPLFree(pszWKT); - } - - return poSR; -} - -/************************************************************************/ -/* MITABSpatialRef2CoordSys() */ -/* */ -/* Converts a OGRSpatialReference object into a MIF COORDSYS */ -/* string. */ -/* */ -/* The function returns a newly allocated string that should be */ -/* CPLFree()'d by the caller. */ -/************************************************************************/ - -char *MITABSpatialRef2CoordSys(const OGRSpatialReference *poSR) - -{ - if (poSR == nullptr) - return nullptr; - - TABProjInfo sTABProj; - int nParamCount = 0; - TABFile::GetTABProjFromSpatialRef(poSR, sTABProj, nParamCount); - - // Do coordsys lookup. - double dXMin = 0.0; - double dYMin = 0.0; - double dXMax = 0.0; - double dYMax = 0.0; - bool bHasBounds = false; - if (sTABProj.nProjId > 1 && - MITABLookupCoordSysBounds(&sTABProj, dXMin, dYMin, dXMax, dYMax, true)) - { - bHasBounds = true; - } - - // Translate the units. - const char *pszMIFUnits = TABUnitIdToString(sTABProj.nUnitsId); - - // Build coordinate system definition. - CPLString osCoordSys; - - if (sTABProj.nProjId != 0) - { - osCoordSys.Printf("Earth Projection %d", sTABProj.nProjId); - } - else - { - osCoordSys.Printf("NonEarth Units"); - } - - // Append Datum. - if (sTABProj.nProjId != 0) - { - osCoordSys += CPLSPrintf(", %d", sTABProj.nDatumId); - - if (sTABProj.nDatumId == 999 || sTABProj.nDatumId == 9999) - { - osCoordSys += - CPLSPrintf(", %d, %.15g, %.15g, %.15g", sTABProj.nEllipsoidId, - sTABProj.dDatumShiftX, sTABProj.dDatumShiftY, - sTABProj.dDatumShiftZ); - } - - if (sTABProj.nDatumId == 9999) - { - osCoordSys += - CPLSPrintf(", %.15g, %.15g, %.15g, %.15g, %.15g", - sTABProj.adDatumParams[0], sTABProj.adDatumParams[1], - sTABProj.adDatumParams[2], sTABProj.adDatumParams[3], - sTABProj.adDatumParams[4]); - } - } - - // Append units. - if (sTABProj.nProjId != 1 && pszMIFUnits != nullptr) - { - if (sTABProj.nProjId != 0) - osCoordSys += ","; - - osCoordSys += CPLSPrintf(" \"%s\"", pszMIFUnits); - } - - // Append Projection Params. - for (int iParam = 0; iParam < nParamCount; iParam++) - osCoordSys += CPLSPrintf(", %.15g", sTABProj.adProjParams[iParam]); - - // Append user bounds. - if (bHasBounds) - { - if (fabs(dXMin - floor(dXMin + 0.5)) < 1e-8 && - fabs(dYMin - floor(dYMin + 0.5)) < 1e-8 && - fabs(dXMax - floor(dXMax + 0.5)) < 1e-8 && - fabs(dYMax - floor(dYMax + 0.5)) < 1e-8) - { - osCoordSys += - CPLSPrintf(" Bounds (%d, %d) (%d, %d)", static_cast(dXMin), - static_cast(dYMin), static_cast(dXMax), - static_cast(dYMax)); - } - else - { - osCoordSys += CPLSPrintf(" Bounds (%f, %f) (%f, %f)", dXMin, dYMin, - dXMax, dYMax); - } - } - - // Report on translation. - char *pszWKT = nullptr; - - poSR->exportToWkt(&pszWKT); - if (pszWKT != nullptr) - { - CPLDebug("MITAB", "This WKT Projection:\n%s\n\ntranslates to:\n%s", - pszWKT, osCoordSys.c_str()); - CPLFree(pszWKT); - } - - return CPLStrdup(osCoordSys.c_str()); -} - -/************************************************************************/ -/* MITABExtractCoordSysBounds */ -/* */ -/* Return true if MIF coordsys string contains a BOUNDS parameter and */ -/* Set x/y min/max values. */ -/************************************************************************/ - -bool MITABExtractCoordSysBounds(const char *pszCoordSys, double &dXMin, - double &dYMin, double &dXMax, double &dYMax) - -{ - if (pszCoordSys == nullptr) - return false; - - char **papszFields = - CSLTokenizeStringComplex(pszCoordSys, " ,()", TRUE, FALSE); - - int iBounds = CSLFindString(papszFields, "Bounds"); - - if (iBounds >= 0 && iBounds + 4 < CSLCount(papszFields)) - { - dXMin = CPLAtof(papszFields[++iBounds]); - dYMin = CPLAtof(papszFields[++iBounds]); - dXMax = CPLAtof(papszFields[++iBounds]); - dYMax = CPLAtof(papszFields[++iBounds]); - CSLDestroy(papszFields); - return true; - } - - CSLDestroy(papszFields); - return false; -} - -/********************************************************************** - * MITABCoordSys2TABProjInfo() - * - * Convert a MIF COORDSYS string into a TABProjInfo structure. - * - * Returns 0 on success, -1 on error. - **********************************************************************/ -int MITABCoordSys2TABProjInfo(const char *pszCoordSys, TABProjInfo *psProj) - -{ - // Set all fields to zero, equivalent of NonEarth Units "mi" - memset(psProj, 0, sizeof(TABProjInfo)); - - if (pszCoordSys == nullptr) - return -1; - - // Parse the passed string into words. - while (*pszCoordSys == ' ') - pszCoordSys++; // Eat leading spaces. - if (STARTS_WITH_CI(pszCoordSys, "CoordSys") && pszCoordSys[8] != '\0') - pszCoordSys += 9; - - char **papszFields = - CSLTokenizeStringComplex(pszCoordSys, " ,", TRUE, FALSE); - - // Clip off Bounds information. - int iBounds = CSLFindString(papszFields, "Bounds"); - - while (iBounds != -1 && papszFields[iBounds] != nullptr) - { - CPLFree(papszFields[iBounds]); - papszFields[iBounds] = nullptr; - iBounds++; - } - - // Fetch the projection. - char **papszNextField = nullptr; - - if (CSLCount(papszFields) >= 3 && EQUAL(papszFields[0], "Earth") && - EQUAL(papszFields[1], "Projection")) - { - int nProjId = atoi(papszFields[2]); - if (nProjId >= 3000) - nProjId -= 3000; - else if (nProjId >= 2000) - nProjId -= 2000; - else if (nProjId >= 1000) - nProjId -= 1000; - - psProj->nProjId = static_cast(nProjId); - papszNextField = papszFields + 3; - } - else if (CSLCount(papszFields) >= 2 && EQUAL(papszFields[0], "NonEarth")) - { - // NonEarth Units "..." Bounds (x, y) (x, y) - psProj->nProjId = 0; - papszNextField = papszFields + 2; - - if (papszNextField[0] != nullptr && EQUAL(papszNextField[0], "Units")) - papszNextField++; - } - else - { - // Invalid projection string ??? - if (CSLCount(papszFields) > 0) - CPLError(CE_Warning, CPLE_IllegalArg, - "Failed parsing CoordSys: '%s'", pszCoordSys); - CSLDestroy(papszFields); - return -1; - } - - // Fetch the datum information. - int nDatum = 0; - - if (psProj->nProjId != 0 && CSLCount(papszNextField) > 0) - { - nDatum = atoi(papszNextField[0]); - papszNextField++; - } - - if ((nDatum == 999 || nDatum == 9999) && CSLCount(papszNextField) >= 4) - { - psProj->nEllipsoidId = static_cast(atoi(papszNextField[0])); - psProj->dDatumShiftX = CPLAtof(papszNextField[1]); - psProj->dDatumShiftY = CPLAtof(papszNextField[2]); - psProj->dDatumShiftZ = CPLAtof(papszNextField[3]); - papszNextField += 4; - - if (nDatum == 9999 && CSLCount(papszNextField) >= 5) - { - psProj->adDatumParams[0] = CPLAtof(papszNextField[0]); - psProj->adDatumParams[1] = CPLAtof(papszNextField[1]); - psProj->adDatumParams[2] = CPLAtof(papszNextField[2]); - psProj->adDatumParams[3] = CPLAtof(papszNextField[3]); - psProj->adDatumParams[4] = CPLAtof(papszNextField[4]); - papszNextField += 5; - } - } - else if (nDatum != 999 && nDatum != 9999) - { - // Find the datum, and collect its parameters if possible. - const MapInfoDatumInfo *psDatumInfo = nullptr; - - int iDatum = 0; // Used after for. - for (; asDatumInfoList[iDatum].nMapInfoDatumID != -1; iDatum++) - { - if (asDatumInfoList[iDatum].nMapInfoDatumID == nDatum) - { - psDatumInfo = asDatumInfoList + iDatum; - break; - } - } - - if (asDatumInfoList[iDatum].nMapInfoDatumID == -1) - { - // Use WGS84. - psDatumInfo = asDatumInfoList + 0; - } - - if (psDatumInfo != nullptr) - { - psProj->nEllipsoidId = static_cast(psDatumInfo->nEllipsoid); - psProj->nDatumId = - static_cast(psDatumInfo->nMapInfoDatumID); - psProj->dDatumShiftX = psDatumInfo->dfShiftX; - psProj->dDatumShiftY = psDatumInfo->dfShiftY; - psProj->dDatumShiftZ = psDatumInfo->dfShiftZ; - psProj->adDatumParams[0] = psDatumInfo->dfDatumParm0; - psProj->adDatumParams[1] = psDatumInfo->dfDatumParm1; - psProj->adDatumParams[2] = psDatumInfo->dfDatumParm2; - psProj->adDatumParams[3] = psDatumInfo->dfDatumParm3; - psProj->adDatumParams[4] = psDatumInfo->dfDatumParm4; - } - } - - // Fetch the units string. - if (CSLCount(papszNextField) > 0) - { - if (isdigit(static_cast(papszNextField[0][0]))) - { - psProj->nUnitsId = static_cast(atoi(papszNextField[0])); - } - else - { - psProj->nUnitsId = - static_cast(TABUnitIdFromString(papszNextField[0])); - } - papszNextField++; - } - - // Finally the projection parameters. - for (int iParam = 0; iParam < 7 && CSLCount(papszNextField) > 0; iParam++) - { - psProj->adProjParams[iParam] = CPLAtof(papszNextField[0]); - papszNextField++; - } - - CSLDestroy(papszFields); - - return 0; -} diff --git a/ogr/ogrsf_frmts/mitab/mitab_priv.h b/ogr/ogrsf_frmts/mitab/mitab_priv.h index 48b9a566f5c7..e9372b9b1e59 100644 --- a/ogr/ogrsf_frmts/mitab/mitab_priv.h +++ b/ogr/ogrsf_frmts/mitab/mitab_priv.h @@ -36,6 +36,7 @@ #include "cpl_conv.h" #include "cpl_string.h" #include "ogr_feature.h" +#include "ogrmitabspatialref.h" #include @@ -256,34 +257,6 @@ typedef struct TABMAPCoordSecHdr_t int nVertexOffset; } TABMAPCoordSecHdr; -/*--------------------------------------------------------------------- - * TABProjInfo - * struct used to store the projection parameters from the .MAP header - *--------------------------------------------------------------------*/ -typedef struct TABProjInfo_t -{ - GByte nProjId; // See MapInfo Ref. Manual, App. F and G - GByte nEllipsoidId; - GByte nUnitsId; - double adProjParams[7]; // params in same order as in .MIF COORDSYS - - GInt16 nDatumId; // Datum Id added in MapInfo 7.8+ (.map V500) - double dDatumShiftX; // Before that, we had to always lookup datum - double dDatumShiftY; // parameters to establish datum id - double dDatumShiftZ; - double adDatumParams[5]; - - // Affine parameters only in .map version 500 and up - GByte nAffineFlag; // 0=No affine param, 1=Affine params - GByte nAffineUnits; - double dAffineParamA; // Affine params - double dAffineParamB; - double dAffineParamC; - double dAffineParamD; - double dAffineParamE; - double dAffineParamF; -} TABProjInfo; - /*--------------------------------------------------------------------- * TABPenDef - Pen definition information *--------------------------------------------------------------------*/ diff --git a/ogr/ogrsf_frmts/mitab/mitab_tabfile.cpp b/ogr/ogrsf_frmts/mitab/mitab_tabfile.cpp index dfca9c020763..108d85491169 100644 --- a/ogr/ogrsf_frmts/mitab/mitab_tabfile.cpp +++ b/ogr/ogrsf_frmts/mitab/mitab_tabfile.cpp @@ -3048,3 +3048,110 @@ CPLErr TABFile::SetMetadataItem(const char *pszName, const char *pszValue, } return IMapInfoFile::SetMetadataItem(pszName, pszValue, pszDomain); } + +/********************************************************************** + * TABFile::GetSpatialRef() + * + * Returns a reference to an OGRSpatialReference for this dataset. + * If the projection parameters have not been parsed yet, then we will + * parse them before returning. + * + * The returned object is owned and maintained by this TABFile and + * should not be modified or freed by the caller. + * + * Returns NULL if the SpatialRef cannot be accessed. + **********************************************************************/ +OGRSpatialReference *TABFile::GetSpatialRef() +{ + if (m_poMAPFile == nullptr) + { + CPLError(CE_Failure, CPLE_AssertionFailed, + "GetSpatialRef() failed: file has not been opened yet."); + return nullptr; + } + + if (GetGeomType() == wkbNone) + return nullptr; + + /*----------------------------------------------------------------- + * If projection params have already been processed, just use them. + *----------------------------------------------------------------*/ + if (m_poSpatialRef != nullptr) + return m_poSpatialRef; + + /*----------------------------------------------------------------- + * Fetch the parameters from the header. + *----------------------------------------------------------------*/ + TABProjInfo sTABProj; + + TABMAPHeaderBlock *poHeader = nullptr; + if ((poHeader = m_poMAPFile->GetHeaderBlock()) == nullptr || + poHeader->GetProjInfo(&sTABProj) != 0) + { + CPLError(CE_Failure, CPLE_FileIO, + "GetSpatialRef() failed reading projection parameters."); + return nullptr; + } + + m_poSpatialRef = TABFileGetSpatialRefFromTABProj(sTABProj); + return m_poSpatialRef; +} + +/********************************************************************** + * TABFile::SetSpatialRef() + * + * Set the OGRSpatialReference for this dataset. + * A reference to the OGRSpatialReference will be kept, and it will also + * be converted into a TABProjInfo to be stored in the .MAP header. + * + * Returns 0 on success, and -1 on error. + **********************************************************************/ +int TABFile::SetSpatialRef(OGRSpatialReference *poSpatialRef) +{ + if (m_eAccessMode != TABWrite) + { + CPLError(CE_Failure, CPLE_NotSupported, + "SetSpatialRef() can be used only with Write access."); + return -1; + } + + if (m_poMAPFile == nullptr) + { + CPLError(CE_Failure, CPLE_AssertionFailed, + "SetSpatialRef() failed: file has not been opened yet."); + return -1; + } + + if (poSpatialRef == nullptr) + { + CPLError(CE_Failure, CPLE_AssertionFailed, + "SetSpatialRef() failed: Called with NULL poSpatialRef."); + return -1; + } + + /*----------------------------------------------------------------- + * Keep a copy of the OGRSpatialReference... + * Note: we have to take the reference count into account... + *----------------------------------------------------------------*/ + if (m_poSpatialRef && m_poSpatialRef->Dereference() == 0) + delete m_poSpatialRef; + + m_poSpatialRef = poSpatialRef->Clone(); + + TABProjInfo sTABProj; + int nParamCount = 0; + TABFileGetTABProjFromSpatialRef(poSpatialRef, sTABProj, nParamCount); + + /*----------------------------------------------------------------- + * Set the new parameters in the .MAP header. + * This will also trigger lookup of default bounds for the projection. + *----------------------------------------------------------------*/ + if (SetProjInfo(&sTABProj) != 0) + { + CPLError(CE_Failure, CPLE_FileIO, + "SetSpatialRef() failed setting projection parameters."); + return -1; + } + + return 0; +} diff --git a/ogr/ogrsf_frmts/mitab/mitab_utils.cpp b/ogr/ogrsf_frmts/mitab/mitab_utils.cpp index f5cbc284f918..f97bfbb27202 100644 --- a/ogr/ogrsf_frmts/mitab/mitab_utils.cpp +++ b/ogr/ogrsf_frmts/mitab/mitab_utils.cpp @@ -536,68 +536,6 @@ char *TABCleanFieldName(const char *pszSrcName, const char *pszEncoding, return pszNewName; } -/********************************************************************** - * MapInfo Units string to numeric ID conversion - **********************************************************************/ -typedef struct -{ - int nUnitId; - const char *pszAbbrev; -} MapInfoUnitsInfo; - -static const MapInfoUnitsInfo gasUnitsList[] = { - {0, "mi"}, {1, "km"}, {2, "in"}, {3, "ft"}, - {4, "yd"}, {5, "mm"}, {6, "cm"}, {7, "m"}, - {8, "survey ft"}, {8, "survey foot"}, // alternate - {13, nullptr}, {9, "nmi"}, {30, "li"}, {31, "ch"}, - {32, "rd"}, {-1, nullptr}}; - -/********************************************************************** - * TABUnitIdToString() - * - * Return the MIF units name for specified units id. - * Return "" if no match found. - * - * The returned string should not be freed by the caller. - **********************************************************************/ -const char *TABUnitIdToString(int nId) -{ - const MapInfoUnitsInfo *psList = gasUnitsList; - - while (psList->nUnitId != -1) - { - if (psList->nUnitId == nId) - return psList->pszAbbrev; - psList++; - } - - return ""; -} - -/********************************************************************** - * TABUnitIdFromString() - * - * Return the units ID for specified MIF units name - * - * Returns -1 if no match found. - **********************************************************************/ -int TABUnitIdFromString(const char *pszName) -{ - if (pszName == nullptr) - return 13; - - const MapInfoUnitsInfo *psList = gasUnitsList; - - while (psList->nUnitId != -1) - { - if (psList->pszAbbrev != nullptr && EQUAL(psList->pszAbbrev, pszName)) - return psList->nUnitId; - psList++; - } - - return -1; -} - /********************************************************************** * TABSaturatedAdd() ***********************************************************************/ diff --git a/ogr/ogrsf_frmts/mitab/mitab_utils.h b/ogr/ogrsf_frmts/mitab/mitab_utils.h index b1be15bfe189..a55ac4e6ba60 100644 --- a/ogr/ogrsf_frmts/mitab/mitab_utils.h +++ b/ogr/ogrsf_frmts/mitab/mitab_utils.h @@ -57,9 +57,6 @@ char *TABUnEscapeString(char *pszString, GBool bSrcIsConst); char *TABCleanFieldName(const char *pszSrcName, const char *pszCharset, bool bStrictLaundering); -const char *TABUnitIdToString(int nId); -int TABUnitIdFromString(const char *pszName); - void TABSaturatedAdd(GInt32 &nVal, GInt32 nAdd); GInt16 TABInt16Diff(int a, int b); diff --git a/ogr/ogrspatialreference.cpp b/ogr/ogrspatialreference.cpp index 25db55399e4f..681ebe0bb532 100644 --- a/ogr/ogrspatialreference.cpp +++ b/ogr/ogrspatialreference.cpp @@ -55,6 +55,7 @@ #include "ogr_p.h" #include "ogr_proj_p.h" #include "ogr_srs_api.h" +#include "ogrmitabspatialref.h" #include "proj.h" #include "proj_experimental.h" @@ -11208,11 +11209,6 @@ OGRErr OSRSetAxes(OGRSpatialReferenceH hSRS, const char *pszTargetKey, eYAxisOrientation); } -#ifdef HAVE_MITAB -char CPL_DLL *MITABSpatialRef2CoordSys(const OGRSpatialReference *); -OGRSpatialReference CPL_DLL *MITABCoordSys2SpatialRef(const char *); -#endif - /************************************************************************/ /* OSRExportToMICoordSys() */ /************************************************************************/ @@ -11251,22 +11247,14 @@ OGRErr OSRExportToMICoordSys(OGRSpatialReferenceH hSRS, char **ppszReturn) * OGRERR_UNSUPPORTED_OPERATION if MITAB library was not linked in. */ -OGRErr OGRSpatialReference::exportToMICoordSys( - [[maybe_unused]] char **ppszResult) const +OGRErr OGRSpatialReference::exportToMICoordSys(char **ppszResult) const { -#ifdef HAVE_MITAB *ppszResult = MITABSpatialRef2CoordSys(this); if (*ppszResult != nullptr && strlen(*ppszResult) > 0) return OGRERR_NONE; return OGRERR_FAILURE; -#else - CPLError(CE_Failure, CPLE_NotSupported, - "MITAB not available, CoordSys support disabled."); - - return OGRERR_UNSUPPORTED_OPERATION; -#endif } /************************************************************************/ @@ -11306,11 +11294,9 @@ OGRErr OSRImportFromMICoordSys(OGRSpatialReferenceH hSRS, * OGRERR_UNSUPPORTED_OPERATION if MITAB library was not linked in. */ -OGRErr OGRSpatialReference::importFromMICoordSys( - [[maybe_unused]] const char *pszCoordSys) +OGRErr OGRSpatialReference::importFromMICoordSys(const char *pszCoordSys) { -#ifdef HAVE_MITAB OGRSpatialReference *poResult = MITABCoordSys2SpatialRef(pszCoordSys); if (poResult == nullptr) @@ -11320,12 +11306,6 @@ OGRErr OGRSpatialReference::importFromMICoordSys( delete poResult; return OGRERR_NONE; -#else - CPLError(CE_Failure, CPLE_NotSupported, - "MITAB not available, CoordSys support disabled."); - - return OGRERR_UNSUPPORTED_OPERATION; -#endif } /************************************************************************/ From be2db8ba65f0741b3d3e8f1a4e4b6e3b2b619328 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 18:51:31 +0200 Subject: [PATCH 213/710] pcidsk.py: add requirements to PNG driver --- autotest/gdrivers/pcidsk.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/autotest/gdrivers/pcidsk.py b/autotest/gdrivers/pcidsk.py index d8c7f24b168c..0a4acba081f0 100755 --- a/autotest/gdrivers/pcidsk.py +++ b/autotest/gdrivers/pcidsk.py @@ -60,6 +60,7 @@ def test_pcidsk_1(): # Test lossless copying (16, multiband) via Create(). +@pytest.mark.require_driver("PNG") def test_pcidsk_2(): tst = gdaltest.GDALTest("PCIDSK", "png/rgba16.png", 2, 2042) @@ -222,6 +223,7 @@ def test_pcidsk_5(tmp_path): # Test FILE interleaving. +@pytest.mark.require_driver("PNG") def test_pcidsk_8(): tst = gdaltest.GDALTest( @@ -284,6 +286,7 @@ def test_pcidsk_10(): # Test INTERLEAVING=TILED interleaving. +@pytest.mark.require_driver("PNG") def test_pcidsk_11(): tst = gdaltest.GDALTest( @@ -297,6 +300,7 @@ def test_pcidsk_11(): tst.testCreate() +@pytest.mark.require_driver("PNG") def test_pcidsk_11_v1(): tst = gdaltest.GDALTest( @@ -310,6 +314,7 @@ def test_pcidsk_11_v1(): tst.testCreate() +@pytest.mark.require_driver("PNG") def test_pcidsk_11_v2(): tst = gdaltest.GDALTest( @@ -327,6 +332,7 @@ def test_pcidsk_11_v2(): # Test INTERLEAVING=TILED interleaving and COMPRESSION=RLE +@pytest.mark.require_driver("PNG") def test_pcidsk_12(): tst = gdaltest.GDALTest( @@ -340,6 +346,7 @@ def test_pcidsk_12(): tst.testCreate() +@pytest.mark.require_driver("PNG") def test_pcidsk_12_v1(): tst = gdaltest.GDALTest( @@ -358,6 +365,7 @@ def test_pcidsk_12_v1(): tst.testCreate() +@pytest.mark.require_driver("PNG") def test_pcidsk_12_v2(): tst = gdaltest.GDALTest( From cd635ee9177f5e78d91e5ad5aa21923db1889e37 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 20:52:35 +0200 Subject: [PATCH 214/710] Raster MEM driver: disable opening a dataset with MEM::: syntax by default ``` Starting with GDAL 3.10, opening a MEM dataset using the above syntax is no longer enabled by default for security reasons. If you want to allow it, define the ``GDAL_MEM_ENABLE_OPEN`` configuration option to ``YES``, or build GDAL with the ``GDAL_MEM_ENABLE_OPEN`` compilation definition. .. config:: GDAL_MEM_ENABLE_OPEN :choices: YES, NO :default: NO :since: 3.10 Whether opening a MEM dataset with the ``MEM:::`` syntax is allowed. ``` --- autotest/gdrivers/mem.py | 19 ++++++++++++++----- doc/source/drivers/raster/mem.rst | 21 ++++++++++++++++++--- frmts/mem/memdataset.cpp | 14 ++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/autotest/gdrivers/mem.py b/autotest/gdrivers/mem.py index 09a3233f9c65..60c5bbb9a864 100755 --- a/autotest/gdrivers/mem.py +++ b/autotest/gdrivers/mem.py @@ -156,7 +156,14 @@ def test_mem_2(mem_native_memory): for i in range(width * height): float_p[i] = 5.0 - dsro = gdal.Open(dsname) + with pytest.raises( + Exception, + match="Opening a MEM dataset with the MEM:::DATAPOINTER= syntax is no longer supported by default for security reasons", + ): + gdal.Open(dsname) + + with gdal.config_option("GDAL_MEM_ENABLE_OPEN", "YES"): + dsro = gdal.Open(dsname) if dsro is None: free(p) pytest.fail("opening MEM dataset failed in read only mode.") @@ -168,7 +175,8 @@ def test_mem_2(mem_native_memory): pytest.fail("checksum failed.") dsro = None - dsup = gdal.Open(dsname, gdal.GA_Update) + with gdal.config_option("GDAL_MEM_ENABLE_OPEN", "YES"): + dsup = gdal.Open(dsname, gdal.GA_Update) if dsup is None: free(p) pytest.fail("opening MEM dataset failed in update mode.") @@ -210,9 +218,10 @@ def test_geotransform(ds_definition, expected_sr, mem_native_memory): proj_crs = "+proj=laea +lon_0=147 +lat_0=-42" ll_crs = """GEOGCS[\\"WGS 84\\",DATUM[\\"WGS_1984\\",SPHEROID[\\"WGS 84\\",6378137,298.257223563,AUTHORITY[\\"EPSG\\",\\"7030\\"]],AUTHORITY[\\"EPSG\\",\\"6326\\"]],PRIMEM[\\"Greenwich\\",0,AUTHORITY[\\"EPSG\\",\\"8901\\"]],UNIT[\\"degree\\",0.0174532925199433,AUTHORITY[\\"EPSG\\",\\"9122\\"]],AXIS[\\"Latitude\\",NORTH],AXIS[\\"Longitude\\",EAST],AUTHORITY[\\"EPSG\\",\\"4326\\"]]""" - dsro = gdal.Open( - ds_definition.format(datapointer=p, proj_crs=proj_crs, ll_crs=ll_crs) - ) + with gdal.config_option("GDAL_MEM_ENABLE_OPEN", "YES"): + dsro = gdal.Open( + ds_definition.format(datapointer=p, proj_crs=proj_crs, ll_crs=ll_crs) + ) if dsro is None: free(p) pytest.fail("opening MEM dataset failed in read only mode.") diff --git a/doc/source/drivers/raster/mem.rst b/doc/source/drivers/raster/mem.rst index 223ead3d0f79..ae18bd9b3220 100644 --- a/doc/source/drivers/raster/mem.rst +++ b/doc/source/drivers/raster/mem.rst @@ -63,14 +63,29 @@ or the next. - GEOTRANSFORM: Set the affine transformation coefficients. 6 real numbers with '/' as separator (optional) -- SPATIALREFERENCE: (GDAL >= 3.7) Set the projection. The coordinate reference - systems that can be passed are anything supported by the - OGRSpatialReference.SetFromUserInput() as per '-a_srs' in +- SPATIALREFERENCE: (GDAL >= 3.7) Set the projection. The coordinate reference + systems that can be passed are anything supported by the + OGRSpatialReference.SetFromUserInput() as per '-a_srs' in :ref:`gdal_translate`. If the passed string includes comma or double-quote characters (typically WKT), it should be surrounded by double-quote characters and the double-quote characters inside it should be escaped with anti-slash. e.g ``SPATIALREFERENCE="GEOGCRS[\"WGS 84\",[... snip ...],ID[\"EPSG\",4326]]"`` +.. warning:: + + Starting with GDAL 3.10, opening a MEM dataset using the above syntax is no + longer enabled by default for security reasons. + If you want to allow it, define the ``GDAL_MEM_ENABLE_OPEN`` configuration + option to ``YES``, or build GDAL with the ``GDAL_MEM_ENABLE_OPEN`` compilation + definition. + + .. config:: GDAL_MEM_ENABLE_OPEN + :choices: YES, NO + :default: NO + :since: 3.10 + + Whether opening a MEM dataset with the ``MEM:::`` syntax is allowed. + Creation Options ---------------- diff --git a/frmts/mem/memdataset.cpp b/frmts/mem/memdataset.cpp index bf1502a42ea5..76ab12833c00 100644 --- a/frmts/mem/memdataset.cpp +++ b/frmts/mem/memdataset.cpp @@ -1155,6 +1155,20 @@ GDALDataset *MEMDataset::Open(GDALOpenInfo *poOpenInfo) poOpenInfo->fpL != nullptr) return nullptr; +#ifndef GDAL_MEM_ENABLE_OPEN + if (!CPLTestBool(CPLGetConfigOption("GDAL_MEM_ENABLE_OPEN", "NO"))) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Opening a MEM dataset with the MEM:::DATAPOINTER= syntax " + "is no longer supported by default for security reasons. " + "If you want to allow it, define the " + "GDAL_MEM_ENABLE_OPEN " + "configuration option to YES, or build GDAL with the " + "GDAL_MEM_ENABLE_OPEN compilation definition"); + return nullptr; + } +#endif + char **papszOptions = CSLTokenizeStringComplex(poOpenInfo->pszFilename + 6, ",", TRUE, FALSE); From b51f137e7da75fbd218f16a76cefa16e0ff4e393 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Sep 2024 00:57:25 +0200 Subject: [PATCH 215/710] Fix Coverity Scan issues (master only) --- frmts/hdf5/s102dataset.cpp | 9 ++++---- gcore/gdaldataset.cpp | 2 ++ gcore/gdalthreadsafedataset.cpp | 4 ++++ .../parquet/ogrparquetwriterlayer.cpp | 21 ++++++++++--------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/frmts/hdf5/s102dataset.cpp b/frmts/hdf5/s102dataset.cpp index 563ca20ce1f2..428ec45cc894 100644 --- a/frmts/hdf5/s102dataset.cpp +++ b/frmts/hdf5/s102dataset.cpp @@ -514,11 +514,10 @@ bool S102Dataset::OpenQuality(GDALOpenInfo *poOpenInfo, // I believe this is non-conformant. // Escape potentials single-quote and double-quote with back-slash - const auto osEscapedCompName = - CPLString(oType.GetComponents()[0]->GetName()) - .replaceAll("\\", "\\\\") - .replaceAll("'", "\\'") - .replaceAll("\"", "\\\""); + CPLString osEscapedCompName(oType.GetComponents()[0]->GetName()); + osEscapedCompName.replaceAll("\\", "\\\\") + .replaceAll("'", "\\'") + .replaceAll("\"", "\\\""); // Gets a view with that single component extracted. poValuesArray = poValuesArray->GetView( diff --git a/gcore/gdaldataset.cpp b/gcore/gdaldataset.cpp index 2f17d2d69757..cc5289d644f5 100644 --- a/gcore/gdaldataset.cpp +++ b/gcore/gdaldataset.cpp @@ -376,11 +376,13 @@ GDALDataset::~GDALDataset() if (m_poPrivate->hMutex != nullptr) CPLDestroyMutex(m_poPrivate->hMutex); + // coverity[missing_lock] CPLFree(m_poPrivate->m_pszWKTCached); if (m_poPrivate->m_poSRSCached) { m_poPrivate->m_poSRSCached->Release(); } + // coverity[missing_lock] CPLFree(m_poPrivate->m_pszWKTGCPCached); if (m_poPrivate->m_poSRSGCPCached) { diff --git a/gcore/gdalthreadsafedataset.cpp b/gcore/gdalthreadsafedataset.cpp index 2ce0de26d31c..3021768ff48c 100644 --- a/gcore/gdalthreadsafedataset.cpp +++ b/gcore/gdalthreadsafedataset.cpp @@ -476,7 +476,10 @@ GDALThreadLocalDatasetCache::~GDALThreadLocalDatasetCache() { // Leak datasets when GDAL has been de-initialized if (!m_poCache->empty()) + { + // coverity[leaked_storage] CPL_IGNORE_RET_VAL(m_poCache.release()); + } return; } @@ -747,6 +750,7 @@ GDALDataset *GDALThreadSafeDataset::RefUnderlyingDataset() const // "Clone" the prototype dataset, which in 99% of the cases, involves // doing a GDALDataset::Open() call to re-open it. Do that by temporarily // dropping the lock that protects poCache->m_oCache. + // coverity[uninit_use_in_call] oLock.unlock(); poTLSDS = m_poPrototypeDS->Clone(GDAL_OF_RASTER, /* bCanShareState=*/true); if (poTLSDS) diff --git a/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp b/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp index f72610e96dc8..b6a1cf493480 100644 --- a/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp +++ b/ogr/ogrsf_frmts/parquet/ogrparquetwriterlayer.cpp @@ -1291,17 +1291,18 @@ bool OGRParquetWriterLayer::IsArrowSchemaSupported( osErrorMsg = "BinaryView not supported"; return false; } - if (schema->format[0] == '+' && schema->format[1] == 'v' && - schema->format[1] == 'l') + if (schema->format[0] == '+' && schema->format[1] == 'v') { - osErrorMsg = "ListView not supported"; - return false; - } - if (schema->format[0] == '+' && schema->format[1] == 'v' && - schema->format[1] == 'L') - { - osErrorMsg = "LargeListView not supported"; - return false; + if (schema->format[2] == 'l') + { + osErrorMsg = "ListView not supported"; + return false; + } + else if (schema->format[2] == 'L') + { + osErrorMsg = "LargeListView not supported"; + return false; + } } for (int64_t i = 0; i < schema->n_children; ++i) { From 453ab825a5589ca8aab07dc530c63998d4aca963 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Sep 2024 01:20:47 +0200 Subject: [PATCH 216/710] nitfdump.c: fix warnings --- frmts/nitf/nitfdump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frmts/nitf/nitfdump.c b/frmts/nitf/nitfdump.c index 4dc24987d42e..8c25e4e08f59 100644 --- a/frmts/nitf/nitfdump.c +++ b/frmts/nitf/nitfdump.c @@ -628,10 +628,10 @@ int main(int nArgc, char **papszArgv) EQUAL(CSLFetchNameValueDef(psDES->papszMetadata, "DESID", ""), "CSSHPA DES")) { - char szFilename[40]; + char szFilename[512]; char szRadix[256]; if (bExtractSHPInMem) - snprintf(szRadix, sizeof(szRadix), + snprintf(szRadix, sizeof(szRadix), "%s", VSIMemGenerateHiddenFilename( CPLSPrintf("nitf_segment_%d", iSegment + 1))); else From 9f8fea14708957b26dff299f32ef74cf56a50c52 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Sep 2024 01:22:35 +0200 Subject: [PATCH 217/710] frmts/nitf/CMakeLists.txt: add missing nitfdes.c to nitfdump --- frmts/nitf/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frmts/nitf/CMakeLists.txt b/frmts/nitf/CMakeLists.txt index 9e4cc0727706..196e4521ae04 100644 --- a/frmts/nitf/CMakeLists.txt +++ b/frmts/nitf/CMakeLists.txt @@ -70,7 +70,7 @@ if (NOT GDAL_USE_TIFF_INTERNAL) gdal_target_link_libraries(gdal_NITF PRIVATE TIFF::TIFF) endif () -add_executable(nitfdump EXCLUDE_FROM_ALL nitfdump.c nitffile.c nitfimage.c rpftocfile.cpp nitfbilevel.cpp nitfaridpcm.cpp mgrs.c) +add_executable(nitfdump EXCLUDE_FROM_ALL nitfdump.c nitffile.c nitfimage.c rpftocfile.cpp nitfbilevel.cpp nitfaridpcm.cpp mgrs.c nitfdes.c) if (GDAL_USE_TIFF_INTERNAL) target_sources(nitfdump PRIVATE $) gdal_add_vendored_lib(nitfdump libtiff) From 3b150490cc043a9159d3d0ea7f38e236d75ee41e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 22 Sep 2024 18:36:29 +0200 Subject: [PATCH 218/710] Allow to partially disable VRT driver with GDAL_ENABLE_VRT_DRIVER=OFF but still enable it by default even if GDAL_BUILD_OPTIONAL_DRIVERS=OFF Creation side is still available, but Open() not Also enable GTI driver to be separately enabled/disabled, and buildable as a plugin (if built as a plugin, gdaladdo --partial-refresh-from-source-timestamp won't work) --- apps/CMakeLists.txt | 5 + apps/gdaladdo.cpp | 12 +++ autotest/alg/cutline.py | 5 + autotest/alg/warp.py | 5 + autotest/cpp/test_gdal.cpp | 6 ++ autotest/cpp/test_gdal_pixelfn.cpp | 9 ++ autotest/gcore/gdal_stats.py | 20 ++++ autotest/gcore/geoloc.py | 4 + autotest/gcore/interpolateatpoint.py | 4 + autotest/gcore/mask.py | 4 + autotest/gcore/numpy_rw.py | 8 ++ autotest/gcore/overviewds.py | 9 ++ autotest/gcore/pixfun.py | 5 + autotest/gcore/testnonboundtoswig.py | 4 + autotest/gcore/tiff_ovr.py | 16 +++ autotest/gcore/tiff_read.py | 4 + autotest/gcore/tiff_write.py | 100 ++++++++++++++++++ autotest/gcore/transformer.py | 60 +++++++++++ autotest/gcore/vrt_read.py | 4 + autotest/gcore/vrtmisc.py | 6 ++ autotest/gcore/vsistdin.py | 4 + autotest/gdrivers/aaigrid.py | 4 + autotest/gdrivers/derived.py | 7 ++ autotest/gdrivers/dimap.py | 5 + autotest/gdrivers/ecw.py | 4 + autotest/gdrivers/ehdr.py | 4 + autotest/gdrivers/ers.py | 4 + autotest/gdrivers/gpkg.py | 4 + autotest/gdrivers/grib.py | 3 +- autotest/gdrivers/gta.py | 4 + autotest/gdrivers/gti.py | 20 +++- autotest/gdrivers/hfa.py | 4 + autotest/gdrivers/jp2openjpeg.py | 12 +++ autotest/gdrivers/jpeg.py | 4 + autotest/gdrivers/kmlsuperoverlay.py | 12 +++ autotest/gdrivers/mrf.py | 4 + autotest/gdrivers/netcdf.py | 12 +++ autotest/gdrivers/netcdf_multidim.py | 4 + autotest/gdrivers/nitf.py | 24 +++++ autotest/gdrivers/pdf.py | 28 +++++ autotest/gdrivers/prf.py | 4 + autotest/gdrivers/sigdem.py | 5 + autotest/gdrivers/test_validate_jp2.py | 8 ++ autotest/gdrivers/vrtderived.py | 4 + autotest/gdrivers/vrtfilt.py | 5 + autotest/gdrivers/vrtlut.py | 5 + autotest/gdrivers/vrtmask.py | 5 + autotest/gdrivers/vrtmultidim.py | 4 + autotest/gdrivers/vrtovr.py | 5 + autotest/gdrivers/vrtpansharpen.py | 5 + autotest/gdrivers/vrtprocesseddataset.py | 5 + autotest/gdrivers/vrtrawlink.py | 5 + autotest/gdrivers/vrtwarp.py | 5 + autotest/gdrivers/wms.py | 4 + autotest/pymod/gdaltest.py | 5 + autotest/pyscripts/gdal2tiles/test_logger.py | 6 ++ autotest/pyscripts/gdal2tiles/test_vsimem.py | 6 ++ autotest/pyscripts/test_gdal2tiles.py | 13 ++- autotest/pyscripts/test_gdal_pansharpen.py | 10 +- autotest/pyscripts/test_gdalattachpct.py | 5 + autotest/pyscripts/test_gdalbuildvrtofvrt.py | 4 + autotest/pyscripts/test_gdalinfo_py.py | 5 + autotest/utilities/test_gdal_create.py | 4 + autotest/utilities/test_gdal_translate.py | 20 ++++ autotest/utilities/test_gdal_translate_lib.py | 12 +++ autotest/utilities/test_gdaladdo.py | 21 ++++ autotest/utilities/test_gdalbuildvrt.py | 14 ++- autotest/utilities/test_gdalbuildvrt_lib.py | 14 +++ autotest/utilities/test_gdaldem.py | 12 +++ autotest/utilities/test_gdaldem_lib.py | 8 ++ autotest/utilities/test_gdalinfo.py | 8 ++ autotest/utilities/test_gdallocationinfo.py | 4 + autotest/utilities/test_gdalmdiminfo.py | 8 ++ autotest/utilities/test_gdalmdimtranslate.py | 8 ++ .../utilities/test_gdalmdimtranslate_lib.py | 48 +++++++++ autotest/utilities/test_gdalwarp.py | 12 +++ autotest/utilities/test_gdalwarp_lib.py | 12 +++ frmts/CMakeLists.txt | 19 +++- frmts/gdalallregister.cpp | 5 +- frmts/gti/CMakeLists.txt | 26 +++++ frmts/{vrt => gti}/data/gdaltileindex.xsd | 0 frmts/{vrt => gti}/gdaltileindexdataset.cpp | 9 +- frmts/vrt/CMakeLists.txt | 6 +- frmts/vrt/vrt_priv.h | 4 +- frmts/vrt/vrtdataset.h | 4 +- frmts/vrt/vrtdriver.cpp | 5 +- gcore/gdal_priv.h | 3 +- ogr/ogrsf_frmts/generic/ogrwarpedlayer.h | 2 +- 88 files changed, 842 insertions(+), 26 deletions(-) create mode 100644 frmts/gti/CMakeLists.txt rename frmts/{vrt => gti}/data/gdaltileindex.xsd (100%) rename frmts/{vrt => gti}/gdaltileindexdataset.cpp (99%) diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 4ce116ae1b01..94bbc93a8694 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -147,6 +147,11 @@ if (BUILD_APPS) target_compile_definitions(${UTILCMD} PRIVATE -DSUPPORTS_WMAIN) endif () target_link_libraries(${UTILCMD} PRIVATE $ utils_common) + + if (NOT GDAL_ENABLE_DRIVER_GTI OR GDAL_ENABLE_DRIVER_GTI_PLUGIN) + target_compile_definitions(${UTILCMD} PRIVATE -DGTI_DRIVER_DISABLED_OR_PLUGIN) + endif() + endforeach () install(TARGETS ${APPS_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/apps/gdaladdo.cpp b/apps/gdaladdo.cpp index 32d1ffa873b7..3f03c122bd18 100644 --- a/apps/gdaladdo.cpp +++ b/apps/gdaladdo.cpp @@ -311,6 +311,17 @@ static bool PartialRefreshFromSourceTimestamp( } } } +#ifdef GTI_DRIVER_DISABLED_OR_PLUGIN + else if (poDS->GetDriver() && + EQUAL(poDS->GetDriver()->GetDescription(), "GTI")) + { + CPLError(CE_Failure, CPLE_NotSupported, + "--partial-refresh-from-source-timestamp only works on a GTI " + "dataset if the GTI driver is not built as a plugin, " + "but in core library"); + return false; + } +#else else if (auto poGTIDS = GDALDatasetCastToGTIDataset(poDS)) { regions = GTIGetSourcesMoreRecentThan(poGTIDS, sStatVRTOvr.st_mtime); @@ -320,6 +331,7 @@ static bool PartialRefreshFromSourceTimestamp( static_cast(region.nDstXSize) * region.nDstYSize; } } +#endif else { CPLError(CE_Failure, CPLE_AppDefined, diff --git a/autotest/alg/cutline.py b/autotest/alg/cutline.py index de27180d5e20..e4bc8bb3c569 100755 --- a/autotest/alg/cutline.py +++ b/autotest/alg/cutline.py @@ -36,6 +36,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + ############################################################################### diff --git a/autotest/alg/warp.py b/autotest/alg/warp.py index fae126885e55..d8ab607e0165 100755 --- a/autotest/alg/warp.py +++ b/autotest/alg/warp.py @@ -40,6 +40,11 @@ import gdaltest import pytest +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + from osgeo import gdal, osr ############################################################################### diff --git a/autotest/cpp/test_gdal.cpp b/autotest/cpp/test_gdal.cpp index c5046797e65f..675ec33396b1 100644 --- a/autotest/cpp/test_gdal.cpp +++ b/autotest/cpp/test_gdal.cpp @@ -3114,6 +3114,12 @@ TEST_F(test_gdal, GDALCachedPixelAccessor) // (https://github.com/OSGeo/gdal/issues/5989) TEST_F(test_gdal, VRTCachingOpenOptions) { + if (GDALGetMetadataItem(GDALGetDriverByName("VRT"), GDAL_DMD_OPENOPTIONLIST, + nullptr) == nullptr) + { + GTEST_SKIP() << "VRT driver Open() missing"; + } + class TestRasterBand : public GDALRasterBand { protected: diff --git a/autotest/cpp/test_gdal_pixelfn.cpp b/autotest/cpp/test_gdal_pixelfn.cpp index 6adfbc567636..cb23e556585f 100644 --- a/autotest/cpp/test_gdal_pixelfn.cpp +++ b/autotest/cpp/test_gdal_pixelfn.cpp @@ -170,6 +170,15 @@ struct test_gdal_pixelfn : public ::testing::Test src_ += SEP; src_ += "pixelfn.vrt"; } + + void SetUp() override + { + if (GDALGetMetadataItem(GDALGetDriverByName("VRT"), + GDAL_DMD_OPENOPTIONLIST, nullptr) == nullptr) + { + GTEST_SKIP() << "VRT driver Open() missing"; + } + } }; // Test constant parameters in a custom pixel function diff --git a/autotest/gcore/gdal_stats.py b/autotest/gcore/gdal_stats.py index 95137a355879..7a7dbe69a3f9 100755 --- a/autotest/gcore/gdal_stats.py +++ b/autotest/gcore/gdal_stats.py @@ -180,6 +180,10 @@ def test_stats_nan_3(): # and complex source nodata (#3576) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_stats_nan_4(): ds = gdal.Open("data/nan32_nodata.vrt") @@ -197,6 +201,10 @@ def test_stats_nan_4(): # and complex source nodata (nan must be translated to 0 then) (#3576) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_stats_nan_5(): ds = gdal.Open("data/nan32_nodata_nan_to_zero.vrt") @@ -213,6 +221,10 @@ def test_stats_nan_5(): # Test reading a warped VRT with nan as src nodata and dest nodata (#3576) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_stats_nan_6(): ds = gdal.Open("data/nan32_nodata_warp.vrt") @@ -229,6 +241,10 @@ def test_stats_nan_6(): # Test reading a warped VRT with nan as src nodata and 0 as dest nodata (#3576) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_stats_nan_7(): ds = gdal.Open("data/nan32_nodata_warp_nan_to_zero.vrt") @@ -245,6 +261,10 @@ def test_stats_nan_7(): # Test reading a warped VRT with zero as src nodata and nan as dest nodata (#3576) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_stats_nan_8(): ds = gdal.Open("data/nan32_nodata_warp_zero_to_nan.vrt") diff --git a/autotest/gcore/geoloc.py b/autotest/gcore/geoloc.py index d27be4319703..dcee7f5f82e4 100755 --- a/autotest/gcore/geoloc.py +++ b/autotest/gcore/geoloc.py @@ -41,6 +41,10 @@ # Verify warped result. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_geoloc_1(): tst = gdaltest.GDALTest("VRT", "warpsst.vrt", 1, 63034) diff --git a/autotest/gcore/interpolateatpoint.py b/autotest/gcore/interpolateatpoint.py index 3dffd4c8b3f1..767a144e54dd 100755 --- a/autotest/gcore/interpolateatpoint.py +++ b/autotest/gcore/interpolateatpoint.py @@ -302,6 +302,10 @@ def test_interpolateatpoint_complex_float(): assert res == pytest.approx((34.433130 - 36.741504j), 1e-4) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_interpolateatpoint_big_complex(): # The purpose of this test is to check that the algorithm implementation # works for bigger values above the first block of 64x64 pixels. diff --git a/autotest/gcore/mask.py b/autotest/gcore/mask.py index 180521e11b65..19570c59dbb2 100755 --- a/autotest/gcore/mask.py +++ b/autotest/gcore/mask.py @@ -72,6 +72,10 @@ def test_mask_1(): # Verify the checksum and flags for "nodata" case. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_mask_2(): ds = gdal.Open("data/byte.vrt") diff --git a/autotest/gcore/numpy_rw.py b/autotest/gcore/numpy_rw.py index d4451ab0dcf7..52258f7dadeb 100755 --- a/autotest/gcore/numpy_rw.py +++ b/autotest/gcore/numpy_rw.py @@ -732,6 +732,10 @@ def test_numpy_rw_18(): # The VRT references a non existing TIF file, but using the proxy pool dataset API (#2837) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_numpy_rw_failure_in_readasarray(): ds = gdal.Open("data/idontexist2.vrt") @@ -964,6 +968,10 @@ def test_numpy_rw_band_read_as_array_error_cases(): # Test that we can get an error (#5374) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_numpy_rw_band_read_as_array_getlasterrormsg(): ds = gdal.Open( diff --git a/autotest/gcore/overviewds.py b/autotest/gcore/overviewds.py index a3a8a11a4540..e4d567c714cd 100755 --- a/autotest/gcore/overviewds.py +++ b/autotest/gcore/overviewds.py @@ -32,6 +32,7 @@ import shutil import struct +import gdaltest import pytest from osgeo import gdal @@ -250,6 +251,10 @@ def test_overviewds_4(tmp_path): # Test GEOLOCATION +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_overviewds_5(tmp_path): shutil.copy("data/sstgeo.tif", tmp_path) @@ -296,6 +301,10 @@ def test_overviewds_5(tmp_path): # Test VRT +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_overviewds_6(tmp_path): shutil.copy("data/byte.tif", tmp_path) diff --git a/autotest/gcore/pixfun.py b/autotest/gcore/pixfun.py index 57fcf99f3fb3..d8dc001eba66 100755 --- a/autotest/gcore/pixfun.py +++ b/autotest/gcore/pixfun.py @@ -35,6 +35,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + # All tests will be skipped if numpy is unavailable. numpy = pytest.importorskip("numpy") diff --git a/autotest/gcore/testnonboundtoswig.py b/autotest/gcore/testnonboundtoswig.py index 8979d26f4512..5d964a2fc0c4 100755 --- a/autotest/gcore/testnonboundtoswig.py +++ b/autotest/gcore/testnonboundtoswig.py @@ -291,6 +291,10 @@ def my_pyDerivedPixelFunc( return 0 +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_testnonboundtoswig_VRTDerivedBands(): DerivedPixelFuncType = ctypes.CFUNCTYPE( diff --git a/autotest/gcore/tiff_ovr.py b/autotest/gcore/tiff_ovr.py index d047ea29843e..7dd49bd0e7cf 100755 --- a/autotest/gcore/tiff_ovr.py +++ b/autotest/gcore/tiff_ovr.py @@ -122,6 +122,10 @@ def mfloat32_tif(tmp_path): yield dst_fname +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_ovr_1(mfloat32_tif, both_endian): ds = gdal.Open(mfloat32_tif) @@ -151,6 +155,10 @@ def test_tiff_ovr_1(mfloat32_tif, both_endian): # Open target file in update mode, and create internal overviews. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_ovr_3(mfloat32_tif, both_endian): src_ds = gdal.Open(mfloat32_tif, gdal.GA_Update) @@ -531,6 +539,10 @@ def test_tiff_ovr_12(tmp_path, both_endian): # Test gaussian resampling +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_ovr_13(mfloat32_tif, both_endian): ds = gdal.Open(mfloat32_tif) @@ -608,6 +620,10 @@ def test_tiff_ovr_15(tmp_path, both_endian): # Test mode resampling on non-byte dataset +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_ovr_16(tmp_path, both_endian): tif_fname = str(tmp_path / "ovr16.tif") diff --git a/autotest/gcore/tiff_read.py b/autotest/gcore/tiff_read.py index 3a18f98579a5..6856094b7b59 100755 --- a/autotest/gcore/tiff_read.py +++ b/autotest/gcore/tiff_read.py @@ -5189,6 +5189,10 @@ def test_tiff_read_webp_lossless_rgba_alpha_fully_opaque(): # Test complex scenario of https://github.com/OSGeo/gdal/issues/9563 +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_creation_option("GTiff", "JPEG") def test_tiff_read_jpeg_cached_multi_range_issue_9563(tmp_vsimem): diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py index c7bcdef00130..88f3caac637f 100755 --- a/autotest/gcore/tiff_write.py +++ b/autotest/gcore/tiff_write.py @@ -212,6 +212,10 @@ def test_tiff_write_4(): # Write a file with GCPs. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_5(): src_ds = gdal.Open("data/gcps.vrt") @@ -434,6 +438,10 @@ def test_tiff_write_14(): # file using the PROFILE creation option with CreateCopy() +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_15(): ds_in = gdal.Open("data/byte.vrt") @@ -484,6 +492,10 @@ def test_tiff_write_15(): # file using the PROFILE creation option with Create() +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_16(): ds_in = gdal.Open("data/byte.vrt") @@ -548,6 +560,10 @@ def test_tiff_write_16(): # Test writing a TIFF with an RPC tag. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_17(): # Translate RPC controlled data to GeoTIFF. @@ -608,6 +624,10 @@ def test_tiff_write_17(): # case (#3996) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_17_disable_readdir(): with gdal.config_option("GDAL_DISABLE_READDIR_ON_OPEN", "TRUE"): test_tiff_write_17() @@ -617,6 +637,10 @@ def test_tiff_write_17_disable_readdir(): # Test writing a TIFF with an RPB file and IMD file. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_18(): # Translate RPC controlled data to GeoTIFF. @@ -713,6 +737,10 @@ def test_tiff_write_imd_with_space_in_values(): # case (#3996) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_18_disable_readdir(): with gdal.config_option("GDAL_DISABLE_READDIR_ON_OPEN", "TRUE"): test_tiff_write_18() @@ -722,6 +750,10 @@ def test_tiff_write_18_disable_readdir(): # Test writing a TIFF with an _RPC.TXT +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_rpc_txt(): # Translate RPC controlled data to GeoTIFF. @@ -774,6 +806,10 @@ def test_tiff_write_rpc_txt(): # Test writing a TIFF with an RPC in .aux.xml +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_rpc_in_pam(): ds_in = gdal.Open("data/rpc.vrt") @@ -1269,6 +1305,10 @@ def test_tiff_write_31(): # Create a rotated image +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_32(): ds_in = gdal.Open("data/byte.vrt") @@ -1309,6 +1349,10 @@ def test_tiff_write_32(): # (BASELINE is tested by tiff_write_15) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_33(): ds_in = gdal.Open("data/byte.vrt") @@ -1464,6 +1508,10 @@ def tiff_write_big_odd_bits(vrtfilename, tmpfilename, nbits, interleaving): # Test copy with NBITS=9, INTERLEAVE=PIXEL +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_36(): return tiff_write_big_odd_bits("data/uint16_3band.vrt", "tmp/tw_36.tif", 9, "PIXEL") @@ -1472,6 +1520,10 @@ def test_tiff_write_36(): # Test copy with NBITS=9, INTERLEAVE=BAND +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_37(): return tiff_write_big_odd_bits("data/uint16_3band.vrt", "tmp/tw_37.tif", 9, "BAND") @@ -1480,6 +1532,10 @@ def test_tiff_write_37(): # Test copy with NBITS=12, INTERLEAVE=PIXEL +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_38(): return tiff_write_big_odd_bits( "data/uint16_3band.vrt", "tmp/tw_38.tif", 12, "PIXEL" @@ -1490,6 +1546,10 @@ def test_tiff_write_38(): # Test copy with NBITS=12, INTERLEAVE=BAND +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_39(): return tiff_write_big_odd_bits("data/uint16_3band.vrt", "tmp/tw_39.tif", 12, "BAND") @@ -1498,6 +1558,10 @@ def test_tiff_write_39(): # Test copy with NBITS=17, INTERLEAVE=PIXEL +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_40(): return tiff_write_big_odd_bits("data/uint32_3band.vrt", "tmp/tw_40tif", 17, "PIXEL") @@ -1506,6 +1570,10 @@ def test_tiff_write_40(): # Test copy with NBITS=17, INTERLEAVE=BAND +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_41(): return tiff_write_big_odd_bits("data/uint32_3band.vrt", "tmp/tw_41.tif", 17, "BAND") @@ -1514,6 +1582,10 @@ def test_tiff_write_41(): # Test copy with NBITS=24, INTERLEAVE=PIXEL +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_42(): return tiff_write_big_odd_bits( "data/uint32_3band.vrt", "tmp/tw_42.tif", 24, "PIXEL" @@ -1524,6 +1596,10 @@ def test_tiff_write_42(): # Test copy with NBITS=24, INTERLEAVE=BAND +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_43(): return tiff_write_big_odd_bits("data/uint32_3band.vrt", "tmp/tw_43.tif", 24, "BAND") @@ -4892,6 +4968,10 @@ def test_tiff_write_120(): # Test error cases of COPY_SRC_OVERVIEWS creation option +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_121(): # Test when the overview band is NULL @@ -5326,6 +5406,10 @@ def test_tiff_write_125(): # Test implicit JPEG-in-TIFF overviews +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_creation_option("GTiff", "JPEG") @pytest.mark.require_driver("JPEG") def test_tiff_write_126(): @@ -6444,6 +6528,10 @@ def test_tiff_write_134(): # Test clearing GCPs (#5945) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_135(): # Simple clear @@ -7876,6 +7964,10 @@ def test_tiff_write_160(): # Test setting GCPs on an image with already a geotransform and vice-versa (#6751) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_161(): ds = gdaltest.tiff_drv.Create("/vsimem/tiff_write_161.tif", 1, 1) @@ -7910,6 +8002,10 @@ def test_tiff_write_161(): # Test creating a JPEG compressed file with big tiles (#6757) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_creation_option("GTiff", "JPEG") def test_tiff_write_162(): @@ -9252,6 +9348,10 @@ def test_tiff_write_compression_create_and_createcopy(): # Attempt at creating a file with more tile arrays larger than 2 GB +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_tiff_write_too_many_tiles(): src_ds = gdal.Open( diff --git a/autotest/gcore/transformer.py b/autotest/gcore/transformer.py index 2be372a75de9..3d291f33b2f7 100644 --- a/autotest/gcore/transformer.py +++ b/autotest/gcore/transformer.py @@ -70,6 +70,10 @@ def test_transformer_1(): # Test GCP based transformer with polynomials. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_2(): ds = gdal.Open("data/gcps.vrt") @@ -98,6 +102,10 @@ def test_transformer_2(): # Test GCP based transformer with thin plate splines. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_3(): ds = gdal.Open("data/gcps.vrt") @@ -126,6 +134,10 @@ def test_transformer_3(): # Test geolocation based transformer. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_4(): ds = gdal.Open("data/sstgeo.vrt") @@ -154,6 +166,10 @@ def test_transformer_4(): # Test RPC based transformer. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_5(): ds = gdal.Open("data/rpc.vrt") @@ -364,6 +380,10 @@ def test_transformer_5(): # Test RPC convergence bug (bug # 5395) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_6(): ds = gdal.Open("data/rpc_5395.vrt") @@ -402,6 +422,10 @@ def test_transformer_7(): # Test handling of nodata in RPC DEM (#5680) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_8(): ds = gdal.Open("data/rpc.vrt") @@ -448,6 +472,10 @@ def test_transformer_8(): # Test RPC DEM line optimization +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_9(): ds = gdal.Open("data/rpc.vrt") @@ -504,6 +532,10 @@ def test_transformer_9(): # Test RPC DEM transform from geoid height to ellipsoidal height +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_driver("GTX") def test_transformer_10(): @@ -637,6 +669,10 @@ def test_transformer_11(): # Test degenerate cases of TPS transformer +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_12(): ds = gdal.Open( @@ -834,6 +870,10 @@ def test_transformer_14(): # beyond +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_15(): ds = gdal.GetDriverByName("MEM").Create("", 6600, 4400) @@ -933,6 +973,10 @@ def test_transformer_15(): # (we mostly test that the parameters are well recognized and serialized) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_16(): gdal.Translate( @@ -966,6 +1010,10 @@ def test_transformer_16(): # Test RPC DEM with unexisting RPC DEM file +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_17(): ds = gdal.Open("data/rpc.vrt") @@ -1018,6 +1066,10 @@ def test_transformer_no_reverse_method(): # Test precision of GCP based transformer with thin plate splines and lots of GCPs (2115). +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_tps_precision(): ds = gdal.Open("data/gcps_2115.vrt") @@ -1099,6 +1151,10 @@ def test_transformer_image_no_srs(): # Test RPC_DEM_SRS by adding vertical component egm 96 geoid +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_dem_overrride_srs(): ds = gdal.Open("data/rpc.vrt") ds_dem = gdal.GetDriverByName("GTiff").Create("/vsimem/dem.tif", 100, 100, 1) @@ -1191,6 +1247,10 @@ def test_transformer_SuggestedWarpOutput_from_options(): # Test GCP antimerdian unwrap (https://github.com/OSGeo/gdal/issues/8371) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_transformer_gcp_antimeridian_unwrap(): ds = gdal.Open("data/test_gcp_antimeridian_unwrap.vrt") diff --git a/autotest/gcore/vrt_read.py b/autotest/gcore/vrt_read.py index 2c4d319f4e33..6c7fc354bd15 100755 --- a/autotest/gcore/vrt_read.py +++ b/autotest/gcore/vrt_read.py @@ -42,6 +42,10 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) ############################################################################### @pytest.fixture(autouse=True, scope="module") diff --git a/autotest/gcore/vrtmisc.py b/autotest/gcore/vrtmisc.py index 472f0c9bdc56..2e4a3fac2869 100755 --- a/autotest/gcore/vrtmisc.py +++ b/autotest/gcore/vrtmisc.py @@ -36,10 +36,16 @@ import tempfile from pathlib import Path +import gdaltest import pytest from osgeo import gdal, osr +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + ############################################################################### # Test linear scaling diff --git a/autotest/gcore/vsistdin.py b/autotest/gcore/vsistdin.py index 1e68741873f3..4dab16e36979 100755 --- a/autotest/gcore/vsistdin.py +++ b/autotest/gcore/vsistdin.py @@ -108,6 +108,10 @@ def test_vsistdin_3(): # Test fix for #6061 +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_vsistdin_4(): if test_cli_utilities.get_gdal_translate_path() is None: pytest.skip() diff --git a/autotest/gdrivers/aaigrid.py b/autotest/gdrivers/aaigrid.py index 8c1fb3f79854..aac8012327bf 100755 --- a/autotest/gdrivers/aaigrid.py +++ b/autotest/gdrivers/aaigrid.py @@ -236,6 +236,10 @@ def test_aaigrid_6bis(): # Verify writing files with non-square pixels. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_aaigrid_7(): tst = gdaltest.GDALTest("AAIGRID", "aaigrid/nonsquare.vrt", 1, 12481) diff --git a/autotest/gdrivers/derived.py b/autotest/gdrivers/derived.py index 00ef1e9ff98f..bb39e8d61632 100755 --- a/autotest/gdrivers/derived.py +++ b/autotest/gdrivers/derived.py @@ -28,6 +28,7 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### +import gdaltest import pytest from osgeo import gdal @@ -159,6 +160,12 @@ def test_derived_test3(): # Raster with zero band gdal.Open("DERIVED_SUBDATASET:LOGAMPLITUDE:data/hdf5/CSK_DGM.h5") + +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) +def test_derived_vrt_errors(): for function in [ "real", "imag", diff --git a/autotest/gdrivers/dimap.py b/autotest/gdrivers/dimap.py index 9758e256362d..1b4b50954e09 100755 --- a/autotest/gdrivers/dimap.py +++ b/autotest/gdrivers/dimap.py @@ -31,6 +31,7 @@ import os import shutil +import gdaltest import pytest from osgeo import gdal @@ -41,6 +42,10 @@ # Open and verify a the GCPs and metadata. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_dimap_1(): shutil.copy("data/dimap/METADATA.DIM", "tmp") diff --git a/autotest/gdrivers/ecw.py b/autotest/gdrivers/ecw.py index d0407441e7fd..000e90c1f450 100755 --- a/autotest/gdrivers/ecw.py +++ b/autotest/gdrivers/ecw.py @@ -1038,6 +1038,10 @@ def test_ecw_31(): # It ignores the content of panBandMap. (#4234) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_ecw_32(): ds = gdal.Open("data/ecw/jrc.ecw") diff --git a/autotest/gdrivers/ehdr.py b/autotest/gdrivers/ehdr.py index 6fb4b75b747f..89876bd50c5c 100755 --- a/autotest/gdrivers/ehdr.py +++ b/autotest/gdrivers/ehdr.py @@ -153,6 +153,10 @@ def test_ehdr_7(): # Test signed 8bit integer support. (#2717) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_ehdr_8(): drv = gdal.GetDriverByName("EHDR") diff --git a/autotest/gdrivers/ers.py b/autotest/gdrivers/ers.py index 701580704ac9..a879e585c3f2 100755 --- a/autotest/gdrivers/ers.py +++ b/autotest/gdrivers/ers.py @@ -140,6 +140,10 @@ def test_ers_7(): # Test GCP support +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_ers_8(): src_ds = gdal.Open("../gcore/data/gcps.vrt") diff --git a/autotest/gdrivers/gpkg.py b/autotest/gdrivers/gpkg.py index 304d0e0b9262..5cc63adbe464 100755 --- a/autotest/gdrivers/gpkg.py +++ b/autotest/gdrivers/gpkg.py @@ -850,6 +850,10 @@ def test_gpkg_10(): # Single band with 32 bit color table +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.parametrize("tile_drv_name", ["JPEG", "WEBP"]) def test_gpkg_11(tile_drv_name): diff --git a/autotest/gdrivers/grib.py b/autotest/gdrivers/grib.py index 66efaa6aef83..1089332651a4 100755 --- a/autotest/gdrivers/grib.py +++ b/autotest/gdrivers/grib.py @@ -1732,7 +1732,8 @@ def test_grib_grib2_write_data_encodings_warnings_and_errors(): tests += [["data/byte.tif", ["JPEG2000_DRIVER=DERIVED"]]] # Read-only driver tests += [["../gcore/data/cfloat32.tif", []]] # complex data type tests += [["data/aaigrid/float64.asc", []]] # no projection - tests += [["data/test_nosrs.vrt", []]] # no geotransform + if gdaltest.vrt_has_open_support(): + tests += [["data/test_nosrs.vrt", []]] # no geotransform tests += [["data/envi/rotation.img", []]] # geotransform with rotation terms gdal.GetDriverByName("GTiff").Create( "/vsimem/huge.tif", 65535, 65535, 1, options=["SPARSE_OK=YES"] diff --git a/autotest/gdrivers/gta.py b/autotest/gdrivers/gta.py index 556dec404f59..89a2423c830c 100755 --- a/autotest/gdrivers/gta.py +++ b/autotest/gdrivers/gta.py @@ -98,6 +98,10 @@ def test_gta_2(): # Test writing and readings GCPs +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gta_3(): src_ds = gdal.Open("../gcore/data/gcps.vrt") diff --git a/autotest/gdrivers/gti.py b/autotest/gdrivers/gti.py index 2bab88273269..a29f282736de 100755 --- a/autotest/gdrivers/gti.py +++ b/autotest/gdrivers/gti.py @@ -820,9 +820,9 @@ def test_gti_invalid_srs(tmp_vsimem): gdal.Open(index_filename) -def test_gti_valid_srs(tmp_vsimem): +def test_gti_valid_srs(tmp_path): - index_filename = str(tmp_vsimem / "index.gti.gpkg") + index_filename = str(tmp_path / "index.gti.gpkg") src_ds = gdal.Open(os.path.join(os.getcwd(), "data", "byte.tif")) index_ds, lyr = create_basic_tileindex(index_filename, src_ds) @@ -1019,6 +1019,10 @@ def test_gti_no_metadata_rgb(tmp_vsimem): check_basic(vrt_ds, src_ds) +@pytest.mark.skipif( + gdal.GetDriverByName("VRT").GetMetadataItem(gdal.DMD_OPENOPTIONLIST) is None, + reason="VRT driver open missing", +) def test_gti_rgb_left_right(tmp_vsimem): index_filename = str(tmp_vsimem / "index.gti.gpkg") @@ -1078,6 +1082,10 @@ def test_gti_rgb_left_right(tmp_vsimem): assert flags == gdal.GDAL_DATA_COVERAGE_STATUS_DATA and pct == 100.0 +@pytest.mark.skipif( + gdal.GetDriverByName("VRT").GetMetadataItem(gdal.DMD_OPENOPTIONLIST) is None, + reason="VRT driver open missing", +) def test_gti_overlapping_sources(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") @@ -1345,6 +1353,10 @@ def test_gti_overlapping_sources(tmp_vsimem): assert vrt_ds.GetRasterBand(1).Checksum() == 2, sort_values +@pytest.mark.skipif( + gdal.GetDriverByName("VRT").GetMetadataItem(gdal.DMD_OPENOPTIONLIST) is None, + reason="VRT driver open missing", +) def test_gti_gap_between_sources(tmp_vsimem): filename1 = str(tmp_vsimem / "one.tif") @@ -1380,6 +1392,10 @@ def test_gti_gap_between_sources(tmp_vsimem): ) +@pytest.mark.skipif( + gdal.GetDriverByName("VRT").GetMetadataItem(gdal.DMD_OPENOPTIONLIST) is None, + reason="VRT driver open missing", +) def test_gti_no_source(tmp_vsimem): index_filename = str(tmp_vsimem / "index.gti.gpkg") diff --git a/autotest/gdrivers/hfa.py b/autotest/gdrivers/hfa.py index e68e2236bbe8..bbdc7eb27324 100755 --- a/autotest/gdrivers/hfa.py +++ b/autotest/gdrivers/hfa.py @@ -666,6 +666,10 @@ def test_hfa_vsimem(): # the .img file. (#2422) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_hfa_proName(): drv = gdal.GetDriverByName("HFA") diff --git a/autotest/gdrivers/jp2openjpeg.py b/autotest/gdrivers/jp2openjpeg.py index cc64df8dabf1..16e14fdc5fb9 100755 --- a/autotest/gdrivers/jp2openjpeg.py +++ b/autotest/gdrivers/jp2openjpeg.py @@ -389,6 +389,10 @@ def test_jp2openjpeg_12(): # Check that PAM overrides internal GCPs (#5279) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_jp2openjpeg_13(): # Create a dataset with GCPs @@ -1446,6 +1450,10 @@ def test_jp2openjpeg_32(): # Test crazy tile size +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_jp2openjpeg_33(): src_ds = gdal.Open( @@ -3818,6 +3826,10 @@ def test_jp2openjpeg_mosaic(): @pytest.mark.require_curl() +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_jp2openjpeg_vrt_protocol(): (webserver_process, webserver_port) = webserver.launch( diff --git a/autotest/gdrivers/jpeg.py b/autotest/gdrivers/jpeg.py index 2dae55fe1e1d..fef684336e0f 100755 --- a/autotest/gdrivers/jpeg.py +++ b/autotest/gdrivers/jpeg.py @@ -800,6 +800,10 @@ def test_jpeg_mask_lsb_order_issue_4351(): # Test correct GCP reading with PAM (#5352) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_jpeg_20(): src_ds = gdal.Open("data/rgb_gcp.vrt") diff --git a/autotest/gdrivers/kmlsuperoverlay.py b/autotest/gdrivers/kmlsuperoverlay.py index 11cfb79e7fa1..8816100c0e6a 100755 --- a/autotest/gdrivers/kmlsuperoverlay.py +++ b/autotest/gdrivers/kmlsuperoverlay.py @@ -127,6 +127,10 @@ def test_kmlsuperoverlay_3(): # Test overviews +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_kmlsuperoverlay_4(): vrt_xml = """ @@ -231,6 +235,10 @@ def test_kmlsuperoverlay_4(): # Test that a raster which crosses the anti-meridian will be able to be displayed correctly (#4528) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_kmlsuperoverlay_5(): from xml.etree import ElementTree @@ -398,6 +406,10 @@ def test_kmlsuperoverlay_gx_latlonquad(): # KML/PNG files in transparent areas +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_kmlsuperoverlay_8(): # a large raster with actual data on each end and blank space in between diff --git a/autotest/gdrivers/mrf.py b/autotest/gdrivers/mrf.py index c46f39bff99f..f7a327d44156 100755 --- a/autotest/gdrivers/mrf.py +++ b/autotest/gdrivers/mrf.py @@ -203,6 +203,10 @@ def cleanup(base="/vsimem/out."): gdal.Unlink(base + ext) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_mrf_zen_test(): expectedCS = 770 diff --git a/autotest/gdrivers/netcdf.py b/autotest/gdrivers/netcdf.py index 56902f5367f1..83ab84e5f25c 100755 --- a/autotest/gdrivers/netcdf.py +++ b/autotest/gdrivers/netcdf.py @@ -1190,6 +1190,10 @@ def test_netcdf_28(tmp_path): # metadata to netcdf file with SetMetadata() and SetMetadataItem()). +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_netcdf_29(tmp_path): # create tif file using gdalwarp @@ -1451,6 +1455,10 @@ def test_netcdf_38(): # Test VRT and NETCDF: +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_netcdf_39(): shutil.copy("data/netcdf/two_vars_scale_offset.nc", "tmp") @@ -1482,6 +1490,10 @@ def test_netcdf_39(): assert cs == 65463 +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_netcdf_39_absolute(): if ( diff --git a/autotest/gdrivers/netcdf_multidim.py b/autotest/gdrivers/netcdf_multidim.py index 46c347a42d99..9714899a4df0 100755 --- a/autotest/gdrivers/netcdf_multidim.py +++ b/autotest/gdrivers/netcdf_multidim.py @@ -1528,6 +1528,10 @@ def copy(): gdal.Unlink(tmpfilename2) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_netcdf_multidim_dims_with_same_name_different_size(): src_ds = gdal.OpenEx( diff --git a/autotest/gdrivers/nitf.py b/autotest/gdrivers/nitf.py index 9ca57a6a3634..582b8622d094 100755 --- a/autotest/gdrivers/nitf.py +++ b/autotest/gdrivers/nitf.py @@ -1506,6 +1506,10 @@ def test_nitf_34(): # Test CreateCopy() writing file with a text segment. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_nitf_35(): src_ds = gdal.Open("data/nitf/text_md.vrt") @@ -1598,6 +1602,10 @@ def test_nitf_37(): # Create and read a NITF file with 999 images +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_nitf_38(): ds = gdal.Open("data/byte.tif") @@ -1978,6 +1986,10 @@ def test_nitf_check_jpeg2000_overviews(driver_to_test): # Check reading of rsets. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_nitf_47(): ds = gdal.Open("data/nitf/rset.ntf.r0") @@ -1997,6 +2009,10 @@ def test_nitf_47(): # Check building of standard overviews in place of rset overviews. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_nitf_48(): try: @@ -2038,6 +2054,10 @@ def test_nitf_48(): # Test TEXT and CGM creation options with CreateCopy() (#3376) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_nitf_49(): options = [ @@ -2759,6 +2779,10 @@ def test_nitf_68(): # Test SetGCPs() support +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_nitf_69(): vrt_txt = """ diff --git a/autotest/gdrivers/pdf.py b/autotest/gdrivers/pdf.py index 7db4786de764..c1c69f3c79d9 100755 --- a/autotest/gdrivers/pdf.py +++ b/autotest/gdrivers/pdf.py @@ -1007,11 +1007,19 @@ def _pdf_update_gcps(poppler_or_pdfium): gdaltest.pdf_drv.Delete(out_filename) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_pdf_update_gcps_iso32000(poppler_or_pdfium): gdal.SetConfigOption("GDAL_PDF_GEO_ENCODING", None) _pdf_update_gcps(poppler_or_pdfium) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_pdf_update_gcps_ogc_bp(poppler_or_pdfium): with gdal.config_option("GDAL_PDF_GEO_ENCODING", "OGC_BP"): _pdf_update_gcps(poppler_or_pdfium) @@ -1021,6 +1029,10 @@ def test_pdf_update_gcps_ogc_bp(poppler_or_pdfium): # Check SetGCPs() but with GCPs that do *not* resolve to a geotransform +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_pdf_set_5_gcps_ogc_bp(poppler_or_pdfium): dpi = 300 out_filename = "tmp/pdf_set_5_gcps_ogc_bp.pdf" @@ -1246,6 +1258,10 @@ def test_pdf_set_neatline_ogc_bp(poppler_or_pdfium): # Check that we can generate identical file +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_pdf_check_identity_iso32000(poppler_or_pdfium): out_filename = "tmp/pdf_check_identity_iso32000.pdf" @@ -1279,6 +1295,10 @@ def test_pdf_check_identity_iso32000(poppler_or_pdfium): # Check that we can generate identical file +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_pdf_check_identity_ogc_bp(poppler_or_pdfium): out_filename = "tmp/pdf_check_identity_ogc_bp.pdf" @@ -1453,6 +1473,10 @@ def test_pdf_custom_layout(poppler_or_pdfium): # Test CLIPPING_EXTENT, EXTRA_RASTERS, EXTRA_RASTERS_LAYER_NAME, OFF_LAYERS, EXCLUSIVE_LAYERS options +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_pdf_extra_rasters(poppler_or_pdfium): subbyte = """ PROJCS["NAD27 / UTM zone 11N",GEOGCS["NAD27",DATUM["North_American_Datum_1927",SPHEROID["Clarke 1866",6378206.4,294.9786982139006,AUTHORITY["EPSG","7008"]],AUTHORITY["EPSG","6267"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4267"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-117],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AUTHORITY["EPSG","26711"]] @@ -1719,6 +1743,10 @@ def test_pdf_jpeg_direct_copy(poppler_or_pdfium): # Test direct copy of source JPEG file within VRT file +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_driver("JPEG") def test_pdf_jpeg_in_vrt_direct_copy(poppler_or_pdfium): diff --git a/autotest/gdrivers/prf.py b/autotest/gdrivers/prf.py index 33e253f69879..e052d9d8b844 100755 --- a/autotest/gdrivers/prf.py +++ b/autotest/gdrivers/prf.py @@ -85,6 +85,10 @@ def test_prf_3(): ds = None +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_prf_4(): tst = gdaltest.GDALTest("prf", "./PRF/dem.x-dem", 1, 0) diff --git a/autotest/gdrivers/sigdem.py b/autotest/gdrivers/sigdem.py index 1670399e2c6c..28c2fb0664b8 100755 --- a/autotest/gdrivers/sigdem.py +++ b/autotest/gdrivers/sigdem.py @@ -31,6 +31,7 @@ import gdaltest +import pytest ############################################################################### # Create simple copy and check. @@ -49,6 +50,10 @@ def test_sigdem_copy_check_prj(): # Verify writing files with non-square pixels. +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_sigdem_non_square(): tst = gdaltest.GDALTest("SIGDEM", "sigdem/nonsquare_nad27_utm11.vrt", 1, 12481) diff --git a/autotest/gdrivers/test_validate_jp2.py b/autotest/gdrivers/test_validate_jp2.py index ad3d7e7c22cc..734d26b7aa65 100755 --- a/autotest/gdrivers/test_validate_jp2.py +++ b/autotest/gdrivers/test_validate_jp2.py @@ -114,6 +114,10 @@ def validate(filename, inspire_tg=True, expected_gmljp2=True, oidoc=None): # Highly corrupted file +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_validate_jp2_2(): import build_jp2_from_xml @@ -284,6 +288,10 @@ def test_validate_jp2_4(): # Also a RGN marker +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_validate_jp2_5(): import build_jp2_from_xml diff --git a/autotest/gdrivers/vrtderived.py b/autotest/gdrivers/vrtderived.py index b5fd42965881..296dfbe0916a 100755 --- a/autotest/gdrivers/vrtderived.py +++ b/autotest/gdrivers/vrtderived.py @@ -37,6 +37,10 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) ############################################################################### @pytest.fixture(autouse=True, scope="module") diff --git a/autotest/gdrivers/vrtfilt.py b/autotest/gdrivers/vrtfilt.py index 579a4003509f..932ae18ffe9e 100755 --- a/autotest/gdrivers/vrtfilt.py +++ b/autotest/gdrivers/vrtfilt.py @@ -34,6 +34,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + ############################################################################### # Verify simple 3x3 averaging filter. diff --git a/autotest/gdrivers/vrtlut.py b/autotest/gdrivers/vrtlut.py index 251e89ae21ff..94fb3fa87380 100755 --- a/autotest/gdrivers/vrtlut.py +++ b/autotest/gdrivers/vrtlut.py @@ -35,6 +35,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + ############################################################################### # Simple test diff --git a/autotest/gdrivers/vrtmask.py b/autotest/gdrivers/vrtmask.py index 13b3822176d9..b24e7915e073 100755 --- a/autotest/gdrivers/vrtmask.py +++ b/autotest/gdrivers/vrtmask.py @@ -36,6 +36,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + ############################################################################### # Test with a global dataset mask band diff --git a/autotest/gdrivers/vrtmultidim.py b/autotest/gdrivers/vrtmultidim.py index 1c701a6b0b54..e1dca6b93182 100755 --- a/autotest/gdrivers/vrtmultidim.py +++ b/autotest/gdrivers/vrtmultidim.py @@ -36,6 +36,10 @@ from osgeo import gdal, osr +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) ############################################################################### @pytest.fixture(autouse=True, scope="module") diff --git a/autotest/gdrivers/vrtovr.py b/autotest/gdrivers/vrtovr.py index da56dc867369..5e63a7397992 100755 --- a/autotest/gdrivers/vrtovr.py +++ b/autotest/gdrivers/vrtovr.py @@ -35,6 +35,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + ############################################################################### # Simple test diff --git a/autotest/gdrivers/vrtpansharpen.py b/autotest/gdrivers/vrtpansharpen.py index e3ba9af028d4..fb650476a0ce 100755 --- a/autotest/gdrivers/vrtpansharpen.py +++ b/autotest/gdrivers/vrtpansharpen.py @@ -36,6 +36,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + @pytest.fixture(autouse=True, scope="module") def startup_and_cleanup(): diff --git a/autotest/gdrivers/vrtprocesseddataset.py b/autotest/gdrivers/vrtprocesseddataset.py index b11483b16df0..76de54f650ce 100755 --- a/autotest/gdrivers/vrtprocesseddataset.py +++ b/autotest/gdrivers/vrtprocesseddataset.py @@ -31,6 +31,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + np = pytest.importorskip("numpy") pytest.importorskip("osgeo.gdal_array") diff --git a/autotest/gdrivers/vrtrawlink.py b/autotest/gdrivers/vrtrawlink.py index 1f4203fc2874..d2ad48e27530 100755 --- a/autotest/gdrivers/vrtrawlink.py +++ b/autotest/gdrivers/vrtrawlink.py @@ -37,6 +37,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + def _xmlsearch(root, nodetype, name): for node in root[2:]: diff --git a/autotest/gdrivers/vrtwarp.py b/autotest/gdrivers/vrtwarp.py index 9b843aa81342..0b412d0e10d1 100755 --- a/autotest/gdrivers/vrtwarp.py +++ b/autotest/gdrivers/vrtwarp.py @@ -38,6 +38,11 @@ from osgeo import gdal +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + ############################################################################### # Verify reading from simple existing warp definition. diff --git a/autotest/gdrivers/wms.py b/autotest/gdrivers/wms.py index 9e7ca4da5508..02c3f8921d9e 100755 --- a/autotest/gdrivers/wms.py +++ b/autotest/gdrivers/wms.py @@ -568,6 +568,10 @@ def test_wms_12(): # Test reading WMS through VRT (test effect of r21866) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @gdaltest.disable_exceptions() def test_wms_13(): diff --git a/autotest/pymod/gdaltest.py b/autotest/pymod/gdaltest.py index 1c1677a3164b..5bedd015acd4 100755 --- a/autotest/pymod/gdaltest.py +++ b/autotest/pymod/gdaltest.py @@ -2199,3 +2199,8 @@ def tell(self): def vsi_open(path, mode="r"): return VSIFile(path, mode) + + +def vrt_has_open_support(): + drv = gdal.GetDriverByName("VRT") + return drv is not None and drv.GetMetadataItem(gdal.DMD_OPENOPTIONLIST) is not None diff --git a/autotest/pyscripts/gdal2tiles/test_logger.py b/autotest/pyscripts/gdal2tiles/test_logger.py index de9cd087b1bd..97b6ebe2711c 100644 --- a/autotest/pyscripts/gdal2tiles/test_logger.py +++ b/autotest/pyscripts/gdal2tiles/test_logger.py @@ -29,11 +29,17 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### +import gdaltest import pytest from osgeo import gdal from osgeo_utils import gdal2tiles +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + def test_gdal2tiles_logger(): diff --git a/autotest/pyscripts/gdal2tiles/test_vsimem.py b/autotest/pyscripts/gdal2tiles/test_vsimem.py index 340e0238f7c4..deaff34fe74a 100644 --- a/autotest/pyscripts/gdal2tiles/test_vsimem.py +++ b/autotest/pyscripts/gdal2tiles/test_vsimem.py @@ -29,11 +29,17 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### +import gdaltest import pytest from osgeo import gdal from osgeo_utils import gdal2tiles +pytestmark = pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) + def test_gdal2tiles_vsimem(): diff --git a/autotest/pyscripts/test_gdal2tiles.py b/autotest/pyscripts/test_gdal2tiles.py index b3da97a4de72..a8c2159df649 100755 --- a/autotest/pyscripts/test_gdal2tiles.py +++ b/autotest/pyscripts/test_gdal2tiles.py @@ -43,10 +43,15 @@ from osgeo import gdal, osr # noqa from osgeo_utils.gdalcompare import compare_db -pytestmark = pytest.mark.skipif( - test_py_scripts.get_py_script("gdal2tiles") is None, - reason="gdal2tiles not available", -) +pytestmark = [ + pytest.mark.skipif( + test_py_scripts.get_py_script("gdal2tiles") is None, + reason="gdal2tiles not available", + ), + pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), reason="VRT driver open missing" + ), +] @pytest.fixture() diff --git a/autotest/pyscripts/test_gdal_pansharpen.py b/autotest/pyscripts/test_gdal_pansharpen.py index 160b68879b94..9883e9a14ad8 100755 --- a/autotest/pyscripts/test_gdal_pansharpen.py +++ b/autotest/pyscripts/test_gdal_pansharpen.py @@ -29,7 +29,7 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### - +import gdaltest import pytest import test_py_scripts @@ -95,6 +95,10 @@ def test_gdal_pansharpen_version(script_path): # Simple test +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_pansharpen_1(script_path, tmp_path, small_world_pan_tif): out_tif = str(tmp_path / "out.tif") @@ -120,6 +124,10 @@ def test_gdal_pansharpen_1(script_path, tmp_path, small_world_pan_tif): # Full options +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_pansharpen_2(script_path, tmp_path, small_world_pan_tif): out_vrt = str(tmp_path / "out.vrt") diff --git a/autotest/pyscripts/test_gdalattachpct.py b/autotest/pyscripts/test_gdalattachpct.py index d81b173eadc4..bc27b4d39bda 100755 --- a/autotest/pyscripts/test_gdalattachpct.py +++ b/autotest/pyscripts/test_gdalattachpct.py @@ -28,6 +28,7 @@ # DEALINGS IN THE SOFTWARE. ############################################################################### +import gdaltest import pytest import test_py_scripts @@ -91,6 +92,10 @@ def test_gdalattachpct_basic(script_path, tmp_path, palette_file): # Test outputting to VRT +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalattachpct_vrt_output(script_path, tmp_path, palette_file): src_filename = str(tmp_path / "src.tif") diff --git a/autotest/pyscripts/test_gdalbuildvrtofvrt.py b/autotest/pyscripts/test_gdalbuildvrtofvrt.py index 19630a21727a..19b8b51fd27c 100755 --- a/autotest/pyscripts/test_gdalbuildvrtofvrt.py +++ b/autotest/pyscripts/test_gdalbuildvrtofvrt.py @@ -30,6 +30,7 @@ import os +import gdaltest import pytest import test_py_scripts @@ -41,6 +42,9 @@ test_py_scripts.get_py_script("gdalbuildvrtofvrt") is None, reason="gdalbuildvrtofvrt.py not available", ), + pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), reason="VRT driver open missing" + ), ] diff --git a/autotest/pyscripts/test_gdalinfo_py.py b/autotest/pyscripts/test_gdalinfo_py.py index 235b40dbdc29..43bf7fe4e28e 100755 --- a/autotest/pyscripts/test_gdalinfo_py.py +++ b/autotest/pyscripts/test_gdalinfo_py.py @@ -31,6 +31,7 @@ import os import shutil +import gdaltest import pytest import test_py_scripts @@ -154,6 +155,10 @@ def test_gdalinfo_py_6(script_path): # Test a dataset with GCPs +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalinfo_py_7(script_path): ret = test_py_scripts.run_py_script( diff --git a/autotest/utilities/test_gdal_create.py b/autotest/utilities/test_gdal_create.py index 13c26b7fb58d..936ca76f9162 100755 --- a/autotest/utilities/test_gdal_create.py +++ b/autotest/utilities/test_gdal_create.py @@ -222,6 +222,10 @@ def test_gdal_create_input_file_overrrides(gdal_create_path, tmp_path): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_create_input_file_gcps(gdal_create_path, tmp_path): output_tif = str(tmp_path / "tmp.tif") diff --git a/autotest/utilities/test_gdal_translate.py b/autotest/utilities/test_gdal_translate.py index 446839a4055f..dad3024e4520 100755 --- a/autotest/utilities/test_gdal_translate.py +++ b/autotest/utilities/test_gdal_translate.py @@ -420,6 +420,10 @@ def test_gdal_translate_15(gdal_translate_path, tmp_path): # Test -of VRT which is a special case +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_translate_16(gdal_translate_path, tmp_path): dst_vrt = str(tmp_path / "test16.vrt") @@ -440,6 +444,10 @@ def test_gdal_translate_16(gdal_translate_path, tmp_path): # Test -expand option to VRT +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_driver("GIF") def test_gdal_translate_17(gdal_translate_path, tmp_path): @@ -483,6 +491,10 @@ def test_gdal_translate_17(gdal_translate_path, tmp_path): # Test translation of a VRT made of VRT +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_driver("BMP") def test_gdal_translate_18(gdal_translate_path, tmp_path): @@ -781,6 +793,10 @@ def test_gdal_translate_28(gdal_translate_path, tmp_path): # Test -r +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_translate_29(gdal_translate_path, tmp_path): dst_tif = str(tmp_path / "test_gdal_translate_29.tif") @@ -962,6 +978,10 @@ def test_gdal_translate_33ter(gdal_translate_path, tmp_path): # Test NBITS is preserved +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_translate_34(gdal_translate_path, tmp_path): dst_vrt = str(tmp_path / "test_gdal_translate_34.vrt") diff --git a/autotest/utilities/test_gdal_translate_lib.py b/autotest/utilities/test_gdal_translate_lib.py index 26aa3c21a438..edd421ff449d 100755 --- a/autotest/utilities/test_gdal_translate_lib.py +++ b/autotest/utilities/test_gdal_translate_lib.py @@ -597,6 +597,10 @@ def test_gdal_translate_lib_104(): # Test GCPs propagation in "VRT path" +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_translate_lib_gcp_vrt_path(): src_ds = gdal.Open("../gcore/data/gcps.vrt") @@ -613,6 +617,10 @@ def test_gdal_translate_lib_gcp_vrt_path(): # Test RPC propagation in "VRT path" +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_translate_lib_rcp_vrt_path(): src_ds = gdal.Open("../gcore/data/rpc.vrt") @@ -624,6 +632,10 @@ def test_gdal_translate_lib_rcp_vrt_path(): # Test GeoLocation propagation in "VRT path" +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdal_translate_lib_geolocation_vrt_path(tmp_vsimem): src_ds = gdal.Open("../gcore/data/sstgeo.vrt") diff --git a/autotest/utilities/test_gdaladdo.py b/autotest/utilities/test_gdaladdo.py index 29f23085d3a8..e68d90072d6f 100755 --- a/autotest/utilities/test_gdaladdo.py +++ b/autotest/utilities/test_gdaladdo.py @@ -54,6 +54,10 @@ def gdaladdo_path(): # Similar to tiff_ovr_1 +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdaladdo_1(gdaladdo_path, tmp_path): shutil.copy("../gcore/data/mfloat32.vrt", f"{tmp_path}/mfloat32.vrt") @@ -230,6 +234,10 @@ def test_gdaladdo_partial_refresh_from_projwin(gdaladdo_path, tmp_path): # Test --partial-refresh-from-source-timestamp +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdaladdo_partial_refresh_from_source_timestamp(gdaladdo_path, tmp_path): left_tif = str(tmp_path / "left.tif") @@ -284,6 +292,10 @@ def test_gdaladdo_partial_refresh_from_source_timestamp(gdaladdo_path, tmp_path) # Test --partial-refresh-from-source-extent +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdaladdo_partial_refresh_from_source_extent(gdaladdo_path, tmp_path): left_tif = str(tmp_path / "left.tif") @@ -330,6 +342,10 @@ def test_gdaladdo_partial_refresh_from_source_extent(gdaladdo_path, tmp_path): # Test reuse of previous resampling method and overview levels +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.parametrize("read_only", [True, False]) def test_gdaladdo_reuse_previous_resampling_and_levels( gdaladdo_path, tmp_path, read_only @@ -382,8 +398,13 @@ def test_gdaladdo_reuse_previous_resampling_and_levels( @pytest.mark.require_driver("GPKG") +@pytest.mark.require_driver("GTI") def test_gdaladdo_partial_refresh_from_source_timestamp_gti(gdaladdo_path, tmp_path): + gti_drv = gdal.GetDriverByName("GTI") + if gti_drv.GetMetadataItem("IS_PLUGIN"): + pytest.skip("Test skipped because GTI driver as a plugin") + left_tif = str(tmp_path / "left.tif") right_tif = str(tmp_path / "right.tif") diff --git a/autotest/utilities/test_gdalbuildvrt.py b/autotest/utilities/test_gdalbuildvrt.py index 27b7756ee074..2088c4626c6a 100755 --- a/autotest/utilities/test_gdalbuildvrt.py +++ b/autotest/utilities/test_gdalbuildvrt.py @@ -35,10 +35,16 @@ from osgeo import gdal, osr -pytestmark = pytest.mark.skipif( - test_cli_utilities.get_gdalbuildvrt_path() is None, - reason="gdalbuildvrt not available", -) +pytestmark = [ + pytest.mark.skipif( + test_cli_utilities.get_gdalbuildvrt_path() is None, + reason="gdalbuildvrt not available", + ), + pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", + ), +] @pytest.fixture(scope="module") diff --git a/autotest/utilities/test_gdalbuildvrt_lib.py b/autotest/utilities/test_gdalbuildvrt_lib.py index 74e41222ed6e..2e1c0e98911d 100755 --- a/autotest/utilities/test_gdalbuildvrt_lib.py +++ b/autotest/utilities/test_gdalbuildvrt_lib.py @@ -97,6 +97,10 @@ def test_gdalbuildvrt_lib_2(): # Test creating overviews +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalbuildvrt_lib_ovr(tmp_vsimem): tmpfilename = tmp_vsimem / "my.vrt" @@ -668,6 +672,12 @@ def test_gdalbuildvrt_lib_strict_mode(): ############################################################################### + + +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalbuildvrt_lib_te_touching_on_edge(tmp_vsimem): tmp_filename = tmp_vsimem / "test_gdalbuildvrt_lib_te_touching_on_edge.vrt" @@ -794,6 +804,10 @@ def test_gdalbuildvrt_lib_nodataMaxMaskThreshold_rgba(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalbuildvrt_lib_nodataMaxMaskThreshold_rgb_mask(tmp_vsimem): # UInt16, VRTNodata=0 diff --git a/autotest/utilities/test_gdaldem.py b/autotest/utilities/test_gdaldem.py index 8372008cd1b4..b42c17bff5c1 100755 --- a/autotest/utilities/test_gdaldem.py +++ b/autotest/utilities/test_gdaldem.py @@ -381,6 +381,10 @@ def test_gdaldem_color_relief_cpt(gdaldem_path, tmp_path): # Test gdaldem color relief to VRT +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdaldem_color_relief_vrt(gdaldem_path, n43_colorrelief_tif, tmp_path): output_vrt = str(tmp_path / "n43_colorrelief.vrt") @@ -518,6 +522,10 @@ def test_gdaldem_color_relief_nearest_color_entry(gdaldem_path, tmp_path): # Test gdaldem color relief with -nearest_color_entry and -of VRT +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdaldem_color_relief_nearest_color_entry_vrt(gdaldem_path, tmp_path): output_vrt = str(tmp_path / "n43_colorrelief_nearest.vrt") @@ -584,6 +592,10 @@ def test_gdaldem_color_relief_nodata_nan(gdaldem_path, tmp_path): # Test gdaldem color relief with entries with repeated DEM values in the color table (#6422) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_driver("AAIGRID") def test_gdaldem_color_relief_repeated_entry(gdaldem_path, tmp_path): diff --git a/autotest/utilities/test_gdaldem_lib.py b/autotest/utilities/test_gdaldem_lib.py index af3170465c1e..76a724691d3b 100755 --- a/autotest/utilities/test_gdaldem_lib.py +++ b/autotest/utilities/test_gdaldem_lib.py @@ -526,6 +526,10 @@ def test_gdaldem_lib_color_relief_nodata_value(tmp_vsimem): gdal.Unlink(colorFilename) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.parametrize( "colorSelection", ["nearest_color_entry", "exact_color_entry", "linear_interpolation"], @@ -563,6 +567,10 @@ def test_gdaldem_lib_color_relief_synthetic(tmp_path, colorSelection, format): assert struct.unpack("B" * 4, ds.GetRasterBand(3).ReadRaster()) == (0, 12, 22, 32) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.parametrize( "colorSelection", ["nearest_color_entry", "exact_color_entry", "linear_interpolation"], diff --git a/autotest/utilities/test_gdalinfo.py b/autotest/utilities/test_gdalinfo.py index c3857c716b0f..815be240a395 100755 --- a/autotest/utilities/test_gdalinfo.py +++ b/autotest/utilities/test_gdalinfo.py @@ -135,6 +135,10 @@ def test_gdalinfo_6(gdalinfo_path): # Test a dataset with GCPs +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalinfo_7(gdalinfo_path): ret = gdaltest.runexternal( @@ -549,6 +553,10 @@ def test_gdalinfo_33(gdalinfo_path): # Test a dataset with GCPs +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalinfo_34(gdalinfo_path): ret = gdaltest.runexternal(gdalinfo_path + " -json ../gcore/data/gcps.vrt") diff --git a/autotest/utilities/test_gdallocationinfo.py b/autotest/utilities/test_gdallocationinfo.py index 575fa42c1c85..e17526f0147f 100755 --- a/autotest/utilities/test_gdallocationinfo.py +++ b/autotest/utilities/test_gdallocationinfo.py @@ -122,6 +122,10 @@ def test_gdallocationinfo_4(gdallocationinfo_path): # Test -lifonly +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdallocationinfo_5(gdallocationinfo_path): ret = gdaltest.runexternal( diff --git a/autotest/utilities/test_gdalmdiminfo.py b/autotest/utilities/test_gdalmdiminfo.py index 9977457513c5..11fa550adb68 100755 --- a/autotest/utilities/test_gdalmdiminfo.py +++ b/autotest/utilities/test_gdalmdiminfo.py @@ -48,6 +48,10 @@ def gdalmdiminfo_path(): # Simple test +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdiminfo_1(gdalmdiminfo_path): (ret, err) = gdaltest.runexternal_out_and_err(gdalmdiminfo_path + " data/mdim.vrt") @@ -59,6 +63,10 @@ def test_gdalmdiminfo_1(gdalmdiminfo_path): # Test -if option +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdiminfo_if_option(gdalmdiminfo_path): (ret, err) = gdaltest.runexternal_out_and_err( diff --git a/autotest/utilities/test_gdalmdimtranslate.py b/autotest/utilities/test_gdalmdimtranslate.py index a6cf1cec7529..7b34e3282149 100755 --- a/autotest/utilities/test_gdalmdimtranslate.py +++ b/autotest/utilities/test_gdalmdimtranslate.py @@ -50,6 +50,10 @@ def gdalmdimtranslate_path(): # Simple test +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_1(gdalmdimtranslate_path, tmp_path): dst_vrt = str(tmp_path / "out.vrt") @@ -65,6 +69,10 @@ def test_gdalmdimtranslate_1(gdalmdimtranslate_path, tmp_path): # Test -if option +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_if(gdalmdimtranslate_path, tmp_path): dst_vrt = str(tmp_path / "out.vrt") diff --git a/autotest/utilities/test_gdalmdimtranslate_lib.py b/autotest/utilities/test_gdalmdimtranslate_lib.py index 333f3a74e8d7..7f17d06d9fae 100755 --- a/autotest/utilities/test_gdalmdimtranslate_lib.py +++ b/autotest/utilities/test_gdalmdimtranslate_lib.py @@ -42,6 +42,10 @@ ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_no_arg(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -53,6 +57,10 @@ def test_gdalmdimtranslate_no_arg(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_multidim_to_mem(): out_ds = gdal.MultiDimTranslate("", "data/mdim.vrt", format="MEM") @@ -67,6 +75,10 @@ def test_gdalmdimtranslate_multidim_to_mem(): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_multidim_to_classic(tmp_vsimem): tmpfile = tmp_vsimem / "out.tif" @@ -84,6 +96,10 @@ def test_gdalmdimtranslate_multidim_to_classic(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_multidim_1d_to_classic(tmp_vsimem): tmpfile = tmp_vsimem / "out.tif" @@ -112,6 +128,10 @@ def test_gdalmdimtranslate_classic_to_classic(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_classic_to_multidim(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -171,6 +191,10 @@ def test_gdalmdimtranslate_classic_to_multidim(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_array(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -252,6 +276,10 @@ def test_gdalmdimtranslate_array(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_array_with_transpose_and_view(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -327,6 +355,10 @@ def test_gdalmdimtranslate_array_with_transpose_and_view(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_group(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -392,6 +424,10 @@ def test_gdalmdimtranslate_group(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_two_groups(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -462,6 +498,10 @@ def test_gdalmdimtranslate_two_groups(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_subset(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -717,6 +757,10 @@ def test_gdalmdimtranslate_subset(tmp_vsimem): ############################################################################### +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_scaleaxes(tmp_vsimem): tmpfile = tmp_vsimem / "out.vrt" @@ -791,6 +835,10 @@ def test_gdalmdimtranslate_scaleaxes(tmp_vsimem): ) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalmdimtranslate_dims_with_same_name_different_size(tmp_vsimem): srcfile = tmp_vsimem / "in.vrt" diff --git a/autotest/utilities/test_gdalwarp.py b/autotest/utilities/test_gdalwarp.py index ee749fed657d..0df412e936ba 100755 --- a/autotest/utilities/test_gdalwarp.py +++ b/autotest/utilities/test_gdalwarp.py @@ -383,6 +383,10 @@ def test_gdalwarp_14(gdalwarp_path, testgdalwarp_gcp_tif, tmp_path): # Test -of VRT which is a special case +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalwarp_16(gdalwarp_path, testgdalwarp_gcp_tif, tmp_path): dst_vrt = str(tmp_path / "testgdalwarp16.vrt") @@ -463,6 +467,10 @@ def test_gdalwarp_19(gdalwarp_path, testgdalwarp_gcp_tif, tmp_path): # Test -of VRT -et 0 which is a special case +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalwarp_20(gdalwarp_path, testgdalwarp_gcp_tif, tmp_path): dst_vrt = str(tmp_path / "testgdalwarp20.vrt") @@ -1047,6 +1055,10 @@ def test_gdalwarp_39(gdalwarp_path, tmp_path): # Test -ovr +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalwarp_40(gdalwarp_path, tmp_path): src_tif = str(tmp_path / "test_gdalwarp_40_src.tif") diff --git a/autotest/utilities/test_gdalwarp_lib.py b/autotest/utilities/test_gdalwarp_lib.py index 2d55e07c212a..6f0a63ca03af 100755 --- a/autotest/utilities/test_gdalwarp_lib.py +++ b/autotest/utilities/test_gdalwarp_lib.py @@ -1599,6 +1599,10 @@ def test_gdalwarp_lib_128(tmp_vsimem): # to an invalid geometry (#6375) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_geos def test_gdalwarp_lib_129(tmp_vsimem): @@ -2069,6 +2073,10 @@ def test_gdalwarp_lib_135h(gdalwarp_135_grid_gtx, gdalwarp_135_grid2_gtx): assert data == pytest.approx(115 / (1200.0 / 3937)), "Bad value" +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) @pytest.mark.require_driver("GTX") def test_gdalwarp_lib_135i( gdalwarp_135_src_ds, gdalwarp_135_grid_gtx, gdalwarp_135_grid2_gtx, tmp_path @@ -3324,6 +3332,10 @@ def test_gdalwarp_lib_src_nodata_with_dstalpha(): # Test warping from a dataset with points outside of Earth (fixes #4934) +@pytest.mark.skipif( + not gdaltest.vrt_has_open_support(), + reason="VRT driver open missing", +) def test_gdalwarp_lib_src_points_outside_of_earth(): class MyHandler: def __init__(self): diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt index 875eaade5385..d89266d00376 100644 --- a/frmts/CMakeLists.txt +++ b/frmts/CMakeLists.txt @@ -39,10 +39,27 @@ gdal_optional_format(raw "Raw formats:EOSAT FAST Format, FARSITE LCP and Vexcel gdal_format(gtiff "GeoTIFF image format") set_package_properties(TIFF PROPERTIES PURPOSE "gdal_GTIFF: GeoTIFF image format") gdal_format(mem "Read/write data in Memory") -gdal_format(vrt "Virtual GDAL Datasets") + +# Exception to the rule: enable the VRT driver by default, even if +# GDAL_BUILD_OPTIONAL_DRIVERS=OFF. +if (NOT DEFINED GDAL_ENABLE_DRIVER_VRT AND + DEFINED GDAL_BUILD_OPTIONAL_DRIVERS AND + NOT GDAL_BUILD_OPTIONAL_DRIVERS) + message(WARNING "Enabling GDAL_ENABLE_DRIVER_VRT=ON, despite GDAL_BUILD_OPTIONAL_DRIVERS=OFF. You can of course override this choice by setting GDAL_ENABLE_DRIVER_VRT=OFF") + option(GDAL_ENABLE_DRIVER_VRT "Set ON to build Virtual GDAL Datasets" ON) +endif() + +gdal_optional_format(vrt "Virtual GDAL Datasets") +if (NOT GDAL_ENABLE_DRIVER_VRT) + # Even if we don't enable the driver (Open method), we still need to compile + # Most of it + add_subdirectory(vrt) +endif() # Note: derived is derived of vrt add_subdirectory(derived) +gdal_optional_format(gti "GDAL Tile Index") + # default formats if (CMAKE_BUILD_TYPE MATCHES "Debug" OR GDAL_ENABLE_DRIVER_NULL) diff --git a/frmts/gdalallregister.cpp b/frmts/gdalallregister.cpp index 376a5fc21199..ee30b082036b 100644 --- a/frmts/gdalallregister.cpp +++ b/frmts/gdalallregister.cpp @@ -329,10 +329,13 @@ void CPL_STDCALL GDALAllRegister() #ifdef FRMT_vrt GDALRegister_VRT(); - GDALRegister_GTI(); GDALRegister_Derived(); #endif +#ifdef FRMT_gti + GDALRegister_GTI(); +#endif + #ifdef FRMT_gtiff GDALRegister_GTiff(); GDALRegister_COG(); diff --git a/frmts/gti/CMakeLists.txt b/frmts/gti/CMakeLists.txt new file mode 100644 index 000000000000..91bc928ea953 --- /dev/null +++ b/frmts/gti/CMakeLists.txt @@ -0,0 +1,26 @@ +add_gdal_driver( + TARGET gdal_GTI + PLUGIN_CAPABLE + NO_DEPS + SOURCES gdaltileindexdataset.cpp + STRONG_CXX_WFLAGS) +gdal_standard_includes(gdal_GTI) +target_include_directories(gdal_GTI PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/vrt + $) + +set(GDAL_DATA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/data/gdaltileindex.xsd +) +set_property( + TARGET ${GDAL_LIB_TARGET_NAME} + APPEND + PROPERTY RESOURCE "${GDAL_DATA_FILES}") + +if (GDAL_USE_GEOS) + gdal_target_link_libraries(gdal_GTI PRIVATE ${GEOS_TARGET}) + target_compile_definitions(gdal_GTI PRIVATE -DHAVE_GEOS=1) +endif () + +if (GDAL_ENABLE_DRIVER_GTI_PLUGIN) + target_compile_definitions(gdal_GTI PRIVATE -DBUILT_AS_PLUGIN) +endif() diff --git a/frmts/vrt/data/gdaltileindex.xsd b/frmts/gti/data/gdaltileindex.xsd similarity index 100% rename from frmts/vrt/data/gdaltileindex.xsd rename to frmts/gti/data/gdaltileindex.xsd diff --git a/frmts/vrt/gdaltileindexdataset.cpp b/frmts/gti/gdaltileindexdataset.cpp similarity index 99% rename from frmts/vrt/gdaltileindexdataset.cpp rename to frmts/gti/gdaltileindexdataset.cpp index 388f45136164..d7d8548d3a07 100644 --- a/frmts/vrt/gdaltileindexdataset.cpp +++ b/frmts/gti/gdaltileindexdataset.cpp @@ -4626,7 +4626,7 @@ void GDALRegister_GTI() if (GDALGetDriverByName("GTI") != nullptr) return; - auto poDriver = std::make_unique(); + auto poDriver = std::make_unique(); poDriver->SetDescription("GTI"); poDriver->SetMetadataItem(GDAL_DCAP_RASTER, "YES"); @@ -4659,7 +4659,10 @@ void GDALRegister_GTI() "default='ALL_CPUS'/>" ""); +#ifdef BUILT_AS_PLUGIN + // Used by gdaladdo and test_gdaladdo.py + poDriver->SetMetadataItem("IS_PLUGIN", "YES"); +#endif + GetGDALDriverManager()->RegisterDriver(poDriver.release()); } - -/*! @endcond */ diff --git a/frmts/vrt/CMakeLists.txt b/frmts/vrt/CMakeLists.txt index 777c49af525a..8e423afd6d3a 100644 --- a/frmts/vrt/CMakeLists.txt +++ b/frmts/vrt/CMakeLists.txt @@ -17,7 +17,6 @@ add_gdal_driver( vrtprocesseddataset.cpp vrtprocesseddatasetfunctions.cpp vrtmultidim.cpp - gdaltileindexdataset.cpp STRONG_CXX_WFLAGS) gdal_standard_includes(gdal_vrt) target_include_directories(gdal_vrt PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/raw @@ -25,7 +24,6 @@ target_include_directories(gdal_vrt PRIVATE ${GDAL_RASTER_FORMAT_SOURCE_DIR}/raw set(GDAL_DATA_FILES ${CMAKE_CURRENT_SOURCE_DIR}/data/gdalvrt.xsd - ${CMAKE_CURRENT_SOURCE_DIR}/data/gdaltileindex.xsd ) set_property( TARGET ${GDAL_LIB_TARGET_NAME} @@ -42,3 +40,7 @@ target_public_header(TARGET gdal HEADERS vrtdataset.h gdal_vrt.h) # because of use of GDALOpenVerticalShiftGrid set_property(SOURCE vrtwarped.cpp PROPERTY SKIP_UNITY_BUILD_INCLUSION ON) + +if (NOT GDAL_ENABLE_DRIVER_VRT) + target_compile_definitions(gdal_vrt PRIVATE -DNO_OPEN) +endif() diff --git a/frmts/vrt/vrt_priv.h b/frmts/vrt/vrt_priv.h index ca2af63383fb..ba53e815ad5d 100644 --- a/frmts/vrt/vrt_priv.h +++ b/frmts/vrt/vrt_priv.h @@ -52,10 +52,10 @@ GDALTileIndexDataset CPL_DLL *GDALDatasetCastToGTIDataset(GDALDataset *poDS); std::vector CPL_DLL GTIGetSourcesMoreRecentThan(GDALTileIndexDataset *poDS, int64_t mTime); -CPLStringList VRTParseCategoryNames(const CPLXMLNode *psCategoryNames); +CPLStringList CPL_DLL VRTParseCategoryNames(const CPLXMLNode *psCategoryNames); std::unique_ptr -VRTParseColorTable(const CPLXMLNode *psColorTable); + CPL_DLL VRTParseColorTable(const CPLXMLNode *psColorTable); #endif diff --git a/frmts/vrt/vrtdataset.h b/frmts/vrt/vrtdataset.h index 6368175e9a3f..f420dae3046a 100644 --- a/frmts/vrt/vrtdataset.h +++ b/frmts/vrt/vrtdataset.h @@ -51,8 +51,8 @@ CPLErr GDALRegisterDefaultPixelFunc(); void GDALVRTRegisterDefaultProcessedDatasetFuncs(); -CPLString VRTSerializeNoData(double dfVal, GDALDataType eDataType, - int nPrecision); +CPLString CPL_DLL VRTSerializeNoData(double dfVal, GDALDataType eDataType, + int nPrecision); #if 0 int VRTWarpedOverviewTransform( void *pTransformArg, int bDstToSrc, diff --git a/frmts/vrt/vrtdriver.cpp b/frmts/vrt/vrtdriver.cpp index 1152f22c25db..7f4b60e86c63 100644 --- a/frmts/vrt/vrtdriver.cpp +++ b/frmts/vrt/vrtdriver.cpp @@ -542,10 +542,12 @@ void GDALRegister_VRT() " " + osCreationOptions += " "; + +#if !defined(JPEG_SUPPORTED) + if (bHasJPEG2000Drivers) +#endif + { + osCreationOptions += + "