From b3badbce6c57b906a0218fc8bf7874fd5c05a359 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 08:06:10 -0600 Subject: [PATCH 1/8] Update deps and fix coord order --- Dockerfile | 4 ++-- openaddr/conform.py | 5 +++++ setup.py | 2 -- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b188e15..e8766a8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.11 +FROM alpine:3.13 RUN apk add nodejs yarn git python3 python3-dev py3-pip \ py3-gdal gdal gdal-dev make bash sqlite-dev zlib-dev \ @@ -6,7 +6,7 @@ RUN apk add nodejs yarn git python3 python3-dev py3-pip \ py3-cairo file # Download and install Tippecanoe -RUN git clone -b 1.35.0 https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe && \ +RUN git clone -b 1.36.0 https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe && \ cd /tmp/tippecanoe && \ make && \ PREFIX=/usr/local make install && \ diff --git a/openaddr/conform.py b/openaddr/conform.py index 01db7b39..46d7854c 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -873,6 +873,8 @@ def _transform_to_4326(srs): in_spatial_ref.ImportFromEPSG(epsg_id) out_spatial_ref = osr.SpatialReference() out_spatial_ref.ImportFromEPSG(4326) + out_spatial_ref.SetAxisMappingStrategy(osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER) + _transform_cache[srs] = osr.CoordinateTransformation(in_spatial_ref, out_spatial_ref) return _transform_cache[srs] @@ -937,6 +939,7 @@ def row_extract_and_reproject(source_config, source_row): out_row[GEOM_FIELDNAME] = None return out_row + # Reproject the coordinates if necessary if "srs" in data_source["conform"]: try: @@ -945,6 +948,8 @@ def row_extract_and_reproject(source_config, source_row): point.Transform(_transform_to_4326(srs)) source_geom = point.ExportToWkt() + print(source_geom) + print('----') except (TypeError, ValueError) as e: if not (source_x == "" or source_y == ""): _L.debug("Could not reproject %s %s in SRS %s", source_x, source_y, srs) diff --git a/setup.py b/setup.py index 02419ca1..9aa984d9 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,6 @@ # Used in openaddr.parcels 'Shapely == 1.7.1', - 'Fiona == 1.8.17', # Used in dotmaps preview to support S3-backed SQLite mbtiles # https://rogerbinns.github.io/apsw/ @@ -77,6 +76,5 @@ 'mapbox-vector-tile==1.2.0', 'future==0.16.0', 'protobuf==3.5.1', - 'pyclipper==1.1.0' ] ) From 1444d50b6b13433c1163a3f5b46dfc6b06ea9991 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 08:33:32 -0600 Subject: [PATCH 2/8] alpine:3.11 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e8766a8e..55e67e28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.13 +FROM alpine:3.11 RUN apk add nodejs yarn git python3 python3-dev py3-pip \ py3-gdal gdal gdal-dev make bash sqlite-dev zlib-dev \ From 4e824c89474a9baca85ecd74fc3d96cb8c9ccf80 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 09:24:13 -0600 Subject: [PATCH 3/8] Remove debug statements --- openaddr/conform.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 46d7854c..e5aa3744 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -948,8 +948,6 @@ def row_extract_and_reproject(source_config, source_row): point.Transform(_transform_to_4326(srs)) source_geom = point.ExportToWkt() - print(source_geom) - print('----') except (TypeError, ValueError) as e: if not (source_x == "" or source_y == ""): _L.debug("Could not reproject %s %s in SRS %s", source_x, source_y, srs) From a91bdb4e3b95cb0ebdae4c3acae1a4f030fee445 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 10:22:10 -0600 Subject: [PATCH 4/8] Update Japan Fixtures --- openaddr/tests/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 2d657c54..1346cde9 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -901,9 +901,9 @@ def test_single_jp_fukushima2(self): self.assertEqual(rows[1]['STREET'], u'田沢字躑躅ケ森') self.assertEqual(rows[2]['NUMBER'], u'22-9') self.assertEqual(rows[2]['STREET'], u'小田字正夫田') - self.assertEqual(rows[0]['GEOM'], 'POINT (140.480007 37.706391)') - self.assertEqual(rows[1]['GEOM'], 'POINT (140.486267 37.707664)') - self.assertEqual(rows[2]['GEOM'], 'POINT (140.41875 37.710239)') + self.assertEqual(rows[0]['GEOM'], 'POINT (37.706391 140.480007)') + self.assertEqual(rows[1]['GEOM'], 'POINT (37.707664 140.486267)') + self.assertEqual(rows[2]['GEOM'], 'POINT (37.710239 140.41875)') def test_single_utah(self): ''' Test complete process_one.process on data that uses file selection with mixed case (issue #104) From 1cf3d7a1a4c8e1001aa538d0d5bae9e6a91f600f Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 11:52:30 -0600 Subject: [PATCH 5/8] Update test fixtures --- openaddr/conform.py | 3 +++ openaddr/tests/conform.py | 2 +- openaddr/tests/conforms/jp-nara.json | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index e5aa3744..ef65fc10 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -869,10 +869,13 @@ def _transform_to_4326(srs): if srs not in _transform_cache: epsg_id = int(srs[5:]) if srs.startswith("EPSG:") else int(srs) # Manufacture a transform object if it's not in the cache + in_spatial_ref = osr.SpatialReference() in_spatial_ref.ImportFromEPSG(epsg_id) out_spatial_ref = osr.SpatialReference() out_spatial_ref.ImportFromEPSG(4326) + + # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546 out_spatial_ref.SetAxisMappingStrategy(osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER) _transform_cache[srs] = osr.CoordinateTransformation(in_spatial_ref, out_spatial_ref) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index d53973fe..721835d2 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -1840,7 +1840,7 @@ def test_lake_man_3740(self): self.assertEqual(0, rc) with open(dest_path) as fp: rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['GEOM'], 'POINT (37.8026123 -122.2592495)') + self.assertEqual(rows[0]['GEOM'], 'POINT (-122.2592627 37.8026173)') self.assertEqual(rows[0]['NUMBER'], '5') self.assertEqual(rows[0]['STREET'], u'PZ ESPA\u00d1A') diff --git a/openaddr/tests/conforms/jp-nara.json b/openaddr/tests/conforms/jp-nara.json index 88c88332..88326755 100644 --- a/openaddr/tests/conforms/jp-nara.json +++ b/openaddr/tests/conforms/jp-nara.json @@ -15,7 +15,7 @@ }, "conform": { "format": "csv", - "lat": "\u7def\u5ea6", + "lon": "\u7def\u5ea6", "srs": "EPSG:4612", "number": "auto_number", "number": { @@ -27,7 +27,7 @@ "separator": "-" }, "street": "\u5927\u5b57\u30fb\u753a\u4e01\u76ee\u540d", - "lon": "\u7d4c\u5ea6", + "lat": "\u7d4c\u5ea6", "encoding": "SHIFT_JISX0213" }, "license": "http://nlftp.mlit.go.jp/ksj/other/yakkan.html" From 98fb3688f832360062158c45de6f522909f20c56 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 12:05:08 -0600 Subject: [PATCH 6/8] More test updates --- openaddr/tests/conform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 721835d2..eac489ac 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -550,7 +550,7 @@ def test_row_extract_and_reproject(self): }), "addresses", "default") r = row_extract_and_reproject(d, {GEOM_FIELDNAME: "POINT (7655634.924 668868.414)"}) - self.assertEqual('POINT (45.4815543938511 -122.630842186651)', r[GEOM_FIELDNAME]) + self.assertEqual('POINT (-122.630856593698 45.4815599174473)', r[GEOM_FIELDNAME]) d = SourceConfig(dict({ "schema": 2, @@ -1840,7 +1840,7 @@ def test_lake_man_3740(self): self.assertEqual(0, rc) with open(dest_path) as fp: rows = list(csv.DictReader(fp)) - self.assertEqual(rows[0]['GEOM'], 'POINT (-122.2592627 37.8026173)') + self.assertEqual(rows[0]['GEOM'], 'POINT (-122.2592495 37.8026123)') self.assertEqual(rows[0]['NUMBER'], '5') self.assertEqual(rows[0]['STREET'], u'PZ ESPA\u00d1A') @@ -2212,7 +2212,7 @@ def test_srs(self): u'3203,SE WOODSTOCK BLVD,7655634.924,668868.414'.encode('ascii')) r = self._convert(c, d) self.assertEqual(r[0], u'n,s,{GEOM_FIELDNAME}'.format(**globals())) - self.assertEqual(r[1], u'3203,SE WOODSTOCK BLVD,POINT (45.4815543938511 -122.630842186651)') + self.assertEqual(r[1], u'3203,SE WOODSTOCK BLVD,POINT (-122.630856593698 45.4815599174473)') def test_too_many_columns(self): "Check that we don't barf on input with too many columns in some rows" From 9bab049daf54b42c88746daea4f2625d1a7389b0 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 18:55:21 -0600 Subject: [PATCH 7/8] Update tests --- openaddr/tests/conform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index eac489ac..625e39ba 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -550,7 +550,7 @@ def test_row_extract_and_reproject(self): }), "addresses", "default") r = row_extract_and_reproject(d, {GEOM_FIELDNAME: "POINT (7655634.924 668868.414)"}) - self.assertEqual('POINT (-122.630856593698 45.4815599174473)', r[GEOM_FIELDNAME]) + self.assertEqual('POINT (-122.630842186651 45.4815599174473)', r[GEOM_FIELDNAME]) d = SourceConfig(dict({ "schema": 2, @@ -2212,7 +2212,7 @@ def test_srs(self): u'3203,SE WOODSTOCK BLVD,7655634.924,668868.414'.encode('ascii')) r = self._convert(c, d) self.assertEqual(r[0], u'n,s,{GEOM_FIELDNAME}'.format(**globals())) - self.assertEqual(r[1], u'3203,SE WOODSTOCK BLVD,POINT (-122.630856593698 45.4815599174473)') + self.assertEqual(r[1], u'3203,SE WOODSTOCK BLVD,POINT (-122.630842186651 45.4815543938511)') def test_too_many_columns(self): "Check that we don't barf on input with too many columns in some rows" From 8110d07c00b17a6b8cfc6cf4bbc80166850301c6 Mon Sep 17 00:00:00 2001 From: ingalls Date: Tue, 15 Jun 2021 19:02:38 -0600 Subject: [PATCH 8/8] Update tests --- openaddr/tests/conform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 625e39ba..0940f620 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -550,7 +550,7 @@ def test_row_extract_and_reproject(self): }), "addresses", "default") r = row_extract_and_reproject(d, {GEOM_FIELDNAME: "POINT (7655634.924 668868.414)"}) - self.assertEqual('POINT (-122.630842186651 45.4815599174473)', r[GEOM_FIELDNAME]) + self.assertEqual('POINT (-122.630842186651 45.4815543938511)', r[GEOM_FIELDNAME]) d = SourceConfig(dict({ "schema": 2,