From bfc0c87dda65fd0950a685730480c666556cbb85 Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Fri, 22 Jan 2021 09:58:24 -0500 Subject: [PATCH 01/10] fix readme links --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 41b7f86..258aea0 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Seabed -[![tests](https://github.com/noaa-ocs-modeling/seabed/workflows/tests/badge.svg)](https://github.com/noaa-ocs-modeling/NEMSpy/actions?query=workflow%3Atests) -[![build](https://github.com/noaa-ocs-modeling/seabed/workflows/build/badge.svg)](https://github.com/noaa-ocs-modeling/NEMSpy/actions?query=workflow%3Abuild) -[![version](https://img.shields.io/pypi/v/seabed)](https://pypi.org/project/nemspy) +[![tests](https://github.com/noaa-ocs-modeling/seabed/workflows/tests/badge.svg)](https://github.com/noaa-ocs-modeling/seabed/actions?query=workflow%3Atests) +[![build](https://github.com/noaa-ocs-modeling/seabed/workflows/build/badge.svg)](https://github.com/noaa-ocs-modeling/seabed/actions?query=workflow%3Abuild) +[![version](https://img.shields.io/pypi/v/seabed)](https://pypi.org/project/seabed) [![license](https://img.shields.io/github/license/noaa-ocs-modeling/seabed)](https://creativecommons.org/share-your-work/public-domain/cc0) [![style](https://sourceforge.net/p/oitnb/code/ci/default/tree/_doc/_static/oitnb.svg?format=raw)](https://sourceforge.net/p/oitnb/code) From c2f8a28e4643f1757095a9e1a3f1e0f6ee5fbe5f Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 26 Jan 2021 15:10:13 -0500 Subject: [PATCH 02/10] workflow updates --- .github/workflows/build.yml | 16 +++++++--------- .github/workflows/tests.yml | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3f1cec..3e38caf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,12 +7,12 @@ on: jobs: build_wheels: - name: build wheel on ${{ matrix.os }} + name: Python ${{ matrix.python-version }} wheel on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ ubuntu-latest ] - python-version: [ 3.8 ] + python-version: [ 3.9 ] steps: - name: Checkout repository uses: actions/checkout@v2 @@ -24,8 +24,8 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('setup.py') }} - restore-keys: ${{ matrix.os }}-pip-${{ matrix.python-version }}- + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('setup.py') }} + restore-keys: ${{ runner.os }}-pip-${{ matrix.python-version }}- - name: Install dependencies run: pip install --upgrade pip setuptools wheel - name: Build wheel @@ -42,8 +42,6 @@ jobs: uses: actions/checkout@v2 - name: Install Python uses: actions/setup-python@v2 - with: - python-version: '3.8' - name: Package source run: python setup.py sdist - name: Save source package @@ -51,16 +49,16 @@ jobs: with: path: ./dist/*.tar.gz upload_pypi: - name: publish wheel to PyPI + name: publish to PyPI needs: [ build_wheels, build_sdist ] runs-on: ubuntu-latest steps: - - name: Retrieve wheel + - name: Retrieve wheel(s) and source uses: actions/download-artifact@v2 with: name: artifact path: dist - - name: Upload wheel + - name: Upload wheel(s) and source uses: pypa/gh-action-pypi-publish@master with: user: __token__ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a403d03..0813cfa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,12 +4,12 @@ on: [ push ] jobs: tests: + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: matrix: os: [ ubuntu-latest, windows-latest ] - python-version: [ 3.8 ] - name: test in Python ${{ matrix.python-version }} on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + python-version: [ 3.8, 3.9 ] steps: - name: Checkout repository uses: actions/checkout@v2 From f1ef4cc2a74c1d2bae9aeadd277062fab0bd1ee5 Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 26 Jan 2021 15:14:29 -0500 Subject: [PATCH 03/10] install Fiona + GDAL on Windows --- setup.py | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 1cf90b1..86ecba5 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,44 @@ #!/usr/bin/env python +import logging +import os from setuptools import config, find_packages, setup -try: - from dunamai import Version -except ImportError: - import sys +if os.name == 'nt': import subprocess + import sys + + try: + import pipwin + except ImportError: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'pipwin']) + + try: + import gdal + except ImportError: + subprocess.check_call([sys.executable, '-m', 'pipwin', 'install', 'gdal==3.1.4']) + + try: + import fiona + except ImportError: + subprocess.check_call([sys.executable, '-m', 'pipwin', 'install', 'fiona']) + +try: + try: + from dunamai import Version + except ImportError: + import sys + import subprocess + + subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'dunamai']) + from dunamai import Version + + version = Version.from_any_vcs().serialize() +except RuntimeError as error: + logging.exception(error) + version = '0.0.0' - subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'dunamai']) - from dunamai import Version +logging.info(f'using version {version}') metadata = config.read_configuration('setup.cfg')['metadata'] From 886102f6263d0b2304c764e72b6e799d3a9fe7d5 Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 9 Mar 2021 14:03:35 -0500 Subject: [PATCH 04/10] update build process --- setup.py | 55 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index 86ecba5..0878cd2 100644 --- a/setup.py +++ b/setup.py @@ -1,32 +1,53 @@ #!/usr/bin/env python +import importlib import logging import os +from pathlib import Path +import subprocess +import sys from setuptools import config, find_packages, setup -if os.name == 'nt': - import subprocess - import sys +BUILT_PACKAGES = {'fiona': ['gdal'], 'geopandas': ['gdal', 'fiona'], 'numpy': [], 'pyproj': ['proj']} +is_conda = (Path(sys.prefix) / 'conda-meta').exists() - try: - import pipwin - except ImportError: - subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'pipwin']) +if is_conda: + conda_packages = [] + for conda_package in BUILT_PACKAGES: + try: + importlib.import_module(conda_package) + except: + conda_packages.append(conda_package) + subprocess.check_call(['conda', 'install', '-y', *conda_packages]) - try: - import gdal - except ImportError: - subprocess.check_call([sys.executable, '-m', 'pipwin', 'install', 'gdal==3.1.4']) +if os.name == 'nt': + for required_package, pipwin_dependencies in BUILT_PACKAGES.items(): + try: + importlib.import_module(required_package) + except: + try: + import pipwin + except: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'pipwin']) - try: - import fiona - except ImportError: - subprocess.check_call([sys.executable, '-m', 'pipwin', 'install', 'fiona']) + failed_pipwin_packages = [] + for pipwin_package in pipwin_dependencies + [required_package]: + try: + subprocess.check_call([sys.executable, '-m', 'pipwin', 'install', pipwin_package.lower()]) + except subprocess.CalledProcessError: + failed_pipwin_packages.append(pipwin_package) + + if len(failed_pipwin_packages) > 0: + raise RuntimeError( + f'failed to download or install non-conda Windows build(s) of {" and ".join(failed_pipwin_packages)}; you can either\n' + '1) install within an Anaconda environment, or\n' + f'2) `pip install .whl`, with `.whl` downloaded from {" and ".join("https://www.lfd.uci.edu/~gohlke/pythonlibs/#" + value.lower() for value in failed_pipwin_packages)} for your Python version' + ) try: try: from dunamai import Version - except ImportError: + except: import sys import subprocess @@ -44,7 +65,7 @@ setup( name=metadata['name'], - version=Version.from_any_vcs().serialize(), + version=version, author=metadata['author'], author_email=metadata['author_email'], description=metadata['description'], From 033637e0654a284dd6053e4769e1cdf1eca0300d Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 9 Mar 2021 14:04:27 -0500 Subject: [PATCH 05/10] example script --- examples/ngdc_seabed.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/ngdc_seabed.py diff --git a/examples/ngdc_seabed.py b/examples/ngdc_seabed.py new file mode 100644 index 0000000..5d321ad --- /dev/null +++ b/examples/ngdc_seabed.py @@ -0,0 +1,18 @@ +import numpy +from pyproj import CRS, Transformer + +from seabed.ngdc import NGDCSeabedDescriptions + +CRS = CRS.from_epsg(32618) + +BOUNDS = numpy.array([[-77, 39], [-75, 40]]) +BOUNDS = numpy.ravel( + numpy.stack( + Transformer.from_crs(CRS.from_epsg(4326), CRS).transform(BOUNDS[:, 0], BOUNDS[:, 1]), + axis=1, + ) +) + +if __name__ == '__main__': + surveys = NGDCSeabedDescriptions.all_surveys()[:5] + seabed = NGDCSeabedDescriptions(bounds=BOUNDS, surveys=surveys, crs=CRS) From d84a6176003b61822612e804077b8946c9c7f99b Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 9 Mar 2021 14:45:22 -0500 Subject: [PATCH 06/10] fix conda error --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0878cd2..a4e8de0 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,8 @@ importlib.import_module(conda_package) except: conda_packages.append(conda_package) - subprocess.check_call(['conda', 'install', '-y', *conda_packages]) + if len(conda_packages) > 0: + subprocess.check_call(['conda', 'install', '-y', *conda_packages]) if os.name == 'nt': for required_package, pipwin_dependencies in BUILT_PACKAGES.items(): From 0c6656275de4e292299b23de729d7e252f26ae3b Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 9 Mar 2021 14:45:48 -0500 Subject: [PATCH 07/10] add validation --- seabed/base.py | 41 ++++++++++++++++++++++++++++++++++------- seabed/ngdc.py | 1 - 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/seabed/base.py b/seabed/base.py index 1d9db42..3af2ce4 100644 --- a/seabed/base.py +++ b/seabed/base.py @@ -1,8 +1,10 @@ from abc import ABC, abstractmethod -import os from os import PathLike +from pathlib import Path +from typing import Union from geopandas import GeoDataFrame +import numpy from pyproj import CRS from .utilities import get_logger @@ -22,8 +24,29 @@ def __init__( crs: CRS = None, ): self.bounds = bounds - self.__surveys = surveys - self.crs = CRS.from_user_input(crs) if crs is not None else None + self.surveys = surveys + self.crs = crs + + @property + def bounds(self) -> (float, float, float, float): + return self.__bounds + + @bounds.setter + def bounds(self, bounds: (float, float, float, float)): + if isinstance(bounds, numpy.ndarray): + if len(bounds.shape) > 1: + bounds = bounds.ravel() + else: + bounds = numpy.asarray(bounds) + self.__bounds = bounds + + @property + def crs(self) -> CRS: + return self.__crs + + @crs.setter + def crs(self, crs: Union[CRS, str]): + self.__crs = CRS.from_user_input(crs) if crs is not None else None @classmethod @abstractmethod @@ -32,10 +55,12 @@ def all_surveys(cls) -> [str]: @property def surveys(self) -> [str]: - if self.__surveys is None: - self.__surveys = self.__class__.all_surveys() return self.__surveys + @surveys.setter + def surveys(self, surveys: [str]): + self.__surveys = surveys if surveys is not None else self.__class__.all_surveys() + def __getitem__(self, survey: str) -> GeoDataFrame: raise NotImplementedError @@ -64,7 +89,9 @@ def write(self, filename: PathLike, **kwargs): '.xml': 'GML', } - extension = os.path.splitext(filename)[-1] - kwargs['driver'] = drivers[extension] + if not isinstance(filename, Path): + filename = Path(filename) + + kwargs['driver'] = drivers[filename.suffix] self.data.to_file(str(filename), **kwargs) diff --git a/seabed/ngdc.py b/seabed/ngdc.py index 1cb0fd4..5d025fd 100644 --- a/seabed/ngdc.py +++ b/seabed/ngdc.py @@ -33,7 +33,6 @@ def __init__( @classmethod def all_surveys(cls) -> [str]: response = requests.get(URL) - print(response.status_code) if response.status_code != 200: raise ConnectionError( f'abnormal status "{response.status_code}" when connecting to {URL}' From b275f63e5161fa49e1ac49369539917e6681cb10 Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 9 Mar 2021 14:45:58 -0500 Subject: [PATCH 08/10] add tests --- tests/test_seabed.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/tests/test_seabed.py b/tests/test_seabed.py index 81560e0..06f1332 100644 --- a/tests/test_seabed.py +++ b/tests/test_seabed.py @@ -3,20 +3,35 @@ from seabed.ngdc import NGDCSeabedDescriptions -CRS = CRS.from_epsg(32618) -BOUNDS = numpy.array([[-77, 39], [-75, 40]]) -BOUNDS = numpy.ravel( - numpy.stack( - Transformer.from_crs(CRS.from_epsg(4326), CRS).transform(BOUNDS[:, 0], BOUNDS[:, 1]), - axis=1, - ) -) +def test_surveys(): + surveys = NGDCSeabedDescriptions.all_surveys() + + assert len(surveys) > 0 + + +def test_seabed_descriptions_within_bounds(): + bounds = numpy.array([[-77, 39], [-75, 40]]) + first_5_surveys = NGDCSeabedDescriptions.all_surveys()[:5] + seabed = NGDCSeabedDescriptions(surveys=first_5_surveys, bounds=bounds) + + assert seabed.data.shape[0] > 0 + assert seabed.data.shape[1] == 14 + assert len(seabed.descriptions) > 0 + assert any(seabed.data['Survey'].isin(['492'])) -def test_seabed_descriptions(): - surveys = NGDCSeabedDescriptions.all_surveys()[:5] - seabed = NGDCSeabedDescriptions(bounds=BOUNDS, surveys=surveys, crs=CRS) +def test_seabed_descriptions_within_transformed_bounds(): + bounds = numpy.array([[-77, 39], [-75, 40]]) + transformed_crs = CRS.from_epsg(32618) + transformed_bounds = numpy.ravel( + numpy.stack( + Transformer.from_crs(CRS.from_epsg(4326), transformed_crs).transform(bounds[:, 0], bounds[:, 1]), + axis=1, + ) + ) + first_5_surveys = NGDCSeabedDescriptions.all_surveys()[:5] + seabed = NGDCSeabedDescriptions(surveys=first_5_surveys, bounds=transformed_bounds, crs=transformed_crs) assert seabed.data.shape[0] > 0 assert seabed.data.shape[1] == 14 From fcc06ef80a6485d9da3b452b433993a9449e83f0 Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 9 Mar 2021 14:46:06 -0500 Subject: [PATCH 09/10] add examples --- examples/descriptions_within_bounds.py | 17 +++++++++++ .../descriptions_within_transformed_bounds.py | 28 +++++++++++++++++++ examples/ngdc_seabed.py | 18 ------------ 3 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 examples/descriptions_within_bounds.py create mode 100644 examples/descriptions_within_transformed_bounds.py delete mode 100644 examples/ngdc_seabed.py diff --git a/examples/descriptions_within_bounds.py b/examples/descriptions_within_bounds.py new file mode 100644 index 0000000..76b5886 --- /dev/null +++ b/examples/descriptions_within_bounds.py @@ -0,0 +1,17 @@ +import numpy + +from seabed.ngdc import NGDCSeabedDescriptions + +if __name__ == '__main__': + # define bounds + bounds = numpy.array([[-77, 39], [-75, 40]]) + + # retrieve a list of the first 5 surveys + first_5_surveys = NGDCSeabedDescriptions.all_surveys()[:5] + + # get surveys within specific bounds + seabed_object = NGDCSeabedDescriptions(surveys=first_5_surveys, bounds=bounds) + surveys_within_bounds = seabed_object.surveys + + print(f'all surveys: {first_5_surveys}') + print(f'surveys within transformed bounds: {surveys_within_bounds}') diff --git a/examples/descriptions_within_transformed_bounds.py b/examples/descriptions_within_transformed_bounds.py new file mode 100644 index 0000000..27a05d5 --- /dev/null +++ b/examples/descriptions_within_transformed_bounds.py @@ -0,0 +1,28 @@ +import numpy +from pyproj import CRS, Transformer + +from seabed.ngdc import NGDCSeabedDescriptions + +if __name__ == '__main__': + # define bounds + bounds = numpy.array([[-77, 39], [-75, 40]]) + + # transform bounds to desired CRS + source_crs = CRS.from_epsg(4326) + transformed_crs = CRS.from_epsg(32618) + transformed_bounds = numpy.ravel( + numpy.stack( + Transformer.from_crs(source_crs, transformed_crs).transform(bounds[:, 0], bounds[:, 1]), + axis=1, + ) + ) + + # retrieve a list of the first 5 surveys + first_5_surveys = NGDCSeabedDescriptions.all_surveys()[:5] + + # get surveys within specific bounds + seabed_object = NGDCSeabedDescriptions(surveys=first_5_surveys, bounds=transformed_bounds, crs=transformed_crs) + surveys_within_bounds = seabed_object.surveys + + print(f'all surveys: {first_5_surveys}') + print(f'surveys within transformed bounds: {surveys_within_bounds}') diff --git a/examples/ngdc_seabed.py b/examples/ngdc_seabed.py deleted file mode 100644 index 5d321ad..0000000 --- a/examples/ngdc_seabed.py +++ /dev/null @@ -1,18 +0,0 @@ -import numpy -from pyproj import CRS, Transformer - -from seabed.ngdc import NGDCSeabedDescriptions - -CRS = CRS.from_epsg(32618) - -BOUNDS = numpy.array([[-77, 39], [-75, 40]]) -BOUNDS = numpy.ravel( - numpy.stack( - Transformer.from_crs(CRS.from_epsg(4326), CRS).transform(BOUNDS[:, 0], BOUNDS[:, 1]), - axis=1, - ) -) - -if __name__ == '__main__': - surveys = NGDCSeabedDescriptions.all_surveys()[:5] - seabed = NGDCSeabedDescriptions(bounds=BOUNDS, surveys=surveys, crs=CRS) From 4fd2fbcdea20570435b56e9ebd92aaaa670a4189 Mon Sep 17 00:00:00 2001 From: "zachary.burnett" Date: Tue, 9 Mar 2021 16:08:01 -0500 Subject: [PATCH 10/10] update workflow --- .github/workflows/tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0813cfa..4d767cc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,10 +25,13 @@ jobs: restore-keys: ${{ runner.os }}-pip-${{ matrix.python-version }}- - name: Update pip run: python -m pip install --upgrade pip + - name: Install wheel + run: pip install wheel + - name: Run setup.py on Windows + if: contains(matrix.os, 'windows') + run: python setup.py install - name: Install dependencies - run: | - pip install wheel - pip install -e .[testing] + run: pip install -e .[testing] - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names