Merge pull request #47 from nasa/HARMONY-1862

HARMONY-1862: Update pystac library dependency

.gitignore

@@ -132,3 +132,6 @@ config.json
 
 # Snyk / Deepcode AI
 .dccache
+
+# VS Code
+.vscode/

dev-requirements.txt

@@ -1,16 +1,16 @@
 autopep8 ~= 1.5
 debugpy ~= 1.2
 Faker ~= 8.1.3
-flake8 ~= 5.0.4
+flake8 >= 6.1.0
 ipython ~= 8.10.0
 jedi ~= 0.17.2
 packaging ~= 24.1
 parameterized ~= 0.7
-pycodestyle ~= 2.9.1
 pytest ~= 7.2.0
 pytest-cov ~=2.11
 pytest-mock ~=3.5
 python-language-server ~= 0.35
 responses ~=0.22.0
+pycodestyle >= 2.9.1
 safety ~= 3.2.7
-setuptools == 70.0.0
+setuptools == 70.0.0

example/source/catalog.json

@@ -1,6 +1,6 @@
 {
-  "stac_version": "1.0.0-beta.2",
-  "stac_extensions": [],
+  "stac_version": "1.0.0",
+  "type": "Catalog",
   "id": "d5addf73-6e95-4ba1-a2a6-528de6d3d22c",
   "links": [
     {

example/source/catalog0.json

@@ -1,6 +1,5 @@
 {
-  "stac_version": "1.0.0-beta.2",
-  "stac_extensions": [],
+  "stac_version": "1.0.0",
   "id": "1bd3924c-c0ee-49b7-b9ac-0d295f4bf909",
   "links": [
     {

example/source/catalog1.json

@@ -1,6 +1,5 @@
 {
-  "stac_version": "1.0.0-beta.2",
-  "stac_extensions": [],
+  "stac_version": "1.0.0",
   "id": "2bd3924c-c0ee-49b7-b9ac-0d296f4af909",
   "links": [
     {

example/source/granule_0_0000000.json

@@ -1,6 +1,5 @@
 {
-  "stac_version": "1.0.0-beta.2",
-  "stac_extensions": [],
+  "stac_version": "1.0.0",
   "id": "3ca8ec5a-6d1e-4a82-8233-313d73769e7c",
   "type": "Feature",
   "links": [],

example/source/granule_0_0000001.json

@@ -1,6 +1,5 @@
 {
-  "stac_version": "1.0.0-beta.2",
-  "stac_extensions": [],
+  "stac_version": "1.0.0",
   "id": "56e7601d-4ff0-445b-85ee-a0174558853e",
   "type": "Feature",
   "links": [],

example/source/source_0.json

@@ -1,7 +1,7 @@
 {
-  "stac_version": "1.0.0-beta.2",
-  "stac_extensions": [],
+  "stac_version": "1.0.0",
   "id": "8bd3924c-c0ee-49b7-b9ac-0d296f4bf909",
+  "type": "Catalog",
   "links": [
     {
       "rel": "harmony_source",

harmony/__init__.py

@@ -12,8 +12,7 @@
 from .adapter import BaseHarmonyAdapter
 from .cli import setup_cli, is_harmony_cli, run_cli
 from .message import Temporal
-from pystac.stac_io import STAC_IO
-from .s3_stac_io import read, write
+from pystac.stac_io import StacIO
+from .s3_stac_io import S3StacIO
 
-STAC_IO.read_text_method = read
-STAC_IO.write_text_method = write
+StacIO.set_default(S3StacIO)

harmony/cli.py

@@ -21,7 +21,7 @@
                           config, create_decrypter)
 from harmony.version import get_version
 from harmony.aws import is_s3, write_s3
-from harmony.s3_stac_io import write
+from harmony.s3_stac_io import S3StacIO
 
 
 class MultiCatalogLayoutStrategy(BestPracticesLayoutStrategy):
@@ -238,6 +238,7 @@ def _invoke(adapter, metadata_dir):
     """
     try:
         logging.info(f'Invoking adapter with harmony-service-lib-py version {get_version()}')
+        s3_io = S3StacIO()
         is_s3_metadata_dir = is_s3(metadata_dir)
         if not is_s3_metadata_dir:
             makedirs(metadata_dir, exist_ok=True)
@@ -246,8 +247,8 @@ def _invoke(adapter, metadata_dir):
             for idx, catalog in enumerate(stac_output):
                 catalog.normalize_and_save(metadata_dir, CatalogType.SELF_CONTAINED, MultiCatalogLayoutStrategy(idx))
             json_str = json.dumps([f'catalog{i}.json' for i, c in enumerate(stac_output)])
-            write(path.join(metadata_dir, 'batch-catalogs.json'), json_str)
-            write(path.join(metadata_dir, 'batch-count.txt'), f'{len(stac_output)}')
+            s3_io.write_text(path.join(metadata_dir, 'batch-catalogs.json'), json_str)
+            s3_io.write_text(path.join(metadata_dir, 'batch-count.txt'), f'{len(stac_output)}')
         else:  # assume stac_output is a single catalog
             stac_output.normalize_and_save(metadata_dir, CatalogType.SELF_CONTAINED)
 
@@ -340,6 +341,7 @@ def run_cli(parser, args, AdapterClass, cfg=None):
             if not successful:
                 raise Exception('Service operation failed')
         else:
+            adapter = None
             try:
                 adapter = _build_adapter(AdapterClass,
                                          args.harmony_input,
@@ -353,8 +355,8 @@ def run_cli(parser, args, AdapterClass, cfg=None):
                 duration_ms = int(round(time_diff.total_seconds() * 1000))
                 duration_logger = build_logger(cfg)
                 extra_fields = {
-                    'user': adapter.message.user,
-                    'requestId': adapter.message.requestId,
+                    'user': adapter.message.user if adapter else '',
+                    'requestId': adapter.message.requestId if adapter else '',
                     'durationMs': duration_ms
                 }
                 duration_logger.info(f'timing.{cfg.app_name}.end', extra=extra_fields)

harmony/s3_stac_io.py

@@ -1,54 +1,57 @@
 from urllib.parse import urlparse
 import boto3
-from pystac import STAC_IO
+from pystac.stac_io import StacIO, DefaultStacIO
 from harmony import util
 from harmony import aws
 from os import environ
 
 """
 Read and write to s3 when STAC links start with s3://.
-https://pystac.readthedocs.io/en/0.5/concepts.html#using-stac-io
+https://pystac.readthedocs.io/en/latest/concepts.html#using-stac-io
 """
 
-
-def read(uri):
-    """
-    Reads STAC files from s3
-    (or via the default method if the protocol is not s3).
-
-    Parameters
-    ----------
-    uri: The STAC file uri.
-
-    Returns
-    -------
-    The file contents
-    """
-    config = util.config(validate=environ.get('ENV') != 'test')
-    service_params = aws.aws_parameters(
-        config.use_localstack, config.localstack_host, config.aws_default_region)
-    parsed = urlparse(uri)
-    if parsed.scheme == 's3':
-        bucket = parsed.netloc
-        key = parsed.path[1:]
-        s3 = boto3.resource('s3', **service_params)
-        obj = s3.Object(bucket, key)
-        return obj.get()['Body'].read().decode('utf-8')
-    else:
-        return STAC_IO.default_read_text_method(uri)
-
-
-def write(uri, txt):
-    """
-    Writes a STAC file to the given uri.
-
-    Parameters
-    ----------
-    uri: The STAC file uri.
-    txt: The STAC contents.
-    """
-    parsed = urlparse(uri)
-    if parsed.scheme == 's3':
-        aws.write_s3(uri, txt)
-    else:
-        STAC_IO.default_write_text_method(uri, txt)
+defaultStacIO = DefaultStacIO()
+
+
+class S3StacIO(StacIO):
+
+    def read_text(self, uri):
+        """
+        Reads STAC files from s3
+        (or via the default method if the protocol is not s3).
+
+        Parameters
+        ----------
+        uri: The STAC file uri.
+
+        Returns
+        -------
+        The file contents
+        """
+        config = util.config(validate=environ.get('ENV') != 'test')
+        service_params = aws.aws_parameters(
+            config.use_localstack, config.localstack_host, config.aws_default_region)
+        parsed = urlparse(uri)
+        if parsed.scheme == 's3':
+            bucket = parsed.netloc
+            key = parsed.path[1:]
+            s3 = boto3.resource('s3', **service_params)
+            obj = s3.Object(bucket, key)
+            return obj.get()['Body'].read().decode('utf-8')
+        else:
+            return defaultStacIO.read_text(uri)
+
+    def write_text(self, uri, txt):
+        """
+        Writes a STAC file to the given uri.
+
+        Parameters
+        ----------
+        uri: The STAC file uri.
+        txt: The STAC contents.
+        """
+        parsed = urlparse(uri)
+        if parsed.scheme == 's3':
+            aws.write_s3(uri, txt)
+        else:
+            defaultStacIO.write_text(uri, txt)

requirements.txt

@@ -1,7 +1,7 @@
 boto3 ~= 1.14
 deprecation ~= 2.1.0
 pynacl ~= 1.4
-pystac ~= 0.5.3
+pystac >= 1.0.0
 python-json-logger ~= 2.0.1
 requests ~= 2.24
 urllib3 ~= 1.26.9

tests/test_adapter_stac.py

@@ -210,7 +210,8 @@ def test_legacy_invocations_create_stac_catalogs(self):
         self.assertEqual(AdapterTester.process_args[2][1], message.sources[1])
         self.assertEqual(AdapterTester.process_args[0][0].to_dict(), {
             'type': 'Feature',
-            'stac_version': '1.0.0-beta.2',
+            'stac_version': '1.0.0',
+            'stac_extensions': [],
             'id': 'G0001-EXAMPLE',
             'properties': {
                 'start_datetime': '2001-01-01T01:01:01Z',
@@ -230,7 +231,8 @@ def test_legacy_invocations_create_stac_catalogs(self):
         })
         self.assertEqual(AdapterTester.process_args[1][0].to_dict(), {
             'type': 'Feature',
-            'stac_version': '1.0.0-beta.2',
+            'stac_version': '1.0.0',
+            'stac_extensions': [],
             'id': 'G0002-EXAMPLE',
             'properties': {
                 'start_datetime': '2003-03-03T03:03:03Z',