diff --git a/.gitignore b/.gitignore index 0d867c0..f649baf 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,6 @@ config.json # Snyk / Deepcode AI .dccache + +# VS Code +.vscode/ diff --git a/dev-requirements.txt b/dev-requirements.txt index 38919d7..f695a72 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,16 +1,16 @@ autopep8 ~= 1.5 debugpy ~= 1.2 Faker ~= 8.1.3 -flake8 ~= 5.0.4 +flake8 >= 6.1.0 ipython ~= 8.10.0 jedi ~= 0.17.2 packaging ~= 24.1 parameterized ~= 0.7 -pycodestyle ~= 2.9.1 pytest ~= 7.2.0 pytest-cov ~=2.11 pytest-mock ~=3.5 python-language-server ~= 0.35 responses ~=0.22.0 +pycodestyle >= 2.9.1 safety ~= 3.2.7 -setuptools == 70.0.0 +setuptools == 70.0.0 \ No newline at end of file diff --git a/example/source/catalog.json b/example/source/catalog.json index 848c58b..640ccf5 100644 --- a/example/source/catalog.json +++ b/example/source/catalog.json @@ -1,6 +1,6 @@ { - "stac_version": "1.0.0-beta.2", - "stac_extensions": [], + "stac_version": "1.0.0", + "type": "Catalog", "id": "d5addf73-6e95-4ba1-a2a6-528de6d3d22c", "links": [ { diff --git a/example/source/catalog0.json b/example/source/catalog0.json index d4398dc..5a6c966 100644 --- a/example/source/catalog0.json +++ b/example/source/catalog0.json @@ -1,6 +1,5 @@ { - "stac_version": "1.0.0-beta.2", - "stac_extensions": [], + "stac_version": "1.0.0", "id": "1bd3924c-c0ee-49b7-b9ac-0d295f4bf909", "links": [ { diff --git a/example/source/catalog1.json b/example/source/catalog1.json index 5cd647b..401ea3f 100644 --- a/example/source/catalog1.json +++ b/example/source/catalog1.json @@ -1,6 +1,5 @@ { - "stac_version": "1.0.0-beta.2", - "stac_extensions": [], + "stac_version": "1.0.0", "id": "2bd3924c-c0ee-49b7-b9ac-0d296f4af909", "links": [ { diff --git a/example/source/granule_0_0000000.json b/example/source/granule_0_0000000.json index 24578be..3710b17 100644 --- a/example/source/granule_0_0000000.json +++ b/example/source/granule_0_0000000.json @@ -1,6 +1,5 @@ { - "stac_version": "1.0.0-beta.2", - "stac_extensions": [], + "stac_version": "1.0.0", "id": "3ca8ec5a-6d1e-4a82-8233-313d73769e7c", "type": "Feature", "links": [], diff --git a/example/source/granule_0_0000001.json b/example/source/granule_0_0000001.json index 6095239..c7d6d04 100644 --- a/example/source/granule_0_0000001.json +++ b/example/source/granule_0_0000001.json @@ -1,6 +1,5 @@ { - "stac_version": "1.0.0-beta.2", - "stac_extensions": [], + "stac_version": "1.0.0", "id": "56e7601d-4ff0-445b-85ee-a0174558853e", "type": "Feature", "links": [], diff --git a/example/source/source_0.json b/example/source/source_0.json index a7b313f..e0a0c8b 100644 --- a/example/source/source_0.json +++ b/example/source/source_0.json @@ -1,7 +1,7 @@ { - "stac_version": "1.0.0-beta.2", - "stac_extensions": [], + "stac_version": "1.0.0", "id": "8bd3924c-c0ee-49b7-b9ac-0d296f4bf909", + "type": "Catalog", "links": [ { "rel": "harmony_source", diff --git a/harmony/__init__.py b/harmony/__init__.py index e62609b..10e8ac8 100644 --- a/harmony/__init__.py +++ b/harmony/__init__.py @@ -12,8 +12,7 @@ from .adapter import BaseHarmonyAdapter from .cli import setup_cli, is_harmony_cli, run_cli from .message import Temporal -from pystac.stac_io import STAC_IO -from .s3_stac_io import read, write +from pystac.stac_io import StacIO +from .s3_stac_io import S3StacIO -STAC_IO.read_text_method = read -STAC_IO.write_text_method = write +StacIO.set_default(S3StacIO) diff --git a/harmony/cli.py b/harmony/cli.py index d59e42f..a0316a0 100644 --- a/harmony/cli.py +++ b/harmony/cli.py @@ -21,7 +21,7 @@ config, create_decrypter) from harmony.version import get_version from harmony.aws import is_s3, write_s3 -from harmony.s3_stac_io import write +from harmony.s3_stac_io import S3StacIO class MultiCatalogLayoutStrategy(BestPracticesLayoutStrategy): @@ -238,6 +238,7 @@ def _invoke(adapter, metadata_dir): """ try: logging.info(f'Invoking adapter with harmony-service-lib-py version {get_version()}') + s3_io = S3StacIO() is_s3_metadata_dir = is_s3(metadata_dir) if not is_s3_metadata_dir: makedirs(metadata_dir, exist_ok=True) @@ -246,8 +247,8 @@ def _invoke(adapter, metadata_dir): for idx, catalog in enumerate(stac_output): catalog.normalize_and_save(metadata_dir, CatalogType.SELF_CONTAINED, MultiCatalogLayoutStrategy(idx)) json_str = json.dumps([f'catalog{i}.json' for i, c in enumerate(stac_output)]) - write(path.join(metadata_dir, 'batch-catalogs.json'), json_str) - write(path.join(metadata_dir, 'batch-count.txt'), f'{len(stac_output)}') + s3_io.write_text(path.join(metadata_dir, 'batch-catalogs.json'), json_str) + s3_io.write_text(path.join(metadata_dir, 'batch-count.txt'), f'{len(stac_output)}') else: # assume stac_output is a single catalog stac_output.normalize_and_save(metadata_dir, CatalogType.SELF_CONTAINED) @@ -340,6 +341,7 @@ def run_cli(parser, args, AdapterClass, cfg=None): if not successful: raise Exception('Service operation failed') else: + adapter = None try: adapter = _build_adapter(AdapterClass, args.harmony_input, @@ -353,8 +355,8 @@ def run_cli(parser, args, AdapterClass, cfg=None): duration_ms = int(round(time_diff.total_seconds() * 1000)) duration_logger = build_logger(cfg) extra_fields = { - 'user': adapter.message.user, - 'requestId': adapter.message.requestId, + 'user': adapter.message.user if adapter else '', + 'requestId': adapter.message.requestId if adapter else '', 'durationMs': duration_ms } duration_logger.info(f'timing.{cfg.app_name}.end', extra=extra_fields) diff --git a/harmony/s3_stac_io.py b/harmony/s3_stac_io.py index 5526f39..b01003a 100644 --- a/harmony/s3_stac_io.py +++ b/harmony/s3_stac_io.py @@ -1,54 +1,57 @@ from urllib.parse import urlparse import boto3 -from pystac import STAC_IO +from pystac.stac_io import StacIO, DefaultStacIO from harmony import util from harmony import aws from os import environ """ Read and write to s3 when STAC links start with s3://. -https://pystac.readthedocs.io/en/0.5/concepts.html#using-stac-io +https://pystac.readthedocs.io/en/latest/concepts.html#using-stac-io """ - -def read(uri): - """ - Reads STAC files from s3 - (or via the default method if the protocol is not s3). - - Parameters - ---------- - uri: The STAC file uri. - - Returns - ------- - The file contents - """ - config = util.config(validate=environ.get('ENV') != 'test') - service_params = aws.aws_parameters( - config.use_localstack, config.localstack_host, config.aws_default_region) - parsed = urlparse(uri) - if parsed.scheme == 's3': - bucket = parsed.netloc - key = parsed.path[1:] - s3 = boto3.resource('s3', **service_params) - obj = s3.Object(bucket, key) - return obj.get()['Body'].read().decode('utf-8') - else: - return STAC_IO.default_read_text_method(uri) - - -def write(uri, txt): - """ - Writes a STAC file to the given uri. - - Parameters - ---------- - uri: The STAC file uri. - txt: The STAC contents. - """ - parsed = urlparse(uri) - if parsed.scheme == 's3': - aws.write_s3(uri, txt) - else: - STAC_IO.default_write_text_method(uri, txt) +defaultStacIO = DefaultStacIO() + + +class S3StacIO(StacIO): + + def read_text(self, uri): + """ + Reads STAC files from s3 + (or via the default method if the protocol is not s3). + + Parameters + ---------- + uri: The STAC file uri. + + Returns + ------- + The file contents + """ + config = util.config(validate=environ.get('ENV') != 'test') + service_params = aws.aws_parameters( + config.use_localstack, config.localstack_host, config.aws_default_region) + parsed = urlparse(uri) + if parsed.scheme == 's3': + bucket = parsed.netloc + key = parsed.path[1:] + s3 = boto3.resource('s3', **service_params) + obj = s3.Object(bucket, key) + return obj.get()['Body'].read().decode('utf-8') + else: + return defaultStacIO.read_text(uri) + + def write_text(self, uri, txt): + """ + Writes a STAC file to the given uri. + + Parameters + ---------- + uri: The STAC file uri. + txt: The STAC contents. + """ + parsed = urlparse(uri) + if parsed.scheme == 's3': + aws.write_s3(uri, txt) + else: + defaultStacIO.write_text(uri, txt) diff --git a/requirements.txt b/requirements.txt index b132d46..206db20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ boto3 ~= 1.14 deprecation ~= 2.1.0 pynacl ~= 1.4 -pystac ~= 0.5.3 +pystac >= 1.0.0 python-json-logger ~= 2.0.1 requests ~= 2.24 urllib3 ~= 1.26.9 \ No newline at end of file diff --git a/tests/test_adapter_stac.py b/tests/test_adapter_stac.py index 6d4b803..6993884 100644 --- a/tests/test_adapter_stac.py +++ b/tests/test_adapter_stac.py @@ -210,7 +210,8 @@ def test_legacy_invocations_create_stac_catalogs(self): self.assertEqual(AdapterTester.process_args[2][1], message.sources[1]) self.assertEqual(AdapterTester.process_args[0][0].to_dict(), { 'type': 'Feature', - 'stac_version': '1.0.0-beta.2', + 'stac_version': '1.0.0', + 'stac_extensions': [], 'id': 'G0001-EXAMPLE', 'properties': { 'start_datetime': '2001-01-01T01:01:01Z', @@ -230,7 +231,8 @@ def test_legacy_invocations_create_stac_catalogs(self): }) self.assertEqual(AdapterTester.process_args[1][0].to_dict(), { 'type': 'Feature', - 'stac_version': '1.0.0-beta.2', + 'stac_version': '1.0.0', + 'stac_extensions': [], 'id': 'G0002-EXAMPLE', 'properties': { 'start_datetime': '2003-03-03T03:03:03Z',