From 5647a672a8dca5aa9a0966814058af5e1c6fe0a8 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 6 Sep 2024 17:22:57 -0400 Subject: [PATCH 1/5] rename package from `harmony` to `harmony_service_lib` --- example/example_service.py | 12 +++---- {harmony => harmony_service_lib}/__init__.py | 0 {harmony => harmony_service_lib}/adapter.py | 28 +++++++-------- {harmony => harmony_service_lib}/aws.py | 4 +-- {harmony => harmony_service_lib}/cli.py | 18 +++++----- {harmony => harmony_service_lib}/earthdata.py | 0 .../exceptions.py | 0 {harmony => harmony_service_lib}/http.py | 8 ++--- {harmony => harmony_service_lib}/logging.py | 2 +- {harmony => harmony_service_lib}/message.py | 0 .../message_utility.py | 4 +-- .../s3_stac_io.py | 4 +-- {harmony => harmony_service_lib}/util.py | 20 +++++------ {harmony => harmony_service_lib}/version.py | 4 +-- setup.py | 4 +-- tests/test_adapter_deprecated.py | 20 +++++------ tests/test_adapter_stac.py | 10 +++--- tests/test_cli.py | 8 ++--- tests/test_cli_stac.py | 4 +-- tests/test_earthdata.py | 2 +- tests/test_http.py | 14 ++++---- tests/test_logging.py | 4 +-- tests/test_message.py | 4 +-- tests/test_message_utilities.py | 4 +-- tests/test_util.py | 34 +++++++++---------- tests/test_util_crypto.py | 2 +- tests/test_util_download.py | 8 ++--- tests/util.py | 2 +- 28 files changed, 112 insertions(+), 112 deletions(-) rename {harmony => harmony_service_lib}/__init__.py (100%) rename {harmony => harmony_service_lib}/adapter.py (97%) rename {harmony => harmony_service_lib}/aws.py (98%) rename {harmony => harmony_service_lib}/cli.py (95%) rename {harmony => harmony_service_lib}/earthdata.py (100%) rename {harmony => harmony_service_lib}/exceptions.py (100%) rename {harmony => harmony_service_lib}/http.py (98%) rename {harmony => harmony_service_lib}/logging.py (99%) rename {harmony => harmony_service_lib}/message.py (100%) rename {harmony => harmony_service_lib}/message_utility.py (98%) rename {harmony => harmony_service_lib}/s3_stac_io.py (94%) rename {harmony => harmony_service_lib}/util.py (97%) rename {harmony => harmony_service_lib}/version.py (72%) diff --git a/example/example_service.py b/example/example_service.py index b608c1b..91a2f87 100644 --- a/example/example_service.py +++ b/example/example_service.py @@ -12,8 +12,8 @@ from tempfile import mkdtemp from pystac import Asset -import harmony -from harmony.util import generate_output_filename, stage, download +import harmony_service_lib +from harmony_service_lib.util import generate_output_filename, stage, download # IMPORTANT: The following line avoids making real calls to a non-existent # Harmony frontend. Service authors should not set this variable to "dev" @@ -21,7 +21,7 @@ os.environ['ENV'] = 'dev' -class ExampleAdapter(harmony.BaseHarmonyAdapter): +class ExampleAdapter(harmony_service_lib.BaseHarmonyAdapter): """ Shows an example of what a service adapter implementation looks like """ @@ -115,12 +115,12 @@ def main(): """ parser = argparse.ArgumentParser(prog='example', description='Run an example service') - harmony.setup_cli(parser) + harmony_service_lib.setup_cli(parser) args = parser.parse_args() - if (harmony.is_harmony_cli(args)): - harmony.run_cli(parser, args, ExampleAdapter) + if (harmony_service_lib.is_harmony_cli(args)): + harmony_service_lib.run_cli(parser, args, ExampleAdapter) else: run_cli(args) diff --git a/harmony/__init__.py b/harmony_service_lib/__init__.py similarity index 100% rename from harmony/__init__.py rename to harmony_service_lib/__init__.py diff --git a/harmony/adapter.py b/harmony_service_lib/adapter.py similarity index 97% rename from harmony/adapter.py rename to harmony_service_lib/adapter.py index d26efa9..16e0cc5 100644 --- a/harmony/adapter.py +++ b/harmony_service_lib/adapter.py @@ -20,11 +20,11 @@ from deprecation import deprecated from pystac import Catalog, Item, Asset, read_file -from harmony.exceptions import CanceledException -from harmony.http import request_context -from harmony.logging import build_logger -from harmony.message import Temporal -from harmony.util import touch_health_check_file +from harmony_service_lib.exceptions import CanceledException +from harmony_service_lib.http import request_context +from harmony_service_lib.logging import build_logger +from harmony_service_lib.message import Temporal +from harmony_service_lib.util import touch_health_check_file from . import util @@ -39,7 +39,7 @@ class BaseHarmonyAdapter(ABC): Attributes ---------- - message : harmony.Message + message : harmony_service_lib.Message The Harmony input which needs acting upon temp_paths : list A list of string paths that should be cleaned up on exit @@ -60,7 +60,7 @@ def __init__(self, message, catalog=None, config=None): Parameters ---------- - message : harmony.Message + message : harmony_service_lib.Message The Harmony input which needs acting upon catalog : pystac.Catalog A STAC catalog containing the files on which to act @@ -111,7 +111,7 @@ def invoke(self): Returns ------- - (harmony.Message, pystac.Catalog | list) + (harmony_service_lib.Message, pystac.Catalog | list) A tuple of the Harmony message, with any processed fields marked as such and in this implementation, a single STAC catalog describing the output. (Services overriding this method may return a list of STAC catalogs if desired.) @@ -319,7 +319,7 @@ def download_granules(self, granules=None): Parameters ---------- granules : list - A list of harmony.message.Granule objects corresponding to the granules to download. Default: + A list of harmony_service_lib.message.Granule objects corresponding to the granules to download. Default: all granules in the incoming message """ temp_dir = mkdtemp() @@ -420,7 +420,7 @@ def completed_with_redirect( The mime type of the file, by default the output mime type requested by Harmony title : string, optional Textual information to provide users along with the link - temporal : harmony.message.Temporal, optional + temporal : harmony_service_lib.message.Temporal, optional The temporal extent of the provided file. If not provided, the source granule's temporal will be used when a source granule is provided bbox : list, optional @@ -480,7 +480,7 @@ def completed_with_local_file( The mime type of the file, by default the output mime type requested by Harmony title : string, optional Textual information to provide users along with the link - temporal : harmony.message.Temporal, optional + temporal : harmony_service_lib.message.Temporal, optional The temporal extent of the provided file. If not provided, the source granule's temporal will be used when a source granule is provided bbox : list, optional @@ -538,7 +538,7 @@ def async_add_local_file_partial_result( The mime type of the file, by default the output mime type requested by Harmony progress : integer, optional Numeric progress of the total request, 0-100 - temporal : harmony.message.Temporal, optional + temporal : harmony_service_lib.message.Temporal, optional The temporal extent of the provided file. If not provided, the source granule's temporal will be used when a source granule is provided bbox : list, optional @@ -576,7 +576,7 @@ def async_add_url_partial_result(self, url, title=None, mime=None, progress=None source_granule : message.Granule, optional The granule from which the file was derived, if it was derived from a single granule. This will be used to produce a canonical filename and assist when temporal and bbox are not specified - temporal : harmony.message.Temporal, optional + temporal : harmony_service_lib.message.Temporal, optional The temporal extent of the provided file. If not provided, the source granule's temporal will be used when a source granule is provided bbox : list, optional @@ -698,7 +698,7 @@ def _build_callback_item_params( source_granule : message.Granule, optional The granule from which the file was derived, if it was derived from a single granule. This will be used to produce a canonical filename and assist when temporal and bbox are not specified - temporal : harmony.message.Temporal, optional + temporal : harmony_service_lib.message.Temporal, optional The temporal extent of the provided file. If not provided, the source granule's temporal will be used when a source granule is provided bbox : list, optional diff --git a/harmony/aws.py b/harmony_service_lib/aws.py similarity index 98% rename from harmony/aws.py rename to harmony_service_lib/aws.py index 8bbda8a..7a39954 100644 --- a/harmony/aws.py +++ b/harmony_service_lib/aws.py @@ -2,14 +2,14 @@ This module includes various AWS-specific functions to stage data in S3 and deal with messages in SQS queues. -This module relies on the harmony.util.config and its environment variables to be +This module relies on the harmony_service_lib.util.config and its environment variables to be set for correct operation. See that module and the project README for details. """ from urllib.parse import urlparse from os import environ import boto3 from botocore.config import Config -from harmony import util +from harmony_service_lib import util def is_s3(url: str) -> bool: diff --git a/harmony/cli.py b/harmony_service_lib/cli.py similarity index 95% rename from harmony/cli.py rename to harmony_service_lib/cli.py index d59e42f..0ef91dd 100644 --- a/harmony/cli.py +++ b/harmony_service_lib/cli.py @@ -14,14 +14,14 @@ from pystac import Catalog, CatalogType from pystac.layout import BestPracticesLayoutStrategy -from harmony.exceptions import CanceledException, HarmonyException -from harmony.message import Message -from harmony.logging import setup_stdout_log_formatting, build_logger -from harmony.util import (receive_messages, delete_message, change_message_visibility, - config, create_decrypter) -from harmony.version import get_version -from harmony.aws import is_s3, write_s3 -from harmony.s3_stac_io import write +from harmony_service_lib.exceptions import CanceledException, HarmonyException +from harmony_service_lib.message import Message +from harmony_service_lib.logging import setup_stdout_log_formatting, build_logger +from harmony_service_lib.util import (receive_messages, delete_message, change_message_visibility, + config, create_decrypter) +from harmony_service_lib.version import get_version +from harmony_service_lib.aws import is_s3, write_s3 +from harmony_service_lib.s3_stac_io import write class MultiCatalogLayoutStrategy(BestPracticesLayoutStrategy): @@ -184,7 +184,7 @@ def _write_error(metadata_dir, message, category='Unknown'): def _build_adapter(AdapterClass, message_string, sources_path, data_location, config): """ - Creates the adapter to be invoked for the given harmony input + Creates the adapter to be invoked for the given harmony_service_lib input Parameters ---------- diff --git a/harmony/earthdata.py b/harmony_service_lib/earthdata.py similarity index 100% rename from harmony/earthdata.py rename to harmony_service_lib/earthdata.py diff --git a/harmony/exceptions.py b/harmony_service_lib/exceptions.py similarity index 100% rename from harmony/exceptions.py rename to harmony_service_lib/exceptions.py diff --git a/harmony/http.py b/harmony_service_lib/http.py similarity index 98% rename from harmony/http.py rename to harmony_service_lib/http.py index 4ae10e5..189c879 100644 --- a/harmony/http.py +++ b/harmony_service_lib/http.py @@ -6,7 +6,7 @@ federated token authentication. It includes an optional fallback authentication that uses an EDL user to download data when the feature is enabled. -This module relies on the harmony.util.config and its environment variables to be +This module relies on the harmony_service_lib.util.config and its environment variables to be set for correct operation. See that module and the project README for details. """ @@ -21,9 +21,9 @@ import requests -from harmony.earthdata import EarthdataAuth, EarthdataSession -from harmony.exceptions import ServerException, ForbiddenException -from harmony.logging import build_logger +from harmony_service_lib.earthdata import EarthdataAuth, EarthdataSession +from harmony_service_lib.exceptions import ServerException, ForbiddenException +from harmony_service_lib.logging import build_logger # Timeout in seconds. Per requests docs, this is not a time limit on # the entire response download; rather, an exception is raised if the diff --git a/harmony/logging.py b/harmony_service_lib/logging.py similarity index 99% rename from harmony/logging.py rename to harmony_service_lib/logging.py index aafe9e6..375f28b 100644 --- a/harmony/logging.py +++ b/harmony_service_lib/logging.py @@ -5,7 +5,7 @@ import copy from pythonjsonlogger import jsonlogger -from harmony import message +from harmony_service_lib import message class HarmonyJsonFormatter(jsonlogger.JsonFormatter): diff --git a/harmony/message.py b/harmony_service_lib/message.py similarity index 100% rename from harmony/message.py rename to harmony_service_lib/message.py diff --git a/harmony/message_utility.py b/harmony_service_lib/message_utility.py similarity index 98% rename from harmony/message_utility.py rename to harmony_service_lib/message_utility.py index 655b229..92c108b 100644 --- a/harmony/message_utility.py +++ b/harmony_service_lib/message_utility.py @@ -1,12 +1,12 @@ """Utilities for acting on Harmony Messages. These are a collection of useful routines for validation and interrogation of -harmony Messages. +harmony_service_lib Messages. """ from typing import Any, List -from harmony.message import Message +from harmony_service_lib.message import Message def has_self_consistent_grid(message: Message) -> bool: diff --git a/harmony/s3_stac_io.py b/harmony_service_lib/s3_stac_io.py similarity index 94% rename from harmony/s3_stac_io.py rename to harmony_service_lib/s3_stac_io.py index 5526f39..079d759 100644 --- a/harmony/s3_stac_io.py +++ b/harmony_service_lib/s3_stac_io.py @@ -1,8 +1,8 @@ from urllib.parse import urlparse import boto3 from pystac import STAC_IO -from harmony import util -from harmony import aws +from harmony_service_lib import util +from harmony_service_lib import aws from os import environ """ diff --git a/harmony/util.py b/harmony_service_lib/util.py similarity index 97% rename from harmony/util.py rename to harmony_service_lib/util.py index 368b7e0..11d5315 100644 --- a/harmony/util.py +++ b/harmony_service_lib/util.py @@ -63,15 +63,15 @@ from nacl.secret import SecretBox -from harmony import aws -from harmony import http +from harmony_service_lib import aws +from harmony_service_lib import http # The following imports are for backwards-compatibility for services -# which import them from `harmony.util`. Though they are not used in +# which import them from `harmony_service_lib.util`. Though they are not used in # this module, importing them here allows applications to work without # modifications. -from harmony.exceptions import (HarmonyException, CanceledException, ForbiddenException) # noqa: F401 -from harmony.logging import build_logger -from harmony.version import get_version +from harmony_service_lib.exceptions import (HarmonyException, CanceledException, ForbiddenException) # noqa: F401 +from harmony_service_lib.logging import build_logger +from harmony_service_lib.version import get_version DEFAULT_SHARED_SECRET_KEY = '_THIS_IS_MY_32_CHARS_SECRET_KEY_' @@ -368,7 +368,7 @@ def stage(local_filename, remote_filename, mime, logger=None, location=None, cfg An s3:// URL to the staged file """ # The implementation of this function has been moved to the - # harmony.aws module. + # harmony_service_lib.aws module. if cfg is None: cfg = config() if logger is None: @@ -399,7 +399,7 @@ def receive_messages(queue_url, visibility_timeout_s=600, logger=None, cfg=None) and the contents of the message """ # The implementation of this function has been moved to the - # harmony.aws module. + # harmony_service_lib.aws module. if cfg is None: cfg = config() if logger is None: @@ -424,7 +424,7 @@ def delete_message(queue_url, receipt_handle, cfg=None): The configuration values for this runtime environment. """ # The implementation of this function has been moved to the - # harmony.aws module. + # harmony_service_lib.aws module. if cfg is None: cfg = config() return aws.delete_message(cfg, queue_url, receipt_handle) @@ -447,7 +447,7 @@ def change_message_visibility(queue_url, receipt_handle, visibility_timeout_s, c The configuration values for this runtime environment. """ # The implementation of this function has been moved to the - # harmony.aws module. + # harmony_service_lib.aws module. if cfg is None: cfg = config() return aws.change_message_visibility(cfg, queue_url, receipt_handle, visibility_timeout_s) diff --git a/harmony/version.py b/harmony_service_lib/version.py similarity index 72% rename from harmony/version.py rename to harmony_service_lib/version.py index 6f6beb1..d485683 100644 --- a/harmony/version.py +++ b/harmony_service_lib/version.py @@ -1,4 +1,4 @@ -import harmony +import harmony_service_lib def get_version(): @@ -10,4 +10,4 @@ def get_version(): string A string representing the current version. """ - return harmony.__version__ + return harmony_service_lib.__version__ diff --git a/setup.py b/setup.py index f2ee348..46fb80f 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def get_version(): - main_file = os.path.join(CURDIR, "harmony", "__init__.py") + main_file = os.path.join(CURDIR, "harmony_service_lib", "__init__.py") _version_re = re.compile(r"__version__\s+=\s+(?P.*)") with open(main_file, "r", encoding="utf8") as f: match = _version_re.search(f.read()) @@ -53,7 +53,7 @@ def get_version(): keywords=[], scripts=[], entry_points={ - "console_scripts": ["harmony-service-lib=harmony.cli.__main__:main"] + "console_scripts": ["harmony-service-lib=harmony_service_lib.cli.__main__:main"] }, zip_safe=False, install_requires=DEPENDENCIES, diff --git a/tests/test_adapter_deprecated.py b/tests/test_adapter_deprecated.py index 23a25eb..1d362fa 100644 --- a/tests/test_adapter_deprecated.py +++ b/tests/test_adapter_deprecated.py @@ -11,10 +11,10 @@ from shutil import rmtree from urllib.error import HTTPError -from harmony.adapter import BaseHarmonyAdapter -from harmony.message import Message, Granule, Variable, Temporal -import harmony.util -from harmony.exceptions import CanceledException +from harmony_service_lib.adapter import BaseHarmonyAdapter +from harmony_service_lib.message import Message, Granule, Variable, Temporal +import harmony_service_lib.util +from harmony_service_lib.exceptions import CanceledException from .example_messages import minimal_message, full_message @@ -37,7 +37,7 @@ def read(self): class TestBaseHarmonyAdapter(unittest.TestCase): def setUp(self): - self.config = harmony.util.config(validate=False) + self.config = harmony_service_lib.util.config(validate=False) def test_cleanup_deletes_temporary_file_paths(self): adapter = AdapterTester(minimal_message, self.config) @@ -129,7 +129,7 @@ def test_completed_with_redirect_when_a_callback_has_been_made_it_throws_an_exce self.assertRaises(Exception, adapter.completed_with_error, 'https://example.com/2') @patch.object(BaseHarmonyAdapter, '_callback_post') - @patch.object(harmony.util, 'stage', return_value='https://example.com/out') + @patch.object(harmony_service_lib.util, 'stage', return_value='https://example.com/out') def test_completed_with_local_file_stages_the_local_file_and_redirects_to_it(self, stage, _callback_post): adapter = AdapterTester(full_message, self.config) adapter.completed_with_local_file('tmp/output.tif', remote_filename='out.tif') @@ -145,7 +145,7 @@ def test_completed_with_local_file_stages_the_local_file_and_redirects_to_it(sel '&status=successful') @patch.object(BaseHarmonyAdapter, '_callback_post') - @patch.object(harmony.util, 'stage', return_value='https://example.com/out') + @patch.object(harmony_service_lib.util, 'stage', return_value='https://example.com/out') def test_completed_with_local_file_uses_granule_file_naming(self, stage, _callback_post): adapter = AdapterTester(full_message, self.config) granule = adapter.message.sources[0].granules[0] @@ -239,7 +239,7 @@ def test_async_completed_successfully_for_complete_requests_throws_an_error(self self.assertRaises(Exception, adapter.async_completed_successfully) @patch.object(BaseHarmonyAdapter, '_callback_post') - @patch.object(harmony.util, 'stage', return_value='https://example.com/out') + @patch.object(harmony_service_lib.util, 'stage', return_value='https://example.com/out') def test_async_add_local_file_partial_result_stages_the_local_file_and_updates_progress(self, stage, _callback_post): adapter = AdapterTester(full_message, self.config) adapter.message.isSynchronous = False @@ -258,7 +258,7 @@ def test_async_add_local_file_partial_result_stages_the_local_file_and_updates_p '&progress=50') @patch.object(BaseHarmonyAdapter, '_callback_post') - @patch.object(harmony.util, 'stage', return_value='https://example.com/out') + @patch.object(harmony_service_lib.util, 'stage', return_value='https://example.com/out') def test_async_add_local_file_partial_result_uses_granule_file_naming(self, stage, _callback_post): adapter = AdapterTester(full_message, self.config) adapter.message.isSynchronous = False @@ -348,7 +348,7 @@ def test_cancel_request(self, urlopen): class TestCallbackPostHealthUpdate(unittest.TestCase): def setUp(self): - self.config = harmony.util.config(validate=False) + self.config = harmony_service_lib.util.config(validate=False) @patch.object(pathlib.Path, '__new__') def test_callback_post_updates_health_check_file(self, mock_path): diff --git a/tests/test_adapter_stac.py b/tests/test_adapter_stac.py index 6d4b803..8906bea 100644 --- a/tests/test_adapter_stac.py +++ b/tests/test_adapter_stac.py @@ -7,8 +7,8 @@ from unittest.mock import patch from pystac import Catalog, Item, Link -from harmony.message import Message -from harmony.adapter import BaseHarmonyAdapter +from harmony_service_lib.message import Message +from harmony_service_lib.adapter import BaseHarmonyAdapter from .example_messages import full_message from .util import config_fixture @@ -52,7 +52,7 @@ def test_invocation_processes_items_with_sources(self): self.assertEqual(AdapterTester.process_args[0][1], message.sources[0]) self.assertEqual(AdapterTester.process_args[1][1], message.sources[0]) - @patch('harmony.adapter.read_file') + @patch('harmony_service_lib.adapter.read_file') def test_invocation_follows_linked_catalogs(self, test_patch): catalog0 = Catalog('0', 'Catalog 0') catalog0.add_link(Link('harmony_source', 'http://example.com/C0001-EXAMPLE')) @@ -111,7 +111,7 @@ def test_invocation_recurses_subcatalogs(self): self.assertEqual(AdapterTester.process_args[2][1], message.sources[1]) self.assertEqual(AdapterTester.process_args[3][1], message.sources[1]) - @patch('harmony.adapter.read_file') + @patch('harmony_service_lib.adapter.read_file') def test_get_all_items_follows_links(self, test_patch): catalog0 = Catalog('0', 'Catalog 0') catalog0.add_link(Link('harmony_source', 'http://example.com/C0001-EXAMPLE')) @@ -139,7 +139,7 @@ def test_get_all_items_follows_links(self, test_patch): no_link_items = list(adapter.get_all_catalog_items(catalog0, False)) self.assertEqual(no_link_items, items_a) - @patch('harmony.adapter.read_file') + @patch('harmony_service_lib.adapter.read_file') def test_get_all_items_handles_children(self, test_patch): catalog = Catalog('0', 'Catalog 0') catalog.add_link(Link('harmony_source', 'http://example.com/C0001-EXAMPLE')) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3de52a1..6e8e10e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,8 +2,8 @@ import unittest from unittest.mock import patch -import harmony.util -from harmony import cli, BaseHarmonyAdapter +import harmony_service_lib.util +from harmony_service_lib import cli, BaseHarmonyAdapter from tests.util import mock_receive, cli_test @@ -50,7 +50,7 @@ def test_when_passing_nothing_it_returns_false(self, parser): class TestCliInvokeAction(unittest.TestCase): def setUp(self): - self.config = harmony.util.config(validate=False) + self.config = harmony_service_lib.util.config(validate=False) with open('/tmp/operation.json', 'w') as f: f.write('{"test": "input"}') @@ -127,7 +127,7 @@ def invoke(self): class TestCliStartAction(unittest.TestCase): def setUp(self): - self.config = harmony.util.config(validate=False) + self.config = harmony_service_lib.util.config(validate=False) def tearDown(self): MockAdapter.messages = [] diff --git a/tests/test_cli_stac.py b/tests/test_cli_stac.py index 2042b92..6a365d3 100644 --- a/tests/test_cli_stac.py +++ b/tests/test_cli_stac.py @@ -7,8 +7,8 @@ from pystac import Catalog, CatalogType, Item -from harmony import cli, BaseHarmonyAdapter -from harmony.exceptions import ForbiddenException +from harmony_service_lib import cli, BaseHarmonyAdapter +from harmony_service_lib.exceptions import ForbiddenException from tests.util import cli_parser, config_fixture diff --git a/tests/test_earthdata.py b/tests/test_earthdata.py index a6f8151..69c190d 100644 --- a/tests/test_earthdata.py +++ b/tests/test_earthdata.py @@ -4,7 +4,7 @@ import pytest from requests import Session -from harmony.earthdata import EarthdataAuth, EarthdataSession +from harmony_service_lib.earthdata import EarthdataAuth, EarthdataSession @dataclass diff --git a/tests/test_http.py b/tests/test_http.py index 5d5cd4a..4ba07d8 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -2,7 +2,7 @@ import responses import os -from harmony.http import (download, is_http, localhost_url) +from harmony_service_lib.http import (download, is_http, localhost_url) from unittest.mock import Mock, patch from tests.util import config_fixture @@ -173,7 +173,7 @@ def test_resource_server_redirects_to_granule_url( assert 'Authorization' not in rs_headers @responses.activate -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_download_validates_token_and_raises_exception( mocker, faker, @@ -243,7 +243,7 @@ def test_when_authn_succeeds_it_writes_to_provided_file( destination_file.write.assert_called() @responses.activate -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_when_given_an_access_token_and_error_occurs_it_falls_back_to_basic_auth_if_enabled( mocker, faker, @@ -277,7 +277,7 @@ def test_when_given_an_access_token_and_error_occurs_it_falls_back_to_basic_auth @responses.activate -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_when_given_an_access_token_and_error_occurs_it_does_not_fall_back_to_basic_auth( mocker, faker, @@ -329,7 +329,7 @@ def test_when_no_access_token_is_provided_it_uses_basic_auth_and_downloads_when_ @responses.activate -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_download_all_retries_failed( mocker, faker, @@ -441,7 +441,7 @@ def test_user_agent_is_passed_to_request_headers_when_using_edl_auth_and_post_pa @responses.activate(registry=responses.registries.OrderedRegistry) @pytest.mark.parametrize('error_code', [RETRY_ERROR_CODES]) -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_retries_on_temporary_errors_edl_auth( mocker, access_token, @@ -463,7 +463,7 @@ def test_retries_on_temporary_errors_edl_auth( @responses.activate(registry=responses.registries.OrderedRegistry) @pytest.mark.parametrize('error_code', RETRY_ERROR_CODES) -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_retries_on_temporary_errors_basic_auth( mocker, faker, diff --git a/tests/test_logging.py b/tests/test_logging.py index f4dabeb..fbf3327 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -2,9 +2,9 @@ import copy from io import StringIO -from harmony.logging import build_logger +from harmony_service_lib.logging import build_logger from tests.util import config_fixture -from harmony.message import Message +from harmony_service_lib.message import Message from .example_messages import minimal_message diff --git a/tests/test_message.py b/tests/test_message.py index b7c3673..f19d6e8 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -1,9 +1,9 @@ from copy import deepcopy import unittest -from harmony.message import Message +from harmony_service_lib.message import Message from .example_messages import minimal_message, minimal_source_message, full_message -from harmony import util +from harmony_service_lib import util class TestMessage(unittest.TestCase): diff --git a/tests/test_message_utilities.py b/tests/test_message_utilities.py index 6bdf41d..c52caeb 100644 --- a/tests/test_message_utilities.py +++ b/tests/test_message_utilities.py @@ -1,8 +1,8 @@ from unittest import TestCase -from harmony.message import Message +from harmony_service_lib.message import Message -from harmony.message_utility import ( +from harmony_service_lib.message_utility import ( _has_all_attributes, _has_consistent_dimension, has_crs, diff --git a/tests/test_util.py b/tests/test_util.py index baf859e..95e7f9a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -4,10 +4,10 @@ from unittest.mock import patch, MagicMock, mock_open, ANY from urllib.error import HTTPError -from harmony import aws -from harmony import util -from harmony.http import request_context -from harmony.message import Variable +from harmony_service_lib import aws +from harmony_service_lib import util +from harmony_service_lib.http import request_context +from harmony_service_lib.message import Variable from tests.test_cli import MockAdapter, cli_test from tests.util import mock_receive, config_fixture @@ -16,9 +16,9 @@ class TestDownload(unittest.TestCase): def setUp(self): self.config = util.config(validate=False) - @patch('harmony.util.get_version') + @patch('harmony_service_lib.util.get_version') @patch('boto3.client') - @patch('harmony.aws.Config') + @patch('harmony_service_lib.aws.Config') def test_s3_download_sets_minimal_user_agent_on_boto_client(self, boto_cfg, client, get_version): fake_lib_version = '0.1.0' get_version.return_value = fake_lib_version @@ -30,9 +30,9 @@ def test_s3_download_sets_minimal_user_agent_on_boto_client(self, boto_cfg, clie boto_cfg.assert_called_with(user_agent_extra=f'harmony (unknown version) harmony-service-lib/{fake_lib_version}') client.assert_called_with(service_name='s3', config=boto_cfg_instance, region_name=ANY) - @patch('harmony.util.get_version') + @patch('harmony_service_lib.util.get_version') @patch('boto3.client') - @patch('harmony.aws.Config') + @patch('harmony_service_lib.aws.Config') def test_s3_download_sets_harmony_user_agent_on_boto_client(self, boto_cfg, client, get_version): fake_lib_version = '0.1.0' get_version.return_value = fake_lib_version @@ -45,9 +45,9 @@ def test_s3_download_sets_harmony_user_agent_on_boto_client(self, boto_cfg, clie boto_cfg.assert_called_with(user_agent_extra=f'{harmony_user_agt} harmony-service-lib/{fake_lib_version}') client.assert_called_with(service_name='s3', config=boto_cfg_instance, region_name=ANY) - @patch('harmony.util.get_version') + @patch('harmony_service_lib.util.get_version') @patch('boto3.client') - @patch('harmony.aws.Config') + @patch('harmony_service_lib.aws.Config') def test_s3_download_sets_app_name_on_boto_client(self, boto_cfg, client, get_version): app_name = 'gdal-subsetter' fake_lib_version = '0.1.0' @@ -60,9 +60,9 @@ def test_s3_download_sets_app_name_on_boto_client(self, boto_cfg, client, get_ve boto_cfg.assert_called_with(user_agent_extra=f'harmony (unknown version) harmony-service-lib/{fake_lib_version} ({app_name})') client.assert_called_with(service_name='s3', config=boto_cfg_instance, region_name=ANY) - @patch('harmony.util.get_version') - @patch('harmony.aws.download') - @patch('harmony.aws.Config') + @patch('harmony_service_lib.util.get_version') + @patch('harmony_service_lib.aws.download') + @patch('harmony_service_lib.aws.Config') def test_s3_download_does_not_set_api_request_uuid(self, boto_cfg, aws_download, get_version): request_context['request_id'] = 'abc123' app_name = 'gdal-subsetter' @@ -75,7 +75,7 @@ def test_s3_download_does_not_set_api_request_uuid(self, boto_cfg, aws_download, util.download('s3://example/file.txt', 'tmp', access_token='', cfg=cfg) aws_download.assert_called_with(ANY, 's3://example/file.txt', ANY, ANY ) - @patch('harmony.util.get_version') + @patch('harmony_service_lib.util.get_version') @patch.object(Session, 'get') def test_http_download_sets_api_request_uuid(self, get, get_version): request_context['request_id'] = 'abc123' @@ -87,7 +87,7 @@ def test_http_download_sets_api_request_uuid(self, get, get_version): util.download('http://example/file.txt', 'tmp', access_token='', cfg=cfg) get.assert_called_with('http://example/file.txt?A-api-request-uuid=abc123', headers={'user-agent': f'harmony (unknown version) harmony-service-lib/{fake_lib_version} (gdal-subsetter)'}, timeout=60, stream=True) - @patch('harmony.util.get_version') + @patch('harmony_service_lib.util.get_version') @patch.object(Session, 'get') def test_https_download_sets_api_request_uuid(self, get, get_version): request_context['request_id'] = 'abc123' @@ -99,7 +99,7 @@ def test_https_download_sets_api_request_uuid(self, get, get_version): util.download('https://example/file.txt', 'tmp', access_token='', cfg=cfg) get.assert_called_with('https://example/file.txt?A-api-request-uuid=abc123', headers={'user-agent': f'harmony (unknown version) harmony-service-lib/{fake_lib_version} (gdal-subsetter)'}, timeout=60, stream=True) - @patch('harmony.util.get_version') + @patch('harmony_service_lib.util.get_version') @patch.object(Session, 'post') def test_http_download_with_post_sets_api_request_uuid(self, post, get_version): request_context['request_id'] = 'abc123' @@ -113,7 +113,7 @@ def test_http_download_with_post_sets_api_request_uuid(self, post, get_version): post.assert_called_with('http://example/file.txt?A-api-request-uuid=abc123', headers={'user-agent': f'harmony (unknown version) harmony-service-lib/{fake_lib_version} (gdal-subsetter)', 'Content-Type': 'application/x-www-form-urlencoded'}, data = { 'foo': 'bar' }, timeout=60, stream=True) - @patch('harmony.util.get_version') + @patch('harmony_service_lib.util.get_version') @patch.object(Session, 'post') def test_https_download_with_post_sets_api_request_uuid(self, post, get_version): request_context['request_id'] = 'abc123' diff --git a/tests/test_util_crypto.py b/tests/test_util_crypto.py index b99fa91..677fea1 100644 --- a/tests/test_util_crypto.py +++ b/tests/test_util_crypto.py @@ -4,7 +4,7 @@ from nacl.secret import SecretBox from nacl.utils import random -from harmony import util +from harmony_service_lib import util class TestDecrypter(unittest.TestCase): diff --git a/tests/test_util_download.py b/tests/test_util_download.py index e35a127..c7b308e 100644 --- a/tests/test_util_download.py +++ b/tests/test_util_download.py @@ -4,8 +4,8 @@ import pytest import responses -from harmony.exceptions import ForbiddenException, ServerException -from harmony import util +from harmony_service_lib.exceptions import ForbiddenException, ServerException +from harmony_service_lib import util from tests.util import config_fixture @@ -44,7 +44,7 @@ def test_when_given_an_http_url_it_downloads_the_url(monkeypatch, mocker, faker) http_download.assert_called() -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_when_given_unknown_url_it_raises_exception(faker): access_token = faker.password(length=40, special_chars=False) config = config_fixture() @@ -148,7 +148,7 @@ def test_when_the_url_returns_a_eula_error_it_returns_a_human_readable_message(f @responses.activate -@patch('harmony.http.get_retry_delay', Mock(return_value = 0)) +@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_when_the_url_returns_a_500_it_does_not_raise_a_forbidden_exception_and_does_not_return_details_to_user(faker): access_token = faker.password(length=43, special_chars=False) url = 'https://example.com/file.txt' diff --git a/tests/util.py b/tests/util.py index afb293e..0e8c109 100644 --- a/tests/util.py +++ b/tests/util.py @@ -3,7 +3,7 @@ from unittest.mock import patch, MagicMock from contextlib import contextmanager -from harmony import cli, util +from harmony_service_lib import cli, util def mock_receive(cfg, client, parser, AdapterClass, *messages): From 71072848dbf86e0f99dfc1cdeac137674f41b4f8 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 6 Sep 2024 17:28:00 -0400 Subject: [PATCH 2/5] update setuptools version --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 57f2bce..1645a2e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -12,4 +12,4 @@ python-language-server ~= 0.35 responses ~=0.22.0 safety ~= 2.3.5 pycodestyle ~= 2.9.1 -setuptools ~= 68.1.2 \ No newline at end of file +setuptools ~= 74.1.2 \ No newline at end of file From 69da9faaacda0163c5658522bccdfc14b3c076d8 Mon Sep 17 00:00:00 2001 From: danielfromearth Date: Fri, 6 Sep 2024 17:30:17 -0400 Subject: [PATCH 3/5] fix name mismatch in flake8 config and Makefile --- .flake8 | 2 +- Makefile | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.flake8 b/.flake8 index 1119687..77da3d7 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] max-line-length = 120 -per-file-ignores = harmony/__init__.py:F401 \ No newline at end of file +per-file-ignores = harmony_service_lib/__init__.py:F401 \ No newline at end of file diff --git a/Makefile b/Makefile index e624f6a..f85fb6f 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ REPO_USER ?= __token__ REPO_PASS ?= unset version: - sed -i.bak "s/__version__ .*/__version__ = \"$(VERSION)\"/" harmony/__init__.py && rm harmony/__init__.py.bak + sed -i.bak "s/__version__ .*/__version__ = \"$(VERSION)\"/" harmony_service_lib/__init__.py && rm harmony_service_lib/__init__.py.bak build: clean version python -m pip install --upgrade --quiet setuptools wheel twine @@ -27,13 +27,13 @@ install: pip install -r requirements.txt lint: - flake8 harmony + flake8 harmony_service_lib test: - pytest --cov=harmony tests + pytest --cov=harmony_service_lib tests test-no-warnings: - pytest --disable-warnings --cov=harmony tests + pytest --disable-warnings --cov=harmony_service_lib tests cve-check: safety check From 189b32299bdf65339aaacc60ccd21e4cd826f192 Mon Sep 17 00:00:00 2001 From: Chris Durbin Date: Fri, 27 Sep 2024 09:19:41 -0400 Subject: [PATCH 4/5] HARMONY-1781: Change references in pyproject.toml from harmony to harmony_service_lib --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9274e5..4790673 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ readme = "README.md" requires-python = ">= 3.8" [project.scripts] -harmony-service-lib = "harmony.cli.__main__:main" +harmony-service-lib = "harmony_service_lib.cli.__main__:main" [project.urls] Homepage = "https://github.com/nasa/harmony-service-lib-py" @@ -30,8 +30,8 @@ Homepage = "https://github.com/nasa/harmony-service-lib-py" [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} optional-dependencies = {dev = {file = ["dev-requirements.txt"]}} -# Will read __version__ from harmony.__init__.py -version = {attr = "harmony.__version__"} +# Will read __version__ from harmony_service_lib.__init__.py +version = {attr = "harmony_service_lib.__version__"} [tool.setuptools.packages.find] exclude = ["contrib", "docs", "tests*"] From d8a3e3120e9855b3c484c696ced835528d86dcda Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Fri, 27 Sep 2024 17:17:35 -0400 Subject: [PATCH 5/5] Change references in documentation strings to harmony_service_lib. --- harmony_service_lib/adapter.py | 6 +++--- harmony_service_lib/aws.py | 12 ++++++------ harmony_service_lib/cli.py | 8 ++++---- harmony_service_lib/http.py | 6 +++--- harmony_service_lib/logging.py | 2 +- harmony_service_lib/util.py | 14 +++++++------- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/harmony_service_lib/adapter.py b/harmony_service_lib/adapter.py index 16e0cc5..864e5ee 100644 --- a/harmony_service_lib/adapter.py +++ b/harmony_service_lib/adapter.py @@ -64,7 +64,7 @@ def __init__(self, message, catalog=None, config=None): The Harmony input which needs acting upon catalog : pystac.Catalog A STAC catalog containing the files on which to act - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration values for this runtime environment. """ if catalog is None: @@ -248,7 +248,7 @@ def process_item(self, item, source): ---------- item : pystac.Item the item that should be processed - source : harmony.message.Source + source : harmony_service_lib.message.Source the input source defining the variables, if any, to subset from the item Returns @@ -279,7 +279,7 @@ def _get_item_source(self, item): Returns ------- - harmony.message.Source + harmony_service_lib.message.Source The source of the input item """ parent = item diff --git a/harmony_service_lib/aws.py b/harmony_service_lib/aws.py index 7a39954..1400f68 100644 --- a/harmony_service_lib/aws.py +++ b/harmony_service_lib/aws.py @@ -73,7 +73,7 @@ def _get_aws_client(config, service, user_agent=None): Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. service : string The AWS service name for which to construct a client, e.g. "s3" or "sqs" @@ -97,7 +97,7 @@ def download(config, url, destination_file, user_agent=None): Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. destination_file : file-like The destination file where the object will be written. Must be @@ -122,7 +122,7 @@ def stage(config, local_filename, remote_filename, mime, logger, location=None): Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. local_filename : string A path and filename to the local file that should be staged @@ -171,7 +171,7 @@ def receive_messages(config, queue_url, visibility_timeout_s, logger): Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. queue_url : string The URL of the queue to receive messages on @@ -212,7 +212,7 @@ def delete_message(config, queue_url, receipt_handle): Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. queue_url : string The queue from which the message originated @@ -229,7 +229,7 @@ def change_message_visibility(config, queue_url, receipt_handle, visibility_time Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. queue_url : string The queue from which the message originated diff --git a/harmony_service_lib/cli.py b/harmony_service_lib/cli.py index 62c6305..041b86c 100644 --- a/harmony_service_lib/cli.py +++ b/harmony_service_lib/cli.py @@ -121,7 +121,7 @@ def _invoke_deprecated(AdapterClass, message_string, config): The BaseHarmonyAdapter subclass to use to handle service invocations message_string : string The Harmony input message - config : harmony.util.Config + config : harmony_service_lib.util.Config A configuration instance for this service Returns ------- @@ -196,7 +196,7 @@ def _build_adapter(AdapterClass, message_string, sources_path, data_location, co A file location containing a STAC catalog corresponding to the input message sources data_location : string The name of the directory where output should be written - config : harmony.util.Config + config : harmony_service_lib.util.Config A configuration instance for this service Returns ------- @@ -279,7 +279,7 @@ def _start(AdapterClass, queue_url, visibility_timeout_s, config): visibility_timeout_s : int The time interval during which the message can't be picked up by other listeners on the queue. - config : harmony.util.Config + config : harmony_service_lib.util.Config A configuration instance for this service """ for receipt, message in receive_messages(queue_url, visibility_timeout_s, cfg=config): @@ -318,7 +318,7 @@ def run_cli(parser, args, AdapterClass, cfg=None): Argument values parsed from the command line, presumably via ArgumentParser.parse_args AdapterClass : class The BaseHarmonyAdapter subclass to use to handle service invocations - cfg : harmony.util.Config + cfg : harmony_service_lib.util.Config A configuration instance for this service """ if cfg is None: diff --git a/harmony_service_lib/http.py b/harmony_service_lib/http.py index 189c879..16eea29 100644 --- a/harmony_service_lib/http.py +++ b/harmony_service_lib/http.py @@ -183,7 +183,7 @@ def _download( Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. url : str The url for the resource to download @@ -272,7 +272,7 @@ def _download_with_fallback_authn(config, url: str, data, user_agent=None, **kwa Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. url : str The url for the resource to download @@ -358,7 +358,7 @@ def download(config, url: str, access_token: str, data, destination_file, Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration for the current runtime environment. url : str The url for the resource to download diff --git a/harmony_service_lib/logging.py b/harmony_service_lib/logging.py index 375f28b..a16a3d7 100644 --- a/harmony_service_lib/logging.py +++ b/harmony_service_lib/logging.py @@ -66,7 +66,7 @@ def build_logger(config, name='harmony-service', stream=None): Builds a logger with appropriate defaults for Harmony Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration values for this runtime environment. name : string The name of the logger diff --git a/harmony_service_lib/util.py b/harmony_service_lib/util.py index 640a0bd..af6d695 100644 --- a/harmony_service_lib/util.py +++ b/harmony_service_lib/util.py @@ -156,7 +156,7 @@ def config(validate=True): Returns ------- - harmony.util.Config + harmony_service_lib.util.Config The configuration values for this runtime environment. """ def str_envvar(name: str, default: str) -> str: @@ -216,7 +216,7 @@ def _build_full_user_agent(config) -> str: Parameters ---------- - config : harmony.util.Config + config : harmony_service_lib.util.Config The configuration values for this runtime environment. Returns @@ -302,7 +302,7 @@ def download(url, destination_dir, logger=None, access_token=None, data=None, cf containing a series of `key=value` pairs, separated by ampersands. If None (the default), urllib.get.urlopen will use the GET method. - cfg : harmony.util.Config + cfg : harmony_service_lib.util.Config The configuration values for this runtime environment. Returns @@ -361,7 +361,7 @@ def stage(local_filename, remote_filename, mime, logger=None, location=None, cfg STAGING_PATH must be set in the environment logger : logging The logger to use - cfg : harmony.util.Config + cfg : harmony_service_lib.util.Config The configuration values for this runtime environment. Returns @@ -391,7 +391,7 @@ def receive_messages(queue_url, visibility_timeout_s=600, logger=None, cfg=None) visibility_timeout_s : int The number of seconds to wait for a received message to be deleted before it is returned to the queue - cfg : harmony.util.Config + cfg : harmony_service_lib.util.Config The configuration values for this runtime environment. Yields @@ -422,7 +422,7 @@ def delete_message(queue_url, receipt_handle, cfg=None): The queue from which the message originated receipt_handle : string The receipt handle of the message, as yielded by `receive_messages` - cfg : harmony.util.Config + cfg : harmony_service_lib.util.Config The configuration values for this runtime environment. """ # The implementation of this function has been moved to the @@ -445,7 +445,7 @@ def change_message_visibility(queue_url, receipt_handle, visibility_timeout_s, c visibility_timeout_s : int The number of additional seconds to wait for a received message to be deleted before it is returned to the queue - cfg : harmony.util.Config + cfg : harmony_service_lib.util.Config The configuration values for this runtime environment. """ # The implementation of this function has been moved to the