From f42f034a277d79260ab5abf06388da3cf94cf868 Mon Sep 17 00:00:00 2001 From: Chris Durbin Date: Tue, 3 Dec 2024 07:59:38 -0500 Subject: [PATCH 1/2] HARMONY-1938: Remove username/password fallback authentication capability and remove all related code and configuration. --- README.md | 22 --- harmony_service_lib/earthdata.py | 51 +------ harmony_service_lib/http.py | 63 +-------- harmony_service_lib/util.py | 44 ------ tests/test_earthdata.py | 60 +------- tests/test_http.py | 233 ++----------------------------- tests/util.py | 14 +- 7 files changed, 23 insertions(+), 464 deletions(-) diff --git a/README.md b/README.md index e235033..b220729 100644 --- a/README.md +++ b/README.md @@ -55,21 +55,6 @@ REQUIRED: `STAGING_BUCKET` under which data will be staged * `ENV`: The name of the environment. If 'dev' or 'test', callbacks to Harmony are not made and data is not staged unless also using localstack -* `OAUTH_UID`, `OAUTH_PASSWORD`: Used to acquire a shared EDL token - needed for downloading granules from EDL token-aware data - sources. Services using data in S3 do not need to set this. - - NOTE: If `FALLBACK_AUTHN_ENABLED` is set to True (CAUTION!) - these credentials will be used to download data *as* the EDL - application user. This may cause problems with metrics and can - result in users getting data for which they've not approved a - EULA. -* `OAUTH_CLIENT_ID`: The Earthdata application client ID. -* `OAUTH_HOST`: Set to the correct Earthdata Login URL, depending on - where the service is being deployed. This should be the same - environment where the `OAUTH_*` credentials are valid. Defaults - to UAT. -* `OAUTH_REDIRECT_URI`: A valid redirect URI for the EDL application. * `SHARED_SECRET_KEY`: The 32-byte encryption key shared between Harmony and backend services. This is used to encrypt & decrypt the `accessToken` in the Harmony operation message. In a production environment, this should be injected into the container running the service @@ -92,13 +77,6 @@ OPTIONAL: * `MAX_DOWNLOAD_RETRIES`: Number of times to retry HTTP download calls that fail due to transient errors. * `POST_URL_LENGTH`: Minimum url length that will be submitted via POST request. -OPTIONAL -- Use with CAUTION: - -* `FALLBACK_AUTHN_ENABLED`: Default: False. Enable the fallback authentication that - uses the EDL application credentials. See CAUTION note above. -* `EDL_USERNAME`: The Earthdata Login username used for fallback authn. -* `EDL_PASSWORD`: The Earthdata Login password used for fallback authn. - ## Development Setup Prerequisites: diff --git a/harmony_service_lib/earthdata.py b/harmony_service_lib/earthdata.py index 94758a5..a6df4e1 100644 --- a/harmony_service_lib/earthdata.py +++ b/harmony_service_lib/earthdata.py @@ -1,19 +1,5 @@ -from base64 import b64encode -import re -from urllib.parse import urlparse - from requests.auth import AuthBase from requests import Session - -EDL_URL_PATTERN = r""".*urs\.earthdata\.nasa\.gov$""" - - -def _edl_url(url: str) -> bool: - """Determine if the given URL is for Earthdata Login.""" - hostname = urlparse(url).hostname - return re.fullmatch(EDL_URL_PATTERN, hostname) is not None - - class EarthdataSession(Session): """Session which ensures the Authorization header is sent to correct servers. @@ -24,18 +10,7 @@ class EarthdataSession(Session): session.auth = EarthdataAuth(...) This lifecycle method on requests.Session is called when handling - redirect requests. There are two cases important for handling - Earthdata Login: - - (A) When handling a redirect from a resource server to Earthdata - Login, the session will use the auth (if provided) to add the - required Authorization to the request. - - (B) When handling a redirect from Earthdata Login back to a - resource server, the session will remove the Authorization header - from the request (which the `requests` package copies from the - request which caused this redirect). - + redirect requests. """ def rebuild_auth(self, prepared_request, response): # If not configured with an EarthdataAuth instance, defer to @@ -43,13 +18,7 @@ def rebuild_auth(self, prepared_request, response): if not self.auth: return super().rebuild_auth(prepared_request, response) - if _edl_url(prepared_request.url): - # (A) Defer to auth to add the Authorization header - self.auth(prepared_request) - else: - # (B) Remove the Authorization header when redirecting away - # from EDL. - prepared_request.headers.pop('Authorization', None) + self.auth(prepared_request) class EarthdataAuth(AuthBase): @@ -63,28 +32,20 @@ class EarthdataAuth(AuthBase): header, and the user's access token as a Bearer auth header. """ - def __init__(self, app_uid: str, app_pwd: str, user_access_token: str): + def __init__(self, user_access_token: str): """Instantiate the Earthdata Auth provider. Parameters ---------- - app_uid: - The Earthdata Login Application `uid`. - - app_pwd: - The Earthdata Login Application `password`. - user_access_token: The EDL-issued token for the user making the request. """ - creds = b64encode(f"{app_uid}:{app_pwd}".encode('utf-8')).decode('utf-8') - self.authorization_header = f'Basic {creds}, Bearer {user_access_token}' + self.authorization_header = f'Bearer {user_access_token}' def __call__(self, r): """The EarthdataAuth is a callable which adds Authorization headers - when handling a request for Earthdata Login. + when handling a request for sites backed by Earthdata Login. """ - if _edl_url(r.url): - r.headers['Authorization'] = self.authorization_header + r.headers['Authorization'] = self.authorization_header return r diff --git a/harmony_service_lib/http.py b/harmony_service_lib/http.py index 16bbd54..cd352e1 100644 --- a/harmony_service_lib/http.py +++ b/harmony_service_lib/http.py @@ -3,8 +3,7 @@ locally. When downloading from an EDL-token aware data source, this module uses EDL shared / -federated token authentication. It includes an optional fallback authentication that -uses an EDL user to download data when the feature is enabled. +federated token authentication. This module relies on the harmony_service_lib.util.config and its environment variables to be set for correct operation. See that module and the project README for details. @@ -178,8 +177,7 @@ def _download( """Implements the download functionality. Using the EarthdataSession and EarthdataAuth extensions to the - `requests` module, this function will download the given url and - perform any necessary Earthdata Login OAuth handshakes. + `requests` module, this function will download the given url. Parameters ---------- @@ -213,7 +211,7 @@ def _download( headers = {} if user_agent is not None: headers['user-agent'] = user_agent - auth = EarthdataAuth(config.oauth_uid, config.oauth_password, access_token) + auth = EarthdataAuth(access_token) tries = 0 retry = True response = None @@ -273,51 +271,6 @@ def _download( return response -def _download_with_fallback_authn(config, url: str, data, user_agent=None, **kwargs_download_agent): - """Downloads the given url using Basic authentication as a fallback - mechanism should the normal EDL Oauth handshake fail. - - This function requires the `edl_username` and `edl_password` - attributes in the config object to be populated with valid - credentials. - - Parameters - ---------- - config : harmony_service_lib.util.Config - The configuration for the current runtime environment. - url : str - The url for the resource to download - data : dict or Tuple[str, str] - Optional parameter for additional data to send to the server - when making an HTTP POST request. These data will be URL - encoded to a query string containing a series of `key=value` - pairs, separated by ampersands. If None (the default), the - request will be sent with an HTTP GET request. - total_retries: int - Upper limit on the number of times to retry the request - user_agent : str - The user agent that is requesting the download. - E.g. harmony/0.0.0 (harmony-sit) harmony-service-lib/4.0 (gdal-subsetter) - kwargs_download_agent: dict - kwargs to be passed to the download agent - E.g. stream=True - - Returns - ------- - requests.Response with the download result - - """ - headers = {} - if user_agent is not None: - headers['user-agent'] = user_agent - auth = requests.auth.HTTPBasicAuth(config.edl_username, config.edl_password) - session = requests.Session() - session.auth = auth - if data is None: - return session.get(url, headers=headers, timeout=TIMEOUT, **kwargs_download_agent) - else: - return session.post(url, headers=headers, data=data, timeout=TIMEOUT) - def _log_download_performance(logger, url, duration_ms, file_size): """Logs a message tracking performance information related to a file download. @@ -364,8 +317,6 @@ def download(config, url: str, access_token: str, data, destination_file, b. Application credentials 4. Error response when downloading 5. Data requires EULA acceptance by user - 6. If fallback authentication enabled, the application credentials are - invalid, or do not have permission to download the data. Parameters ---------- @@ -422,14 +373,6 @@ def download(config, url: str, access_token: str, data, destination_file, config, url, access_token, data, config.max_download_retries, logger, user_agent, stream=stream ) - if response is None or not response.ok: - if config.fallback_authn_enabled: - msg = ('No valid user access token in request or EDL OAuth authentication failed.' - 'Fallback authentication enabled: retrying with Basic auth.') - logger.warning(msg) - response = _download_with_fallback_authn( - config, url, data, user_agent, stream=stream) - if response.ok: if not stream: destination_file.write(response.content) diff --git a/harmony_service_lib/util.py b/harmony_service_lib/util.py index 9646bf1..661aa52 100644 --- a/harmony_service_lib/util.py +++ b/harmony_service_lib/util.py @@ -17,24 +17,9 @@ STAGING_BUCKET: The bucket where staged files should be placed STAGING_PATH: The base path under which staged files should be placed -Required when using HTTPS, allowing Earthdata Login auth: - OAUTH_HOST: The Earthdata Login (EDL) environment to connect to - OAUTH_CLIENT_ID: The EDL application client id used to acquire an EDL shared access token - OAUTH_UID: The EDL application UID used to acquire an EDL shared access token - OAUTH_PASSWORD: The EDL application password used to acquire an EDL shared access token - OAUTH_REDIRECT_URI: A valid redirect URI for the EDL application (NOTE: the redirect URI is - not followed or used; it does need to be in the app's redirect URI list) - Always provided by newer versions of the Harmony frontend: USER_AGENT: The Harmony user agent string. E.g. harmony/0.0.0 (harmony-sit) -Optional, if support is needed for downloading data from an endpoint that is not -EDL-share-token aware: - - FALLBACK_AUTHN_ENABLED: Whether to try downloading with the EDL_* credentials. - EDL_USERNAME: An valid EDL user entity username. - EDL_PASSWORD: The password belonging to EDL_USERNAME. - Optional when reading from or staging to S3: USE_LOCALSTACK: 'true' if the S3 client should connect to a LocalStack instance instead of Amazon S3 (for testing) @@ -79,14 +64,6 @@ Config = namedtuple( 'Config', [ 'app_name', - 'oauth_host', - 'oauth_client_id', - 'oauth_uid', - 'oauth_password', - 'oauth_redirect_uri', - 'fallback_authn_enabled', - 'edl_username', - 'edl_password', 'use_localstack', 'backend_host', 'localstack_host', @@ -108,10 +85,6 @@ def _validated_config(config): """ required = [ 'shared_secret_key', - 'oauth_client_id', - 'oauth_uid', - 'oauth_password', - 'oauth_redirect_uri', 'staging_path', 'staging_bucket', 'max_download_retries' @@ -119,12 +92,6 @@ def _validated_config(config): unset = [var.upper() for var in required if getattr(config, var) is None] - # Conditionally required - if config.fallback_authn_enabled and getattr(config, 'edl_username') is None: - unset.append("EDL_USERNAME") - if config.fallback_authn_enabled and getattr(config, 'edl_password') is None: - unset.append("EDL_PASSWORD") - if len(unset) > 0: msg = f"Required environment variables are not set: {', '.join(unset)}" raise Exception(msg) @@ -168,22 +135,11 @@ def int_envvar(name: str, default: int) -> int: value = environ.get(name) return int(value) if value is not None else default - oauth_redirect_uri = str_envvar('OAUTH_REDIRECT_URI', None) - if oauth_redirect_uri is not None: - oauth_redirect_uri = parse.quote(oauth_redirect_uri) backend_host = str_envvar('BACKEND_HOST', 'localhost') localstack_host = str_envvar('LOCALSTACK_HOST', backend_host) config = Config( app_name=str_envvar('APP_NAME', sys.argv[0]), - oauth_host=str_envvar('OAUTH_HOST', 'https://uat.urs.earthdata.nasa.gov'), - oauth_client_id=str_envvar('OAUTH_CLIENT_ID', None), - oauth_uid=str_envvar('OAUTH_UID', None), - oauth_password=str_envvar('OAUTH_PASSWORD', None), - oauth_redirect_uri=oauth_redirect_uri, - fallback_authn_enabled=bool_envvar('FALLBACK_AUTHN_ENABLED', False), - edl_username=str_envvar('EDL_USERNAME', None), - edl_password=str_envvar('EDL_PASSWORD', None), use_localstack=bool_envvar('USE_LOCALSTACK', False), backend_host=backend_host, localstack_host=localstack_host, diff --git a/tests/test_earthdata.py b/tests/test_earthdata.py index 69c190d..4d51e03 100644 --- a/tests/test_earthdata.py +++ b/tests/test_earthdata.py @@ -9,31 +9,24 @@ @dataclass class FakeRequest: - url: str = 'https://uat.urs.earthdata.nasa.gov/oauth' + url: str = 'https://fake.download.earthdata.nasa.gov/data' headers: dict = field(default_factory=dict) @pytest.fixture def earthdata_auth(faker): - uid = faker.simple_profile('username') - pwd = faker.password(length=12, special_chars=False) token = faker.password(length=40, special_chars=False) - return EarthdataAuth(uid, pwd, token) + return EarthdataAuth(token) def test_authdata_auth_creates_correct_header(faker): - uid = faker.simple_profile('username') - pwd = faker.password(length=12, special_chars=False) token = faker.password(length=40, special_chars=False) - auth = EarthdataAuth(uid, pwd, token) + auth = EarthdataAuth(token) request = FakeRequest() auth(request) assert 'Authorization' in request.headers - expected_creds = b64encode(f'{uid}:{pwd}'.encode('utf-8')).decode('utf-8') - assert 'Basic' in request.headers['Authorization'] - assert expected_creds in request.headers['Authorization'] assert 'Bearer' in request.headers['Authorization'] assert token in request.headers['Authorization'] @@ -45,29 +38,6 @@ def test_earthdata_auth_given_edl_url_adds_auth_header(earthdata_auth): assert 'Authorization' in request.headers - -def test_earthdata_auth_given_non_edl_url_does_not_add_header(earthdata_auth): - request = FakeRequest() - request.url = 'https://github.com/acme/foobar' - - earthdata_auth(request) - - assert 'Authorization' not in request.headers - - -# Expected EarthdataSession behavior: -# -# Auth: False => delegates to super() -# Auth: True => -# | Request Has Header | Request for EDL | Auth Header? | -# | False | False | False | -# | False | True | True | -# | True | False | False | -# | True | True | True (*) | -# Note: -# (*) Replace pre-existing Authorization header with new header -# - def test_earthdata_session_given_no_auth_delegates_to_super(monkeypatch): called = False @@ -82,17 +52,7 @@ def mock_rebuild_auth(self, prepared_request, response): assert called -def test_earthdata_session_given_no_header_and_non_edl_url_request_does_not_contain_header(earthdata_auth): - session = EarthdataSession() - session.auth = earthdata_auth - request = FakeRequest('https://duckduckgo.com/') - - session.rebuild_auth(request, None) - - assert 'Authorization' not in request.headers - - -def test_earthdata_session_given_no_header_and_edl_url_request_contains_new_header(earthdata_auth): +def test_earthdata_session_given_no_auth_header_sets_auth_header(earthdata_auth): session = EarthdataSession() session.auth = earthdata_auth request = FakeRequest() @@ -102,17 +62,7 @@ def test_earthdata_session_given_no_header_and_edl_url_request_contains_new_head assert 'Authorization' in request.headers -def test_earthdata_session_given_header_and_non_edl_url_request_does_not_contain_header(earthdata_auth): - session = EarthdataSession() - session.auth = earthdata_auth - request = FakeRequest('https://duckduckgo.com/', {'Authorization': 'PreExistingValue'}) - - session.rebuild_auth(request, None) - - assert 'Authorization' not in request.headers - - -def test_earthdata_session_given_header_and_edl_url_request_contains_existing_header(earthdata_auth): +def test_earthdata_session_given_auth_header_replaces_auth_header(earthdata_auth): session = EarthdataSession() session.auth = earthdata_auth request = FakeRequest(headers={'Authorization': 'PreExistingValue'}) diff --git a/tests/test_http.py b/tests/test_http.py index 4ba07d8..5a7a0ea 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -38,13 +38,11 @@ def test_when_given_urls_localhost_url_returns_correct_url(url, expected): def access_token(faker): return faker.password(length=40, special_chars=False) - @pytest.fixture def validate_access_token_url(): return (f'{EDL_URL}/oauth/tokens/user' '?token={token}&client_id={client_id}') - @pytest.fixture def resource_server_granule_url(): return 'https://resource.server.daac.com/foo/bar/granule.nc' @@ -61,7 +59,6 @@ def resource_server_redirect_url(faker): f'?code={faker.password(length=64, special_chars=False)}' f'&state={faker.password(length=128, special_chars=False)}') - @pytest.fixture def edl_redirect_url(faker): return ('https://uat.urs.earthdata.nasa.gov/oauth/authorize' @@ -70,12 +67,13 @@ def edl_redirect_url(faker): '&redirect_uri=https%3A%2F%2Fn5eil11u.ecs.nsidc.org%2FTS1_redirect' f'&state={faker.password(length=128, special_chars=False)}') + @pytest.fixture(autouse=True) def getsize_patched(monkeypatch): monkeypatch.setattr(os.path, "getsize", lambda a: 0) @responses.activate -def test_download_follows_redirect_to_edl_and_adds_auth_headers( +def test_download_follows_redirect_and_uses_auth_headers( mocker, access_token, resource_server_granule_url, @@ -97,81 +95,20 @@ def test_download_follows_redirect_to_edl_and_adds_auth_headers( response = download(cfg, resource_server_granule_url, access_token, None, destination_file) - # We should get redirected to EDL + # We should get redirected assert response.status_code == 302 assert len(responses.calls) == 2 - # We shouldn't have Auth headers in the request, but they should - # be added on the redirect to EDL + # We should include auth headers in both requests request_headers = responses.calls[0].request.headers redirect_headers = responses.calls[1].request.headers - assert 'Authorization' not in request_headers + assert 'Authorization' in request_headers assert 'Authorization' in redirect_headers - assert 'Basic' in redirect_headers['Authorization'] + assert 'Bearer' in request_headers['Authorization'] assert 'Bearer' in redirect_headers['Authorization'] -@responses.activate -def test_download_follows_redirect_to_resource_server_with_code( - mocker, - access_token, - edl_redirect_url, - resource_server_redirect_url): - - responses.add( - responses.GET, - edl_redirect_url, - status=302, - headers=[('Location', resource_server_redirect_url)] - ) - - responses.add( - responses.GET, - resource_server_redirect_url, - status=302 - ) - destination_file = mocker.Mock() - cfg = config_fixture() - - response = download(cfg, edl_redirect_url, access_token, None, destination_file) - - assert response.status_code == 302 - assert len(responses.calls) == 2 - edl_headers = responses.calls[0].request.headers - assert 'Authorization' in edl_headers - rs_headers = responses.calls[1].request.headers - assert 'Authorization' not in rs_headers - - -@responses.activate -def test_resource_server_redirects_to_granule_url( - mocker, - access_token, - resource_server_redirect_url, - resource_server_granule_url): - - responses.add( - responses.GET, - resource_server_redirect_url, - status=301, - headers=[('Location', resource_server_granule_url)] - ) - responses.add( - responses.GET, - resource_server_granule_url, - status=303 - ) - destination_file = mocker.Mock() - cfg = config_fixture() - - response = download(cfg, resource_server_redirect_url, access_token, None, destination_file) - - assert response.status_code == 303 - assert len(responses.calls) == 2 - rs_headers = responses.calls[0].request.headers - assert 'Authorization' not in rs_headers - @responses.activate @patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) def test_download_validates_token_and_raises_exception( @@ -181,7 +118,7 @@ def test_download_validates_token_and_raises_exception( client_id = faker.password(length=22, special_chars=False) access_token = faker.password(length=42, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id) + cfg = config_fixture() url = validate_access_token_url.format( token=access_token, client_id=client_id @@ -242,91 +179,6 @@ def test_when_authn_succeeds_it_writes_to_provided_file( assert len(responses.calls) == 1 destination_file.write.assert_called() -@responses.activate -@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) -def test_when_given_an_access_token_and_error_occurs_it_falls_back_to_basic_auth_if_enabled( - mocker, - faker, - resource_server_granule_url, - response_body_from_granule_url): - - client_id = faker.password(length=22, special_chars=False) - access_token = faker.password(length=42, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True) - - responses.add( - responses.GET, - resource_server_granule_url, - status=401 - ) - responses.add( - responses.GET, - resource_server_granule_url, - body=response_body_from_granule_url, - status=200 - ) - destination_file = mocker.Mock() - - response = download(cfg, resource_server_granule_url, access_token, None, destination_file) - - assert response.status_code == 200 - assert len(responses.calls) == 2 - assert 'Authorization' in responses.calls[1].request.headers - assert 'Basic' in responses.calls[1].request.headers['Authorization'] - destination_file.write.assert_called() - - -@responses.activate -@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) -def test_when_given_an_access_token_and_error_occurs_it_does_not_fall_back_to_basic_auth( - mocker, - faker, - resource_server_granule_url): - - client_id = faker.password(length=22, special_chars=False) - access_token = faker.password(length=42, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False) - - responses.add( - responses.GET, - resource_server_granule_url, - status=401 - ) - destination_file = mocker.Mock() - - with pytest.raises(Exception): - download(cfg, resource_server_granule_url, access_token, None, destination_file) - - assert len(responses.calls) == 1 - assert 'Authorization' not in responses.calls[0].request.headers - - -@responses.activate -def test_when_no_access_token_is_provided_it_uses_basic_auth_and_downloads_when_enabled( - mocker, - faker, - resource_server_granule_url, - response_body_from_granule_url): - - client_id = faker.password(length=22, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True) - - responses.add( - responses.GET, - resource_server_granule_url, - body=response_body_from_granule_url, - status=200 - ) - destination_file = mocker.Mock() - - response = download(cfg, resource_server_granule_url, None, None, destination_file) - - assert response.status_code == 200 - assert len(responses.calls) == 1 - assert 'Authorization' in responses.calls[0].request.headers - assert 'Basic' in responses.calls[0].request.headers['Authorization'] - destination_file.write.assert_called() - @responses.activate @patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) @@ -337,7 +189,7 @@ def test_download_all_retries_failed( client_id = faker.password(length=22, special_chars=False) access_token = faker.password(length=42, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=False) + cfg = config_fixture() responses.add( responses.GET, @@ -351,51 +203,6 @@ def test_download_all_retries_failed( assert len(responses.calls) == 5 -@responses.activate -def test_user_agent_is_passed_to_request_headers_when_using_basic_auth( - mocker, - faker, - resource_server_granule_url): - - client_id = faker.password(length=22, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True) - - responses.add( - responses.GET, - resource_server_granule_url, - status=200 - ) - destination_file = mocker.Mock() - - user_agent = 'test-agent/0.0.0' - download(cfg, resource_server_granule_url, None, None, destination_file, user_agent=user_agent) - - assert 'User-Agent' in responses.calls[0].request.headers - assert user_agent in responses.calls[0].request.headers['User-Agent'] - -@responses.activate -def test_user_agent_is_passed_to_request_headers_when_using_basic_auth_and_post_param( - mocker, - faker, - resource_server_granule_url): - - client_id = faker.password(length=22, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True) - data = {'param': 'value'} - - responses.add( - responses.POST, - resource_server_granule_url, - status=200 - ) - destination_file = mocker.Mock() - - user_agent = 'test-agent/0.0.0' - download(cfg, resource_server_granule_url, None, data, destination_file, user_agent=user_agent) - - assert 'User-Agent' in responses.calls[0].request.headers - assert user_agent in responses.calls[0].request.headers['User-Agent'] - @responses.activate def test_user_agent_is_passed_to_request_headers_when_using_edl_auth( mocker, @@ -460,27 +267,3 @@ def test_retries_on_temporary_errors_edl_auth( assert rsp1.call_count == 1 assert rsp2.call_count == 1 assert rsp3.call_count == 1 - -@responses.activate(registry=responses.registries.OrderedRegistry) -@pytest.mark.parametrize('error_code', RETRY_ERROR_CODES) -@patch('harmony_service_lib.http.get_retry_delay', Mock(return_value = 0)) -def test_retries_on_temporary_errors_basic_auth( - mocker, - faker, - access_token, - resource_server_granule_url, - error_code): - rsp1 = responses.get(resource_server_granule_url, body="Error", status=error_code) - rsp2 = responses.get(resource_server_granule_url, body="Error", status=error_code) - rsp3 = responses.get(resource_server_granule_url, body="OK", status=200) - - destination_file = mocker.Mock() - client_id = faker.password(length=22, special_chars=False) - cfg = config_fixture(oauth_client_id=client_id, fallback_authn_enabled=True, max_download_retries=5) - - response = download(cfg, resource_server_granule_url, access_token, None, destination_file) - - assert response.status_code == 200 - assert rsp1.call_count == 1 - assert rsp2.call_count == 1 - assert rsp3.call_count == 1 diff --git a/tests/util.py b/tests/util.py index da54958..0183df1 100644 --- a/tests/util.py +++ b/tests/util.py @@ -35,13 +35,9 @@ def cli_parser(*cli_args): yield parser -def config_fixture(fallback_authn_enabled=False, - edl_username='yoda', - edl_password='password_this_is', - use_localstack=False, +def config_fixture(use_localstack=False, staging_bucket='UNKNOWN', staging_path='UNKNOWN', - oauth_client_id=None, user_agent=None, app_name=None, text_logger=False, @@ -50,23 +46,15 @@ def config_fixture(fallback_authn_enabled=False, c = util.config(validate=False) return util.Config( # Override - fallback_authn_enabled=fallback_authn_enabled, - edl_username=edl_username, - edl_password=edl_password, use_localstack=use_localstack, staging_path=staging_path, staging_bucket=staging_bucket, - oauth_client_id=oauth_client_id, app_name=app_name, text_logger=text_logger, max_download_retries=max_download_retries, post_url_length=post_url_length, # Default env=c.env, - oauth_host=c.oauth_host, - oauth_uid=c.oauth_uid, - oauth_password=c.oauth_password, - oauth_redirect_uri=c.oauth_redirect_uri, backend_host=c.backend_host, localstack_host=c.localstack_host, aws_default_region=c.aws_default_region, From 6cb15e88811521921243fd7245cdf383df53ed2f Mon Sep 17 00:00:00 2001 From: Chris Durbin Date: Tue, 3 Dec 2024 09:05:08 -0500 Subject: [PATCH 2/2] HARMONY-1938: Fix linter issues. --- harmony_service_lib/earthdata.py | 2 ++ harmony_service_lib/http.py | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/harmony_service_lib/earthdata.py b/harmony_service_lib/earthdata.py index a6df4e1..e317581 100644 --- a/harmony_service_lib/earthdata.py +++ b/harmony_service_lib/earthdata.py @@ -1,5 +1,7 @@ from requests.auth import AuthBase from requests import Session + + class EarthdataSession(Session): """Session which ensures the Authorization header is sent to correct servers. diff --git a/harmony_service_lib/http.py b/harmony_service_lib/http.py index cd352e1..bd84712 100644 --- a/harmony_service_lib/http.py +++ b/harmony_service_lib/http.py @@ -17,9 +17,6 @@ import sys import os import re - -import requests - from harmony_service_lib.earthdata import EarthdataAuth, EarthdataSession from harmony_service_lib.exceptions import ServerException, ForbiddenException from harmony_service_lib.logging import build_logger @@ -271,7 +268,6 @@ def _download( return response - def _log_download_performance(logger, url, duration_ms, file_size): """Logs a message tracking performance information related to a file download.