diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ac6849..c8f67e6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.14.2] +## [0.15.0] ### Added -* `--publish-bucket` option has been added to the HyP3 entry point to publish product to the ITS_LIVE AWS Open Data bucket, `s3://its-live-data` or the test bucket `s3://its-live-data-test`. -* `upload_file_to_s3_with_upload_access_keys` to perform S3 uploads with credentialed S3 clients. -* use of `UPLOAD_ACCESS_KEY_ID` and `UPLOAD_ACCESS_KEY_SECRET` to upload products to write-protected bucket. +* `--publish-bucket` option has been added to the HyP3 entry point to additionally publish products an AWS bucket, such as the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`. +* `upload_file_to_s3_with_publish_access_keys` to perform S3 uploads using credentials from the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment vairables. ## [0.14.1] ### Changed diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 0926c286..de807959 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -27,7 +27,7 @@ from hyp3_autorift import geometry, image, io from hyp3_autorift.crop import crop_netcdf_product -from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_upload_access_keys +from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_publish_access_keys log = logging.getLogger(__name__) @@ -48,8 +48,6 @@ DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \ 'autorift_parameters/v001/autorift_landice_0120m.shp' -OPEN_DATA_BUCKET = 'its-live-data' -OPEN_DATA_BUCKET_TEST = 'its-live-data-test' PLATFORM_SHORTNAME_LONGNAME_MAPPING = { 'S1': 'sentinel1', 'S2': 'sentinel2', @@ -337,24 +335,24 @@ def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]: return var.latitude, var.longitude -def point_to_prefix(lat: float, lon: float) -> str: +def point_to_region(lat: float, lon: float) -> str: """ - Returns a string (for example, N78W124) for directory name based on - granule centerpoint lat,lon + Returns a string (for example, N78W124) of a region name based on + granule center point lat,lon """ - NShemi_str = 'N' if lat >= 0.0 else 'S' - EWhemi_str = 'E' if lon >= 0.0 else 'W' + nw_hemisphere = 'N' if lat >= 0.0 else 'S' + ew_hemisphere = 'E' if lon >= 0.0 else 'W' - outlat = int(10*np.trunc(np.abs(lat/10.0))) - if outlat == 90: # if you are exactly at a pole, put in lat = 80 bin - outlat = 80 + region_lat = int(10*np.trunc(np.abs(lat/10.0))) + if region_lat == 90: # if you are exactly at a pole, put in lat = 80 bin + region_lat = 80 - outlon = int(10*np.trunc(np.abs(lon/10.0))) + region_lon = int(10*np.trunc(np.abs(lon/10.0))) - if outlon >= 180: # if you are at the dateline, back off to the 170 bin - outlon = 170 + if region_lon >= 180: # if you are at the dateline, back off to the 170 bin + region_lon = 170 - return f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}' + return f'{nw_hemisphere}{region_lat:02d}{ew_hemisphere}{region_lon:03d}' def get_opendata_prefix(file: Path): @@ -363,11 +361,14 @@ def get_opendata_prefix(file: Path): platform_shortname = get_platform(scene) lat, lon = get_lat_lon_from_ncfile(file) - lat_lon_prefix_component = point_to_prefix(lat, lon) + region = point_to_region(lat, lon) - dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' - prefix = os.path.join(dir_path, lat_lon_prefix_component) - return prefix + return '/'.join([ + 'velocity_image_pair', + PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname], + 'v02', + region + ]) def process( @@ -572,8 +573,8 @@ def main(): parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') parser.add_argument('--publish-bucket', default='', - help='Bucket to publish the product to. ' - f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') + help='Additionally, publish products to this bucket. Necessary credentials must be provided ' + 'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.') parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -600,12 +601,7 @@ def main(): upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix) if args.publish_bucket: - - if args.publish_bucket not in [OPEN_DATA_BUCKET, OPEN_DATA_BUCKET_TEST]: - raise ValueError(f'Invalid publish bucket: {args.publish}. ' - f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') - prefix = get_opendata_prefix(product_file) - upload_file_to_s3_with_upload_access_keys(product_file, args.publish_bucket, prefix) - upload_file_to_s3_with_upload_access_keys(browse_file, args.publish_bucket, prefix) - upload_file_to_s3_with_upload_access_keys(thumbnail_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(product_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(browse_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(thumbnail_file, args.publish_bucket, prefix) diff --git a/src/hyp3_autorift/utils.py b/src/hyp3_autorift/utils.py index 7fda6eb4..6bcb6244 100644 --- a/src/hyp3_autorift/utils.py +++ b/src/hyp3_autorift/utils.py @@ -34,14 +34,14 @@ def get_esa_credentials() -> Tuple[str, str]: ) -def upload_file_to_s3_with_upload_access_keys(path_to_file: Path, bucket: str, prefix: str = ''): - if 'UPLOAD_ACCESS_KEY_ID' in os.environ and 'UPLOAD_ACCESS_KEY_SECRET' in os.environ: - access_key_id = os.environ['UPLOAD_ACCESS_KEY_ID'] - access_key_secret = os.environ['UPLOAD_ACCESS_KEY_SECRET'] - else: +def upload_file_to_s3_with_publish_access_keys(path_to_file: Path, bucket: str, prefix: str = ''): + try: + access_key_id = os.environ['PUBLISH_ACCESS_KEY_ID'] + access_key_secret = os.environ['PUBLISH_SECRET_ACCESS_KEY'] + except KeyError: raise ValueError( 'Please provide S3 Bucket upload access key credentials via the ' - 'UPLOAD_ACCESS_KEY_ID and UPLOAD_ACCESS_KEY_SECRET environment variables' + 'PUBLISH_ACCESS_KEY_ID and PUBLISH_SECRET_ACCESS_KEY environment variables' ) s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret) diff --git a/tests/test_process.py b/tests/test_process.py index 1f93b7eb..d9040e33 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -419,15 +419,15 @@ def mock_apply_filter_function(scene, _): def test_point_to_prefix(): - assert process.point_to_prefix(63.0, 128.0) == 'N60E120' - assert process.point_to_prefix(-63.0, 128.0) == 'S60E120' - assert process.point_to_prefix(63.0, -128.0) == 'N60W120' - assert process.point_to_prefix(-63.0, -128.0) == 'S60W120' - assert process.point_to_prefix(0.0, 128.0) == 'N00E120' - assert process.point_to_prefix(0.0, -128.0) == 'N00W120' - assert process.point_to_prefix(63.0, 0.0) == 'N60E000' - assert process.point_to_prefix(-63.0, 0.0) == 'S60E000' - assert process.point_to_prefix(0.0, 0.0) == 'N00E000' + assert process.point_to_region(63.0, 128.0) == 'N60E120' + assert process.point_to_region(-63.0, 128.0) == 'S60E120' + assert process.point_to_region(63.0, -128.0) == 'N60W120' + assert process.point_to_region(-63.0, -128.0) == 'S60W120' + assert process.point_to_region(0.0, 128.0) == 'N00E120' + assert process.point_to_region(0.0, -128.0) == 'N00W120' + assert process.point_to_region(63.0, 0.0) == 'N60E000' + assert process.point_to_region(-63.0, 0.0) == 'S60E000' + assert process.point_to_region(0.0, 0.0) == 'N00E000' def test_get_lat_lon_from_ncfile(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 24b5f0df..e640d411 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_upload_access_keys +from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_publish_access_keys def test_get_esa_credentials_env(tmp_path, monkeypatch): @@ -49,15 +49,15 @@ def test_get_esa_credentials_missing(tmp_path, monkeypatch): def test_upload_file_to_s3_credentials_missing(tmp_path, monkeypatch): with monkeypatch.context() as m: - m.delenv('UPLOAD_ACCESS_KEY_ID', raising=False) - m.setenv('UPLOAD_ACCESS_KEY_SECRET', 'upload_access_key_secret') + m.delenv('PUBLISH_ACCESS_KEY_ID', raising=False) + m.setenv('PUBLISH_SECRET_ACCESS_KEY', 'publish_access_key_secret') msg = 'Please provide.*' with pytest.raises(ValueError, match=msg): - upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket') + upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket') with monkeypatch.context() as m: - m.setenv('UPLOAD_ACCESS_KEY_ID', 'upload_access_key_id') - m.delenv('UPLOAD_ACCESS_KEY_SECRET', raising=False) + m.setenv('PUBLISH_ACCESS_KEY_ID', 'publish_access_key_id') + m.delenv('PUBLISH_SECRET_ACCESS_KEY', raising=False) msg = 'Please provide.*' with pytest.raises(ValueError, match=msg): - upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket') + upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')