Merge pull request #251 from ASFHyP3/publish-changes

Refactor publishing open data

CHANGELOG.md

@@ -6,11 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.14.2]
+## [0.15.0]
 ### Added
-* `--publish-bucket` option has been added to the HyP3 entry point to publish product to the ITS_LIVE AWS Open Data bucket, `s3://its-live-data` or the test bucket `s3://its-live-data-test`.
-* `upload_file_to_s3_with_upload_access_keys` to perform S3 uploads with credentialed S3 clients.
-* use of `UPLOAD_ACCESS_KEY_ID` and `UPLOAD_ACCESS_KEY_SECRET` to upload products to write-protected bucket.
+* `--publish-bucket` option has been added to the HyP3 entry point to additionally publish products an AWS bucket, such as the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`.
+* `upload_file_to_s3_with_publish_access_keys` to perform S3 uploads using credentials from the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment vairables.
 
 ## [0.14.1]
 ### Changed

src/hyp3_autorift/process.py

@@ -27,7 +27,7 @@
 
 from hyp3_autorift import geometry, image, io
 from hyp3_autorift.crop import crop_netcdf_product
-from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_upload_access_keys
+from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_publish_access_keys
 
 log = logging.getLogger(__name__)
 
@@ -48,8 +48,6 @@
 DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \
                          'autorift_parameters/v001/autorift_landice_0120m.shp'
 
-OPEN_DATA_BUCKET = 'its-live-data'
-OPEN_DATA_BUCKET_TEST = 'its-live-data-test'
 PLATFORM_SHORTNAME_LONGNAME_MAPPING = {
     'S1': 'sentinel1',
     'S2': 'sentinel2',
@@ -337,24 +335,24 @@ def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]:
         return var.latitude, var.longitude
 
 
-def point_to_prefix(lat: float, lon: float) -> str:
+def point_to_region(lat: float, lon: float) -> str:
     """
-    Returns a string (for example, N78W124) for directory name based on
-    granule centerpoint lat,lon
+    Returns a string (for example, N78W124) of a region name based on
+    granule center point lat,lon
     """
-    NShemi_str = 'N' if lat >= 0.0 else 'S'
-    EWhemi_str = 'E' if lon >= 0.0 else 'W'
+    nw_hemisphere = 'N' if lat >= 0.0 else 'S'
+    ew_hemisphere = 'E' if lon >= 0.0 else 'W'
 
-    outlat = int(10*np.trunc(np.abs(lat/10.0)))
-    if outlat == 90:  # if you are exactly at a pole, put in lat = 80 bin
-        outlat = 80
+    region_lat = int(10*np.trunc(np.abs(lat/10.0)))
+    if region_lat == 90:  # if you are exactly at a pole, put in lat = 80 bin
+        region_lat = 80
 
-    outlon = int(10*np.trunc(np.abs(lon/10.0)))
+    region_lon = int(10*np.trunc(np.abs(lon/10.0)))
 
-    if outlon >= 180:  # if you are at the dateline, back off to the 170 bin
-        outlon = 170
+    if region_lon >= 180:  # if you are at the dateline, back off to the 170 bin
+        region_lon = 170
 
-    return f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}'
+    return f'{nw_hemisphere}{region_lat:02d}{ew_hemisphere}{region_lon:03d}'
 
 
 def get_opendata_prefix(file: Path):
@@ -363,11 +361,14 @@ def get_opendata_prefix(file: Path):
 
     platform_shortname = get_platform(scene)
     lat, lon = get_lat_lon_from_ncfile(file)
-    lat_lon_prefix_component = point_to_prefix(lat, lon)
+    region = point_to_region(lat, lon)
 
-    dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02'
-    prefix = os.path.join(dir_path, lat_lon_prefix_component)
-    return prefix
+    return '/'.join([
+        'velocity_image_pair',
+        PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname],
+        'v02',
+        region
+    ])
 
 
 def process(
@@ -572,8 +573,8 @@ def main():
     parser.add_argument('--bucket', help='AWS bucket to upload product files to')
     parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files')
     parser.add_argument('--publish-bucket', default='',
-                        help='Bucket to publish the product to. '
-                        f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}')
+                        help='Additionally, publish products to this bucket. Necessary credentials must be provided '
+                             'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.')
     parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE,
@@ -600,12 +601,7 @@ def main():
         upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix)
 
     if args.publish_bucket:
-
-        if args.publish_bucket not in [OPEN_DATA_BUCKET, OPEN_DATA_BUCKET_TEST]:
-            raise ValueError(f'Invalid publish bucket: {args.publish}. '
-                             f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}')
-
         prefix = get_opendata_prefix(product_file)
-        upload_file_to_s3_with_upload_access_keys(product_file, args.publish_bucket, prefix)
-        upload_file_to_s3_with_upload_access_keys(browse_file, args.publish_bucket, prefix)
-        upload_file_to_s3_with_upload_access_keys(thumbnail_file, args.publish_bucket, prefix)
+        upload_file_to_s3_with_publish_access_keys(product_file, args.publish_bucket, prefix)
+        upload_file_to_s3_with_publish_access_keys(browse_file, args.publish_bucket, prefix)
+        upload_file_to_s3_with_publish_access_keys(thumbnail_file, args.publish_bucket, prefix)

src/hyp3_autorift/utils.py

@@ -34,14 +34,14 @@ def get_esa_credentials() -> Tuple[str, str]:
     )
 
 
-def upload_file_to_s3_with_upload_access_keys(path_to_file: Path, bucket: str, prefix: str = ''):
-    if 'UPLOAD_ACCESS_KEY_ID' in os.environ and 'UPLOAD_ACCESS_KEY_SECRET' in os.environ:
-        access_key_id = os.environ['UPLOAD_ACCESS_KEY_ID']
-        access_key_secret = os.environ['UPLOAD_ACCESS_KEY_SECRET']
-    else:
+def upload_file_to_s3_with_publish_access_keys(path_to_file: Path, bucket: str, prefix: str = ''):
+    try:
+        access_key_id = os.environ['PUBLISH_ACCESS_KEY_ID']
+        access_key_secret = os.environ['PUBLISH_SECRET_ACCESS_KEY']
+    except KeyError:
         raise ValueError(
             'Please provide S3 Bucket upload access key credentials via the '
-            'UPLOAD_ACCESS_KEY_ID and UPLOAD_ACCESS_KEY_SECRET environment variables'
+            'PUBLISH_ACCESS_KEY_ID and PUBLISH_SECRET_ACCESS_KEY environment variables'
         )
 
     s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret)

tests/test_process.py

@@ -419,15 +419,15 @@ def mock_apply_filter_function(scene, _):
 
 
 def test_point_to_prefix():
-    assert process.point_to_prefix(63.0, 128.0) == 'N60E120'
-    assert process.point_to_prefix(-63.0, 128.0) == 'S60E120'
-    assert process.point_to_prefix(63.0, -128.0) == 'N60W120'
-    assert process.point_to_prefix(-63.0, -128.0) == 'S60W120'
-    assert process.point_to_prefix(0.0, 128.0) == 'N00E120'
-    assert process.point_to_prefix(0.0, -128.0) == 'N00W120'
-    assert process.point_to_prefix(63.0, 0.0) == 'N60E000'
-    assert process.point_to_prefix(-63.0, 0.0) == 'S60E000'
-    assert process.point_to_prefix(0.0, 0.0) == 'N00E000'
+    assert process.point_to_region(63.0, 128.0) == 'N60E120'
+    assert process.point_to_region(-63.0, 128.0) == 'S60E120'
+    assert process.point_to_region(63.0, -128.0) == 'N60W120'
+    assert process.point_to_region(-63.0, -128.0) == 'S60W120'
+    assert process.point_to_region(0.0, 128.0) == 'N00E120'
+    assert process.point_to_region(0.0, -128.0) == 'N00W120'
+    assert process.point_to_region(63.0, 0.0) == 'N60E000'
+    assert process.point_to_region(-63.0, 0.0) == 'S60E000'
+    assert process.point_to_region(0.0, 0.0) == 'N00E000'
 
 
 def test_get_lat_lon_from_ncfile():

tests/test_utils.py

@@ -1,6 +1,6 @@
 import pytest
 
-from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_upload_access_keys
+from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_publish_access_keys
 
 
 def test_get_esa_credentials_env(tmp_path, monkeypatch):
@@ -49,15 +49,15 @@ def test_get_esa_credentials_missing(tmp_path, monkeypatch):
 
 def test_upload_file_to_s3_credentials_missing(tmp_path, monkeypatch):
     with monkeypatch.context() as m:
-        m.delenv('UPLOAD_ACCESS_KEY_ID', raising=False)
-        m.setenv('UPLOAD_ACCESS_KEY_SECRET', 'upload_access_key_secret')
+        m.delenv('PUBLISH_ACCESS_KEY_ID', raising=False)
+        m.setenv('PUBLISH_SECRET_ACCESS_KEY', 'publish_access_key_secret')
         msg = 'Please provide.*'
         with pytest.raises(ValueError, match=msg):
-            upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket')
+            upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')
 
     with monkeypatch.context() as m:
-        m.setenv('UPLOAD_ACCESS_KEY_ID', 'upload_access_key_id')
-        m.delenv('UPLOAD_ACCESS_KEY_SECRET', raising=False)
+        m.setenv('PUBLISH_ACCESS_KEY_ID', 'publish_access_key_id')
+        m.delenv('PUBLISH_SECRET_ACCESS_KEY', raising=False)
         msg = 'Please provide.*'
         with pytest.raises(ValueError, match=msg):
-            upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket')
+            upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')