From ac20db20509beff75f5229c4a0c006c8bc13d39b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Jan 2024 16:53:06 +0000 Subject: [PATCH 01/28] Bump ASFHyP3/actions from 0.10.0 to 0.11.0 Bumps [ASFHyP3/actions](https://github.com/asfhyp3/actions) from 0.10.0 to 0.11.0. - [Release notes](https://github.com/asfhyp3/actions/releases) - [Changelog](https://github.com/ASFHyP3/actions/blob/develop/CHANGELOG.md) - [Commits](https://github.com/asfhyp3/actions/compare/v0.10.0...v0.11.0) --- updated-dependencies: - dependency-name: ASFHyP3/actions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-jira-issue.yml | 2 +- .github/workflows/labeled-pr.yml | 2 +- .github/workflows/release-template-comment.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/static-analysis.yml | 4 ++-- .github/workflows/tag-version.yml | 2 +- .github/workflows/test-and-build.yml | 6 +++--- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 1d001cc5..3ce6f429 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,6 +13,6 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0 secrets: USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 8436b4ed..0b69efec 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 48fc9e84..66ba502e 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -12,4 +12,4 @@ on: jobs: call-labeled-pr-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml index 64197b00..cae89e6f 100644 --- a/.github/workflows/release-template-comment.yml +++ b/.github/workflows/release-template-comment.yml @@ -7,6 +7,6 @@ on: jobs: call-release-checklist-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0 secrets: USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f0d6ffc..c8fed248 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0 with: release_prefix: HyP3 autoRIFT secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index a142be7f..3b07f93b 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,10 +4,10 @@ on: push jobs: call-flake8-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.0 with: local_package_names: hyp3_autorift excludes: src/hyp3_autorift/vend call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index eeaf346f..4714c1f1 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -7,6 +7,6 @@ on: jobs: call-bump-version-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 1e215924..cee1a1e2 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -12,18 +12,18 @@ on: jobs: call-pytest-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0 with: local_package_name: hyp3_autorift python_versions: >- ["3.9"] call-version-info-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.0 call-docker-ghcr-workflow: needs: call-version-info-workflow - uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.0 with: version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} secrets: From 5cf67333b4003d5450329583c067e23fa62f120c Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 17 Jan 2024 09:01:14 -0900 Subject: [PATCH 02/28] Delete .trufflehog.txt --- .trufflehog.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .trufflehog.txt diff --git a/.trufflehog.txt b/.trufflehog.txt deleted file mode 100644 index 4ed920df..00000000 --- a/.trufflehog.txt +++ /dev/null @@ -1,3 +0,0 @@ -.*gitleaks.toml$ -.*hyp3_autorift/vend/workflow/.* -.*hyp3_autorift/vend/README.md$ From 4fc259478721ab4106deae6025eaf255e167536f Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Mon, 18 Mar 2024 15:39:41 -0800 Subject: [PATCH 03/28] add an option to upload the opendata --- src/hyp3_autorift/process.py | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 99ecaccb..5a5f5aa4 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -49,6 +49,16 @@ 'autorift_parameters/v001/autorift_landice_0120m.shp' +PLATFORM_SHORTNAME_LONGNAME_MAPPING = { + 'S1': 'sentinel1', + 'S2': 'sentinel2', + 'L4': 'landsatOLI', + 'L5': 'landsatOLI', + 'L7': 'landsatOLI', + 'L8': 'landsatOLI', + 'L9': 'landsatOLI', +} + def get_lc2_stac_json_key(scene_name: str) -> str: platform = get_platform(scene_name) year = scene_name[17:21] @@ -319,6 +329,42 @@ def apply_landsat_filtering(reference_path: str, secondary_path: str) \ return reference_path, reference_zero_path, secondary_path, secondary_zero_path +def get_lon_lat_from_ncfile(ncfile): + with Dataset(ncfile) as ds: + var = ds.variables['img_pair_info'] + return var.latitude, var.longitude + + +def point_to_prefix(dir_path: str, lat: float, lon: float) -> str: + """ + Returns a string (for example, N78W124) for directory name based on + granule centerpoint lat,lon + """ + NShemi_str = 'N' if lat >= 0.0 else 'S' + EWhemi_str = 'E' if lon >= 0.0 else 'W' + + outlat = int(10*np.trunc(np.abs(lat/10.0))) + if outlat == 90: # if you are exactly at a pole, put in lat = 80 bin + outlat = 80 + + outlon = int(10*np.trunc(np.abs(lon/10.0))) + + if outlon >= 180: # if you are at the dateline, back off to the 170 bin + outlon = 170 + + dirstring = os.path.join(dir_path, f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}') + return dirstring + + +def upload_opendata(product_file: str, bucket: str, scene: str): + # bucket = 's3://its-live-data' + platform_shortname = get_platform(scene) + dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}' + lat, lon = get_lon_lat_from_ncfile(product_file) + bucket_prefix = point_to_prefix(dir_path, lat, lon) + upload_file_to_s3(product_file, bucket, bucket_prefix) + + def process( reference: str, secondary: str, @@ -520,6 +566,7 @@ def main(): ) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') + parser.add_argument('--opendata-upload', type=bool, default=Ture, help="If or not upload to its-live-data bucket") parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -539,6 +586,10 @@ def main(): product_file, browse_file = process(g1, g2, parameter_file=args.parameter_file, naming_scheme=args.naming_scheme) + if args.opendata_upload: + bucket = 'its-live-data' + upload_opendata(product_file, bucket, args.granules[0]) + if args.bucket: upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix) From 8af8a549b8ee2789321443942bd4276ce08491a0 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Mon, 18 Mar 2024 15:45:14 -0800 Subject: [PATCH 04/28] modify code style --- src/hyp3_autorift/process.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 5a5f5aa4..f1f7b160 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -59,6 +59,7 @@ 'L9': 'landsatOLI', } + def get_lc2_stac_json_key(scene_name: str) -> str: platform = get_platform(scene_name) year = scene_name[17:21] @@ -344,12 +345,12 @@ def point_to_prefix(dir_path: str, lat: float, lon: float) -> str: EWhemi_str = 'E' if lon >= 0.0 else 'W' outlat = int(10*np.trunc(np.abs(lat/10.0))) - if outlat == 90: # if you are exactly at a pole, put in lat = 80 bin + if outlat == 90: # if you are exactly at a pole, put in lat = 80 bin outlat = 80 outlon = int(10*np.trunc(np.abs(lon/10.0))) - if outlon >= 180: # if you are at the dateline, back off to the 170 bin + if outlon >= 180: # if you are at the dateline, back off to the 170 bin outlon = 170 dirstring = os.path.join(dir_path, f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}') @@ -566,7 +567,7 @@ def main(): ) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') - parser.add_argument('--opendata-upload', type=bool, default=Ture, help="If or not upload to its-live-data bucket") + parser.add_argument('--opendata-upload', type=bool, default=True, help="If or not upload to its-live-data bucket") parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, From 9ad03cc74d3c9369c8a3c18c4906faa7cb40d277 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Mon, 18 Mar 2024 16:42:51 -0800 Subject: [PATCH 05/28] refactor the upload open data code --- CHANGELOG.md | 3 +++ src/hyp3_autorift/process.py | 15 +++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c6b257..dfa9d5da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.14.2] +### Added +* the option upload-opendata to upload the produdct to s3://its-live-data bucket ## [0.14.1] ### Changed diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index f1f7b160..02c950ae 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -357,13 +357,11 @@ def point_to_prefix(dir_path: str, lat: float, lon: float) -> str: return dirstring -def upload_opendata(product_file: str, bucket: str, scene: str): - # bucket = 's3://its-live-data' +def get_opendata_prefix(file: Path, scene: str): platform_shortname = get_platform(scene) dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}' - lat, lon = get_lon_lat_from_ncfile(product_file) - bucket_prefix = point_to_prefix(dir_path, lat, lon) - upload_file_to_s3(product_file, bucket, bucket_prefix) + lat, lon = get_lon_lat_from_ncfile(str(file)) + return point_to_prefix(dir_path, lat, lon) def process( @@ -586,10 +584,15 @@ def main(): g1, g2 = sorted(args.granules, key=get_datetime) product_file, browse_file = process(g1, g2, parameter_file=args.parameter_file, naming_scheme=args.naming_scheme) + thumbnail_file = create_thumbnail(browse_file) if args.opendata_upload: bucket = 'its-live-data' - upload_opendata(product_file, bucket, args.granules[0]) + prefix = get_opendata_prefix(product_file, args.granules[0]) + upload_file_to_s3(product_file, bucket, prefix) + upload_file_to_s3(browse_file, bucket, prefix) + upload_file_to_s3(thumbnail_file, bucket, prefix) + if args.bucket: upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) From 5a11809f74caf0ac64298fb6cd6f8948bf2966d7 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Mon, 18 Mar 2024 16:45:06 -0800 Subject: [PATCH 06/28] modify code style --- src/hyp3_autorift/process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 02c950ae..ab89b069 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -593,7 +593,6 @@ def main(): upload_file_to_s3(browse_file, bucket, prefix) upload_file_to_s3(thumbnail_file, bucket, prefix) - if args.bucket: upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix) From 1ff476975d381c4275c3cde864daf639b3cdc6ac Mon Sep 17 00:00:00 2001 From: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> Date: Tue, 19 Mar 2024 07:43:34 -0500 Subject: [PATCH 07/28] Apply suggestions from code review Co-authored-by: Joseph H Kennedy --- src/hyp3_autorift/process.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index ab89b069..203e6803 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -330,7 +330,7 @@ def apply_landsat_filtering(reference_path: str, secondary_path: str) \ return reference_path, reference_zero_path, secondary_path, secondary_zero_path -def get_lon_lat_from_ncfile(ncfile): +def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]: with Dataset(ncfile) as ds: var = ds.variables['img_pair_info'] return var.latitude, var.longitude @@ -359,8 +359,8 @@ def point_to_prefix(dir_path: str, lat: float, lon: float) -> str: def get_opendata_prefix(file: Path, scene: str): platform_shortname = get_platform(scene) - dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}' - lat, lon = get_lon_lat_from_ncfile(str(file)) + dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' + lat, lon = get_lon_lat_from_ncfile(file) return point_to_prefix(dir_path, lat, lon) @@ -565,7 +565,7 @@ def main(): ) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') - parser.add_argument('--opendata-upload', type=bool, default=True, help="If or not upload to its-live-data bucket") + parser.add_argument('--publish', type=string_is_true, default=False, help=f'Additionally publish the product to the ITS_LIVE AWS Open Data bucket: s3://{OPEN_DATA_BUCKET}') parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -587,11 +587,10 @@ def main(): thumbnail_file = create_thumbnail(browse_file) if args.opendata_upload: - bucket = 'its-live-data' - prefix = get_opendata_prefix(product_file, args.granules[0]) - upload_file_to_s3(product_file, bucket, prefix) - upload_file_to_s3(browse_file, bucket, prefix) - upload_file_to_s3(thumbnail_file, bucket, prefix) + prefix = get_opendata_prefix(product_file, g1) + upload_file_to_s3(product_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3(browse_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3(thumbnail_file, OPEN_DATA_BUCKET, prefix) if args.bucket: upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) From af16ede720609023e3d91e91e2f54e93172af954 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 19 Mar 2024 13:08:51 +0000 Subject: [PATCH 08/28] finish up code review additions --- src/hyp3_autorift/process.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 203e6803..c0de21ee 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -22,6 +22,7 @@ from hyp3lib.get_orb import downloadSentinelOrbitFile from hyp3lib.image import create_thumbnail from hyp3lib.scene import get_download_url +from hyp3lib.util import string_is_true from netCDF4 import Dataset from osgeo import gdal @@ -48,7 +49,7 @@ DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \ 'autorift_parameters/v001/autorift_landice_0120m.shp' - +OPEN_DATA_BUCKET = 'its-live-data' PLATFORM_SHORTNAME_LONGNAME_MAPPING = { 'S1': 'sentinel1', 'S2': 'sentinel2', @@ -336,7 +337,7 @@ def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]: return var.latitude, var.longitude -def point_to_prefix(dir_path: str, lat: float, lon: float) -> str: +def point_to_prefix(platform_shortname: str, lat: float, lon: float) -> str: """ Returns a string (for example, N78W124) for directory name based on granule centerpoint lat,lon @@ -353,15 +354,18 @@ def point_to_prefix(dir_path: str, lat: float, lon: float) -> str: if outlon >= 180: # if you are at the dateline, back off to the 170 bin outlon = 170 + dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' dirstring = os.path.join(dir_path, f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}') return dirstring -def get_opendata_prefix(file: Path, scene: str): +def get_opendata_prefix(file: Path): + # filenames have form GRANULE1_X_GRANULE2 + scene = file.name.split('_X_')[0] + platform_shortname = get_platform(scene) - dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' - lat, lon = get_lon_lat_from_ncfile(file) - return point_to_prefix(dir_path, lat, lon) + lat, lon = get_lat_lon_from_ncfile(file) + return point_to_prefix(platform_shortname, lat, lon) def process( @@ -565,7 +569,9 @@ def main(): ) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') - parser.add_argument('--publish', type=string_is_true, default=False, help=f'Additionally publish the product to the ITS_LIVE AWS Open Data bucket: s3://{OPEN_DATA_BUCKET}') + parser.add_argument('--publish', type=string_is_true, default=False, + help='Additionally publish the product to ' + f'the ITS_LIVE AWS Open Data bucket: s3://{OPEN_DATA_BUCKET}') parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -587,7 +593,7 @@ def main(): thumbnail_file = create_thumbnail(browse_file) if args.opendata_upload: - prefix = get_opendata_prefix(product_file, g1) + prefix = get_opendata_prefix(product_file) upload_file_to_s3(product_file, OPEN_DATA_BUCKET, prefix) upload_file_to_s3(browse_file, OPEN_DATA_BUCKET, prefix) upload_file_to_s3(thumbnail_file, OPEN_DATA_BUCKET, prefix) From c584df14ee29d0c45deb3e444ed87194bdabab61 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 19 Mar 2024 13:20:22 +0000 Subject: [PATCH 09/28] mild refactor --- src/hyp3_autorift/process.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index c0de21ee..e50d51ad 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -354,9 +354,7 @@ def point_to_prefix(platform_shortname: str, lat: float, lon: float) -> str: if outlon >= 180: # if you are at the dateline, back off to the 170 bin outlon = 170 - dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' - dirstring = os.path.join(dir_path, f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}') - return dirstring + return f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}' def get_opendata_prefix(file: Path): @@ -365,7 +363,11 @@ def get_opendata_prefix(file: Path): platform_shortname = get_platform(scene) lat, lon = get_lat_lon_from_ncfile(file) - return point_to_prefix(platform_shortname, lat, lon) + lat_lon_prefix_component = point_to_prefix(platform_shortname, lat, lon) + + dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' + prefix = os.path.join(dir_path, lat_lon_prefix_component) + return prefix def process( From 587e2ce07b478949269033b374c1752157334f2c Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 11:02:15 -0800 Subject: [PATCH 10/28] correct one statement in process.py --- src/hyp3_autorift/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index e50d51ad..572934cc 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -594,7 +594,7 @@ def main(): product_file, browse_file = process(g1, g2, parameter_file=args.parameter_file, naming_scheme=args.naming_scheme) thumbnail_file = create_thumbnail(browse_file) - if args.opendata_upload: + if args.publish: prefix = get_opendata_prefix(product_file) upload_file_to_s3(product_file, OPEN_DATA_BUCKET, prefix) upload_file_to_s3(browse_file, OPEN_DATA_BUCKET, prefix) From b97aecdafbf5b92810d1d04d828d2c730a2a0fd5 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 15:41:00 -0800 Subject: [PATCH 11/28] add test functions in test_process.py --- src/hyp3_autorift/process.py | 4 ++-- tests/test_process.py | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 572934cc..ca54d724 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -337,7 +337,7 @@ def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]: return var.latitude, var.longitude -def point_to_prefix(platform_shortname: str, lat: float, lon: float) -> str: +def point_to_prefix(lat: float, lon: float) -> str: """ Returns a string (for example, N78W124) for directory name based on granule centerpoint lat,lon @@ -363,7 +363,7 @@ def get_opendata_prefix(file: Path): platform_shortname = get_platform(scene) lat, lon = get_lat_lon_from_ncfile(file) - lat_lon_prefix_component = point_to_prefix(platform_shortname, lat, lon) + lat_lon_prefix_component = point_to_prefix(lat, lon) dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' prefix = os.path.join(dir_path, lat_lon_prefix_component) diff --git a/tests/test_process.py b/tests/test_process.py index f5d6d5c6..14bc1d14 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -1,5 +1,6 @@ import io from datetime import datetime +from pathlib import Path from re import match from unittest import mock from unittest.mock import MagicMock, patch @@ -11,7 +12,7 @@ from hyp3_autorift import process - +import pdb def test_get_platform(): assert process.get_platform('S1B_IW_GRDH_1SSH_20201203T095903_20201203T095928_024536_02EAB3_6D81') == 'S1' assert process.get_platform('S1A_IW_SLC__1SDV_20180605T233148_20180605T233215_022228_0267AD_48B2') == 'S1' @@ -415,3 +416,23 @@ def mock_apply_filter_function(scene, _): process.apply_landsat_filtering('LT04', 'LE07') assert process.apply_landsat_filtering('LT04', 'LT05') == ('LT04', None, 'LT05', None) assert process.apply_landsat_filtering('LT04', 'LT04') == ('LT04', None, 'LT04', None) + + +def test_point_to_prefix(): + assert process.point_to_prefix(63.0, 128.0) == 'N60E120' + assert process.point_to_prefix(-63.0, 128.0) == 'S60E120' + assert process.point_to_prefix(63.0, -128.0) == 'N60W120' + assert process.point_to_prefix(-63.0, -128.0) == 'S60W120' + assert process.point_to_prefix(0.0, 0.0) == 'N00E000' + + +def test_get_lat_lon_from_ncfile(): + file = Path('tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + assert process.get_lat_lon_from_ncfile(file) == (-81.49, -128.28) + + +def test_get_opendata_prefix(): + file = Path('tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120' + + From 0787b01fad8ebfe0993c4bcab07da20118c8eba9 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 15:43:23 -0800 Subject: [PATCH 12/28] add an example product file in tests/data --- ...20_19850124_20200918_02_T2_G0120V02_P000.nc | Bin 0 -> 309968 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc diff --git a/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc b/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc new file mode 100644 index 0000000000000000000000000000000000000000..487be3bcd2c2c5719fdeb9094b274a32dfb7e702 GIT binary patch literal 309968 zcmeF)2b>z!zwrOrtV)Qnts2CNu@E$6mlQ%Ynofvmva19WT{pXgV75ed6M{7w6|uV( z#NH9Rs|GQa7z<*JYeDQCvAY(;`ai?xkYw|VUj04Kz0dvM3$K@X?{_$JICJKlGiL^7 z3{9OhWvhNW_v_QQuTpybAzy1B+^o2$TE6l6<70=yX}>;7g}P+NT@^L6%59Xh`;cO{N}MQF^2LNK{L_E8*g;V=n|${#-r4!j)*+)U zcKp-#|LUzjFUf!KdCFFQXG=G^k^fT{UUJB8KTL5ZVQ-B@zvGJF&QF&?@=a+bE?^ z*=?W5>|!ddnrbUjZB8ERQz;cQo2Jx?yp*}Tr6w-EO3F|aX|}RV86-l=^y9v~vyV(r z^7dbUe0@UQ^eLirXR6&{vZ8D)XdeFd5am2IQ?H_IDd-rm^Au&3nwQ8^O?b&|2icSX z;tyr8`j4X2A9wgdWrDiObVcbWc+2pnPtmCh_E!|GAiCR0%apvj;C)3=`}>=p-Fu31 zjrc{r&{Lm0O?g#aFoIgW=EsHCD-+ZOMz$ndJ>)j!e6<|nW4938zwM-Zl@HYg zVik4QUA^nJ-O2!Um-s4+75UY%)4LU|n&J#aQLFSD?-4Jn+g1EnwK%tZQ}xTP2Mv8k zUG%^3RowsDUOTf_o%gD%Ui+)pzUo!Sz4lSBhSO_j^=cozT2HUm*Q?d^+PS@2?W%|7 z)cr5qfB&bOyy56nAx5W_dyr#=vb)TB+Sku~Ii>@>NV+3WRb5EOuFSahDvCN@J-Uh1V@tJFfaFEnm48u{s8*uhQ~T zKBZ&(#>-1_$x=S*0HwZ4%S-vFBboZzc)2<#s;|=WQa>PIRY++;*>Vk(YoJ^Mn{lQke7m*O6{uD z1WT7Vq@{~v>$3inKSYNpUDT32K>dPX+%S??C|x6ceKx-Izj8O;-F@99xYxeO4dc@P zA#|g^ed&J}ap_!r$qQ`jyx6+KZ}QUriRx9fnTn!*s{f3a^vcER3X|1uP{h^#L*Lv- z$*2p)aoO&!7q_ogK2o=7p!hycudc(oWJavKCO0_zFJftu| znI-;EGgZs~@|}HVamCrHE>)DLmko0$W7Kj?RFqu>H@)}f93`$^_8YG#YL~w)d;C1* zCw0LDMN#+7l}F7!iV}%c)XPa@@AtU5IxJGC7sRqp=@ux(?*&B)^-9~zD-FjeTc~*q zREi(Jc;WqvhrhK}e8m6C*M^t=<&NmJn|d{bUX4Hu?4rg*wHDjlNi385A$g zKBNY-<@cif)g~c=Vh`G|eeh+jXe`Cctq-cn_Tr1DUhS&a-tM(yl`|^EI%b70)Ehp0kt< zJGyVqL3j7+k^NjZ-LGueTTb}?v_!AYxZ*qU%a08^*B3uee!JJOu;__H-%&Q~r9VCJ z;5)r~-HBC$KIk?2cn@3jt+L^0;+XNyZwKvu(yIT;I{xqcSXOXjw=*0pC+Xvq5nC!F{tyG~=36t1FHMWYSMRoM#faNq`KEXTP?R3I>7}{*)=Fts zzp?$tt4S-B9mLwADD^Yl9gR&3MoesL^EG)}8#>(W%C_QjipLtopr{2^k1Oh{r22P% zF)&w%Y_<@E6=^3Lk=2Z6mt{d!or`-1?|M|XGl-rkz zPY|!~VcqYw{!{n8x@G^@_kBQQx~DJlaq)}q7r$nEDqaw?4E5tyf3W$Qsv*VqO1~tk z7^En#t1}O!Vz{C_U#!ggzC5S8n5c9%Q6auC{ftB-E0kx8bEHo{d1~^MsgowGJGCt~ z^cPp;e!J)*`uyr!+36~uHQe+4ISQ&B~9+Xox}NSei$^bzHW7;m^2rQ z(A?JQ`Da;}=7~)D9xbxivDD1G^-CL?y-oFt-9EJ%8pIjZKWcKKuF9`}qXv5Se>H=N zP5t}-#bHGK;f2~Vy)9xN)h}-FivHBPpv~7J+EH7p+AfR753wuP^$jsT6<3*FS3Rs#J^-R zfRZ#c9=pWb?kyhhioIoBxzuY2>pNNXsJa^Sn33i&HP)fSEtX1go?hINO|4InHt3|q z64Z$vI(+1q;Uk8QtQNg=RHZm4D`wHy*4!))xs9T~dfI$}^|hmxK>bhM?!QjYr*4>D zNB7wv|JEff^WAcY`-dG!j0u00-#;C}6_KB;!p1hAPi}v+d#Ss5{l=>^D|Lg^R`Hj~ z+4kPax1x4iyzTAkY-ZhlEgc=Et3Sjn?tl%FkkJN7h`%8xP9(cu_eWWnL`}$)Sc{H=x#ME zQ8&)evC!LY5L?(V*s!?S-O%nfG&C-1ZSyy~n>=puuG`SwwnQ8W41?O;ZiA<}ZGJ;@ zX;DK*htE5INr&5H5OZ8ZqnH_t>9?Q3CT6>>Zr@*B8y7aTdc>QP z8+yIz6`R)NZ5QwQ#P+r=b^F98sGHrarn49V?gn4`P(z(qGSL@kYhS!jROnzsjak&! z(2?evp@xMW9gEw?3>((e<{jGR^9-x09y+qdI$~Hy|2A3%B zh83h%+<1?#p=GdPc5zeGEXObDXlre2Xg3Rbl9iVnPk1#JK7E77cXx1is4OkMT0mVtWRwh+d9eHzPP!e(cL1B?L$TGEo~ic zL#>#OiR{E?4IebZq;8pIgmtLMaueG&e7LoGn6JHNxENVQs($^(H#HT<^oHiyytBS+V)J5JZnoH*OpCMHiEUfJmeiVmlCI?=S$(Q&MKaBK18cI=3uqleZE zUf8r?;^jYn_3r=b>iWCm73(Ixwn^is*N=Bh9$!DpQ9pZ@eP&&894L+6 z<3)#AH{+hz*1A+|u_z*gg?eV9mV4rqAvHtAJXen3#nDjhX&vsSVrJ@<12vs!?9&=r zMPoM)H&l-tV>XW&E{@&uBvedon>qVcuPmy`OS{?7zQngcG)RLu9JPrdL{wX)xZKYX zU$*Z0_BNVMy{yFI2dkG`iyu66lb+CXS?JYYO@wSR-6|P zY4$F44^hXzq3y$mHncPx*VfwLZy(y&)-p^jq0hUZL(HvYPVK{%R#(@of1_FSUNL~w zi&;-~%h2|Piz}6FDMh=OJIhH~M_@5quAUF7d~sWmF=P`i>Fn^tx_)m ztEr}oyu~{PandBWOmtQC=-MQlU`L6H0k6V1!uL0OUIBFBPzFOJ_bU4Wh`J zbrrX;%~vc#wJgK7l6jGDUrDJZm^_zsoLGmuJ5=J*Jxf3N=#Qm&C>|(DFIjK0{{? zJ)fdKi@{GY{4qv9!uW@n`~cJMWA;7Fzl$2F?j2&|+h}Ru!^zJQkJ(e@lVpGD6z=uctrX$&VZ`V_{W#N-p0ejKxF zF#i~8q`K9_#z)ce2-+S-=R@dOh5iRIn85G@7>#3mB_{93^nIAU7xP`Hk?QUtHr|bv zyU=zgI`2Tw?dZP^gSTQhhS6Iv9>wI%n7#?KH)8$<)JS#L6C1BX%e82`2Ax-<=PLAH ziNPx{d^tuV7{3gYVN73&*-J2gF>0i`i-?V#Xt@w=7ohWe^qhzOb1`@hhR??6Sr`vt z@=QzzF?$B)Pe+YZcN(#A1zJu;+bQT=j-Hdze-Z{y#PA6iJs#u7VKRW}WtjD2ekp3C zx+TQM4z#qR&4!SuJ7{RZ=2qeiOxirDxi zTE0Nr=ji+lJvsD$ioq;~Kf&n782<>9A7c6g%)XEL_fR9%y-RF-2Q6=-?Jab^iJmvm zpTXek7*1pKHH^QC$yYG_GG@Cm{}O7Xy0yf{7t!(p+MY+}bLe>%{m)=9h2f_$n#A~1 zn0yk`Phj?O%&$R>RQDLMaWz^VMcX6jd>B0sp??(yAH;A1qYq#_j>(mnz8|yqVg6p! zNOfJr#(U6mH`?w(=bh-e1O2yS@HPzJiqROxZ^2{~(>G)GCd}W68maCEV&nB_xejgD zqVpQ`T#f##FnA?~ufXW#7>{7`GE9dtdnx8GL5);*F|qL?v~;5FLUdk$p7YUv9tO|F z@HrSg8{=nTGKA?fF&o7E8K{x!PA4{=hL#m*I~AR$pl3PyPsZR$7(NlBCt&<|Odf~n z0A`nA-j5ooZYi;G30gYP){agedX7c^VhpxnxD}%<7;nbpB1|8HSuf@nqDHFo5E~bu z#f`QmbT*=AKKdIlcr=FVF?tlnkHq9WOwYya5tw(OMyi`bY;>aKaI_tU&N}qeqThkR zLoqxXqq8tR6O%JAJsq=$V162Eq&hpXaVlDDXq$r0$>^Dc{)rfzfZ_2N9f$FQF?kTC z$71$C%pZUnscsCh(TbMQXtSVm6naLYe*^}HW7v$*8jM$Cau}wEVs;4T2ct%++n?CD zA6iUk8-z|HdiF*CJ{a5^!+T+LAjbE^qyf`=V0L%R?}i$wZdYRCE@;^qZF+R>gdQFG z2VihV4DW!^?J-`3$?Y(`EoQgD{MM+E>b4>__D9Q>XzPbgEqeN*UxUFdFuXZNH^X=( zCi`Hz0<#L{|1cKkAsVUfcVgpjX!#Xwzo7GH^c2wl69)4b{t=@;VElVbeuwEE%zlgc zZ%`xEeNAlq3N2rv?F)2%j-JoZpTpp%7|vq!6O4b1$&WDoA!a|o{QIbp>fR$ZzKfQ3 z(DpVu-$Kuu=zjx)84SOU(KN1>FnBwLZ^P)V7>{A{7EDJmdo$*5LXA{+ zBeC%Yv|Nw2>(F^EdagnL)fl`A!&hST3XETl$q1$|!)zGym!d|hyM)+yF|D$rff}jKMQogd7AM*c zN9SSasY8D)1|1ka6r-~-J`0mGF+Bsb(=mStYNWbp#6~+>rlQS;&MD}bjQ&X&oQUBG z7#)xCahN<9(+6R8EaneHjZ}94v2hGqtY{mJP78WQp?@R>M__n3M$H(n!DKb2hhcUo z=7*q0svAsf+#fCbq0NNOLFh4}e_sslgWE=66SpRJR+kaaXkL zg0`K}sYlOF=+|Lz0ETzO=nfd)9+Op=-VU?dVtyOcNOfBi8@EDBf3$6h&VK08qQ5T& zH5lFkqnl%VGfY-ux({Y6Ft4CSs{3Q#;(WyTJ6e82+pp;S1wB8bzktD?Fr3Hej~M>} zliy?dJIwZA{#(>Yb>9#hzedYfX!{bKU!do6^nZrI9ELx|Xcpt2VDe*3e}vf&G5-N- zq`LQsjqjo5U9`P}&bQI?7W&`B;2RjuVDxp2r!n~&reDSEE0}*7HBwzSvGFCetVP?4 z=zIY^&!hi23_gqDXE2(=_|uq7V)`k}K8g7!P$ShnPHbF*mdDVx8l8`#=MnTjjKPO6 zyb7ZaVmyJ#2QVGS>`Khvj~c1&K4Rm&Xz4=RJ?Oj}J$Ir1P7L0G;oC8K8^&+NWDL`{ zU^a^Rn^7ax-9&7>5iK{M?Rs=xhn{QEe+>q&#_&}by%OVBVDfTIM=*OC=EJCw>MkWV zUV@g3(RL9!JJE9?`Y*uX`4~PAqvvA$988{#>9a5!!u*-2k?MlP#xu}zI@(S{=L+9R_PL?7--u7@v*FS(u)Q*%_FhjvA@%5MtvrwAj%$6`eNp zOhNx-3{JxEM2t?r_;^f?!}P(JJqYt-Q6tqINNhX+Eo0DTMdxVrSkONTgCj9K0;9t* zZpLH{rmHbK4D&-#Bh?KdHV#J1{%G3|ohI}QLcbA%`(k(>jP8x`y)Zct(|cmpfcZU8 zBh~FrY}^ekyP|Cubnc8EJ^FXTpboF+W79p-ybBh`INZ2Sf-U!(0Sbbg7RFVO!v20z1a4x^u9Jd4Rs zF#R!RKf?TnsFCVEAU3{_miN&1E;`>q&)evK3xjWB_zjF^F#b9w)0ln@v#(R>Fds#YRChD6@g}s~h_)Nhc|CfrL;tlHyavNpWArMFUx~>pFnu{@ zBbdJoHBw!e*mx;gE>{*x(p+>4Zlh_zU z%Nb}p9i6A4X9fCC#o#FzUXIa|F@6#zPsH>Im^~iz$Du~53lJNZp~a83rRZFOo(}Z4 zW6+1;V==lI<87F1#dHg1n=!u#HB#L%#6~Y#7NX6A&IRakqrVA*jToMf(FTkkjmdgU zABEW?F+UGAQr%o);}K|ap=}O2o#;6n{fA+&4#Tw=bzuBZOwPviEX>Zt{0!7cb<>HB zhoEH|+U)3@iXI#Kr(ke0h9_ZkBE~0Ray+KTVfJ9mAA}mIZY;6!K(riywlV0mqGvSv zEf^ey;gJ{}f$`y(G-J93v(=a%h8n4ED6w$}S_Y$Se{}AL9uxWpVbF--eKEQZ#`nhL zUYH(;**!6DK#f$l2eEN?wCskqUD3G@AE8F7`;gf90b1Ti+k5DI7d`Kw|7{Guh2b|b`Ub``n0y`6 zY0SQc`BzaR)xAP&d>Jj>XnP5rYti!}`d`4{^B8^(qt9ae8BC@y{WNBin12d2Qr(lp z#wXD7INH{r^D*?SM*pK2d<4S}WAq`6ufpVmm`-5!0nEoyBh{@WHr|hx`_OhTI=j$w z5Bl%M;9VHL6Qg%v{B}&^3?F@9Dj-J!dzXF4&V)zt{F30%Em^=y7Ct~&l%pZ>$sqQ#pV*oA7 z(B?+Wu^3;B$u>;4VzveI&8U&;77-hdL5mk{3(@I8&jR$jG1!FR zMvTtKcmpPn#&kVqkHY+ssFCXC5gX^CZG+LdKYI2I}rjJApIOzw#3 z9Wc8+=BrR6)on*?+!ig{plxe(ZiSxy=-(29{V=S>XkUzLFu4V$H^=N|n6E^QRM&^t zSb-J=ZGY@loQF7nN6&BQ{}qG3VEAW@7BK!3Ci9s75wkyF{(IC&b>9&id(iSN+P*>O z*Xa2Q{a<463k-jb(a$iR!{n!!&SLfx%zum;sqQ0U$ z!sweAe*=>lOuvrVH0EDJja2t4vGEnOyo|PPbiRb1wdj8lgD+tCd5k`X@n>^BDS9WAIT7KZ4PRG5!!HS7G`=%qB4Z0BWSVII(dh zTJA^NedxRwJzeO(2ZMKG_%4jziSauyc{`?W!|bh?kD*4YyM@>oMa#`-y9u2)qUQ$m zUys4-FnldWufh1$n7j(pS7P=G%wLWgsV+inybLX2v|WnMOVD#M`Y*y@Cx$P?=mi)* zACu={`drMOgZZ;jBh{TnYz(30Otb~jc?Nn;NB?OUT!G^~ma%9%5S<60XAJtS7#xjZ3r0s_d?Y4EV0t)a&6ux2jZ{}nY#fG`p=cX| z&cW!}AN~7b(1hVZ7&T&iUrg?U>Af+#7v=||MylJB*l0k@9%$Pgox7oDSM=|K!JRRz z$LLNN*I{x1rgy~b4w&B_HBwy_v2i=JY>T#S(782wwnBe@3~q_xei+qaye}p-nBD@j zn`3@6)JS!e#Kt~osX&{8&OZhg=OLco(f=C;f5q@G82uUJ1x)^g={#nC#QYDak?OuD zHhzbe9<+Un&Tr83HTu88;FlQw0;8W}{4-4EF#RcJvzY$`HB#Nj#Kw=%@*&zjKXV(=Xdzm3tiF#aYc-@tSRv#(=5jT))$HDcqdXn6%~FQc;?JujhuEe2o2@Cz7y z9^=np@>xtjgV_}3pGJ*Tmn1emg_bAL_5?Z~N6#AcKZe287=9F^k6`>^Og@C^RhWGc z^9j^Qbq^35<7iolw)@d}AA0UZe-{St!SLM}y$j=aV)715-;UYaFn=p*q`DZf@fNg1 z(RMRBZ$i(F=)VDj*JJoPj9!cJYcP2=rmw>6m6*Q*HB#N>#Ks6(E<;-wotL8L67*k; z!HY25iO~x&egP)W$MksvADcCC>LEO-^3#FpvliArsZxE2;NO1QV})4p-+9_&;GaM3 zF(95BPjS6KB8_I!)o-(lAe*qKLP z0Uf`f`8TZk1ACNRi=Wrs2fH@I&MnZ_7ajf3+#jp9#-44kdpqpf9y@nLpAH>*H1C2{ zyJ64n*lobBf!Mh>`u0V~AT;lXRfDl-2zC#{t{Utdj=quTu%OwBRR>_tf!KWzc8$Z% z3Fw=IjwxuKidECF=Me0kfnBq(^HB8FqT?_$JF&`zJx5^oJnT9OJC8=+d~`ISc>z`} z#2znpFT$=C>}*5dvFK<=^AfD`W6v_|J`TH1z|ND&Vhdu9O_lMZ^F?MFrmqW+rX#NtbzQ&$!u)7DlzQ@iV(f1QNen#`J zSoJ&h{DIvSyA{Wgu1f6O9DN#eXwkeSR&9knTVwaO*j0s{JD_g>I(9R80OJ-^fjQP5zTI_@?g(G>^=s& znz6GLeT&iILvsgKEyW%`b_cNQcdXud(}E?D`Hne?VUz9R)Q1f>pm^&+pi+ z?9TYn2Rk=I-xlcTi{^e<)gODd!tQObYdh@R9(_BaLx*NPR_%g4yJGk5*k!=Zf#};C z9s8nr5LWGnJ^N$#5bPR;oi*qij*gLNwqTVNd&Xe*f!K8rc8){e1awS7^AxO_iamDh zJ_NgFVCO9K9g2=xG#`dlPVAY3-A7>8JnTFQeMh5XKAM}bY613mu-l7Wi?Fi=eQoGC z7R~KgwFG;XV)ruaIu1KeK;KE|SdQjXvFbGJIUT!$*cHOgv(a}hI?hM)g;;eF_FRnJ zmtxmt*m*hnu0+SxXucM!uE(Amu=^(LiehIBeYc_G4m96|Rrg>|7k1x=T`RHk0rWkH zj)&0v2v)7ep2x8JaqM~$JCo>3q2pOJKaW)}V$WLa?#8ZHu=6$ay^fAI(EJuwy@NgP zV)y&l^&xhCjJ_;7a%lb>tG>jZudw?Y?CQbJ@6q=oI(|a)&sg;<_WXw3e_&U|9-Kc_ zqHl9_Xwa<1sx7gnKXz}8UE5-375a8S#{e|%gjG9Z&o0=#8+Pr1oqM8hFLdmKW+PUa zuxCH)9*kW>v9lU|W^{}|^C+wujXhTEJ^;JMV&}o=8;_2OXr7E!Htd;--P5pZI(E)P z-)wX^&|HUAhhvWuyIt5d7dwwcUp+b+(A6DUn@EmquGa59oVx3 zyZzV|z|P~*cOp7YM)N6HwE}xi!|pS%>rCuC3w`IH<2*E9fK{E?a}jo5f?Z+kjG*rd zbXVE4V)bw75-(U(BSDl|WgRgYrN zYV2NvT~A==Q|Nmd9nYZoIjnjCdtSuum$2()?0gk{X>?@J{3cetjXm#R_j}m&0d{_b zzE9BcDVjgSsxPqTOYHs{yS~NF@6h)HI`U{PVAU_!^DB1$jxNPe96#(8XzhchO4M(L z+Rd>b&22%P(O{}CCbSsqhmkEY)E@&|p?7O^ZG-l0(YhU)s!+c@YInebG`AyhW&oyi znAizpdW`Ihpo7D`qrL{UW-Lf^!-+E^Ff|erqcCQ{$Y=~%F)#+b2cYXfw2wvW zL1;P{_2W=G9t+ak1mes@OijYXWQh5EBmdkz+) zxpRp#=V9u6Ok9Am3o+7(p^GqZF?ugS*QIC=qxCX0MNoe^YOlb8G0W`KY-c<7Nofci8HG(^$;c=#@Hhmc@#sdG4L3A*P!chv_FB?C(-m2 z>XWE_8Vk}~ia7HOrk=&ba~OLbBQId+MGUM(?@Q?FM*GWXeFaUgqW(41rm-N+y-u9T zVCoG_yos^5F!DBr-oe1T=zR}e@1y+#w0?-Dk5KvjwI!nCOc!Ek^obXiE(ANAFhX+8XWKpmkd`ZHM|Q z)NYRjX>JGN%#N5EfC(MOcEX4rLpx(&7xeCmuHDeSJ6iWZlL7U6qIMt_q`AF_Gkard zA583vF(XC>VaSAm{m{EVx(1_t2wI1tX&CCOQCourY0gZX8IGwDm>7w%Q5dmcXfy__ z=pBQu1JHgTTF0X4Ak-g>+HqKr=Ef6eCSYnJCMIEQGDfCg$cBNb=(VG38rly*>vS~D zK>bYA&ccE;H=8(fD5e~ksKrn2bI~*p^+%%iC@e^G^~9N@ zG1Y*H`50@&NE3$K7+8Q_54skj-HX;^(6k8k&8Tg`f;87koN2?E^DuEf#xB6fg&69@z(wf27+sg3{Zh1s(R3N=BdEO`3)0*b z#F;BGbrmMA#@IC&xfVm$Vc>f7-hi$f(S8$JZ$?uT^|zolh6QQvR^rTUn7SPkcVO&J zjNFBxyD@MNdb`kdFWT=z>-}h2iTXHdAHae%mmtnOh^bYWcnD(;W8@JGJ&J+V=zR=b zYta5UTAx7Elc;|RwMi^Ub59dzQkZ%M6VGDoIgC7yp%*alB6`=N>m{^zqxEGpy@L8z zQTrMeq`5S4=5d^V?mnxfjILcrt+Bh31bC}{EVSr zFz_pSe?!;rX#WGP3ctxQRiM5PYAdlI&22`U*&I__U_yhjz8KMBs2>KlL~nm|ZH4x& z(Yg(qwnhDRsI9_+G`Br*W(Q2|h=~Ch(_v&M4CyhjGkSMH*RE*a4XwMQX%EyJP`f7< zq`85_nY}QzHzxMM*uEGsVrUQsOz7PYUHhYbFj|M8X(;N4p|%7Yv zkr)|;AqxgZqt}YAF=#&ktp}oMEb0$J?ZH@(=Ef0c#$##%CMIHR5=JItXbJ{w=$(o# zJKCqA^$;{oNBs=c&cuQ=H;Xtk8&ij3!hx|`jMQQ1Fbo`yUMITdpxuSmBhWM#_481B zBo?H(qlh#0m^vC04H%n`kwy$PVZe>v1?cjieIZ)CXgUV;D8x)KvtVeD#*T!W!&F>oDvuSeGnXulDyH=*ff)JIW!3l^lg7;)xS zOx=cw+c9Kt)=v|Gj$I!k8t&gMW3DiG{+NZD}%_WI5Ph%>DiDxkOEJmKg(DN900lhDxYc1Me zLTfjgUPk>ZsC^X+(%fsrnKY(e$3zBWZ(!t2484Vcx6%6!y52?mduV+hO&_5CL)3nR z1!?YM;>;(Q%3|VEjO8%$8HPT`z!&KK5?x=R{cE&-gQjm$--FukuprHSPn`JyQ$J!N zkFlRHQozv982AOfzoP3mwEvFQKhPw9+pm|u_1DVZ{uiVw69t7mQ2wr(_f=k}3V3+qUSmnJ8CV6jzUf%1VmG?drq&ay{L`L2tk&^dL zB;>smF?nxAMBZx=lJ{N&YPd2fbG-m77k_ik9_y&NWaZ--vq>!FqReiWoRc~3}2 z-XoHd_l_jwy(BStZ%IVnYZ8+8o&@B*C|-GQic8+BVwd->SmnJeCV6j*Uf%1XmG{0B zq&ay{Oh(=#lalw&B;>s`F?nxIMBZx?lK0*O!X$T{uHD+c~4Mg9QS~v#$#dv#wKE95{4#YU7g z3DiD`1!?Xn;!F}#Ph%p5v1c&yEQX%L!1L&R0bMVmeJxsFLQ^;DUqdeF*eKNO@n`Amq6d`3h{K06{IpCu8K&z6YDXHA6UvnK-b zSrlIRYzmirR)t+YyTU4;Wnq%fw$RIGU1;UAFACC}d?rRlJ|iO~pPiAA&(es=XKO^{ zvo=EV*&6}*EDo=HHit_-tHUmz-C>o_@-WF~d+6n}KD6@L9|dVnJ`*G(pAnLh&kjk* zXNkn*vqd8EStB9&?2&+c7Kv9ro5UrbRbrRVF0smInV96WP4x0vCtCUJlY%rSpNW!@ z&qzs)1Ge#*W5F1BT{fpb@=I=yIcd z0a`t1T8MftYLCH!G`EO2(~PMWOtfOG4I_&&bSwsZ=xs+=2iljQbt#(ss9%QK02ZXV zwXPuEfYy7`hq**P!=W zbX|w`>(P1xnr=k>O{l#Y3({PaICBf8VwkuUW4B@Cb`0HtfjiNA7rO37`#oswLessd zzYn$dV?ml*Nt}se>H$n7F!mruR$=HN3_Oh9N6_^s+E=6XF*L0~{o|;80t?dIlf;>) zFqOo_(-=!(LemU5lodP~VN(m$4w6HFc7s&sN)2R;d5e`r`i? zy>bVoLi~U3D=NhQY+Wh-CqnV+t86|%eW!9ib$RjC7u0)v`&OxcS5}n%f7qooJMOBe zIahA2EV*Azp(vV_I7F&k$UVX=E&p4tf&UgY(63Nwn&Ox`TeN$%UzFOj2ue+Sl!k)V zmKt&?O@Fxo(ye9qA0hflu@kE4=yHREiMLpb6~s;x-%0C}uWW!mBF3#QbHyp-vcyeAR2=l6bXrp#8BDTBm9%CyfD zygy7PE4gBi)anU!)2E2ijZ^bDNZh6l5h@u>F%N&D}QWy9YWlO;a z_eT#BpCMlLe1@y#47+ldy1>SWG-=*t)07|81yj|>2Pn$gonAUzd0#D$T~W3Wym$2Y zxynK60wCD`20SKR&eC|@1>Lh z;tyqv`j4VK{$TnwWq?}dqxvZQ1n(G+dtKR2EqbjO#suFSsr&YnW1edif0kdO28#P% z+p7|L?b%*?wbvf)wex!IxL*6YSMB#|R=sv*uf5i*vGr=3z1l#&KnTby(R}bTYMDsndIPS_gCYJQTYC>J@!`W)y+AWX~4AT0b`Y&)x-TnMcGC$U-RI2N(6a+s~b+oS{~~$YTe=-Vdyt$I+#Q2}{Ry-`t?Q zrRE_rRi|CQ_8lHmwo}WokE~|z)%PgZshJ+iL`^qh`<2Q8YNm%M%JzcKZkfJPxlJvR znCPgp5A%^74=6{d6=hS3la0*=KbuglRx^E^$;MWD+`CE{r7kcrrFr&)CmvE7)I7wL zMxCZacDrPaa-EvTBqm>mt~H->ESx7sM6tZ_q6>5E&B-EC7d*cx=fY#6lHe* zL(Z8}TRT~)`CMHr-q=M^J}#!NuqaAOEx1x~UZvtwJF3!O*Y~(7Q|%L!+RxNbzU(KE zoqd_K14}2I^R7r=Ui^@&@27ATt16Vq#Xmc4{^Q<{KQF#nIvuSTErvTvaAt+FOdXAs zigPNI4)s(3(#XHb*5+dB;Ak^86*cl5bCd|vlb@tI5A&25d|4zIhtQd}wPua08v z9S!rF-Sy(b7IYNPOBtTV6~`}zV=;Jcv`Z?*%<*5=u3ymH;Q0qx?9P^LSb$;{|FVJV zOj8{})%*<$+I)tF#fzH*YVFDG2^i*k8`{MqEqy+Bqgr)i4Aq9QhSs(X79C)y5sTI* zF)Z-4wG^wXS!AbXMnMeiOMD9&8r_DLhW158H-b67r+F|NWER|~gE)#`u zH@3AltJ2uQvy>Rh&8^|U~CTi~AC{ypYCd;H|^><6NQS1G?VekI0uJy_FC&em7g%|6n zex|#lv1!4GiR+%E+FmS?I#8(7r$3zlh<7W+#EeX)c$=FKZ)jfPF0RW&VcSjaq>AO0 z;_Rz98i-xo+SK4{s&8#*aTkXhG5ad^k!JBmy|+wRMp~)+|Ri(K0T+CrftGA=Q__^xbN-gyI2Ct4R#Y7&Tx2e8` z&Q(myWLBhQnVMU%jTDE*zuhh_kBC>oGfo|Yil4(cyzxd|DGN4V9SFsL-WDI*SSh~LSYHDh_OpN9ad+FWGS&|P|1y}KDC=c| zqebxuP#X8z2de|&|Jj&l@HUCY<6YoY>q#AE8(IzSWnx5ZZD=lzt7=2uV8i7h@ZV?x zTW*+*I*R`7JVZ5!*H0(oCjTv-u#~Q*7E`Qvj)W_omOg={6PDj?bE=A?M(Mj>u~Wns zJ|DXts}z3~TWh1Qz23j5qu3$Fi=Lx)p4pCB(x8b*&W3^R;x?P&0cBUz(i zc3X3U&rmDs&fVtmH7s7}Z5%k*HhGq9efoieCylQ=Y({bQ`Wox#5d#O;&Yn8iG0Q%w zl;yy|qlQ~X*9^A|Hd{vzHIKBI&BI5I8a{aZVRchyIqY?=xdSK9uC)yuJkU}-yn5h~ zM{eSsQNzun)OTh(#CebSG@H-uZuNT`7Y-a;y@}+bt<^PZ^227@>*fw@a(hH_k#cBt zjb+5h;byCOWR2B2V)&-YV68EmH(iDi>&h@|+T@9K;@oTQz+w&6_pTvP%JGizGbYzf zcFY~9&Zat+G`Z{B7SxM_u*cgvaPSaIO|LYKqP_YWn(JHKV$kw7h!oYm-e1tr-0rS- zH;8eh)wAh0Tibjc3rp{^)nZWeiH)w=MD|)1HE@eD(mZNT}sQu#7nd@>a zE&i9?HN1MbTFbP;^1uHbYM^)jS0@9f7$*F(?%>n zq2vBvDnsx7uVy;#_7>;5+3JR~;`P&a^(zu}Ca_pN+>43e28a7fWnU&a?cxaK^(^cV z6MCPR8i+BZxvka1p-r7-)r*zg|8V}Jo{7k*Sg%9ezmd&0l#*;p2g83OpWVr4!=tvC zcOS!vy*z;bvuu|izPeJ3wZ#eNhS@S1_-A>oK2+qTUj9|jy4@{aae{MitUd6M9b@P8Uhs5jTlXyiczlAQHHGDT_>OOUj$&)KY=}XH>_8-hTfA;S_ z-|$W;-EwskH}R_3l_pY3SNx6GCen?aT53wXxxDyA4nH$dm;6cgo5-oWs9Xc(8YtI5 zxdzHLP_BV;4U}u(KcfczAN>N^zMN^II^F!&Um%x$6v-|4_kMwVgDlT5xsfHmubvn#{{p%E3uN(I?P7y3{{ng4@8kd1`~tZ+ z?rg4J@=-6YH4XaIz}25E*Zr!v@6~UAUre=ynrfS3s{Z$GoGQQLl${uzO21kzU1KUK z2OoKVgh7P7^Txisv$IT4a`T72&2QJFduiEGU#_iC?>D5QCUS3hT zMgN`3f0Qes+!D&Ypxg_}y`bC+%Dtf63(CEq+zZORpxg_}y`bC+%Dtf63(CEq+zZOR zpxg`oNA!Xdw%Vb5)wg`5Ds(_wDSe)KW^C^|PV-vMo9{&Hd3&B`p83su9?vr~ z>!1JyC^*v!co&!(Xxu>KK_3cGfC3bt00k&O0SZun0u-PC1t>rP3Ytv;?*j9OkNNA* z{0VFnpa2CZKmiI+fC3bt00k&O0SZun0u-Dp3V0Wo=R7>;X}0G)d}rcJe`kVLf>wf7 z0tF~Q0SZun0u-PC1t>rP3Q(~06eRBg_xAPdlHvEn3~4pm`@sA6+peWmoEHR@AXlmsyrNgg6}(zL@T;L;Da!>a>5b~6*@1(*a^6+$QQR035@ZQQTgsyUgrv0AaRf5Dcninla$%W&4I zOnQN%lXxFbsTeQf+h6D^~J;=zBF=IwYf8v0CWfUG#ElW22#Tsz%7P zLp!7PD{7xq@yS?CMuRQqFU*}AExKbTQN$%?&*oPkDq_J7u2P%Vq15F&qm{sMtJUJP zj$XV!)6U(#TXXW2An@`!bUU1_&J9kvGH=nBoom#hxg^eLP$@;>DyrgiVWuKZZ6X+T z^PW?3Yxg?cYN0OB4W=EnVIA13M%9lfy-8Se_WR-ZTvfi}Tywos@Qn>268Pct<}W+f9-|r!2TUKtq*il>OjX9qSIULC!faWm4q?MV%*|C0nFcM5f?KiA?q>7q`wHfHxu z`e(g=oqhPp)<@F+&P0WZuKMz`&*RDyzER;X)HjdVjH;w*jvpw`;b9v9N<7mK&K z#R)IUYc3Qzk9m-+8CzD&nkg6DppYxOB`<1jE>?x=%tg0c$oY8%+MJ!a>0E-uR$GkP zlE$X$3vu2gp;A(+R?1#Ebo{b2?G*f&l`YFn2@_>MtVP>BFU~u6!t0hL+RRws7jh+o zD@v=ccyt}(WxBNJ9EqC76Fx5g4f9eH|y=%(`XAn^G-e*)2xxsy>nD4t?@F^b}5k_W8x=R`i6+P+?HRr}vrh zF@KH>WwVx|dbDB4Y&mA~*@YtynszlwH)B-Ku1_Yu`C@%KX-U@9kBje^bji{g4M`1$ zvglCcUYY>TuuQaK+X*Z)fx^7TujS z*K(cGlGaP?EA_lJ)4oQ}rBW$1vZ6G4Y^KushO^iyd+L0XSzm7IOG7=kdcD3=C3fqA zi`e9vHj|CAtLx~~?$lpMlK*0s%iMOXMZ9bA2TUGLF?4sOx`%Uq|s)r)p=&yVvb z$yGmOO!ZkUU8&;{=h-ACRzW?dVLhrP z3Q&Lo6rcbFC_n)UP|)lO__Hv4rDf@TrG<8>+1n*lqLQy$q5uUbKmiI+fC3bt00k&O z0SZuXx(bp%3&Xp>Jm=wlD)&=SfC3bt00k&O0SZun0u-PC1t>rP3Q*AO3V0Wo=PEo` zIo;rP3YMM%-UZHZ+Ah7*mRqgdYURN! z3Q&Lo6rcbFC_n)UP=Epypa2CZK*1>~;9X#D2y;W2r+Fwq0SZun0u-PC1t>rP3Q&Lo z6rcbFC}=hX$-BTKL)ok)wj6C3GF#-h!!w&w(fTCajG0}K=jFZi>7->rl4!x}e^_6@ zOsh$EI3CxpJ$4x^B?3f%2oM1xKm>>Y5jcAh80_mAN_E_ovNq{$q*9i3d&3o5UdM=epsgV_>(PJ}}UaPa%Df^%Q%4F7;oBGmF&#hiB z%ks(#Tr zk<4~IZA*JxRIgtw-sTo3yt+NpB_ug!$1|C0ba}R{khHFGEp*ETHz?%FZpo7i)CN|l zkKXX4SFGgya60D&K_$q1N$s5amoeE2UM(N^)zGh$qfF}>GC3u8tn7yq1P54qxk2{c5sH9=ha{v^)?c*!8cIe=5IJlb6ap_q08x zvS_1Q|M3S;%ac*s|Gs1AkL6A^d6B$6^!Fdh^U>trytDB)(xnd0CGwZ|KmA{FeU$cR zxqqsv35L%r0>{<-3*8Fn_T)@yj(;UdGCp0+Z*Q#E{o$ed`N<=OV-nrgk>5O;O#lCm zca8M*sAgAHZrP3Q&N8rdGgr$ho1%4Lxq? zp#TLaKmiI+fC3bt00k&O0SZun0u-R2*%Tz-A?ICS9&t4FBMusG8t-OnyaCITd46r7 z00k&O0SZun0u-PC1t>rP3QkD@?*enHkXwb^DntPaP=Epypa2CZKmiI+fC3bt00k&O zL9;2~UEmC?Vcl%3c@B0;&%tO4XbNZwP=Epypa2CZKmiI+fC3bt00rlQ0^SAYJU#RM^lN6t#qs)N+;yO~GCIf8a)(~E-DLRq;KohzuwHPjh~Br? zr>-872X$F0nLcSh{?4k0q)#vCmFT42ap$-GSZuvueY_X`Y zdci)4Vw!*C;dfBJ+)IJRyV)!{m-($`^pBrh~X?nAUghSbqAvE zWT-n4>JCr+@t%LI>rRcj6Q}MB_|=cU@egNduPF9JBQV(4GnDEO%i5%Kq*9i3d&6Cc z#(6uVn-$$4ns%>KTGD!neWjUCuhk{lDf`BMGfkJiZK7{+=+&#&M~$4_CLau%wfF5c z-R-fC3bt00k&O0SZun0u-PC1t|C} zRgk<3+}qc)OES-=WOn$UAbqPXU6N5B$qjg6zA(CNccl^({IVN*HR()?EQ^vmlCmr- z$(xL=xF}oN<3f7oN)m+X0wwK%Q7s7E{Z6(T4@M`(MuZDi59HrkFr;k!Ol~JeWxj}y13B6LasCLEa z)+KGWWx2^eQTD@Hlv7tnSF>bGhgmlk_=Q}_tycZ=Sd^qSNn)1igk3sfQ@N@?IM@;V4x)kjy&FS%o$Qw=Kj zczM0Y&X`vzd0{ZU!!i4NgEO_o8Tb6L@zA-!nYLBLmd5*< zT*)4aJ1KAMjV~>e&PXzj(8VemRBbv{hejl@D`wFj5T|Fd!q{~vtbXgyN zXVpXUHNBu$)SVKM^?&xW$D~CsSRY5kfBaebf?lv$qU-xNe*N0Z^1fcM&)ihVKl1Rq z>ehxDvigTe{>``EmDlvtSM@C)k(*!m{%5c5m7nTG3}-0<(eZDqI}mjzL*0o`cX;ZL z_xxjBcWTs~ICW>hxE-_jfOe5;>t z_4DWGQGfyzpa2CZKmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&Lo6rcbFC_n)UP=Epy zpa2CZK!H~9crP3Q&Lo6f7wPyb8=K;e0WIFGipM1t>rP3Q&Lo z6rcbFC_n)UP=Epypa2CZKmiI+fC3bt00k&O0SZun0u-PC1t>rP3Q&N8C8;2O6}Y#r zXP101Xv^$yaAn8Kwsc8GeIz&Fh55qhw%wIVQ1Huc=+&e%EwU_1?nuhAtW=ye8C&tZ zE$wk3y?(KHn_Halq*JXk={lplN!nyAV#^9i>+)-%TQ0akAy;-wo?M_ZCYkf&el=I~ zzwPD5J%4OGwAE&!ECsKY5BzHASISWaC*GP#=2S+Vn&$@jaVPXj)uP%Nr(2h`*_P#| zfQhmn)}pMsLb{?QTRP0TvA{3nN^Z64m&c+ctw|EIOegHpF`Ld+{VA^))lgPBH>vD; z*eToEW!q5|X&tm=hm&!XL~h(^%Nnz}xt+|c-t!I|3P#QT1OGi}RC6+%_Pp8i|11H=7;1Gzo@!y|H`DoclUt@`My zE_zJOe$0j^-Y;Du!%~gF$WS(GDRf60hW0JT-=mw~GfhU4ZpNrykZEX*X)2O*(y|~) zwBXKNCS9_2M(a|;;ds1A@g@;Ji2xBG0z`la5CI}U1kQQ{2K#!3QXPMnvNq{WrBt8c z_J%9A#(6sgh^go?U|I-(KY27suJ7tfT%~hhl z+SFHuI@#*=QE${vFSv$HE@?B_D7)&YoOYi!bG_c^Vm;evM^v}ym>s;#4&7}#PWM*t zq~2S<9C|^uqWhrP3Q&Lo6rcbF zC_n)UP{4nWv?RaEPUA)6MdO766rcbFC_n)UP=Epypa2CZKmiI+fC3bt00k&O0SZun z0u-PC1t>rP3Q&Lo6rcbFC_n)Ucompef%(7P_`lsyfC3bt00k&O0SZun0u-PC1t>rP z3Q&Lo6rcbFC_n)UP=Epypa2CZKmiI+fC3bt00k&O!ID%EzY07ul+9XV%h86RbB!E# zcw%Z Date: Tue, 19 Mar 2024 15:47:51 -0800 Subject: [PATCH 13/28] re-orangenize the workflow --- src/hyp3_autorift/process.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index ca54d724..453b1d61 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -594,14 +594,14 @@ def main(): product_file, browse_file = process(g1, g2, parameter_file=args.parameter_file, naming_scheme=args.naming_scheme) thumbnail_file = create_thumbnail(browse_file) - if args.publish: - prefix = get_opendata_prefix(product_file) - upload_file_to_s3(product_file, OPEN_DATA_BUCKET, prefix) - upload_file_to_s3(browse_file, OPEN_DATA_BUCKET, prefix) - upload_file_to_s3(thumbnail_file, OPEN_DATA_BUCKET, prefix) - if args.bucket: upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix) thumbnail_file = create_thumbnail(browse_file) upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix) + + if args.publish: + prefix = get_opendata_prefix(product_file) + upload_file_to_s3(product_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3(browse_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3(thumbnail_file, OPEN_DATA_BUCKET, prefix) From 533eb1574ae2b1007018a4216f8e2b42018ff6f4 Mon Sep 17 00:00:00 2001 From: cirrusasf <62269400+cirrusasf@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:51:58 -0800 Subject: [PATCH 14/28] Update CHANGELOG.md Co-authored-by: Joseph H Kennedy --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfa9d5da..ac66b0f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.14.2] ### Added -* the option upload-opendata to upload the produdct to s3://its-live-data bucket +* `--publish` option has been added to the HyP3 entry point to publish product to the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`. ## [0.14.1] ### Changed From cbe5d8e962973a43ce95513010f50922e1c7d8c7 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 15:53:10 -0800 Subject: [PATCH 15/28] clean up the code --- src/hyp3_autorift/process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 453b1d61..00578810 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -597,7 +597,6 @@ def main(): if args.bucket: upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix) - thumbnail_file = create_thumbnail(browse_file) upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix) if args.publish: From 56eb37f80687b1082fb40ca3af05c0e06a95747d Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 15:56:48 -0800 Subject: [PATCH 16/28] clean up the debug code --- tests/test_process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_process.py b/tests/test_process.py index 14bc1d14..eff86ef2 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -12,7 +12,6 @@ from hyp3_autorift import process -import pdb def test_get_platform(): assert process.get_platform('S1B_IW_GRDH_1SSH_20201203T095903_20201203T095928_024536_02EAB3_6D81') == 'S1' assert process.get_platform('S1A_IW_SLC__1SDV_20180605T233148_20180605T233215_022228_0267AD_48B2') == 'S1' From ce20b83102d75f0ef33e5d64f8e8d52590bfd46f Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 15:59:44 -0800 Subject: [PATCH 17/28] modify code style --- tests/test_process.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_process.py b/tests/test_process.py index eff86ef2..5ef12da2 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -12,6 +12,7 @@ from hyp3_autorift import process + def test_get_platform(): assert process.get_platform('S1B_IW_GRDH_1SSH_20201203T095903_20201203T095928_024536_02EAB3_6D81') == 'S1' assert process.get_platform('S1A_IW_SLC__1SDV_20180605T233148_20180605T233215_022228_0267AD_48B2') == 'S1' @@ -426,12 +427,14 @@ def test_point_to_prefix(): def test_get_lat_lon_from_ncfile(): - file = Path('tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + file = Path('tests/data/' + 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') assert process.get_lat_lon_from_ncfile(file) == (-81.49, -128.28) def test_get_opendata_prefix(): - file = Path('tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + file = Path('tests/data/' + 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120' From 0be1a65e4a4251dfd167a63c0341d100d95358e0 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 16:00:48 -0800 Subject: [PATCH 18/28] modify code style --- tests/test_process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_process.py b/tests/test_process.py index 5ef12da2..b51ec0f6 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -437,4 +437,3 @@ def test_get_opendata_prefix(): 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120' - From dbf50884462122408625f3babb1b2c400c658421 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 19 Mar 2024 16:03:13 -0800 Subject: [PATCH 19/28] modify code style --- tests/test_process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_process.py b/tests/test_process.py index b51ec0f6..641e1cd7 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -436,4 +436,3 @@ def test_get_opendata_prefix(): file = Path('tests/data/' 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120' - From deb80cfe3904d6bb7927cc68214ca1defe003e03 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Wed, 20 Mar 2024 15:11:20 +0000 Subject: [PATCH 20/28] set up access key for open data upload --- src/hyp3_autorift/process.py | 8 ++++---- src/hyp3_autorift/utils.py | 26 ++++++++++++++++++++++++++ tests/test_utils.py | 18 +++++++++++++++++- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 00578810..c31825e4 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -28,7 +28,7 @@ from hyp3_autorift import geometry, image, io from hyp3_autorift.crop import crop_netcdf_product -from hyp3_autorift.utils import get_esa_credentials +from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_upload_access_keys log = logging.getLogger(__name__) @@ -601,6 +601,6 @@ def main(): if args.publish: prefix = get_opendata_prefix(product_file) - upload_file_to_s3(product_file, OPEN_DATA_BUCKET, prefix) - upload_file_to_s3(browse_file, OPEN_DATA_BUCKET, prefix) - upload_file_to_s3(thumbnail_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3_with_upload_access_keys(product_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3_with_upload_access_keys(browse_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3_with_upload_access_keys(thumbnail_file, OPEN_DATA_BUCKET, prefix) diff --git a/src/hyp3_autorift/utils.py b/src/hyp3_autorift/utils.py index 61845eb8..7fda6eb4 100644 --- a/src/hyp3_autorift/utils.py +++ b/src/hyp3_autorift/utils.py @@ -1,9 +1,13 @@ +import logging import netrc import os from pathlib import Path from platform import system from typing import Tuple +import boto3 +from hyp3lib.aws import get_content_type, get_tag_set + ESA_HOST = 'dataspace.copernicus.eu' @@ -28,3 +32,25 @@ def get_esa_credentials() -> Tuple[str, str]: "Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the " "ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file." ) + + +def upload_file_to_s3_with_upload_access_keys(path_to_file: Path, bucket: str, prefix: str = ''): + if 'UPLOAD_ACCESS_KEY_ID' in os.environ and 'UPLOAD_ACCESS_KEY_SECRET' in os.environ: + access_key_id = os.environ['UPLOAD_ACCESS_KEY_ID'] + access_key_secret = os.environ['UPLOAD_ACCESS_KEY_SECRET'] + else: + raise ValueError( + 'Please provide S3 Bucket upload access key credentials via the ' + 'UPLOAD_ACCESS_KEY_ID and UPLOAD_ACCESS_KEY_SECRET environment variables' + ) + + s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret) + key = str(Path(prefix) / path_to_file.name) + extra_args = {'ContentType': get_content_type(key)} + + logging.info(f'Uploading s3://{bucket}/{key}') + s3_client.upload_file(str(path_to_file), bucket, key, extra_args) + + tag_set = get_tag_set(path_to_file.name) + + s3_client.put_object_tagging(Bucket=bucket, Key=key, Tagging=tag_set) diff --git a/tests/test_utils.py b/tests/test_utils.py index f3263d06..24b5f0df 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from hyp3_autorift.utils import ESA_HOST, get_esa_credentials +from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_upload_access_keys def test_get_esa_credentials_env(tmp_path, monkeypatch): @@ -45,3 +45,19 @@ def test_get_esa_credentials_missing(tmp_path, monkeypatch): msg = 'Please provide.*' with pytest.raises(ValueError, match=msg): get_esa_credentials() + + +def test_upload_file_to_s3_credentials_missing(tmp_path, monkeypatch): + with monkeypatch.context() as m: + m.delenv('UPLOAD_ACCESS_KEY_ID', raising=False) + m.setenv('UPLOAD_ACCESS_KEY_SECRET', 'upload_access_key_secret') + msg = 'Please provide.*' + with pytest.raises(ValueError, match=msg): + upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket') + + with monkeypatch.context() as m: + m.setenv('UPLOAD_ACCESS_KEY_ID', 'upload_access_key_id') + m.delenv('UPLOAD_ACCESS_KEY_SECRET', raising=False) + msg = 'Please provide.*' + with pytest.raises(ValueError, match=msg): + upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket') From 9415247689f8591c489cd41249ced28c88e1392b Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Wed, 20 Mar 2024 09:27:19 -0800 Subject: [PATCH 21/28] add 4 more test cases in the test_point_to_prefix function in test_process.py --- tests/test_process.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_process.py b/tests/test_process.py index 641e1cd7..1f93b7eb 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -423,6 +423,10 @@ def test_point_to_prefix(): assert process.point_to_prefix(-63.0, 128.0) == 'S60E120' assert process.point_to_prefix(63.0, -128.0) == 'N60W120' assert process.point_to_prefix(-63.0, -128.0) == 'S60W120' + assert process.point_to_prefix(0.0, 128.0) == 'N00E120' + assert process.point_to_prefix(0.0, -128.0) == 'N00W120' + assert process.point_to_prefix(63.0, 0.0) == 'N60E000' + assert process.point_to_prefix(-63.0, 0.0) == 'S60E000' assert process.point_to_prefix(0.0, 0.0) == 'N00E000' From 30d6f17a1b236951282bd51f9a112b2ef521dde9 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Wed, 20 Mar 2024 18:34:35 +0000 Subject: [PATCH 22/28] change to a publish-bucket parameter to enable testing --- src/hyp3_autorift/process.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index c31825e4..deb9a2e8 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -50,6 +50,7 @@ 'autorift_parameters/v001/autorift_landice_0120m.shp' OPEN_DATA_BUCKET = 'its-live-data' +OPEN_DATA_BUCKET_TEST = 'its-live-data-test' PLATFORM_SHORTNAME_LONGNAME_MAPPING = { 'S1': 'sentinel1', 'S2': 'sentinel2', @@ -571,9 +572,10 @@ def main(): ) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') - parser.add_argument('--publish', type=string_is_true, default=False, - help='Additionally publish the product to ' - f'the ITS_LIVE AWS Open Data bucket: s3://{OPEN_DATA_BUCKET}') + parser.add_argument('--publish-bucket', default='', + help='Bucket to publish the product to. ' + f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') + parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -599,8 +601,13 @@ def main(): upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix) upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix) - if args.publish: + if args.publish_bucket: + + if args.publish_bucket not in [OPEN_DATA_BUCKET, OPEN_DATA_BUCKET_TEST]: + raise ValueError(f'Invalid publish bucket: {args.publish}. ' + f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') + prefix = get_opendata_prefix(product_file) - upload_file_to_s3_with_upload_access_keys(product_file, OPEN_DATA_BUCKET, prefix) - upload_file_to_s3_with_upload_access_keys(browse_file, OPEN_DATA_BUCKET, prefix) - upload_file_to_s3_with_upload_access_keys(thumbnail_file, OPEN_DATA_BUCKET, prefix) + upload_file_to_s3_with_upload_access_keys(product_file, args.publish_bucket, prefix) + upload_file_to_s3_with_upload_access_keys(browse_file, args.publish_bucket, prefix) + upload_file_to_s3_with_upload_access_keys(thumbnail_file, args.publish_bucket, prefix) From cae745508ba5c384caf0aad8159a92dd3237a65c Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Wed, 20 Mar 2024 18:36:36 +0000 Subject: [PATCH 23/28] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac66b0f7..75726f69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.14.2] ### Added -* `--publish` option has been added to the HyP3 entry point to publish product to the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`. +* `--publish-bucket` option has been added to the HyP3 entry point to publish product to the ITS_LIVE AWS Open Data bucket, `s3://its-live-data` or the test bucket `s3://its-live-data-test`. ## [0.14.1] ### Changed From 61879bc91e0ad077e166e9f6e2431bc870a370fd Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Wed, 20 Mar 2024 18:37:32 +0000 Subject: [PATCH 24/28] fix flake8 --- src/hyp3_autorift/process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index deb9a2e8..f4af7633 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -22,7 +22,6 @@ from hyp3lib.get_orb import downloadSentinelOrbitFile from hyp3lib.image import create_thumbnail from hyp3lib.scene import get_download_url -from hyp3lib.util import string_is_true from netCDF4 import Dataset from osgeo import gdal From 279e62f80335206c98ca45cb37a5799dc1b71cfe Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Wed, 20 Mar 2024 18:42:33 +0000 Subject: [PATCH 25/28] fix flake8 --- src/hyp3_autorift/process.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index f4af7633..0926c286 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -574,7 +574,6 @@ def main(): parser.add_argument('--publish-bucket', default='', help='Bucket to publish the product to. ' f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') - parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -604,7 +603,7 @@ def main(): if args.publish_bucket not in [OPEN_DATA_BUCKET, OPEN_DATA_BUCKET_TEST]: raise ValueError(f'Invalid publish bucket: {args.publish}. ' - f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') + f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') prefix = get_opendata_prefix(product_file) upload_file_to_s3_with_upload_access_keys(product_file, args.publish_bucket, prefix) From 9ad126d34b43fc11fa2ba25ffdda277d593c4c16 Mon Sep 17 00:00:00 2001 From: cirrusasf <62269400+cirrusasf@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:55:02 -0800 Subject: [PATCH 26/28] Apply suggestions from code review Co-authored-by: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75726f69..89ac6849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.14.2] ### Added * `--publish-bucket` option has been added to the HyP3 entry point to publish product to the ITS_LIVE AWS Open Data bucket, `s3://its-live-data` or the test bucket `s3://its-live-data-test`. +* `upload_file_to_s3_with_upload_access_keys` to perform S3 uploads with credentialed S3 clients. +* use of `UPLOAD_ACCESS_KEY_ID` and `UPLOAD_ACCESS_KEY_SECRET` to upload products to write-protected bucket. ## [0.14.1] ### Changed From 553eeab1dd496abf43d9ddc977606af11cf3842e Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 22 Mar 2024 16:33:33 -0800 Subject: [PATCH 27/28] Refactor publishing open data --- CHANGELOG.md | 7 ++--- src/hyp3_autorift/process.py | 54 +++++++++++++++++------------------- src/hyp3_autorift/utils.py | 12 ++++---- tests/test_utils.py | 14 +++++----- 4 files changed, 41 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ac6849..c8f67e6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.14.2] +## [0.15.0] ### Added -* `--publish-bucket` option has been added to the HyP3 entry point to publish product to the ITS_LIVE AWS Open Data bucket, `s3://its-live-data` or the test bucket `s3://its-live-data-test`. -* `upload_file_to_s3_with_upload_access_keys` to perform S3 uploads with credentialed S3 clients. -* use of `UPLOAD_ACCESS_KEY_ID` and `UPLOAD_ACCESS_KEY_SECRET` to upload products to write-protected bucket. +* `--publish-bucket` option has been added to the HyP3 entry point to additionally publish products an AWS bucket, such as the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`. +* `upload_file_to_s3_with_publish_access_keys` to perform S3 uploads using credentials from the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment vairables. ## [0.14.1] ### Changed diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 0926c286..de807959 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -27,7 +27,7 @@ from hyp3_autorift import geometry, image, io from hyp3_autorift.crop import crop_netcdf_product -from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_upload_access_keys +from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_publish_access_keys log = logging.getLogger(__name__) @@ -48,8 +48,6 @@ DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \ 'autorift_parameters/v001/autorift_landice_0120m.shp' -OPEN_DATA_BUCKET = 'its-live-data' -OPEN_DATA_BUCKET_TEST = 'its-live-data-test' PLATFORM_SHORTNAME_LONGNAME_MAPPING = { 'S1': 'sentinel1', 'S2': 'sentinel2', @@ -337,24 +335,24 @@ def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]: return var.latitude, var.longitude -def point_to_prefix(lat: float, lon: float) -> str: +def point_to_region(lat: float, lon: float) -> str: """ - Returns a string (for example, N78W124) for directory name based on - granule centerpoint lat,lon + Returns a string (for example, N78W124) of a region name based on + granule center point lat,lon """ - NShemi_str = 'N' if lat >= 0.0 else 'S' - EWhemi_str = 'E' if lon >= 0.0 else 'W' + nw_hemisphere = 'N' if lat >= 0.0 else 'S' + ew_hemisphere = 'E' if lon >= 0.0 else 'W' - outlat = int(10*np.trunc(np.abs(lat/10.0))) - if outlat == 90: # if you are exactly at a pole, put in lat = 80 bin - outlat = 80 + region_lat = int(10*np.trunc(np.abs(lat/10.0))) + if region_lat == 90: # if you are exactly at a pole, put in lat = 80 bin + region_lat = 80 - outlon = int(10*np.trunc(np.abs(lon/10.0))) + region_lon = int(10*np.trunc(np.abs(lon/10.0))) - if outlon >= 180: # if you are at the dateline, back off to the 170 bin - outlon = 170 + if region_lon >= 180: # if you are at the dateline, back off to the 170 bin + region_lon = 170 - return f'{NShemi_str}{outlat:02d}{EWhemi_str}{outlon:03d}' + return f'{nw_hemisphere}{region_lat:02d}{ew_hemisphere}{region_lon:03d}' def get_opendata_prefix(file: Path): @@ -363,11 +361,14 @@ def get_opendata_prefix(file: Path): platform_shortname = get_platform(scene) lat, lon = get_lat_lon_from_ncfile(file) - lat_lon_prefix_component = point_to_prefix(lat, lon) + region = point_to_region(lat, lon) - dir_path = f'velocity_image_pair/{PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname]}/v02' - prefix = os.path.join(dir_path, lat_lon_prefix_component) - return prefix + return '/'.join([ + 'velocity_image_pair', + PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname], + 'v02', + region + ]) def process( @@ -572,8 +573,8 @@ def main(): parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') parser.add_argument('--publish-bucket', default='', - help='Bucket to publish the product to. ' - f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') + help='Additionally, publish products to this bucket. Necessary credentials must be provided ' + 'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.') parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -600,12 +601,7 @@ def main(): upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix) if args.publish_bucket: - - if args.publish_bucket not in [OPEN_DATA_BUCKET, OPEN_DATA_BUCKET_TEST]: - raise ValueError(f'Invalid publish bucket: {args.publish}. ' - f'Must be one of {OPEN_DATA_BUCKET} or {OPEN_DATA_BUCKET_TEST}') - prefix = get_opendata_prefix(product_file) - upload_file_to_s3_with_upload_access_keys(product_file, args.publish_bucket, prefix) - upload_file_to_s3_with_upload_access_keys(browse_file, args.publish_bucket, prefix) - upload_file_to_s3_with_upload_access_keys(thumbnail_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(product_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(browse_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(thumbnail_file, args.publish_bucket, prefix) diff --git a/src/hyp3_autorift/utils.py b/src/hyp3_autorift/utils.py index 7fda6eb4..6bcb6244 100644 --- a/src/hyp3_autorift/utils.py +++ b/src/hyp3_autorift/utils.py @@ -34,14 +34,14 @@ def get_esa_credentials() -> Tuple[str, str]: ) -def upload_file_to_s3_with_upload_access_keys(path_to_file: Path, bucket: str, prefix: str = ''): - if 'UPLOAD_ACCESS_KEY_ID' in os.environ and 'UPLOAD_ACCESS_KEY_SECRET' in os.environ: - access_key_id = os.environ['UPLOAD_ACCESS_KEY_ID'] - access_key_secret = os.environ['UPLOAD_ACCESS_KEY_SECRET'] - else: +def upload_file_to_s3_with_publish_access_keys(path_to_file: Path, bucket: str, prefix: str = ''): + try: + access_key_id = os.environ['PUBLISH_ACCESS_KEY_ID'] + access_key_secret = os.environ['PUBLISH_SECRET_ACCESS_KEY'] + except KeyError: raise ValueError( 'Please provide S3 Bucket upload access key credentials via the ' - 'UPLOAD_ACCESS_KEY_ID and UPLOAD_ACCESS_KEY_SECRET environment variables' + 'PUBLISH_ACCESS_KEY_ID and PUBLISH_SECRET_ACCESS_KEY environment variables' ) s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret) diff --git a/tests/test_utils.py b/tests/test_utils.py index 24b5f0df..e640d411 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_upload_access_keys +from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_publish_access_keys def test_get_esa_credentials_env(tmp_path, monkeypatch): @@ -49,15 +49,15 @@ def test_get_esa_credentials_missing(tmp_path, monkeypatch): def test_upload_file_to_s3_credentials_missing(tmp_path, monkeypatch): with monkeypatch.context() as m: - m.delenv('UPLOAD_ACCESS_KEY_ID', raising=False) - m.setenv('UPLOAD_ACCESS_KEY_SECRET', 'upload_access_key_secret') + m.delenv('PUBLISH_ACCESS_KEY_ID', raising=False) + m.setenv('PUBLISH_SECRET_ACCESS_KEY', 'publish_access_key_secret') msg = 'Please provide.*' with pytest.raises(ValueError, match=msg): - upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket') + upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket') with monkeypatch.context() as m: - m.setenv('UPLOAD_ACCESS_KEY_ID', 'upload_access_key_id') - m.delenv('UPLOAD_ACCESS_KEY_SECRET', raising=False) + m.setenv('PUBLISH_ACCESS_KEY_ID', 'publish_access_key_id') + m.delenv('PUBLISH_SECRET_ACCESS_KEY', raising=False) msg = 'Please provide.*' with pytest.raises(ValueError, match=msg): - upload_file_to_s3_with_upload_access_keys('file.zip', 'myBucket') + upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket') From 437fdc9eeda82c480b085597639ceed8d83bf452 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 25 Mar 2024 12:16:05 -0800 Subject: [PATCH 28/28] fix function name change missed in refactor --- tests/test_process.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_process.py b/tests/test_process.py index 1f93b7eb..d9040e33 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -419,15 +419,15 @@ def mock_apply_filter_function(scene, _): def test_point_to_prefix(): - assert process.point_to_prefix(63.0, 128.0) == 'N60E120' - assert process.point_to_prefix(-63.0, 128.0) == 'S60E120' - assert process.point_to_prefix(63.0, -128.0) == 'N60W120' - assert process.point_to_prefix(-63.0, -128.0) == 'S60W120' - assert process.point_to_prefix(0.0, 128.0) == 'N00E120' - assert process.point_to_prefix(0.0, -128.0) == 'N00W120' - assert process.point_to_prefix(63.0, 0.0) == 'N60E000' - assert process.point_to_prefix(-63.0, 0.0) == 'S60E000' - assert process.point_to_prefix(0.0, 0.0) == 'N00E000' + assert process.point_to_region(63.0, 128.0) == 'N60E120' + assert process.point_to_region(-63.0, 128.0) == 'S60E120' + assert process.point_to_region(63.0, -128.0) == 'N60W120' + assert process.point_to_region(-63.0, -128.0) == 'S60W120' + assert process.point_to_region(0.0, 128.0) == 'N00E120' + assert process.point_to_region(0.0, -128.0) == 'N00W120' + assert process.point_to_region(63.0, 0.0) == 'N60E000' + assert process.point_to_region(-63.0, 0.0) == 'S60E000' + assert process.point_to_region(0.0, 0.0) == 'N00E000' def test_get_lat_lon_from_ncfile():