diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 1d001cc5..3ce6f429 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,6 +13,6 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0 secrets: USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 8436b4ed..0b69efec 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 48fc9e84..66ba502e 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -12,4 +12,4 @@ on: jobs: call-labeled-pr-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml index 64197b00..cae89e6f 100644 --- a/.github/workflows/release-template-comment.yml +++ b/.github/workflows/release-template-comment.yml @@ -7,6 +7,6 @@ on: jobs: call-release-checklist-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0 secrets: USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f0d6ffc..c8fed248 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0 with: release_prefix: HyP3 autoRIFT secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index a142be7f..3b07f93b 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,10 +4,10 @@ on: push jobs: call-flake8-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.0 with: local_package_names: hyp3_autorift excludes: src/hyp3_autorift/vend call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index eeaf346f..4714c1f1 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -7,6 +7,6 @@ on: jobs: call-bump-version-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 1e215924..cee1a1e2 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -12,18 +12,18 @@ on: jobs: call-pytest-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0 with: local_package_name: hyp3_autorift python_versions: >- ["3.9"] call-version-info-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.0 call-docker-ghcr-workflow: needs: call-version-info-workflow - uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.10.0 + uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.0 with: version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} secrets: diff --git a/.trufflehog.txt b/.trufflehog.txt deleted file mode 100644 index 4ed920df..00000000 --- a/.trufflehog.txt +++ /dev/null @@ -1,3 +0,0 @@ -.*gitleaks.toml$ -.*hyp3_autorift/vend/workflow/.* -.*hyp3_autorift/vend/README.md$ diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c6b257..c8f67e6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.15.0] +### Added +* `--publish-bucket` option has been added to the HyP3 entry point to additionally publish products an AWS bucket, such as the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`. +* `upload_file_to_s3_with_publish_access_keys` to perform S3 uploads using credentials from the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment vairables. ## [0.14.1] ### Changed diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 99ecaccb..de807959 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -27,7 +27,7 @@ from hyp3_autorift import geometry, image, io from hyp3_autorift.crop import crop_netcdf_product -from hyp3_autorift.utils import get_esa_credentials +from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_publish_access_keys log = logging.getLogger(__name__) @@ -48,6 +48,16 @@ DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \ 'autorift_parameters/v001/autorift_landice_0120m.shp' +PLATFORM_SHORTNAME_LONGNAME_MAPPING = { + 'S1': 'sentinel1', + 'S2': 'sentinel2', + 'L4': 'landsatOLI', + 'L5': 'landsatOLI', + 'L7': 'landsatOLI', + 'L8': 'landsatOLI', + 'L9': 'landsatOLI', +} + def get_lc2_stac_json_key(scene_name: str) -> str: platform = get_platform(scene_name) @@ -319,6 +329,48 @@ def apply_landsat_filtering(reference_path: str, secondary_path: str) \ return reference_path, reference_zero_path, secondary_path, secondary_zero_path +def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]: + with Dataset(ncfile) as ds: + var = ds.variables['img_pair_info'] + return var.latitude, var.longitude + + +def point_to_region(lat: float, lon: float) -> str: + """ + Returns a string (for example, N78W124) of a region name based on + granule center point lat,lon + """ + nw_hemisphere = 'N' if lat >= 0.0 else 'S' + ew_hemisphere = 'E' if lon >= 0.0 else 'W' + + region_lat = int(10*np.trunc(np.abs(lat/10.0))) + if region_lat == 90: # if you are exactly at a pole, put in lat = 80 bin + region_lat = 80 + + region_lon = int(10*np.trunc(np.abs(lon/10.0))) + + if region_lon >= 180: # if you are at the dateline, back off to the 170 bin + region_lon = 170 + + return f'{nw_hemisphere}{region_lat:02d}{ew_hemisphere}{region_lon:03d}' + + +def get_opendata_prefix(file: Path): + # filenames have form GRANULE1_X_GRANULE2 + scene = file.name.split('_X_')[0] + + platform_shortname = get_platform(scene) + lat, lon = get_lat_lon_from_ncfile(file) + region = point_to_region(lat, lon) + + return '/'.join([ + 'velocity_image_pair', + PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname], + 'v02', + region + ]) + + def process( reference: str, secondary: str, @@ -520,6 +572,9 @@ def main(): ) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') + parser.add_argument('--publish-bucket', default='', + help='Additionally, publish products to this bucket. Necessary credentials must be provided ' + 'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.') parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem") parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, @@ -538,9 +593,15 @@ def main(): g1, g2 = sorted(args.granules, key=get_datetime) product_file, browse_file = process(g1, g2, parameter_file=args.parameter_file, naming_scheme=args.naming_scheme) + thumbnail_file = create_thumbnail(browse_file) if args.bucket: upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix) - thumbnail_file = create_thumbnail(browse_file) upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix) + + if args.publish_bucket: + prefix = get_opendata_prefix(product_file) + upload_file_to_s3_with_publish_access_keys(product_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(browse_file, args.publish_bucket, prefix) + upload_file_to_s3_with_publish_access_keys(thumbnail_file, args.publish_bucket, prefix) diff --git a/src/hyp3_autorift/utils.py b/src/hyp3_autorift/utils.py index 61845eb8..6bcb6244 100644 --- a/src/hyp3_autorift/utils.py +++ b/src/hyp3_autorift/utils.py @@ -1,9 +1,13 @@ +import logging import netrc import os from pathlib import Path from platform import system from typing import Tuple +import boto3 +from hyp3lib.aws import get_content_type, get_tag_set + ESA_HOST = 'dataspace.copernicus.eu' @@ -28,3 +32,25 @@ def get_esa_credentials() -> Tuple[str, str]: "Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the " "ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file." ) + + +def upload_file_to_s3_with_publish_access_keys(path_to_file: Path, bucket: str, prefix: str = ''): + try: + access_key_id = os.environ['PUBLISH_ACCESS_KEY_ID'] + access_key_secret = os.environ['PUBLISH_SECRET_ACCESS_KEY'] + except KeyError: + raise ValueError( + 'Please provide S3 Bucket upload access key credentials via the ' + 'PUBLISH_ACCESS_KEY_ID and PUBLISH_SECRET_ACCESS_KEY environment variables' + ) + + s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret) + key = str(Path(prefix) / path_to_file.name) + extra_args = {'ContentType': get_content_type(key)} + + logging.info(f'Uploading s3://{bucket}/{key}') + s3_client.upload_file(str(path_to_file), bucket, key, extra_args) + + tag_set = get_tag_set(path_to_file.name) + + s3_client.put_object_tagging(Bucket=bucket, Key=key, Tagging=tag_set) diff --git a/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc b/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc new file mode 100644 index 00000000..487be3bc Binary files /dev/null and b/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc differ diff --git a/tests/test_process.py b/tests/test_process.py index f5d6d5c6..d9040e33 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -1,5 +1,6 @@ import io from datetime import datetime +from pathlib import Path from re import match from unittest import mock from unittest.mock import MagicMock, patch @@ -415,3 +416,27 @@ def mock_apply_filter_function(scene, _): process.apply_landsat_filtering('LT04', 'LE07') assert process.apply_landsat_filtering('LT04', 'LT05') == ('LT04', None, 'LT05', None) assert process.apply_landsat_filtering('LT04', 'LT04') == ('LT04', None, 'LT04', None) + + +def test_point_to_prefix(): + assert process.point_to_region(63.0, 128.0) == 'N60E120' + assert process.point_to_region(-63.0, 128.0) == 'S60E120' + assert process.point_to_region(63.0, -128.0) == 'N60W120' + assert process.point_to_region(-63.0, -128.0) == 'S60W120' + assert process.point_to_region(0.0, 128.0) == 'N00E120' + assert process.point_to_region(0.0, -128.0) == 'N00W120' + assert process.point_to_region(63.0, 0.0) == 'N60E000' + assert process.point_to_region(-63.0, 0.0) == 'S60E000' + assert process.point_to_region(0.0, 0.0) == 'N00E000' + + +def test_get_lat_lon_from_ncfile(): + file = Path('tests/data/' + 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + assert process.get_lat_lon_from_ncfile(file) == (-81.49, -128.28) + + +def test_get_opendata_prefix(): + file = Path('tests/data/' + 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120' diff --git a/tests/test_utils.py b/tests/test_utils.py index f3263d06..e640d411 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from hyp3_autorift.utils import ESA_HOST, get_esa_credentials +from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_publish_access_keys def test_get_esa_credentials_env(tmp_path, monkeypatch): @@ -45,3 +45,19 @@ def test_get_esa_credentials_missing(tmp_path, monkeypatch): msg = 'Please provide.*' with pytest.raises(ValueError, match=msg): get_esa_credentials() + + +def test_upload_file_to_s3_credentials_missing(tmp_path, monkeypatch): + with monkeypatch.context() as m: + m.delenv('PUBLISH_ACCESS_KEY_ID', raising=False) + m.setenv('PUBLISH_SECRET_ACCESS_KEY', 'publish_access_key_secret') + msg = 'Please provide.*' + with pytest.raises(ValueError, match=msg): + upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket') + + with monkeypatch.context() as m: + m.setenv('PUBLISH_ACCESS_KEY_ID', 'publish_access_key_id') + m.delenv('PUBLISH_SECRET_ACCESS_KEY', raising=False) + msg = 'Please provide.*' + with pytest.raises(ValueError, match=msg): + upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')