Merge pull request #90 from ASFHyP3/develop

Release direct bucket lookup for Landsat

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 <!--
 If this is a pull request for a new release, please use the release template:
-   https://github.com/ASFHyP3/hyp3-autorift/compare/master...develop?template=release.md
+   https://github.com/ASFHyP3/hyp3-autorift/compare/main...develop?template=release.md
  
 NOTE: Pull requests should only be opened for merges to protected branches (required) and any 
 changes which you'd like reviewed. Do not open a pull request to update a feature or personal

.github/workflows/changelog.yml

@@ -7,7 +7,7 @@ on:
       - unlabeled
       - synchronize
     branches:
-      - master
+      - main
       - develop
 jobs:
   changelog-udated:

.github/workflows/labeled-pr.yml

@@ -8,7 +8,7 @@ on:
       - unlabeled
       - synchronize
     branches:
-      - master
+      - main
 
 jobs:
   bump-labeled:

.github/workflows/release.yml

@@ -22,21 +22,21 @@ jobs:
           ALLOW_TAG_PREFIX: "true"
           RELEASE_NAME_PREFIX: "HyP3 autoRIFT "
 
-      - name: Attempt fast-forward develop from master
+      - name: Attempt fast-forward develop from main
         run: |
           git fetch --prune
           git checkout develop && git pull --ff-only
-          git merge --ff-only origin/master
+          git merge --ff-only origin/main
           git push
 
-      - name: Open PR to bring master back to develop
+      - name: Open PR to bring main back to develop
         if: ${{ failure() }}
         uses: repo-sync/pull-request@v2
         with:
-          source_branch: master
+          source_branch: main
           destination_branch: develop
           pr_title: Pulling ${{ github.ref }} into default
-          pr_body: Fast-forward of master to default failed!
+          pr_body: Fast-forward of main to default failed!
           pr_assignee: ${{ github.actor }}
           pr_label: tools-bot
           pr_draft: false

.github/workflows/static-analysis.yml

@@ -47,12 +47,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install trufflehog gitdb2=="2.*"
+          python -m pip install trufflehog
 
       - name: Scan for secrets with trufflehog
         run: |
           git fetch origin +refs/tags/*:refs/tags/*
-          export LAST_TAG_HASH=$(git show-ref --hash -- $(git describe --abbrev=0))
+          export LAST_TAG_HASH=$(git rev-list -1 $(git describe --abbrev=0))
           trufflehog --regex --entropy True --since_commit "${LAST_TAG_HASH}" \
               --exclude_paths .trufflehog.txt file://"${PWD}"
 

.github/workflows/test-and-build.yml

@@ -3,11 +3,11 @@ name: Test and build
 on:
   push:
     branches:
-      - master
+      - main
       - develop
   pull_request:
     branches:
-      - master
+      - main
       - develop
 
 env:
@@ -49,7 +49,7 @@ jobs:
           token: ${{ secrets.TOOLS_BOT_PAK }}
 
       - name: Get associated PR
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         uses: helaili/[email protected]
         env:
           GITHUB_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}
@@ -61,7 +61,7 @@ jobs:
           sha: ${{ github.sha }}
 
       - name: Get PR labels
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         uses: helaili/[email protected]
         env:
           GITHUB_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}
@@ -72,7 +72,7 @@ jobs:
           name: hyp3-autorift
 
       - name: Upload a Build Artifact
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         uses: actions/upload-artifact@v2
         with:
           name: query-responces
@@ -89,7 +89,7 @@ jobs:
           python -m pip install bump2version
 
       - name: Tag version
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         run: |
           git fetch origin +refs/tags/*:refs/tags/*
           git config user.email "[email protected]"
@@ -163,23 +163,23 @@ jobs:
           docker push ${HYP3_REGISTRY}/${GITHUB_REPOSITORY##*/}:test
 
       - name: Add latest tag
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         run: |
           export SDIST_VERSION=${{ needs.package.outputs.SDIST_VERSION }}
           docker tag ${HYP3_REGISTRY}/${GITHUB_REPOSITORY##*/}:${SDIST_VERSION/+/_} \
               ${HYP3_REGISTRY}/${GITHUB_REPOSITORY##*/}:latest
           docker push ${HYP3_REGISTRY}/${GITHUB_REPOSITORY##*/}:latest
 
       - name: Login to GitHub Container Registry
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: tools-bot
           password: ${{ secrets.TOOLS_BOT_PAK }}
 
       - name: Publish on github container registry
-        if: github.ref == 'refs/heads/master'
+        if: github.ref == 'refs/heads/main'
         run: |
           export SDIST_VERSION=${{ needs.package.outputs.SDIST_VERSION }}
           docker tag ${HYP3_REGISTRY}/${GITHUB_REPOSITORY##*/}:${SDIST_VERSION/+/_} \

.trufflehog.txt

@@ -1,6 +1,3 @@
 .*gitleaks.toml$
-.gitlab-ci.yml
-Dockerfile
-.*install_autoRIFT_ISCE_conda.sh$
-.*single_scene.*_full.py$
+.*hyp3_autorift/vend/workflow/.*
 .*hyp3_autorift/vend/README.md$

CHANGELOG.md

@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.2](https://github.com/ASFHyP3/hyp3-autorift/compare/v0.5.1...v0.5.2)
+
+### Changed
+* `process.get_lc2_metadata()` now fetches STAC metadata from the `usgs-landsat` S3 bucket instead of the
+  https://landsatlook.usgs.gov/ API.
+
 ## [0.5.1](https://github.com/ASFHyP3/hyp3-autorift/compare/v0.5.0...v0.5.1)
 
 ### Changed

hyp3_autorift/etc/entrypoint.sh

@@ -1,4 +1,4 @@
 #!/bin/bash --login
 set -e
 conda activate hyp3-autorift
-exec hyp3_autorift "$@"
+exec python -um hyp3_autorift "$@"

hyp3_autorift/process.py

@@ -3,6 +3,7 @@
 """
 
 import argparse
+import json
 import logging
 import os
 import shutil
@@ -11,6 +12,7 @@
 from secrets import token_hex
 from typing import Tuple
 
+import boto3
 import numpy as np
 import requests
 from hyp3lib.fetch import download_file
@@ -27,17 +29,25 @@
 
 gdal.UseExceptions()
 
+S3_CLIENT = boto3.client('s3')
 S2_SEARCH_URL = 'https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l1c/items'
-LC2_SEARCH_URL = 'https://landsatlook.usgs.gov/sat-api/collections/landsat-c2l1/items'
+LANDSAT_BUCKET = 'usgs-landsat'
 
 DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.jpl.nasa.gov.s3.amazonaws.com/' \
                          'autorift_parameters/v001/autorift_landice_0120m.shp'
 
 
+def get_lc2_stac_json_key(scene_name):
+    year = scene_name[17:21]
+    path = scene_name[10:13]
+    row = scene_name[13:16]
+    return f'collection02/level-1/standard/oli-tirs/{year}/{path}/{row}/{scene_name}/{scene_name}_stac.json'
+
+
 def get_lc2_metadata(scene_name):
-    response = requests.get(f'{LC2_SEARCH_URL}/{scene_name}')
-    response.raise_for_status()
-    return response.json()
+    key = get_lc2_stac_json_key(scene_name)
+    obj = S3_CLIENT.get_object(Bucket=LANDSAT_BUCKET, Key=key, RequestPayer='requester')
+    return json.load(obj['Body'])
 
 
 def get_s2_metadata(scene_name):
@@ -156,9 +166,9 @@ def process(reference: str, secondary: str, parameter_file: str = DEFAULT_PARAME
 
         orbits = Path('Orbits').resolve()
         orbits.mkdir(parents=True, exist_ok=True)
-        reference_state_vec, reference_provider = downloadSentinelOrbitFile(reference, directory=orbits)
+        reference_state_vec, reference_provider = downloadSentinelOrbitFile(reference, directory=str(orbits))
         log.info(f'Downloaded orbit file {reference_state_vec} from {reference_provider}')
-        secondary_state_vec, secondary_provider = downloadSentinelOrbitFile(secondary, directory=orbits)
+        secondary_state_vec, secondary_provider = downloadSentinelOrbitFile(secondary, directory=str(orbits))
         log.info(f'Downloaded orbit file {secondary_state_vec} from {secondary_provider}')
 
         polarization = get_s1_primary_polarization(reference)
@@ -188,11 +198,11 @@ def process(reference: str, secondary: str, parameter_file: str = DEFAULT_PARAME
             band = 'B8'
         reference_metadata = get_lc2_metadata(reference)
         reference_path = reference_metadata['assets'][f'{band}.TIF']['href']
-        reference_path = reference_path.replace('https://landsatlook.usgs.gov/data/', '/vsis3/usgs-landsat/')
+        reference_path = reference_path.replace('https://landsatlook.usgs.gov/data/', f'/vsis3/{LANDSAT_BUCKET}/')
 
         secondary_metadata = get_lc2_metadata(secondary)
         secondary_path = secondary_metadata['assets'][f'{band}.TIF']['href']
-        secondary_path = secondary_path.replace('https://landsatlook.usgs.gov/data/', '/vsis3/usgs-landsat/')
+        secondary_path = secondary_path.replace('https://landsatlook.usgs.gov/data/', f'/vsis3/{LANDSAT_BUCKET}/')
 
         bbox = reference_metadata['bbox']
         lat_limits = (bbox[1], bbox[3])

tests/test_process.py

@@ -1,13 +1,22 @@
+import io
 from datetime import datetime
 from re import match
+from unittest import mock
 
 import pytest
 import responses
-from requests import HTTPError
+from botocore.stub import Stubber
 
 from hyp3_autorift import process
 
 
+@pytest.fixture
+def s3_stubber():
+    with Stubber(process.S3_CLIENT) as stubber:
+        yield stubber
+        stubber.assert_no_pending_responses()
+
+
 def test_get_platform():
     assert process.get_platform('S1B_IW_GRDH_1SSH_20201203T095903_20201203T095928_024536_02EAB3_6D81') == 'S1'
     assert process.get_platform('S1A_IW_SLC__1SDV_20180605T233148_20180605T233215_022228_0267AD_48B2') == 'S1'
@@ -26,25 +35,34 @@ def test_get_platform():
         process.get_platform('foobar')
 
 
-@responses.activate
-def test_get_lc2_metadata_not_found():
-    responses.add(responses.GET, f'{process.LC2_SEARCH_URL}/foo', status=404)
-    with pytest.raises(HTTPError):
-        process.get_lc2_metadata('foo')
+def test_get_lc2_stac_json_key():
+    expected = 'collection02/level-1/standard/oli-tirs/2019/041/001/LC08_L1TP_041001_20191005_20200825_02_T1/' \
+               'LC08_L1TP_041001_20191005_20200825_02_T1_stac.json'
+    assert process.get_lc2_stac_json_key('LC08_L1TP_041001_20191005_20200825_02_T1') == expected
+
+    expected = 'collection02/level-1/standard/oli-tirs/2020/226/012/LO08_L1TP_226012_20201108_20201120_02_T2/' \
+               'LO08_L1TP_226012_20201108_20201120_02_T2_stac.json'
+    assert process.get_lc2_stac_json_key('LO08_L1TP_226012_20201108_20201120_02_T2') == expected
 
 
 @responses.activate
-def test_get_lc2_metadata():
-    responses.add(
-        responses.GET, f'{process.LC2_SEARCH_URL}/LC08_L1TP_009011_20200703_20200913_02_T1',
-        body='{"foo": "bar"}', status=200,
-    )
-    assert process.get_lc2_metadata('LC08_L1TP_009011_20200703_20200913_02_T1') == {'foo': 'bar'}
+def test_get_lc2_metadata(s3_stubber):
+    params = {
+        'Bucket': process.LANDSAT_BUCKET,
+        'Key': 'foo.json',
+        'RequestPayer': 'requester',
+    }
+    s3_response = {
+        'Body': io.StringIO('{"foo": "bar"}')
+    }
+    s3_stubber.add_response(method='get_object', expected_params=params, service_response=s3_response)
+
+    with mock.patch('hyp3_autorift.process.get_lc2_stac_json_key', return_value='foo.json'):
+        assert process.get_lc2_metadata('LC08_L1TP_009011_20200703_20200913_02_T1') == {'foo': 'bar'}
 
 
 @responses.activate
 def test_get_s2_metadata_not_found():
-
     responses.add(responses.GET, f'{process.S2_SEARCH_URL}/foo', status=404)
     responses.add(
         responses.POST, process.S2_SEARCH_URL,