Merge pull request #37 from ASFHyP3/develop

Release: v0.3.0

.github/workflows/test-and-build.yml

@@ -23,7 +23,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - uses: goanpeca/setup-miniconda@v1
+      - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
           python-version: 3.7

.trufflehog.txt

@@ -2,3 +2,4 @@
 .gitlab-ci.yml
 Dockerfile
 .*install_autoRIFT_ISCE_conda.sh$
+.*single_scene.*_full.py$

CHANGELOG.md

@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0](https://github.com/ASFHyP3/hyp3-autorift/compare/v0.1.0...v0.2.0)
+
+### Added
+* Installed autoRIFT v1.0.8 for processing optical scenes (in addition to v1.0.7 already installed as part of ISCE)
+* Added support for processing Sentinel-2 scene pairs
+
+### Changed
+* Upgraded to isce2 [v2.4.2](https://github.com/isce-framework/isce2/releases/tag/v2.4.2) from v2.4.1
+* Upgraded to hyp3lib [v1.6.2](https://github.com/ASFHyP3/hyp3-lib/blob/develop/CHANGELOG.md#162) from v1.6.1
+
+### Removed
+* Removed the "include intermediate files" option when running jobs via HyP3 v1
+
 ## [0.2.0](https://github.com/ASFHyP3/hyp3-autorift/compare/v0.1.0...v0.2.0)
 
 ### Changed

conda-env.yml

@@ -11,18 +11,21 @@ dependencies:
   - pytest
   - pytest-console-scripts
   - pytest-cov
+  - responses
   - setuptools
   - setuptools_scm
   - wheel
   # For running
   - gdal>=3
-  - hyp3lib=1.6.1
-  - isce2>=2.4.1
+  - hyp3lib=1.6.2
+  - isce2=2.4.2
+  - autorift=1.0.8
   - boto3
   - importlib_metadata
   - netCDF4
   - numpy
   - psycopg2  # missing hyp3proclib dep
+  - requests
   - scikit-image  # missing autoRIFT dep
   - scipy
   - pip:

hyp3_autorift/__init__.py

@@ -2,8 +2,6 @@
 
 from importlib.metadata import PackageNotFoundError, version
 
-from hyp3_autorift.process import process
-
 try:
     __version__ = version(__name__)
 except PackageNotFoundError:
@@ -15,5 +13,4 @@
 
 __all__ = [
     '__version__',
-    'process'
 ]

hyp3_autorift/__main__.py

@@ -11,7 +11,6 @@
 from hyp3lib.fetch import write_credentials_to_netrc_file
 from hyp3lib.image import create_thumbnail
 from hyp3proclib import (
-    earlier_granule_first,
     extra_arg_is,
     failure,
     success,
@@ -25,6 +24,7 @@
 from pkg_resources import load_entry_point
 
 import hyp3_autorift
+from hyp3_autorift.process import get_datetime, process
 
 
 def entry():
@@ -43,8 +43,8 @@ def entry():
 
 def main_v2():
     parser = ArgumentParser()
-    parser.add_argument('--username', required=True)
-    parser.add_argument('--password', required=True)
+    parser.add_argument('--username')
+    parser.add_argument('--password')
     parser.add_argument('--bucket')
     parser.add_argument('--bucket-prefix', default='')
     parser.add_argument('granules', type=str.split, nargs='+')
@@ -54,11 +54,12 @@ def main_v2():
     if len(args.granules) != 2:
         parser.error('Must provide exactly two granules')
 
-    write_credentials_to_netrc_file(args.username, args.password)
+    if args.username and args.password:
+        write_credentials_to_netrc_file(args.username, args.password)
 
-    g1, g2 = earlier_granule_first(args.granules[0], args.granules[1])
+    g1, g2 = sorted(args.granules, key=get_datetime)
 
-    product_file = hyp3_autorift.process(f'{g1}.zip', f'{g2}.zip', download=True)
+    product_file = process(g1, g2)
 
     browse_file = product_file.with_suffix('.png')
 
@@ -73,7 +74,7 @@ def hyp3_process(cfg, n):
     try:
         log.info(f'Processing autoRIFT-ISCE pair "{cfg["sub_name"]}" for "{cfg["username"]}"')
 
-        g1, g2 = earlier_granule_first(cfg['granule'], cfg['other_granules'][0])
+        g1, g2 = sorted((cfg['granule'], cfg['other_granules'][0]), key=get_datetime)
 
         d1 = datetime.strptime(g1[17:25], '%Y%m%d')
         d2 = datetime.strptime(g2[17:25], '%Y%m%d')
@@ -84,17 +85,7 @@ def hyp3_process(cfg, n):
         cfg['ftd'] = '_'.join([g1[17:17+15], g2[17:17+15]])
         log.debug(f'FTD dir is: {cfg["ftd"]}')
 
-        autorift_args = [f'{g1}.zip', f'{g2}.zip', '--process-dir', f'{cfg["ftd"]}', '--download']
-        if not extra_arg_is(cfg, 'intermediate_files', 'no'):  # handle processes b4 option added
-            autorift_args.append('--product')
-
-        product_file = hyp3_autorift.process(
-            reference=f'{g1}.zip',
-            secondary=f'{g2}.zip',
-            download=True,
-            process_dir=cfg["ftd"],
-            product=extra_arg_is(cfg, 'intermediate_files', 'yes')
-        )
+        product_file = process(g1, g2)
         cfg['attachment'] = str(product_file.with_suffix('.png'))
         cfg['email_text'] = ' '  # fix line break in email
 

hyp3_autorift/geometry.py

@@ -91,10 +91,10 @@ def bounding_box(safe, priority='reference', polarization='hh', orbits='Orbits',
 
 def find_jpl_dem(lat_limits, lon_limits, z_limits=(-200, 4000)):
 
-    dems = ['GRE240m_h.tif', 'ANT240m_h.tif']
+    dems = ['GRE240m', 'ANT240m']
     bounding_dem = None
     for dem in dems:
-        dem_file = f'/vsicurl/http://{ITS_LIVE_BUCKET}.s3.amazonaws.com/{AUTORIFT_PREFIX}/{dem}'
+        dem_file = f'/vsicurl/http://{ITS_LIVE_BUCKET}.s3.amazonaws.com/{AUTORIFT_PREFIX}/{dem}_h.tif'
         log.info(f'Checking DEM: {dem_file}')
         dem_ds = gdal.Open(dem_file, gdal.GA_ReadOnly)
         dem_sr = dem_ds.GetSpatialRef()

hyp3_autorift/io.py

@@ -19,12 +19,6 @@
 AUTORIFT_PREFIX = 'isce_autoRIFT'
 
 
-def _list_s3_files(bucket, prefix):
-    response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)
-    keys = [item['Key'] for item in response['Contents']]
-    return keys
-
-
 def _download_s3_files(target_dir, bucket, keys, chunk_size=50*1024*1024):
     transfer_config = TransferConfig(multipart_threshold=chunk_size, multipart_chunksize=chunk_size)
     for key in keys:
@@ -33,14 +27,35 @@ def _download_s3_files(target_dir, bucket, keys, chunk_size=50*1024*1024):
         s3_client.download_file(Bucket=bucket, Key=key, Filename=filename, Config=transfer_config)
 
 
-def fetch_jpl_tifs(ice_sheet='GRE', target_dir='DEM', bucket=ITS_LIVE_BUCKET, prefix=AUTORIFT_PREFIX):
-    log.info(f"Downloading {ice_sheet} tifs from JPL's AWS bucket")
+def _get_s3_keys_for_dem(prefix=AUTORIFT_PREFIX, dem='GRE240m'):
+    tags = [
+        'h',
+        'StableSurface',
+        'dhdx',
+        'dhdy',
+        'dhdxs',
+        'dhdys',
+        'vx0',
+        'vy0',
+        'vxSearchRange',
+        'vySearchRange',
+        'xMinChipSize',
+        'yMinChipSize',
+        'xMaxChipSize',
+        'yMaxChipSize',
+        'masks',
+    ]
+    keys = [f'{prefix}/{dem}_{tag}.tif' for tag in tags]
+    return keys
+
+
+def fetch_jpl_tifs(dem='GRE240m', target_dir='DEM', bucket=ITS_LIVE_BUCKET, prefix=AUTORIFT_PREFIX):
+    log.info(f"Downloading {dem} tifs from JPL's AWS bucket")
 
     for logger in ('botocore', 's3transfer'):
         logging.getLogger(logger).setLevel(logging.WARNING)
 
-    full_prefix = f'{prefix}/{ice_sheet}'
-    keys = _list_s3_files(bucket, full_prefix)
+    keys = _get_s3_keys_for_dem(prefix, dem)
     _download_s3_files(target_dir, bucket, keys)
 
 
@@ -52,14 +67,14 @@ def format_tops_xml(reference, secondary, polarization, dem, orbits, xml_file='t
                 <property name="orbit directory">{orbits}</property>
                 <property name="auxiliary data directory">{orbits}</property>
                 <property name="output directory">reference</property>
-                <property name="safe">['{reference}']</property>
+                <property name="safe">['{reference}.zip']</property>
                 <property name="polarization">{polarization}</property>
             </component>
             <component name="secondary">
                 <property name="orbit directory">{orbits}</property>
                 <property name="auxiliary data directory">{orbits}</property>
                 <property name="output directory">secondary</property>
-                <property name="safe">['{secondary}']</property>
+                <property name="safe">['{secondary}.zip']</property>
                 <property name="polarization">{polarization}</property>
             </component>
             <property name="demfilename">{dem}</property>