From a9c1b71e2b8ba1e6416a37d90126dece7c692f65 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 18 Sep 2020 11:30:29 -0800 Subject: [PATCH 1/9] Create browse and thumbnail images for product --- conda-env.yml | 1 + hyp3_autorift/__main__.py | 22 ++++++++++++++++++---- hyp3_autorift/process.py | 30 ++++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/conda-env.yml b/conda-env.yml index 6df84450..4dee1271 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -18,6 +18,7 @@ dependencies: - isce2>=2.4 - importlib_metadata - numpy + - pillow - psycopg2 # missing hyp3proclib dep - requests - scikit-image # missing autoRIFT dep diff --git a/hyp3_autorift/__main__.py b/hyp3_autorift/__main__.py index d04ac8af..c48cd042 100644 --- a/hyp3_autorift/__main__.py +++ b/hyp3_autorift/__main__.py @@ -26,6 +26,7 @@ from hyp3proclib.file_system import cleanup_workdir from hyp3proclib.logger import log from hyp3proclib.proc_base import Processor +from PIL import Image from pkg_resources import load_entry_point import hyp3_autorift @@ -49,6 +50,16 @@ def entry(): # v2 functions +def create_thumbnail(input_image, size=(100, 100)): + filename, ext = os.path.splitext(input_image) + thumbnail_name = f'{filename}_thumb{ext}' + + output_image = Image.open(input_image) + output_image.thumbnail(size) + output_image.save(thumbnail_name) + return thumbnail_name + + def write_netrc_file(username, password): netrc_file = os.path.join(os.environ['HOME'], '.netrc') if os.path.isfile(netrc_file): @@ -110,9 +121,16 @@ def main_v2(): product_name = f'{outname}.nc' netcdf_file = glob.glob('*nc')[0] os.rename(netcdf_file, product_name) + browse_name = f'{outname}.png' + browse_file = glob.glob('*.png')[0] + os.rename(browse_file, browse_name) if args.bucket: upload_file_to_s3(product_name, 'product', args.bucket, args.bucket_prefix) + upload_file_to_s3(browse_name, 'product', args.bucket, args.bucket_prefix) + thumbnail_name = create_thumbnail(browse_name) + upload_file_to_s3(thumbnail_name, 'product', args.bucket, args.bucket_prefix) +# End v2 functions def hyp3_process(cfg, n): @@ -139,10 +157,6 @@ def hyp3_process(cfg, n): out_name = build_output_name_pair(g1, g2, cfg['workdir'], cfg['suffix']) log.info(f'Output name: {out_name}') - # TODO: - # * browse images - # * citation - # * phase_png (?) if extra_arg_is(cfg, 'intermediate_files', 'no'): product_glob = os.path.join(cfg['workdir'], cfg['ftd'], '*.nc') netcdf_files = glob.glob(product_glob) diff --git a/hyp3_autorift/process.py b/hyp3_autorift/process.py index 1fd3f09a..d46a82a7 100644 --- a/hyp3_autorift/process.py +++ b/hyp3_autorift/process.py @@ -10,9 +10,12 @@ import shutil from pathlib import Path +import numpy as np from hyp3lib.execute import execute from hyp3lib.file_subroutines import mkdir_p from hyp3lib.get_orb import downloadSentinelOrbitFile +from hyp3lib.makeAsfBrowse import makeAsfBrowse +from osgeo import gdal from hyp3_autorift import geometry from hyp3_autorift import io @@ -22,10 +25,18 @@ _PRODUCT_LIST = [ 'offset.tif', 'velocity.tif', + 'velocity_browse.tif', + 'velocity_browse.kmz', + 'velocity_browse.png', + 'velocity_browse.png.aux.xml', + 'window_chip_size_max.tif', + 'window_chip_size_min.tif', 'window_location.tif', 'window_offset.tif', 'window_rdr_off2vel_x_vec.tif', 'window_rdr_off2vel_y_vec.tif', + 'window_search_range.tif', + 'window_stable_surface_mask.tif', ] @@ -124,6 +135,25 @@ def process(reference, secondary, download=False, polarization='hh', orbits=None f' -csmax window_chip_size_max.tif -nc S' execute(cmd, logfile=f, uselogging=True) + velocity_tif = gdal.Open('velocity.tif') + x_velocity = np.ma.masked_invalid(velocity_tif.GetRasterBand(1).ReadAsArray()) + y_velocity = np.ma.masked_invalid(velocity_tif.GetRasterBand(2).ReadAsArray()) + velocity = np.sqrt(x_velocity**2 + y_velocity**2) + + browse_file = Path('velocity_browse.tif') + driver = gdal.GetDriverByName('GTiff') + browse_tif = driver.Create( + str(browse_file), velocity.shape[1], velocity.shape[0], 1, gdal.GDT_Float32, ['COMPRESS=LZW'] + ) + browse_tif.SetProjection(velocity_tif.GetProjection()) + browse_tif.SetGeoTransform(velocity_tif.GetGeoTransform()) + velocity_band = browse_tif.GetRasterBand(1) + velocity_band.WriteArray(velocity) + + del velocity_band, browse_tif, velocity_tif + + makeAsfBrowse(browse_file, browse_file.stem) + if product: mkdir_p(product_dir) for f in _PRODUCT_LIST: diff --git a/setup.py b/setup.py index c530e8d7..0b46b7e9 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ 'hyp3lib==1.5', 'hyp3proclib', 'importlib_metadata', + 'pillow', 'numpy', 'scipy', ], From 056a33de82abb4049431cf2ca1bd78e1b930ef80 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 18 Sep 2020 11:32:59 -0800 Subject: [PATCH 2/9] Fix GDAL warnings --- hyp3_autorift/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyp3_autorift/process.py b/hyp3_autorift/process.py index d46a82a7..33f2e998 100644 --- a/hyp3_autorift/process.py +++ b/hyp3_autorift/process.py @@ -143,7 +143,7 @@ def process(reference, secondary, download=False, polarization='hh', orbits=None browse_file = Path('velocity_browse.tif') driver = gdal.GetDriverByName('GTiff') browse_tif = driver.Create( - str(browse_file), velocity.shape[1], velocity.shape[0], 1, gdal.GDT_Float32, ['COMPRESS=LZW'] + str(browse_file), velocity.shape[1], velocity.shape[0], 1, gdal.GDT_Byte, ['COMPRESS=LZW'] ) browse_tif.SetProjection(velocity_tif.GetProjection()) browse_tif.SetGeoTransform(velocity_tif.GetGeoTransform()) From ee65f630a695d8d8fa5640645b54e2ec23629556 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 18 Sep 2020 11:36:31 -0800 Subject: [PATCH 3/9] Fix lint --- hyp3_autorift/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyp3_autorift/__main__.py b/hyp3_autorift/__main__.py index c48cd042..818e967a 100644 --- a/hyp3_autorift/__main__.py +++ b/hyp3_autorift/__main__.py @@ -11,6 +11,7 @@ from mimetypes import guess_type import boto3 +from PIL import Image from hyp3proclib import ( build_output_name_pair, earlier_granule_first, @@ -26,7 +27,6 @@ from hyp3proclib.file_system import cleanup_workdir from hyp3proclib.logger import log from hyp3proclib.proc_base import Processor -from PIL import Image from pkg_resources import load_entry_point import hyp3_autorift From 90f2fbbc413b7ffa29a218b50c80b61e2bf67743 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 18 Sep 2020 11:45:34 -0800 Subject: [PATCH 4/9] Fix upload type --- hyp3_autorift/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hyp3_autorift/__main__.py b/hyp3_autorift/__main__.py index 818e967a..aa7cdc3a 100644 --- a/hyp3_autorift/__main__.py +++ b/hyp3_autorift/__main__.py @@ -127,9 +127,9 @@ def main_v2(): if args.bucket: upload_file_to_s3(product_name, 'product', args.bucket, args.bucket_prefix) - upload_file_to_s3(browse_name, 'product', args.bucket, args.bucket_prefix) + upload_file_to_s3(browse_name, 'browse', args.bucket, args.bucket_prefix) thumbnail_name = create_thumbnail(browse_name) - upload_file_to_s3(thumbnail_name, 'product', args.bucket, args.bucket_prefix) + upload_file_to_s3(thumbnail_name, 'thumbnail', args.bucket, args.bucket_prefix) # End v2 functions From 270040e33a6aa5877e8cc1b82fb9fe1c5c326251 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 18 Sep 2020 11:53:36 -0800 Subject: [PATCH 5/9] Restrict ISCE for autorift 1.0.6 --- conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-env.yml b/conda-env.yml index 4dee1271..ff4c958c 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -15,7 +15,7 @@ dependencies: - wheel # For running - hyp3lib=1.5 - - isce2>=2.4 + - isce2=2.4.1 - importlib_metadata - numpy - pillow From 43e4168651b280d7389db2fbc4c50b9d6086ded4 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 18 Sep 2020 14:10:14 -0800 Subject: [PATCH 6/9] Correct ISCE version for autoRIFT 1.0.6 --- conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-env.yml b/conda-env.yml index ff4c958c..3ac17b23 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -15,7 +15,7 @@ dependencies: - wheel # For running - hyp3lib=1.5 - - isce2=2.4.1 + - isce2=2.4.0 - importlib_metadata - numpy - pillow From 7ec2dfe55cb958e6273f9f08e39d854ee290d83c Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 23 Sep 2020 11:13:52 -0800 Subject: [PATCH 7/9] makeAsfBrowse expects a string --- hyp3_autorift/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyp3_autorift/process.py b/hyp3_autorift/process.py index 33f2e998..40a34e11 100644 --- a/hyp3_autorift/process.py +++ b/hyp3_autorift/process.py @@ -152,7 +152,7 @@ def process(reference, secondary, download=False, polarization='hh', orbits=None del velocity_band, browse_tif, velocity_tif - makeAsfBrowse(browse_file, browse_file.stem) + makeAsfBrowse(str(browse_file), browse_file.stem) if product: mkdir_p(product_dir) From b4f2e193deecf91231bf47f8ca52cc7d8d1fa165 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 23 Sep 2020 11:18:44 -0800 Subject: [PATCH 8/9] update changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcbd04d4..ee216028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.1](https://github.com/ASFHyP3/hyp3-autorift/compare/v0.1.0...v0.1.1) + +### Added +* Browse and thumbnail images are now created and uploaded for hyp3v2 + +### Fixes +* Restrict ISCE version to 2.4.0 which include autoRIFT 1.0.6 + ## [0.1.0](https://github.com/ASFHyP3/hyp3-autorift/compare/v0.0.0...v0.1.0) Initial release of hyp3-autorift, a HyP3 plugin for feature tracking processing From 8f07a6b90c878541170ecc26b56c08f20604a633 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 23 Sep 2020 11:20:55 -0800 Subject: [PATCH 9/9] Fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee216028..b8fcda04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * Browse and thumbnail images are now created and uploaded for hyp3v2 ### Fixes -* Restrict ISCE version to 2.4.0 which include autoRIFT 1.0.6 +* Restrict ISCE version to 2.4.0 which includes autoRIFT 1.0.6 ## [0.1.0](https://github.com/ASFHyP3/hyp3-autorift/compare/v0.0.0...v0.1.0)