From 24028ecdb97da59797022e11613f1548757cc2d9 Mon Sep 17 00:00:00 2001 From: James Rine Date: Thu, 9 Jul 2020 14:08:52 -0800 Subject: [PATCH 01/10] update browse image to s3 --- hyp3_rtc_gamma/__main__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hyp3_rtc_gamma/__main__.py b/hyp3_rtc_gamma/__main__.py index 3439fffc..e2ea357b 100644 --- a/hyp3_rtc_gamma/__main__.py +++ b/hyp3_rtc_gamma/__main__.py @@ -1,7 +1,7 @@ """ rtc_gamma processing for HyP3 """ - +import glob import os import shutil import sys @@ -133,9 +133,12 @@ def main_v2(): product_name = build_output_name(args.granule, '.', '-30m-power-rtc-gamma') os.rename(output_folder, product_name) output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) + browse_images = glob.glob(product_name + '/*large.png') if args.bucket: upload_file_to_s3(output_zip, args.bucket, args.bucket_prefix) - + print(browse_images) + for image in browse_images: + upload_file_to_s3(image, args.bucket, args.bucket_prefix + '_DISPLAY') # end v2 functions From 8d536002d45931a2098a0d760e38dd55be0e8b2d Mon Sep 17 00:00:00 2001 From: James Rine Date: Thu, 9 Jul 2020 14:25:48 -0800 Subject: [PATCH 02/10] remove debugging print --- hyp3_rtc_gamma/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hyp3_rtc_gamma/__main__.py b/hyp3_rtc_gamma/__main__.py index e2ea357b..abefcce5 100644 --- a/hyp3_rtc_gamma/__main__.py +++ b/hyp3_rtc_gamma/__main__.py @@ -136,7 +136,6 @@ def main_v2(): browse_images = glob.glob(product_name + '/*large.png') if args.bucket: upload_file_to_s3(output_zip, args.bucket, args.bucket_prefix) - print(browse_images) for image in browse_images: upload_file_to_s3(image, args.bucket, args.bucket_prefix + '_DISPLAY') # end v2 functions From a644d4f4c855d6396f8168d40e0ef0062e2db341 Mon Sep 17 00:00:00 2001 From: James Rine Date: Thu, 9 Jul 2020 14:59:23 -0800 Subject: [PATCH 03/10] make browse an extention of existing prefix instead of new --- hyp3_rtc_gamma/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hyp3_rtc_gamma/__main__.py b/hyp3_rtc_gamma/__main__.py index abefcce5..bf0ecc4b 100644 --- a/hyp3_rtc_gamma/__main__.py +++ b/hyp3_rtc_gamma/__main__.py @@ -133,11 +133,11 @@ def main_v2(): product_name = build_output_name(args.granule, '.', '-30m-power-rtc-gamma') os.rename(output_folder, product_name) output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) - browse_images = glob.glob(product_name + '/*large.png') if args.bucket: upload_file_to_s3(output_zip, args.bucket, args.bucket_prefix) + browse_images = glob.glob(f'{product_name}/*.png') for image in browse_images: - upload_file_to_s3(image, args.bucket, args.bucket_prefix + '_DISPLAY') + upload_file_to_s3(image, args.bucket, args.bucket_prefix + '/browse') # end v2 functions From ec70061e7fc674c7f09c2ed235b075dd5a371a6d Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 13 Jul 2020 11:07:00 -0800 Subject: [PATCH 04/10] upgrade to hyp3-lib v1.3.0 --- hyp3_rtc_gamma/rtc_sentinel.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hyp3_rtc_gamma/rtc_sentinel.py b/hyp3_rtc_gamma/rtc_sentinel.py index 269c8555..766fb05f 100755 --- a/hyp3_rtc_gamma/rtc_sentinel.py +++ b/hyp3_rtc_gamma/rtc_sentinel.py @@ -723,7 +723,7 @@ def rtc_sentinel_gamma(in_file, create_iso_xml(rtc_name, out_name, pol, cpol, in_file, dem_type, log_file, gamma_ver) create_arc_xml(in_file, out_name, input_type, gamma_flag, pwr_flag, filter_flag, looks, pol, cpol, dem_type, res, hyp3_rtc_gamma.__version__, gamma_ver, rtc_name) - cogify_dir(res=res) + cogify_dir(directory='PRODUCT') clean_prod_dir() perform_sanity_checks() logging.info("===================================================================") diff --git a/setup.py b/setup.py index 6729d408..62d48ad9 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires=[ 'boto3', - 'hyp3lib>=1.2.3,<2', + 'hyp3lib>=1.3.0,<2', 'hyp3proclib>=1.0.1,<2', 'importlib_metadata', 'lxml', From 75e5283f1728a83a428d680450df86cd5130b177 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 13 Jul 2020 11:21:23 -0800 Subject: [PATCH 05/10] update hyp3-lib version to 1.3.0 for conda --- conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-env.yml b/conda-env.yml index 0db52430..ba7d293e 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -36,5 +36,5 @@ dependencies: # For running - --trusted-host hyp3-pypi.s3-website-us-east-1.amazonaws.com --extra-index-url http://hyp3-pypi.s3-website-us-east-1.amazonaws.com - - hyp3lib>=1.2.3,<2 + - hyp3lib>=1.3.0,<2 - hyp3proclib>=1.0.1,<2 From 222263d6889afb2942f2eae10e401c398cacb48a Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 13 Jul 2020 12:28:17 -0800 Subject: [PATCH 06/10] Revise description of browse images in readme --- hyp3_rtc_gamma/etc/README_RTC_GAMMA.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hyp3_rtc_gamma/etc/README_RTC_GAMMA.txt b/hyp3_rtc_gamma/etc/README_RTC_GAMMA.txt index 90721f68..cce01ebc 100644 --- a/hyp3_rtc_gamma/etc/README_RTC_GAMMA.txt +++ b/hyp3_rtc_gamma/etc/README_RTC_GAMMA.txt @@ -65,13 +65,13 @@ Each RTC image [FILT] been speckle filtered. The default is to not apply a speck ------------- ## 2. Browse images in grayscale and color -PNG files are generated in two different resolutions for quick visualization of the backscatter data. Each png browse image is accompanied by an aux file containing the projection and geocoding information for the file. +PNG files are generated for quick visualization of the backscatter data. Each png browse image is accompanied by an aux file containing the projection and geocoding information for the file. -All products will include a grayscale png browse image in both resolutions. It is a rendering of the primary polarization data, scaled to an ASF standard to display nicely in grayscale. The low-resolution image is designated by a simple .png extension, while the tag _large.png indicates the medium-resolution image. +All products will include a grayscale png browse image. It is a rendering of the primary polarization data, scaled to an ASF standard to display nicely in grayscale. The image is designated by a simple .png extension. -For dual-pol products, a false-color png browse image is generated in both resolutions. It is a rendering of the primary and cross-polarization data, scaled to an ASF standard to display nicely in color. These files are additionally tagged with _rgb, but otherwise have the same tags/extensions as the grayscale browse images. +For dual-pol products, a false-color png browse image is generated. It is a rendering of the primary and cross-polarization data, scaled to an ASF standard to display nicely in color. These files are additionally tagged with _rgb, but otherwise have the same tags/extensions as the grayscale browse images. -KMZ files are generated in the higher resolution for use in Google Earth and other compatible applications. All products will include a grayscale kmz image, and dual-pol products will also include a color browse kmz image. +KMZ files are generated for use in Google Earth and other compatible applications. All products will include a grayscale kmz image, and dual-pol products will also include a color browse kmz image. ------------- ## 3. DEM used to correct the data From fd033eca1f7a612259e3e893ff17918f4097f841 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 13 Jul 2020 13:17:42 -0800 Subject: [PATCH 07/10] remove obsolete references to large png image --- hyp3_rtc_gamma/__main__.py | 4 ++-- hyp3_rtc_gamma/create_metadata.py | 6 ------ hyp3_rtc_gamma/etc/RTC_GAMMA_Template_color_png.xml | 4 ++-- hyp3_rtc_gamma/etc/RTC_GAMMA_Template_grayscale_png.xml | 4 ++-- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/hyp3_rtc_gamma/__main__.py b/hyp3_rtc_gamma/__main__.py index 3439fffc..b109bcc9 100644 --- a/hyp3_rtc_gamma/__main__.py +++ b/hyp3_rtc_gamma/__main__.py @@ -144,7 +144,7 @@ def find_png(dir_): for subdir, dirs, files in os.walk(dir_): for file in files: filepath = os.path.join(subdir, file) - if filepath.endswith(".png") and 'rgb' in filepath and 'large' not in filepath: + if filepath.endswith(".png") and 'rgb' in filepath: log.info('Browse image: ' + filepath) return filepath @@ -152,7 +152,7 @@ def find_png(dir_): for subdir, dirs, files in os.walk(dir_): for file in files: filepath = os.path.join(subdir, file) - if filepath.endswith(".png") and 'large' not in filepath: + if filepath.endswith(".png"): log.info('Browse image: ' + filepath) return filepath diff --git a/hyp3_rtc_gamma/create_metadata.py b/hyp3_rtc_gamma/create_metadata.py index fb6a095b..c69c6c87 100644 --- a/hyp3_rtc_gamma/create_metadata.py +++ b/hyp3_rtc_gamma/create_metadata.py @@ -186,11 +186,6 @@ def create_arc_xml(infile, outfile, input_type, gamma_flag, pwr_flag, filter_fla scale = 'grayscale' encoded_jpg = pngtothumb("{}.png".format(outfile)) - if "large" in myfile: - res = "medium" - else: - res = "low" - with open("{}/RTC_GAMMA_Template_{}_png.xml".format(etc_dir, scale), "rb") as f: for line in f: line = line.replace(b"[DATE]", bytes(date, 'utf-8')) @@ -202,7 +197,6 @@ def create_arc_xml(infile, outfile, input_type, gamma_flag, pwr_flag, filter_fla line = line.replace(b"[FULL_TYPE]", bytes(full_type, 'utf-8')) line = line.replace(b"[THUMBNAIL_BINARY_STRING]", encoded_jpg) line = line.replace(b"[GRAN_NAME]", bytes(granulename, 'utf-8')) - line = line.replace(b"[RES]", bytes(res, 'utf-8')) line = line.replace(b"[SPACING]", bytes("{}".format(spacing), 'utf-8')) line = line.replace(b"[DEM]", bytes("{}".format(dem_type), 'utf-8')) line = line.replace(b"[FORMAT]", bytes(format_type, 'utf-8')) diff --git a/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_color_png.xml b/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_color_png.xml index 2317d755..8b790fd1 100644 --- a/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_color_png.xml +++ b/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_color_png.xml @@ -19,8 +19,8 @@ [TIME] - This is a [RES]-resolution browse image of a radiometric terrain corrected granule of [TYPE] SAR data from the Sentinel-1 mission, processed using GAMMA software. The values are scaled from a combination of the primary and cross-polarization values to maximize color contrast in the appearance of the image. To access actual [FORMAT] values, refer to the VV/VH/HV/HH TIFF files included in the same folder as this image. - <DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>This file is a [RES]-resolution browse image of a radiometric terrain corrected granule of [FULL_TYPE] ([TYPE]) Synthetic Aperture Radar data from the Copernicus Sentinel-1 mission (European Space Agency). It is provided in [PCS] coordinates and is corrected for terrain using the [DEM] Digital Elevation Model.</SPAN></P><P><SPAN>The actual [FORMAT] data is contained in the VV/HH (primary polarization) and/or VH/HV (cross-polarization) TIFF files included in the same downloaded folder as this file. This image is a [RES]-resolution rendering of a combination of the primary and cross-polarization data, scaled to an ASF standard to display nicely in color for visualization purposes. </SPAN></P><P><SPAN>The name of the SAR granule used to generate the files in this folder is: [GRAN_NAME]. For a detailed description of the Sentinel-1 file naming convention, refer to </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention</SPAN></A><SPAN>.</SPAN></P><P><SPAN>For areas where there is not a publicly-available digital elevation model (i.e. sea ice), geocoded products without terrain correction are also available from </SPAN><A href="http://hyp3.asf.alaska.edu"><SPAN>http://hyp3.asf.alaska.edu</SPAN></A><SPAN>.</SPAN></P><P><SPAN>The Sentinel-1A satellite was launched April 3, 2014, and the Sentinel-1B satellite was launched April 25, 2016. The satellites each have a 12-day repeat cycle. More information about the mission is available at </SPAN><A href="https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1"><SPAN>https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1</SPAN></A><SPAN>.</SPAN></P><P><SPAN>Additional information about Sentinel-1 data, imagery, tools and applications is available at </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1</SPAN></A><SPAN>.</SPAN></P></DIV></DIV></DIV> + This is a browse image of a radiometric terrain corrected granule of [TYPE] SAR data from the Sentinel-1 mission, processed using GAMMA software. The values are scaled from a combination of the primary and cross-polarization values to maximize color contrast in the appearance of the image. To access actual [FORMAT] values, refer to the VV/VH/HV/HH TIFF files included in the same folder as this image. + <DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>This file is a browse image of a radiometric terrain corrected granule of [FULL_TYPE] ([TYPE]) Synthetic Aperture Radar data from the Copernicus Sentinel-1 mission (European Space Agency). It is provided in [PCS] coordinates and is corrected for terrain using the [DEM] Digital Elevation Model.</SPAN></P><P><SPAN>The actual [FORMAT] data is contained in the VV/HH (primary polarization) and/or VH/HV (cross-polarization) TIFF files included in the same downloaded folder as this file. This image is a rendering of a combination of the primary and cross-polarization data, scaled to an ASF standard to display nicely in color for visualization purposes. </SPAN></P><P><SPAN>The name of the SAR granule used to generate the files in this folder is: [GRAN_NAME]. For a detailed description of the Sentinel-1 file naming convention, refer to </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention</SPAN></A><SPAN>.</SPAN></P><P><SPAN>For areas where there is not a publicly-available digital elevation model (i.e. sea ice), geocoded products without terrain correction are also available from </SPAN><A href="http://hyp3.asf.alaska.edu"><SPAN>http://hyp3.asf.alaska.edu</SPAN></A><SPAN>.</SPAN></P><P><SPAN>The Sentinel-1A satellite was launched April 3, 2014, and the Sentinel-1B satellite was launched April 25, 2016. The satellites each have a 12-day repeat cycle. More information about the mission is available at </SPAN><A href="https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1"><SPAN>https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1</SPAN></A><SPAN>.</SPAN></P><P><SPAN>Additional information about Sentinel-1 data, imagery, tools and applications is available at </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1</SPAN></A><SPAN>.</SPAN></P></DIV></DIV></DIV> ASF DAAC [YEARPROCESSED] using GAMMA software. Contains modified Copernicus Sentinel data [YEARACQUIRED], processed by ESA. diff --git a/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_grayscale_png.xml b/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_grayscale_png.xml index bbc1ec0f..42a73073 100644 --- a/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_grayscale_png.xml +++ b/hyp3_rtc_gamma/etc/RTC_GAMMA_Template_grayscale_png.xml @@ -19,8 +19,8 @@ [TIME] - This is a [RES]-resolution browse image of a radiometric terrain corrected granule of [TYPE] SAR data from the Sentinel-1 mission, processed using GAMMA software. The values are scaled from primary polarization values to improve the appearance of the image; to access actual [FORMAT] values, refer to the VV/VH/HV/HH TIFF files included in the same folder as this image. - <DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>This file is a [RES]-resolution browse image of a radiometric terrain corrected granule of [FULL_TYPE] ([TYPE]) Synthetic Aperture Radar data from the Copernicus Sentinel-1 mission (European Space Agency). It is provided in [PCS] coordinates and is corrected for terrain using the [DEM] Digital Elevation Model.</SPAN></P><P><SPAN>The actual [FORMAT] data is contained in the VV/HH (primary polarization) and/or VH/HV (cross-polarization) TIFF files included in the same downloaded folder as this file. This image is a [RES]-resolution rendering of the primary polarization data, scaled to an ASF standard to display nicely in grayscale for visualization purposes.</SPAN></P><P><SPAN>The name of the SAR granule used to generate the files in this folder is: [GRAN_NAME]. For a detailed description of the Sentinel-1 file naming convention, refer to </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention</SPAN></A><SPAN>.</SPAN></P><P><SPAN>For areas where there is not a publicly-available digital elevation model (i.e. sea ice), geocoded products without terrain correction are also available from </SPAN><A href="http://hyp3.asf.alaska.edu"><SPAN>http://hyp3.asf.alaska.edu</SPAN></A><SPAN>.</SPAN></P><P><SPAN>The Sentinel-1A satellite was launched April 3, 2014, and the Sentinel-1B satellite was launched April 25, 2016. The satellites each have a 12-day repeat cycle. More information about the mission is available at </SPAN><A href="https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1"><SPAN>https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1</SPAN></A><SPAN>.</SPAN></P><P><SPAN>Additional information about Sentinel-1 data, imagery, tools and applications is available at </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1</SPAN></A><SPAN>.</SPAN></P></DIV></DIV></DIV> + This is a browse image of a radiometric terrain corrected granule of [TYPE] SAR data from the Sentinel-1 mission, processed using GAMMA software. The values are scaled from primary polarization values to improve the appearance of the image; to access actual [FORMAT] values, refer to the VV/VH/HV/HH TIFF files included in the same folder as this image. + <DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>This file is a browse image of a radiometric terrain corrected granule of [FULL_TYPE] ([TYPE]) Synthetic Aperture Radar data from the Copernicus Sentinel-1 mission (European Space Agency). It is provided in [PCS] coordinates and is corrected for terrain using the [DEM] Digital Elevation Model.</SPAN></P><P><SPAN>The actual [FORMAT] data is contained in the VV/HH (primary polarization) and/or VH/HV (cross-polarization) TIFF files included in the same downloaded folder as this file. This image is a rendering of the primary polarization data, scaled to an ASF standard to display nicely in grayscale for visualization purposes.</SPAN></P><P><SPAN>The name of the SAR granule used to generate the files in this folder is: [GRAN_NAME]. For a detailed description of the Sentinel-1 file naming convention, refer to </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1/sentinel-1-data-and-imagery/#Naming_Convention</SPAN></A><SPAN>.</SPAN></P><P><SPAN>For areas where there is not a publicly-available digital elevation model (i.e. sea ice), geocoded products without terrain correction are also available from </SPAN><A href="http://hyp3.asf.alaska.edu"><SPAN>http://hyp3.asf.alaska.edu</SPAN></A><SPAN>.</SPAN></P><P><SPAN>The Sentinel-1A satellite was launched April 3, 2014, and the Sentinel-1B satellite was launched April 25, 2016. The satellites each have a 12-day repeat cycle. More information about the mission is available at </SPAN><A href="https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1"><SPAN>https://earth.esa.int/web/guest/missions/esa-operational-eo-missions/sentinel-1</SPAN></A><SPAN>.</SPAN></P><P><SPAN>Additional information about Sentinel-1 data, imagery, tools and applications is available at </SPAN><A href="https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1"><SPAN>https://asf.alaska.edu/data-sets/sar-data-sets/sentinel-1</SPAN></A><SPAN>.</SPAN></P></DIV></DIV></DIV> ASF DAAC [YEARPROCESSED] using GAMMA software. Contains modified Copernicus Sentinel data [YEARACQUIRED], processed by ESA. From c5f5f0a23f946c15de7fbf9b1e204e14b7493f2a Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 13 Jul 2020 20:53:19 -0800 Subject: [PATCH 08/10] asfdmin --> ASFHyP3 --- .github/pull_request_template.md | 2 +- .github/workflows/release.yml | 8 ++++---- .github/workflows/test-and-build.yml | 4 ++-- CHANGELOG.md | 12 ++++++------ Dockerfile | 4 ++-- setup.py | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6e1b4151..c94b9395 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,6 @@