From 3e5293e7275ea71151d819ef94cbaf13f83c69af Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 22 Jul 2024 20:51:26 +0300 Subject: [PATCH 1/6] Fix str when a Path is expected --- src/hyp3_autorift/s1_isce2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hyp3_autorift/s1_isce2.py b/src/hyp3_autorift/s1_isce2.py index 58c873ac..19e507a8 100644 --- a/src/hyp3_autorift/s1_isce2.py +++ b/src/hyp3_autorift/s1_isce2.py @@ -234,7 +234,7 @@ def create_conversion_matrices( epsg: int = 4326, parameter_file: str = DEFAULT_PARAMETER_FILE, **kwargs, -) -> str: +) -> Path: xGrid, tran, _, srs, nodata = utils.load_geospatial(grid_location, band=1) offset2vy_1, _, _, _, _ = utils.load_geospatial(offset2vy, band=1) @@ -274,14 +274,14 @@ def create_conversion_matrices( dr_2_vr_factor=dr_2_vr_factor, ChunkSize=ChunkSize, noDataMask=noDataMask, parameter_file=parameter_file, ) - return conversion_nc + return Path(conversion_nc) def generate_correction_data( scene: str, buffer: int = 0, parameter_file: str = DEFAULT_PARAMETER_FILE, -) -> (dict, str): +) -> (dict, Path): from hyp3_autorift.vend.testGeogrid_ISCE import loadParsedata, runGeogrid scene_path = Path(f'{scene}.zip') if not scene_path.exists(): From d050ae81170e6088b2b27107e61afdc32bd6aea9 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 22 Jul 2024 20:58:06 +0300 Subject: [PATCH 2/6] add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ad1b937..ea93d792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.18.0] +### Fixed +* `s2_isce2.generate_correction_data` now returns a Path instead of a str as expected by `hyp3lib.aws.upload_file_to_s3`. + ## [0.18.0] ### Added * The Sentinel-1 correction workflow will now calculate and write the M11/M12 conversion matrices to a netCDF file. From 696c76d1692cca294055607523fa908200fc1a01 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 24 Jul 2024 23:49:50 +0300 Subject: [PATCH 3/6] ensure nc file is flushed to disk and closed --- src/hyp3_autorift/s1_isce2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hyp3_autorift/s1_isce2.py b/src/hyp3_autorift/s1_isce2.py index 19e507a8..9935e1a1 100644 --- a/src/hyp3_autorift/s1_isce2.py +++ b/src/hyp3_autorift/s1_isce2.py @@ -221,6 +221,9 @@ def write_conversion_file( M12[noDataMask] = NoDataValue * np.float32(1 / C[0]) + np.float32(-C[1] / C[0]) var[:] = M12 + nc_outfile.sync() + nc_outfile.close() + return file_name From b8c4d1825af078c5de6e38c10b1640a00c9c80e8 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 24 Jul 2024 23:50:30 +0300 Subject: [PATCH 4/6] don't include scene name in nc filename (feedback from JPL) --- src/hyp3_autorift/s1_isce2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_autorift/s1_isce2.py b/src/hyp3_autorift/s1_isce2.py index 9935e1a1..a68d0623 100644 --- a/src/hyp3_autorift/s1_isce2.py +++ b/src/hyp3_autorift/s1_isce2.py @@ -273,7 +273,7 @@ def create_conversion_matrices( dr_2_vr_factor = np.median(offset2vr[np.logical_not(np.isnan(offset2vr))]) conversion_nc = write_conversion_file( - file_name=f'{scene}_conversion_matrices.nc', srs=srs, epsg=epsg, tran=tran, x=x, y=y, M11=M11, M12=M12, + file_name='conversion_matrices.nc', srs=srs, epsg=epsg, tran=tran, x=x, y=y, M11=M11, M12=M12, dr_2_vr_factor=dr_2_vr_factor, ChunkSize=ChunkSize, noDataMask=noDataMask, parameter_file=parameter_file, ) From daeca0192c1fb93b43bd4c58fb91c57e42475653 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 25 Jul 2024 09:35:26 -0800 Subject: [PATCH 5/6] fix transform for upper-left -> cell center --- src/hyp3_autorift/s1_isce2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hyp3_autorift/s1_isce2.py b/src/hyp3_autorift/s1_isce2.py index a68d0623..77f529ee 100644 --- a/src/hyp3_autorift/s1_isce2.py +++ b/src/hyp3_autorift/s1_isce2.py @@ -258,6 +258,9 @@ def create_conversion_matrices( scale_factor_1, _, _, _, _ = utils.load_geospatial(scale_factor, band=1) scale_factor_1[scale_factor_1 == nodata] = np.nan + # GDAL using upper-left of pixel -> netCDF using center of pixel + tran = [tran[0] + tran[1] / 2, tran[1], 0.0, tran[3] + tran[5] / 2, 0.0, tran[5]] + dimidY, dimidX = xGrid.shape noDataMask = xGrid == nodata From 49adb8a8de795faf60e270ce0ed0a484fbdbf9d0 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Thu, 25 Jul 2024 10:39:34 -0800 Subject: [PATCH 6/6] Update changelog for release --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea93d792..401bfcc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.18.0] +## [0.18.1] +### Changed +* The conversion matrices netCDF file created bt the S1 correction workflow is now called `conversion_matricies.nc` and no longer includes the scene name per feedback from JPL. + ### Fixed * `s2_isce2.generate_correction_data` now returns a Path instead of a str as expected by `hyp3lib.aws.upload_file_to_s3`. +* `s2_isce2.create_conversion_matricies` now uses the pixel-center instead of the upper-left corner for the x,y dimensions. +* `s2_isce2.create_conversion_matricies` now explicitly syncs data to and closes the netCDF file to prevent corrupt files from being uploaded. ## [0.18.0] ### Added