Merge branch 'dev' into dev-fix-inundate-nation

config/params_template.env

@@ -33,48 +33,33 @@ export levee_id_attribute=SYSTEM_ID
 
 #### apply bathymetry adjustment to rating curve ####
 export bathymetry_adjust=True
-export bathymetry_file="$inputsDir/bathymetry/bathymetry_adjustment_data.gpkg"
 
 #### estimating bankfull stage in SRCs ####
 # Toggle to run identify_bankfull routine (True=on; False=off)
 export src_bankfull_toggle="True"
-# input file location with nwm feature_id and recurrence flow values
-export bankfull_flows_file="$inputsDir/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv"
 
 #### applying channel/overbank subdivision routine to SRCs ####
 # Toggle to run composite roughness src routine (True=on; False=off)
 export src_subdiv_toggle="True"
 # text to append to output log and hydrotable file names (use for testing/debugging)
 export vrough_suffix=""
-# input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv"
 
 #### SRC calibration variables
-# input file location with nwm feature_id and recurrence flow values
-export nwm_recur_file="$inputsDir/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv"
-
 #### apply SRC adjustments using USGS rating curve database ####
 # Toggle to run src adjustment routine with USGS rating data (True=on; False=off)
 export src_adjust_usgs="True"
-# input file location with usgs rating curve database
-export usgs_rating_curve_csv="$inputsDir/usgs_gages/usgs_rating_curves.csv"
 
 #### apply SRC adjustments using ras2fim rating curve database ####
 # Toggle to run src adjustment routine with ras2fim data (True=on; False=off)
 export src_adjust_ras2fim="True"
-# input file location with nwm feature_id and recurrence flow values
-export ras_rating_curve_csv="$inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table.csv"
 
 #### apply SRC adjustments using observed FIM/flow point database (parquet files)####
 # Toggle to run src adjustment routine with flood edge point db (True=on; False=off)
 export src_adjust_spatial="True"
-export fim_obs_pnt_data="$inputsDir/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg"
 
 #### apply manual calibration ####
 # Toggle to run manual calibration routine (True=on; False=off)
 export manual_calb_toggle="True"
-# Input file location with HUC, nwm feature_id and manual calibration coefficients
-export man_calb_file="$inputsDir/rating_curve/manual_calibration_coefficients.csv"
 
 #### computational parameters ####
 export ncores_gw=1 # mpi number of cores for gagewatershed

docs/CHANGELOG.md

@@ -1,8 +1,7 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
-
-## v4.4.x.x - 2023-11-14 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026)
+## v4.4.7.2 - 2023-12-08 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026)
 
 A couple of directly related issues were fixed in this PR.
 The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundation-mapping/issues/1025) which was about a pathing issue for the outputs directory. In testing that fix, it exposed a few other pathing and file cleanup issues which are now fixed. We also added more console output to help view variables and pathing.
@@ -16,6 +15,25 @@ The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundati
 
 <br/><br/>
 
+## v4.4.7.1 - 2023-12-01 - [PR#1036](https://github.com/NOAA-OWP/inundation-mapping/pull/1036)
+
+Quick update to match incoming ras2fim calibration output files being feed into FIM was the initial change.
+
+There is no FIM issue card for this, but this is related to a ras2fim [PR #205](https://github.com/NOAA-OWP/ras2fim/pull/205) which also made changes to ensure compatibility. New copies of both the `reformat_ras_rating_curve_table_rel_101.csv` and `reformat_ras_rating_curve_points_rel_101.gpkg` were generated from ras2fim but retained the version of `rel_101`.
+
+Originally, was planning to update just the two locations for newer versions of the two `reformat_ras_rating_surve...` files. Both had been update to recognize the ras2release version rel_101.
+
+In the process of doing that, we took the opportunity to move all inputs files from params_template.env and put them into bash_variables.env as per precedence set recently.
+
+### Changes
+
+- `config`/`params_template.env`: moved input variables into `src/bash_variables.env`
+- `src`
+    - `bash_variablles.env`: Added all input variables from `params_template.env` to here and added one new one from `run_unit_wb.sh` for ras_rating_curve_points_gpkg.
+    - `run_unit_wb.sh`:   Updated an input param to the usgs_gage_unit_setup.py file to point the -ras param to the updated rel_101 value now in the `src/bash_variables.env`.
+    - `usgs_gage_unit_setup.py`:  Changed to drop a column no longer going to be coming from ras2fim calibration files.
+
+<br/><br/>
 
 ## v4.4.7.0 - 2023-11-13 - [PR#1030](https://github.com/NOAA-OWP/inundation-mapping/pull/1030)
 
@@ -62,7 +80,6 @@ A small fix was also included where `src_adjust_ras2fim_rating.py` which sometim
 
 <br/><br/>
 
-
 ## v4.4.6.0 - 2023-11-17 - [PR#1031](https://github.com/NOAA-OWP/inundation-mapping/pull/1031)
 
 Upgrade our acquire 3Dep DEMs script to pull down South Alaska HUCS with its own CRS.
@@ -85,7 +102,6 @@ This issue closes [1028](https://github.com/NOAA-OWP/inundation-mapping/issues/1
         - Changed date/times to utc.
         - Upgraded error handing for the gdal "processing" call.
 
-
 <br/><br/>
 
 

src/bash_variables.env

@@ -16,6 +16,30 @@ export                   input_WBD_gdb=${inputsDir}/wbd/WBD_National_EPSG_5070_W
 export          input_calib_points_dir=${inputsDir}/rating_curve/water_edge_database/calibration_points/
 export                pre_clip_huc_dir=${inputsDir}/pre_clip_huc8/23_10_17
 
+export                 bathymetry_file=${inputsDir}/bathymetry/bathymetry_adjustment_data.gpkg
+
+# input file location with nwm feature_id and recurrence flow values
+export             bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
+
+# input file location with nwm feature_id and channel roughness and overbank roughness attributes
+export                vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_025_05.csv
+
+# input file location with nwm feature_id and recurrence flow values
+export                  nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
+
+# input file location with usgs rating curve database
+export           usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv
+
+# input file locations for ras2fim locations and rating curve data
+export            ras_rating_curve_csv=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_table_rel_101.csv
+export    ras_rating_curve_points_gpkg=${inputsDir}/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points_rel_101.gpkg
+
+export                fim_obs_pnt_data=${inputsDir}/rating_curve/water_edge_database/usgs_nws_benchmark_points_cleaned.gpkg
+
+# Input file location with HUC, nwm feature_id and manual calibration coefficients
+export                   man_calb_file=${inputsDir}/rating_curve/manual_calibration_coefficients.csv
+
+
 # Styling
 export startDiv="\n-----------------------------------------------------------------\n"
 export  stopDiv="\n-----------------------------------------------------------------\n"

src/run_unit_wb.sh

@@ -276,7 +276,7 @@ if [ -f $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg ]; then
     python3 $srcDir/usgs_gage_unit_setup.py \
         -gages $inputsDir/usgs_gages/usgs_gages.gpkg \
         -nwm $tempHucDataDir/nwm_subset_streams_levelPaths.gpkg \
-        -ras $inputsDir/rating_curve/ras2fim_exports/reformat_ras_rating_curve_points.gpkg \
+        -ras $ras_rating_curve_points_gpkg \
         -o $tempHucDataDir/usgs_subset_gages.gpkg \
         -huc $hucNumber \
         -ahps $inputsDir/ahps_sites/nws_lid.gpkg \

src/usgs_gage_crosswalk.py

@@ -117,8 +117,8 @@ def write(self, output_directory):
         elev_table = elev_table[elev_table['location_id'].notna()]
         elev_table.source = elev_table.source.apply(str.lower)
 
-        # filter for just ras2fim entries
-        ras_elev_table = elev_table[elev_table['source'] == 'ras2fim']
+        # filter for just ras2fim entries (note that source column includes suffix with version number)
+        ras_elev_table = elev_table[elev_table['source'].str.contains('ras2fim')]
         ras_elev_table = ras_elev_table[
             [
                 "location_id",
@@ -134,9 +134,15 @@ def write(self, output_directory):
         ]
         if not ras_elev_table.empty:
             ras_elev_table.to_csv(join(output_directory, 'ras_elev_table.csv'), index=False)
+        else:
+            print(
+                'INFO: there were no ras2fim points located in this huc'
+                ' (note that most hucs do not have ras2fim data)'
+            )
 
         # filter for just usgs entries
-        usgs_elev_table = elev_table[elev_table['source'] != 'ras2fim']
+        # look for source attributes that do not contain "ras2fim"
+        usgs_elev_table = elev_table[~elev_table['source'].str.contains('ras2fim')]
         if not usgs_elev_table.empty:
             usgs_elev_table.to_csv(join(output_directory, 'usgs_elev_table.csv'), index=False)
 

src/usgs_gage_unit_setup.py

@@ -33,9 +33,7 @@ def load_gages(self):
         # !!! Geopandas is not honoring the dtype arg with this read_file below (huc8 being read as int64).
         # Need the raw data to store the 'huc8' attribute as an object to avoid issues with integers truncating the leading zero from some hucs
         ras_locs = gpd.read_file(self.ras_locs_filename, dtype={'huc8': 'object'})
-        ras_locs = ras_locs[
-            ['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'wrds_timestamp', 'geometry']
-        ]
+        ras_locs = ras_locs[['feature_id', 'huc8', 'stream_stn', 'fid_xs', 'source', 'geometry']]
         ras_locs['location_id'] = ras_locs['fid_xs']
 
         # Convert ras locs crs to match usgs gage crs