diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6d85756e..3099c008 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,19 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.5.14.7 - 2025-02-14 - [PR#1426](https://github.com/NOAA-OWP/inundation-mapping/pull/1426) + +Added two new input args to add hand version and product version as output columns to all four output files of FB sites and library plus SB sites and library. This includes the new "model_version" and "product_version". The model verion field will be similar to "HAND 4_5_11_1" and the product version will be similar to "CatFIM 2_2" + +### Changes + +- `tools\catfim` + - `generate_categorical_fim.py' : as described above. + - `generate_categorical_mapping.py' : as described above. + +

+ + ## v4.5.14.6 - 2025-02-14 - [PR#1418](https://github.com/NOAA-OWP/inundation-mapping/pull/1418) Previously, stage-based CatFIM would inundate areas that we know to be lakes based on our FIM data. This update masks out lakes from stage-based CatFIM inundation. diff --git a/tools/catfim/generate_categorical_fim.py b/tools/catfim/generate_categorical_fim.py index 1cd75bdc..a187abad 100755 --- a/tools/catfim/generate_categorical_fim.py +++ b/tools/catfim/generate_categorical_fim.py @@ -82,6 +82,8 @@ def process_generate_categorical_fim( overwrite, search, lst_hucs, + catfim_version, + model_version, job_number_intervals, past_major_interval_cap, step_num, @@ -146,7 +148,13 @@ def process_generate_categorical_fim( # ================================ # Define default arguments. Modify these if necessary - fim_version = os.path.split(fim_run_dir)[1] + + if model_version != "": + model_version = "HAND " + model_version + model_version = model_version.replace(".", "_") + if catfim_version != "": + catfim_version = "CatFIM " + catfim_version + catfim_version = catfim_version.replace(".", "_") # ================================ # TODO: Aug 2024: Job values are not well used. There are some times where not @@ -282,7 +290,7 @@ def process_generate_categorical_fim( ahps_jobs = job_number_huc * 5 post_process_cat_fim_for_viz( - catfim_method, output_catfim_dir, ahps_jobs, fim_version, FLOG.LOG_FILE_PATH + catfim_method, output_catfim_dir, ahps_jobs, catfim_version, model_version, FLOG.LOG_FILE_PATH ) else: FLOG.lprint("post_process_cat_fim_for_viz step skipped") @@ -329,6 +337,8 @@ def process_generate_categorical_fim( output_flows_dir, output_catfim_dir, catfim_method, + catfim_version, + model_version, job_number_huc, job_number_inundate, FLOG.LOG_FILE_PATH, @@ -338,12 +348,14 @@ def process_generate_categorical_fim( # end if else FLOG.lprint("") + + # This is done for SB and FB if ( step_num <= 3 ): # can later be changed to is_flow_based and step_num > 3, so stage can have it's own numbers # Updating mapping status FLOG.lprint('Updating mapping status...') - update_flow_mapping_status(output_mapping_dir, catfim_sites_file_path) + update_sites_mapping_status(output_mapping_dir, catfim_sites_file_path, catfim_version, model_version) FLOG.lprint('Updating mapping status complete') else: FLOG.lprint("Updating mapping status step skipped") @@ -381,7 +393,8 @@ def get_list_ahps_with_library_gpkgs(output_mapping_dir): return ahps_ids_with_gpkgs -def update_flow_mapping_status(output_mapping_dir, catfim_sites_file_path): +# This is used by both Stage Based and Flow Based +def update_sites_mapping_status(output_mapping_dir, catfim_sites_file_path, catfim_version, model_version): ''' Overview: - Gets a list of valid ahps that have at least one gkpg file. If we have at least one, then the site mapped something @@ -439,6 +452,9 @@ def update_flow_mapping_status(output_mapping_dir, catfim_sites_file_path): # sites_gdf.reset_index(inplace=True, drop=True) + sites_gdf["model_version"] = model_version + sites_gdf["product_version"] = catfim_version + # We are re-saving the sites files sites_gdf.to_file(catfim_sites_file_path, driver='GPKG', crs=VIZ_PROJECTION, engine="fiona") @@ -1771,7 +1787,7 @@ def set_start_files_folders( Sample python /foss_fim/tools/generate_categorical_fim.py -f /outputs/Rob_catfim_test_1 -jh 1 -jn 10 -ji 8 -e /data/config/catfim.env -t /data/catfim/rob_test/docker_test_1 - -me '/data/catfim/rob_test/nwm_metafile.pkl' -sb -step 2 + -me '/data/catfim/rob_test/nwm_metafile.pkl' -sb -cv "2.2" -hv "4.5.11.1" -step 2 ''' # Parse arguments @@ -1779,15 +1795,16 @@ def set_start_files_folders( parser.add_argument( '-f', '--fim_run_dir', - help='Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_5_2_11', + help='REQUIRED: Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_5_2_11', required=True, ) parser.add_argument( '-e', '--env_file', - help='Docker mount path to the catfim environment file. ie) data/config/catfim.env', + help='REQUIRED: Docker mount path to the catfim environment file. ie) data/config/catfim.env', required=True, ) + parser.add_argument( '-jh', '--job_number_huc', @@ -1887,6 +1904,29 @@ def set_start_files_folders( default="", ) + parser.add_argument( + '-cv', + '--catfim-version', + help='OPTIONAL: The version of the code that was used to run the product. This value is included' + ' in the output gpkgs and csvs in a field named product_version. If you put in a value here,' + ' we will add the phrase CatFIM to the front of it.' + ' ie) 2.0 becomes CatFIM, 2.2 becomes CatFIM, etc. Defaults to blank', + required=False, + default="", + ) + + parser.add_argument( + '-hv', + '--model-version', + help='OPTIONAL: The version of the HAND data outputs that was used to run the product.' + ' This value is included in the output gpkgs and csvs in a field named model_version.' + ' If you put in a value here, we will change dots to underscores only.' + ' This shoudl be a HAND version number only and not include the word HAND_' + ' ie) 4.5.11.1 becomes 4_5_11_1, etc. Defaults to blank', + required=False, + default="", + ) + parser.add_argument( '-o', '--overwrite', help='OPTIONAL: Overwrite files', required=False, action="store_true" ) diff --git a/tools/catfim/generate_categorical_fim_mapping.py b/tools/catfim/generate_categorical_fim_mapping.py index 65e2b3ad..e0d66837 100755 --- a/tools/catfim/generate_categorical_fim_mapping.py +++ b/tools/catfim/generate_categorical_fim_mapping.py @@ -618,7 +618,6 @@ def post_process_huc( ahps_dir_list, huc_dir, gpkg_dir, - fim_version, huc, parent_log_output_file, child_log_file_prefix, @@ -715,7 +714,6 @@ def post_process_huc( ahps_lid, tif_to_process, gpkg_dir, - fim_version, huc, magnitude, nws_lid_attributes_filename, @@ -745,7 +743,7 @@ def post_process_huc( # This is not part of an MP process, but does need FLOG carried into it so it can use FLOG directly def post_process_cat_fim_for_viz( - catfim_method, output_catfim_dir, job_huc_ahps, fim_version, log_output_file + catfim_method, output_catfim_dir, job_huc_ahps, catfim_version, model_version, log_output_file ): # Adding a pointer in this file coming from generate_categorial_fim so they can share the same log file @@ -801,7 +799,6 @@ def post_process_cat_fim_for_viz( ahps_dir_list, huc_dir, gpkg_dir, - fim_version, huc, log_output_file, child_log_file_prefix, @@ -873,6 +870,9 @@ def post_process_cat_fim_for_viz( output_file_name = f"{catfim_method}_catfim_library" + merged_layers_gdf["model_version"] = model_version + merged_layers_gdf["product_version"] = catfim_version + # TODO: Aug 2024: gpkg are not opening in qgis now? project, wkt, non defined geometry columns? gpkg_file_path = os.path.join(output_mapping_dir, f'{output_file_name}.gpkg') FLOG.lprint(f"Saving catfim library gpkg version to {gpkg_file_path}") @@ -892,7 +892,6 @@ def reformat_inundation_maps( ahps_lid, tif_to_process, gpkg_dir, - fim_version, huc, magnitude, nws_lid_attributes_filename, @@ -941,7 +940,6 @@ def reformat_inundation_maps( extent_poly_diss = extent_poly_diss.reset_index(drop=True) extent_poly_diss['ahps_lid'] = ahps_lid extent_poly_diss['magnitude'] = magnitude - extent_poly_diss['version'] = fim_version extent_poly_diss['huc'] = huc extent_poly_diss['interval_stage'] = interval_stage extent_poly_diss['is_interval'] = is_interval @@ -1010,6 +1008,8 @@ def manage_catfim_mapping( output_flows_dir, output_catfim_dir, catfim_method, + catfim_version, + model_version, job_number_huc, job_number_inundate, log_output_file, @@ -1039,10 +1039,6 @@ def manage_catfim_mapping( FLOG.lprint("Skip running Inundation as Step > 1") # FLOG.lprint("Aggregating Categorical FIM") - # Get fim_version. - - fim_version = os.path.basename(os.path.normpath(fim_run_dir)) - # Step 2 # TODO: Aug 2024, so we need to clean it up # This step does not need a job_number_inundate as it can't really use it. @@ -1050,7 +1046,7 @@ def manage_catfim_mapping( # for now, we will manually multiple the huc * 5 (max number of ahps types) ahps_jobs = job_number_huc * 5 post_process_cat_fim_for_viz( - catfim_method, output_catfim_dir, ahps_jobs, fim_version, str(FLOG.LOG_FILE_PATH) + catfim_method, output_catfim_dir, ahps_jobs, catfim_version, model_version, str(FLOG.LOG_FILE_PATH) ) end = time.time()