[1pt] PR: Add code version data to outputs

docs/CHANGELOG.md

@@ -1,6 +1,19 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.x.x - 2025-02-09 - [PR#1426](https://github.com/NOAA-OWP/inundation-mapping/pull/1426)
+
+Added two new input args to add hand version and product version as output columns to all four output files of FB sites and library plus SB sites and library. This includes the new "model_version" and "product_version". The model verion field will be similar to "HAND 4_5_11_1" and the product version will be similar to "CatFIM 2_2"
+
+### Changes
+
+- `tools\catfim`
+    - `generate_categorical_fim.py' : as described above.
+     - `generate_categorical_mapping.py' : as described above.
+
+<br/><br/>
+
+
 ## v4.5.14.5 - 2025-01-31 - [PR#1401](https://github.com/NOAA-OWP/inundation-mapping/pull/1401)
 
 This PR improves the current HUC processing duration system by saving the processing time for each HUC separately. This helps prevent collisions that can happen during parallel processing and ensures more accurate, comprehensive results. The new Python script reads all the processing time files and combines them into a CSV. It also adds a summary line at the end with the total runtime, as well as the number of HUCs and branches.

tools/catfim/generate_categorical_fim.py

@@ -82,6 +82,8 @@ def process_generate_categorical_fim(
     overwrite,
     search,
     lst_hucs,
+    catfim_version,
+    model_version,
     job_number_intervals,
     past_major_interval_cap,
     step_num,
@@ -146,7 +148,13 @@ def process_generate_categorical_fim(
 
     # ================================
     # Define default arguments. Modify these if necessary
-    fim_version = os.path.split(fim_run_dir)[1]
+
+    if model_version != "":
+        model_version = "HAND " + model_version
+        model_version = model_version.replace(".", "_")
+    if catfim_version != "":
+        catfim_version = "CatFIM " + catfim_version
+        catfim_version = catfim_version.replace(".", "_")
 
     # ================================
     # TODO: Aug 2024: Job values are not well used. There are some times where not
@@ -282,7 +290,7 @@ def process_generate_categorical_fim(
 
             ahps_jobs = job_number_huc * 5
             post_process_cat_fim_for_viz(
-                catfim_method, output_catfim_dir, ahps_jobs, fim_version, FLOG.LOG_FILE_PATH
+                catfim_method, output_catfim_dir, ahps_jobs, catfim_version, model_version, FLOG.LOG_FILE_PATH
             )
         else:
             FLOG.lprint("post_process_cat_fim_for_viz step skipped")
@@ -329,6 +337,8 @@ def process_generate_categorical_fim(
                 output_flows_dir,
                 output_catfim_dir,
                 catfim_method,
+                catfim_version,
+                model_version,
                 job_number_huc,
                 job_number_inundate,
                 FLOG.LOG_FILE_PATH,
@@ -338,12 +348,14 @@ def process_generate_categorical_fim(
     # end if else
 
     FLOG.lprint("")
+
+    # This is done for SB and FB
     if (
         step_num <= 3
     ):  # can later be changed to is_flow_based and step_num > 3, so stage can have it's own numbers
         # Updating mapping status
         FLOG.lprint('Updating mapping status...')
-        update_flow_mapping_status(output_mapping_dir, catfim_sites_file_path)
+        update_sites_mapping_status(output_mapping_dir, catfim_sites_file_path, catfim_version, model_version)
         FLOG.lprint('Updating mapping status complete')
     else:
         FLOG.lprint("Updating mapping status step skipped")
@@ -381,7 +393,8 @@ def get_list_ahps_with_library_gpkgs(output_mapping_dir):
     return ahps_ids_with_gpkgs
 
 
-def update_flow_mapping_status(output_mapping_dir, catfim_sites_file_path):
+# This is used by both Stage Based and Flow Based
+def update_sites_mapping_status(output_mapping_dir, catfim_sites_file_path, catfim_version, model_version):
     '''
     Overview:
         - Gets a list of valid ahps that have at least one gkpg file. If we have at least one, then the site mapped something
@@ -439,6 +452,9 @@ def update_flow_mapping_status(output_mapping_dir, catfim_sites_file_path):
 
         # sites_gdf.reset_index(inplace=True, drop=True)
 
+        sites_gdf["model_version"] = model_version
+        sites_gdf["product_version"] = catfim_version
+
         # We are re-saving the sites files
         sites_gdf.to_file(catfim_sites_file_path, driver='GPKG', crs=VIZ_PROJECTION, engine="fiona")
 
@@ -1771,23 +1787,24 @@ def set_start_files_folders(
     Sample
     python /foss_fim/tools/generate_categorical_fim.py -f /outputs/Rob_catfim_test_1 -jh 1 -jn 10 -ji 8
     -e /data/config/catfim.env -t /data/catfim/rob_test/docker_test_1
-    -me '/data/catfim/rob_test/nwm_metafile.pkl' -sb -step 2
+    -me '/data/catfim/rob_test/nwm_metafile.pkl' -sb -cv "2.2" -hv "4.5.11.1" -step 2
     '''
 
     # Parse arguments
     parser = argparse.ArgumentParser(description='Run Categorical FIM')
     parser.add_argument(
         '-f',
         '--fim_run_dir',
-        help='Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_5_2_11',
+        help='REQUIRED: Path to directory containing HAND outputs, e.g. /data/previous_fim/fim_4_5_2_11',
         required=True,
     )
     parser.add_argument(
         '-e',
         '--env_file',
-        help='Docker mount path to the catfim environment file. ie) data/config/catfim.env',
+        help='REQUIRED: Docker mount path to the catfim environment file. ie) data/config/catfim.env',
         required=True,
     )
+
     parser.add_argument(
         '-jh',
         '--job_number_huc',
@@ -1887,6 +1904,29 @@ def set_start_files_folders(
         default="",
     )
 
+    parser.add_argument(
+        '-cv',
+        '--catfim-version',
+        help='OPTIONAL: The version of the code that was used to run the product. This value is included'
+        ' in the output gpkgs and csvs in a field named product_version. If you put in a value here,'
+        ' we will add the phrase CatFIM to the front of it.'
+        ' ie) 2.0 becomes CatFIM, 2.2 becomes CatFIM, etc. Defaults to blank',
+        required=False,
+        default="",
+    )
+
+    parser.add_argument(
+        '-hv',
+        '--model-version',
+        help='OPTIONAL: The version of the HAND data outputs that was used to run the product.'
+        ' This value is included in the output gpkgs and csvs in a field named model_version.'
+        ' If you put in a value here, we will change dots to underscores only.'
+        ' This shoudl be a HAND version number only and not include the word HAND_'
+        ' ie) 4.5.11.1 becomes 4_5_11_1, etc. Defaults to blank',
+        required=False,
+        default="",
+    )
+
     parser.add_argument(
         '-o', '--overwrite', help='OPTIONAL: Overwrite files', required=False, action="store_true"
     )

tools/catfim/generate_categorical_fim_mapping.py

@@ -585,7 +585,6 @@ def post_process_huc(
     ahps_dir_list,
     huc_dir,
     gpkg_dir,
-    fim_version,
     huc,
     parent_log_output_file,
     child_log_file_prefix,
@@ -682,7 +681,6 @@ def post_process_huc(
                         ahps_lid,
                         tif_to_process,
                         gpkg_dir,
-                        fim_version,
                         huc,
                         magnitude,
                         nws_lid_attributes_filename,
@@ -712,7 +710,7 @@ def post_process_huc(
 
 # This is not part of an MP process, but does need FLOG carried into it so it can use FLOG directly
 def post_process_cat_fim_for_viz(
-    catfim_method, output_catfim_dir, job_huc_ahps, fim_version, log_output_file
+    catfim_method, output_catfim_dir, job_huc_ahps, catfim_version, model_version, log_output_file
 ):
 
     # Adding a pointer in this file coming from generate_categorial_fim so they can share the same log file
@@ -768,7 +766,6 @@ def post_process_cat_fim_for_viz(
                 ahps_dir_list,
                 huc_dir,
                 gpkg_dir,
-                fim_version,
                 huc,
                 log_output_file,
                 child_log_file_prefix,
@@ -840,6 +837,9 @@ def post_process_cat_fim_for_viz(
 
     output_file_name = f"{catfim_method}_catfim_library"
 
+    merged_layers_gdf["model_version"] = model_version
+    merged_layers_gdf["product_version"] = catfim_version
+
     # TODO: Aug 2024: gpkg are not opening in qgis now? project, wkt, non defined geometry columns?
     gpkg_file_path = os.path.join(output_mapping_dir, f'{output_file_name}.gpkg')
     FLOG.lprint(f"Saving catfim library gpkg version to {gpkg_file_path}")
@@ -859,7 +859,6 @@ def reformat_inundation_maps(
     ahps_lid,
     tif_to_process,
     gpkg_dir,
-    fim_version,
     huc,
     magnitude,
     nws_lid_attributes_filename,
@@ -914,7 +913,6 @@ def reformat_inundation_maps(
         extent_poly_diss = extent_poly_diss.reset_index(drop=True)
         extent_poly_diss['ahps_lid'] = ahps_lid
         extent_poly_diss['magnitude'] = magnitude
-        extent_poly_diss['version'] = fim_version
         extent_poly_diss['huc'] = huc
         extent_poly_diss['interval_stage'] = interval_stage
         extent_poly_diss['is_interval'] = is_interval
@@ -983,6 +981,8 @@ def manage_catfim_mapping(
     output_flows_dir,
     output_catfim_dir,
     catfim_method,
+    catfim_version,
+    model_version,
     job_number_huc,
     job_number_inundate,
     log_output_file,
@@ -1012,18 +1012,14 @@ def manage_catfim_mapping(
         FLOG.lprint("Skip running Inundation as Step > 1")
 
     # FLOG.lprint("Aggregating Categorical FIM")
-    # Get fim_version.
-
-    fim_version = os.path.basename(os.path.normpath(fim_run_dir))
-
     # Step 2
     # TODO: Aug 2024, so we need to clean it up
     # This step does not need a job_number_inundate as it can't really use it.
     # It processes primarily hucs and ahps in multiproc
     # for now, we will manually multiple the huc * 5 (max number of ahps types)
     ahps_jobs = job_number_huc * 5
     post_process_cat_fim_for_viz(
-        catfim_method, output_catfim_dir, ahps_jobs, fim_version, str(FLOG.LOG_FILE_PATH)
+        catfim_method, output_catfim_dir, ahps_jobs, catfim_version, model_version, str(FLOG.LOG_FILE_PATH)
     )
 
     end = time.time()