[1pt] Fix inundation nation pathing errors

config/params_template.env

@@ -47,7 +47,7 @@ export src_subdiv_toggle="True"
 # text to append to output log and hydrotable file names (use for testing/debugging)
 export vrough_suffix=""
 # input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_06_12.csv"
+export vmann_input_file="$inputsDir/rating_curve/variable_roughness/mannings_global_025_05.csv"
 
 #### SRC calibration variables
 # input file location with nwm feature_id and recurrence flow values

docs/CHANGELOG.md

@@ -1,6 +1,21 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.4.x.x - 2023-11-14 - [PR#1026](https://github.com/NOAA-OWP/inundation-mapping/pull/1026)
+
+A couple of directly related issues were fixed in this PR.
+The initial problem came from Issue #[1025](https://github.com/NOAA-OWP/inundation-mapping/issues/1025) which was about a pathing issue for the outputs directory. In testing that fix, it exposed a few other pathing and file cleanup issues which are now fixed. We also added more console output to help view variables and pathing.
+
+### Changes
+
+- `config`/`params_template.env`:  Updated for a newer mannings global file. Changed and tested by Ryan Spies.
+- `tools`
+    - `inundate_mosiac_wrapper.py`:  Took out a misleading and non-required print statement.
+    - `inundate_nation.py`: As mentioned above.
+
+<br/><br/>
+
+
 ## v4.4.5.0 - 2023-10-26 - [PR#1018](https://github.com/NOAA-OWP/inundation-mapping/pull/1018)
 
 During a recent BED attempt which added the new pre-clip system, it was erroring out on a number of hucs. It was issuing an error in the add_crosswalk.py script. While a minor bug does exist there, after a wide number of tests, the true culprit is the memory profile system embedded throughout FIM. This system has been around for at least a few years but not in use. It is not 100% clear why it became a problem with the addition of pre-clip, but that changes how records are loaded which likely affected memory at random times.

tools/inundate_mosaic_wrapper.py

@@ -92,8 +92,6 @@ def produce_mosaicked_inundation(
             "Please lower the num_workers.".format(num_workers, total_cpus_available)
         )
 
-    fh.vprint("Running inundate for " + huc + "...", verbose)
-
     # Call Inundate_gms
     map_file = Inundate_gms(
         hydrofabric_dir=hydrofabric_dir,

tools/inundate_nation.py

@@ -12,7 +12,6 @@
 
 import rasterio
 from inundate_mosaic_wrapper import produce_mosaicked_inundation
-from osgeo import gdal, ogr
 from rasterio.merge import merge
 
 from utils.shared_functions import FIM_Helpers as fh
@@ -26,6 +25,9 @@
 # DEFAULT_OUTPUT_DIR = '/data/inundation_review/inundate_nation/mosaic_output/'
 
 
+# TODO: Nov 2023, Logging system appears to be not working correctly.
+
+
 def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list, inc_mosaic, job_number):
     assert os.path.exists(flow_file), f"ERROR: could not find the flow file: {flow_file}"
 
@@ -37,10 +39,12 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
             + " max jobs will be used instead."
         )
 
+    print()
+    print("Inundation Nation script starting...")
+
     fim_version = os.path.basename(os.path.normpath(fim_run_dir))
     logging.info(f"Using fim version: {fim_version}")
     output_base_file_name = magnitude_key + "_" + fim_version
-    # print(output_base_file_name)
 
     __setup_logger(output_dir, output_base_file_name)
 
@@ -65,7 +69,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
         shutil.rmtree(magnitude_output_dir, ignore_errors=True)
         os.mkdir(magnitude_output_dir)
 
-    if huc_list is None:
+    if huc_list == 'all' or len(huc_list) == 0:
         huc_list = []
         for huc in os.listdir(fim_run_dir):
             # if (
@@ -78,22 +82,27 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
                 huc_list.append(huc)
     else:
         for huc in huc_list:
-            assert os.path.isdir(
-                fim_run_dir + os.sep + huc
-            ), f'ERROR: could not find the input fim_dir location: {fim_run_dir + os.sep + huc}'
+            huc_path = os.path.join(fim_run_dir, huc)
+            assert os.path.isdir(huc_path), f'ERROR: could not find the input fim_dir location: {huc_path}'
 
-    print("Inundation raw mosaic outputs here: " + magnitude_output_dir)
+    huc_list.sort()
+
+    print("Inundation raw mosaic outputs will saved here: " + magnitude_output_dir)
+    print()
 
     run_inundation([fim_run_dir, huc_list, magnitude_key, magnitude_output_dir, flow_file, job_number])
 
+    print()
+
     # Perform mosaic operation
     if inc_mosaic:
         fh.print_current_date_time()
         logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S"))
         print("Performing bool mosaic process...")
         logging.info("Performing bool mosaic process...")
-
         output_bool_dir = os.path.join(output_dir, "bool_temp")
+        print(f"output_bool_dir is {output_bool_dir}")
+
         if not os.path.exists(output_bool_dir):
             os.mkdir(output_bool_dir)
         else:
@@ -105,7 +114,7 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
         for rasfile in os.listdir(magnitude_output_dir):
             if rasfile.endswith(".tif") and "extent" in rasfile:
                 # p = magnitude_output_dir + rasfile
-                procs_list.append([magnitude_output_dir, rasfile, output_bool_dir])
+                procs_list.append([magnitude_output_dir, rasfile, output_bool_dir, fim_version])
 
         # Multiprocess --> create boolean inundation rasters for all hucs
         if len(procs_list) > 0:
@@ -116,11 +125,14 @@ def inundate_nation(fim_run_dir, output_dir, magnitude_key, flow_file, huc_list,
             print(msg)
             logging.info(msg)
 
-        # now cleanup the raw mosiac directories
+        # now cleanup the temp bool directory
         shutil.rmtree(output_bool_dir, ignore_errors=True)
 
+    else:
+        print("Skipping mosiaking")
+
     # now cleanup the raw mosiac directories
-    shutil.rmtree(magnitude_output_dir, ignore_errors=True)
+    # shutil.rmtree(magnitude_output_dir, ignore_errors=True)
 
     fh.print_current_date_time()
     logging.info(logging.info(datetime.now().strftime("%Y_%m_%d-%H_%M_%S")))
@@ -150,6 +162,12 @@ def run_inundation(args):
     inundation_raster = os.path.join(magnitude_output_dir, magnitude + "_inund_extent.tif")
 
     print("Running the NWM recurrence intervals for HUC inundation (extent) for magnitude: " + str(magnitude))
+    print(
+        "This will take a long time depending on the number of HUCs. No progress bar will be shown."
+        " Once is gets to mosiacing (if applicable), screen output will exist. To see if the script has frozen,"
+        " you should be able to watch the file system for some changes."
+    )
+    print()
 
     produce_mosaicked_inundation(
         fim_run_dir,
@@ -158,7 +176,7 @@ def run_inundation(args):
         inundation_raster=inundation_raster,
         num_workers=job_number,
         remove_intermediate=True,
-        verbose=True,
+        verbose=False,
         is_mosaic_for_branches=True,
     )
 
@@ -167,6 +185,7 @@ def create_bool_rasters(args):
     in_raster_dir = args[0]
     rasfile = args[1]
     output_bool_dir = args[2]
+    fim_version = args[3]
 
     print("Calculating boolean inundate raster: " + rasfile)
     p = in_raster_dir + os.sep + rasfile
@@ -189,7 +208,9 @@ def create_bool_rasters(args):
         dtype="int8",
         compress="lzw",
     )
-    with rasterio.open(output_bool_dir + os.sep + "bool_" + rasfile, "w", **profile) as dst:
+    with rasterio.open(
+        output_bool_dir + os.sep + rasfile[:-4] + '_' + fim_version + '.tif', "w", **profile
+    ) as dst:
         dst.write(array.astype(rasterio.int8))