Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[1pt] Remove deprecated references to LID selection functionality in CatFIM #1405

Merged
merged 5 commits into from
Jan 24, 2025
Merged

[1pt] Remove deprecated references to LID selection functionality in CatFIM #1405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b8f68e5
Remove references to lid_to_run vestigial variable.
EmilyDeardorff Jan 16, 2025
360e62f
Remove references to lid_to_run in CatFIM.
EmilyDeardorff Jan 16, 2025
9bbc28e
Update CHANGELOG.md
EmilyDeardorff Jan 16, 2025
51faf85
Linting.
EmilyDeardorff Jan 16, 2025
3dfec6f
update changelog
Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
All notable changes to this project will be documented in this file.
We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.

## v4.5.13.8 - 2025-01-24 - [PR#1405](https://github.com/NOAA-OWP/inundation-mapping/pull/1405)

Removing the references to lid_to_run from CatFIM in order to keep the CatFIM scripts cleaner.

### Changes
- `tools/catfim/generate_categorical_fim.py`: Remove references to `lid_to_run` variable.
- ` tools/catfim/generate_categorical_fim_flows.py`: Remove references to `lid_to_run` variable.

<br/><br/>

## v4.5.13.7 - 2025-01-10 - [PR#1379](https://github.com/NOAA-OWP/inundation-mapping/pull/1379)

There are many sites in non-CONUS regions (AK, PR, HI) where we would like to run CatFIM but they are being excluded because they are not NWM forecast points. This update brings back the double API pull and adds in some code to filter out duplicate (and NULL) lids from the metadata lists.
Expand Down
19 changes: 0 additions & 19 deletions tools/catfim/generate_categorical_fim.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ def process_generate_categorical_fim(
output_folder,
overwrite,
search,
# lid_to_run,
lst_hucs,
job_number_intervals,
past_major_interval_cap,
Expand Down Expand Up @@ -225,10 +224,6 @@ def process_generate_categorical_fim(
'USGS_METADATA_URL, USGS_DOWNLOAD_URL'
)

# TODO: lid_to_run functionality... remove? for now, just hard code lid_to_run as "all"
# single lid, not multiple
lid_to_run = "all"

# Check that fim_inputs.csv exists and raise error if necessary
fim_inputs_csv_path = os.path.join(fim_run_dir, 'fim_inputs.csv')
if not os.path.exists(fim_inputs_csv_path):
Expand Down Expand Up @@ -265,7 +260,6 @@ def process_generate_categorical_fim(
fim_run_dir,
nwm_us_search,
nwm_ds_search,
lid_to_run,
env_file,
job_number_inundate,
job_number_huc,
Expand Down Expand Up @@ -310,7 +304,6 @@ def process_generate_categorical_fim(
output_catfim_dir,
nwm_us_search,
nwm_ds_search,
lid_to_run,
env_file,
job_flows,
is_stage_based,
Expand Down Expand Up @@ -1451,7 +1444,6 @@ def generate_stage_based_categorical_fim(
fim_run_dir,
nwm_us_search,
nwm_ds_search,
lid_to_run,
env_file,
job_number_inundate,
job_number_huc,
Expand Down Expand Up @@ -1497,7 +1489,6 @@ def generate_stage_based_categorical_fim(
output_catfim_dir,
nwm_us_search,
nwm_ds_search,
lid_to_run,
env_file,
job_flows,
True,
Expand Down Expand Up @@ -1800,16 +1791,6 @@ def set_start_files_folders(
default='5',
)

## Deprecated, use lst_hucs instead
# TODO: lid_to_run functionality... remove? for now, just hard code lid_to_run as "all"
# parser.add_argument(
# '-l',
# '--lid_to_run',
# help='OPTIONAL: NWS LID, lowercase, to produce CatFIM for. Currently only accepts one. Defaults to all sites',
# required=False,
# default='all',
# )

# NOTE: The HUCs you put in this, MUST be a HUC that is valid in your -f/ --fim_run_dir (HAND output folder)
parser.add_argument(
'-lh',
Expand Down
110 changes: 48 additions & 62 deletions tools/catfim/generate_categorical_fim_flows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,12 +353,10 @@ def generate_flows_for_huc(


# This is called from within this script and is not MP, so it can use FLOG directly
# lid_to_run is temp disabled
def generate_flows(
output_catfim_dir,
nwm_us_search,
nwm_ds_search,
lid_to_run,
env_file,
job_number_huc,
is_stage_based,
Expand Down Expand Up @@ -432,7 +430,7 @@ def generate_flows(
# TODO: Aug 2024:
# Filter the meta list to just HUCs in the fim run output or huc if sent in as a param
all_meta_lists = __load_nwm_metadata(
output_catfim_dir, metadata_url, nwm_us_search, nwm_ds_search, lid_to_run, nwm_metafile
output_catfim_dir, metadata_url, nwm_us_search, nwm_ds_search, nwm_metafile
)

end_dt = datetime.now(timezone.utc)
Expand Down Expand Up @@ -640,9 +638,7 @@ def generate_flows(


# local script calls __load_nwm_metadata so FLOG is already setup
def __load_nwm_metadata(
output_catfim_dir, metadata_url, nwm_us_search, nwm_ds_search, lid_to_run, nwm_metafile
):
def __load_nwm_metadata(output_catfim_dir, metadata_url, nwm_us_search, nwm_ds_search, nwm_metafile):

FLOG.trace(metadata_url)

Expand All @@ -662,72 +658,62 @@ def __load_nwm_metadata(

FLOG.lprint(f"Meta file will be downloaded and saved at {meta_file}")

if lid_to_run != "all": # TODO: Deprecate LID options (in favor of HUC list functionlity)
# Single lid for now (deprecated)
output_meta_list, ___ = get_metadata(
metadata_url,
select_by='nws_lid',
selector=[lid_to_run],
must_include='nws_data.rfc_forecast_point',
upstream_trace_distance=nwm_us_search,
downstream_trace_distance=nwm_ds_search,
)
# Jan 2025: Removed lid_to_run functionality, so it is no longer needed as an input.

else:
# Dec 2024: Running two API calls: one to get all forecast points, and another
# to get all points (non-forecast and forecast) for the OCONUS regions. Then,
# duplicate LIDs are removed.

# Get all forecast points
forecast_point_meta_list, ___ = get_metadata(
metadata_url,
select_by='nws_lid',
selector=['all'],
must_include='nws_data.rfc_forecast_point',
upstream_trace_distance=nwm_us_search,
downstream_trace_distance=nwm_ds_search,
)
# Dec 2024: Running two API calls: one to get all forecast points, and another
# to get all points (non-forecast and forecast) for the OCONUS regions. Then,
# duplicate LIDs are removed.

# Get all points for OCONUS regions (HI, PR, and AK)
oconus_meta_list, ___ = get_metadata(
metadata_url,
select_by='state',
selector=['HI', 'PR', 'AK'],
must_include=None,
upstream_trace_distance=nwm_us_search,
downstream_trace_distance=nwm_ds_search,
)
# Get all forecast points
forecast_point_meta_list, ___ = get_metadata(
metadata_url,
select_by='nws_lid',
selector=['all'],
must_include='nws_data.rfc_forecast_point',
upstream_trace_distance=nwm_us_search,
downstream_trace_distance=nwm_ds_search,
)

# Append the lists
unfiltered_meta_list = forecast_point_meta_list + oconus_meta_list
# Get all points for OCONUS regions (HI, PR, and AK)
oconus_meta_list, ___ = get_metadata(
metadata_url,
select_by='state',
selector=['HI', 'PR', 'AK'],
must_include=None,
upstream_trace_distance=nwm_us_search,
downstream_trace_distance=nwm_ds_search,
)

# print(f"len(all_meta_lists) is {len(all_meta_lists)}")
# Append the lists
unfiltered_meta_list = forecast_point_meta_list + oconus_meta_list

# Filter the metadata list
output_meta_list = []
unique_lids, duplicate_lids = [], [] # TODO: remove?
duplicate_meta_list = []
nonelid_metadata_list = [] # TODO: remove
# print(f"len(all_meta_lists) is {len(all_meta_lists)}")

for i, site in enumerate(unfiltered_meta_list):
nws_lid = site['identifiers']['nws_lid']
# Filter the metadata list
output_meta_list = []
unique_lids, duplicate_lids = [], [] # TODO: remove?
duplicate_meta_list = []
nonelid_metadata_list = [] # TODO: remove

if nws_lid is None:
# No LID available
nonelid_metadata_list.append(site) # TODO: replace with Continue
for i, site in enumerate(unfiltered_meta_list):
nws_lid = site['identifiers']['nws_lid']

elif nws_lid in unique_lids:
# Duplicate LID
duplicate_lids.append(nws_lid)
duplicate_meta_list.append(site) # TODO: remove extra lists
if nws_lid is None:
# No LID available
nonelid_metadata_list.append(site) # TODO: replace with Continue

else:
# Unique/unseen LID that's not None
unique_lids.append(nws_lid)
output_meta_list.append(site)
elif nws_lid in unique_lids:
# Duplicate LID
duplicate_lids.append(nws_lid)
duplicate_meta_list.append(site) # TODO: remove extra lists

else:
# Unique/unseen LID that's not None
unique_lids.append(nws_lid)
output_meta_list.append(site)

FLOG.lprint(f'{len(duplicate_lids)} duplicate points removed.')
FLOG.lprint(f'Filtered metadatada downloaded for {len(output_meta_list)} points.')
FLOG.lprint(f'{len(duplicate_lids)} duplicate points removed.')
FLOG.lprint(f'Filtered metadatada downloaded for {len(output_meta_list)} points.')

# ----------

Expand Down
Loading