Merge branch 'master' into add_config_notes

readme.md

@@ -111,6 +111,120 @@ python3 -m nwm_routing -f -V4 test_AnA_V4_HYFeature.yaml
 
 **Note**: The following instructions are for setting up T-Route on a Linux environment (standalone, no MPI). If you are using Windows, please install WSL (Windows Subsystem for Linux) before proceeding.
 
+### T-Route Setup and Testing Guide for Windows Users WITHOUT conda [based on pip and venv - only widely available dependencies].
+### WARNING: INSTALLATION WITHIN EXISTING MINICONDA/CONDA VIRTUAL ENVIRONMENT NOT RECOMMENDED, PIP AND CONDA DO NOT MIX WELL, AND YOU MAY BREAK YOUR CONDA ENVIRONMENT!
+
+1. **Install Recommended distro:**
+   - Download and install WSL2 for your Windows OS
+   - We recommend long-term stable (LTS) Ubuntu distribution 22.04 or 20.04 (24.04 not recommended yet)
+   - Open Windows Power Shell and issue
+      ``` shell
+      wsl --install Ubuntu-22.04
+      ```
+   - Enter (root) username and password (2x) of your choice
+
+2. **Set up venv-based virtual environment:**
+   - From root (the username you created under 1):
+      - Update Linux distro:
+        ```shell
+        sudo apt update
+        ```
+      - Install pip (package manager):
+        ```shell
+        sudo apt install python3-pip
+        ```
+      - Install venv:
+         ```shell
+         sudo apt install python3.10-venv
+         ```
+      - Create a virtual environment for T-route (named here 'troute-env1'):
+         ```shell
+         python3 -m venv troute_env1
+         ```
+      - Activate your shiny new virtual environment:
+         ```shell
+         source troute_env1/bin/activate
+         ```
+      - Now, the command prompts in the Shell window should start with (troute-env1)
+
+3. **Clone T-route:**
+   - Go to a folder of your choice (here, your home folder) and create a T-route directory
+      ```shell
+      mkdir troute1
+      cd troute1
+      ```
+   - Clone a T-route repository (the current main branch is used as an example):
+      ```shell
+      git clone --progress --single-branch --branch master http://github.com/NOAA-OWP/t-route.git
+      cd troute1
+      ```
+   - Install python packages per requirements file
+      ```shell
+      pip install -r requirements.txt
+      ```
+
+4. **Download & build netcdf fortran libraries from UCAR:**
+   - Go to a folder of your choice (here, your home folder) and download the source code:
+      ```shell
+      cd ~
+      wget https://downloads.unidata.ucar.edu/netcdf-fortran/4.6.1/netcdf-fortran-4.6.1.tar.gz
+      ```
+   - Unzip it:
+      ```shell
+      tar xvf netcdf-fortran-4.6.1.tar.gz
+      ```
+   - Enter the directory:
+      ```shell
+      cd netcdf-fortran-4.6.1/
+      ```
+   - Install some prerequisites (Fortran compiler, build essentials, standard C-netcdf library):
+      ```shell
+      sudo apt install gfortran
+      sudo apt install build-essential
+      sudo apt-get install libnetcdf-dev
+      ```
+   - Configure the fortran-netcdf libraries:
+      ```shell
+      ./configure
+      ```
+   - There should be no error message, and the output log should end up with something like:
+     ![image](https://github.com/user-attachments/assets/48268212-0b74-4f75-9d52-97f68e6c80d0)
+   - Check the installation (running two sets of examples):
+      ```shell
+      make check
+      ```
+   - Again, there should be no error message (expect some warnings, though), and output should end with "passing" of two sets:
+     ![image](https://github.com/user-attachments/assets/83745989-9f14-4c1b-a2a1-675aa94e5181)
+   - Finally, install the libraries:
+      ```shell
+      sudo make install
+      ```
+   - Output should be something like:
+      ![image](https://github.com/user-attachments/assets/57e48501-18f4-4004-9b10-5a9245186e38)
+
+5. **Build and test T-route:**
+   - Go to your T-route folder:
+      ```shell
+      cd ~/troute1
+      ```
+   - Compile T-route (may take a few minutes, depending on the machine):
+      ```shell
+      ./compiler.sh
+      ```
+   - Set path to runtime netcdf-Fortran library [recommend including this in the .bashrc file or your equivalent]:
+      ```shell
+      export LD_LIBRARY_PATH=/usr/local/lib/
+      ```
+   - Run one of the demo examples provided:
+      ```shell
+      cd test/LowerColorado_TX
+      python -m nwm_routing -f -V3 test_AnA.yaml
+      ```
+   - The latter is a hybrid (MC + diffusive) routing example that should run within a few minutes at most
+
+
+### T-Route Setup Instructions and Troubleshooting Guide for Windows Users - Legacy Conda Version [may have to be built with compiler.sh no-e option]
+
 1. **Install Required Components:**
    - Open the WSL terminal.
    - Install Miniconda, Python, Pip, and Git.

requirements.txt

@@ -1,7 +1,7 @@
 pip>=22.1.0
 setuptools>=64.0,<69.0
 wheel
-numpy
+numpy~=1.0
 Cython>3,!=3.0.4
 pandas >=1.1.5, <=2.2.0
 xarray

src/troute-config/troute/config/output_parameters.py

@@ -16,6 +16,7 @@ class OutputParameters(BaseModel):
     """
     chanobs_output: Optional["ChanobsOutput"] = None
     csv_output: Optional["CsvOutput"] = None
+    parquet_output: Optional["ParquetOutput"] = None
     chrtout_output: Optional["ChrtoutOutput"] = None
     lite_restart: Optional["LiteRestart"] = None
     hydro_rst_output: Optional["HydroRstOutput"] = None
@@ -55,6 +56,14 @@ class CsvOutput(BaseModel):
     """
 
 
+class ParquetOutput(BaseModel):
+    # NOTE: required if writing results to parquet
+    parquet_output_folder: Optional[DirectoryPath] = None
+    parquet_output_segments: Optional[List[str]] = None
+    configuration: str = 'None'
+    prefix_ids: str = 'wb'
+
+
 class ChrtoutOutput(BaseModel):
     """
     CHRTOUT files are outputs from WRF-Hydro containing full channel network output. This replicates that behavior.
@@ -121,6 +130,7 @@ class ParityCheckCompareFileSet(BaseModel):
     """
     """
 
+
 class StreamOutput(BaseModel):
     """
     t-route's most recent output file type. This will output channel network values (flow, velocity, depth, and nudge values). 
@@ -134,6 +144,10 @@ class StreamOutput(BaseModel):
     """
     Directory to save flowveldepth outputs. If this is not None, this form of output will be written.
     """
+    mask_output: Optional[FilePath] = None
+    """
+    Yaml file specifying flowpath/nexus IDs to include in output files.
+    """
     stream_output_time: int = 1
     """
     Value is in simulation time hours. This tells t-route how frequently to make output files. '1' would be 1 file per hour 

src/troute-network/pyproject.toml

@@ -8,6 +8,7 @@ version = "0.0.0"
 dependencies = [
     "deprecated",
     "geopandas",
+    "fiona",
     "joblib",
     "netcdf4",
     "numpy",

src/troute-network/troute/AbstractNetwork.py

@@ -27,14 +27,16 @@ class AbstractNetwork(ABC):
     __slots__ = ["_dataframe", "_waterbody_connections", "_gages",  
                 "_terminal_codes", "_connections", "_waterbody_df", 
                 "_waterbody_types_df", "_waterbody_type_specified", "_link_gage_df",
+                "_canadian_gage_link_df",
                 "_independent_networks", "_reaches_by_tw", "_flowpath_dict",
                 "_reverse_network", "_q0", "_t0", "_link_lake_crosswalk",
                 "_usgs_lake_gage_crosswalk", "_usace_lake_gage_crosswalk", "_rfc_lake_gage_crosswalk",
                 "_qlateral", "_break_segments", "_segment_index", "_coastal_boundary_depth_df",
                 "supernetwork_parameters", "waterbody_parameters","data_assimilation_parameters",
                 "restart_parameters", "compute_parameters", "forcing_parameters",
                 "hybrid_parameters", "preprocessing_parameters", "output_parameters",
-                "verbose", "showtiming", "break_points", "_routing"]
+                "verbose", "showtiming", "break_points", "_routing", "_gl_climatology_df", "_nexus_dict", "_poi_nex_dict"]
+
 
     def __init__(self, from_files=True, value_dict={}):
 
@@ -371,6 +373,14 @@ def gages(self):
     @property
     def dataframe(self):
         return self._dataframe
+
+    @property
+    def nexus_dict(self):
+        return self._nexus_dict
+
+    @property
+    def poi_nex_dict(self):
+        return self._poi_nex_dict
 
     @property
     def terminal_codes(self):
@@ -402,6 +412,14 @@ def refactored_reaches(self):
     @property
     def unrefactored_topobathy_df(self):
         return self._routing.unrefactored_topobathy_df
+
+    @property
+    def great_lakes_climatology_df(self):
+        return self._gl_climatology_df
+
+    @property
+    def canadian_gage_df(self):
+        return self._canadian_gage_link_df
 
 
     def set_synthetic_wb_segments(self, synthetic_wb_segments, synthetic_wb_id_offset):