Merge pull request #4 from openclimatefix/dataloader_bug

Dataloader bug
openclimatefix · Jul 5, 2024 · 173c567 · 173c567
2 parents fc64df9 + 35e191a
commit 173c567
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 3 deletions.
diff --git a/sat_pred/dataloader.py b/sat_pred/dataloader.py
@@ -55,9 +55,13 @@ def find_valid_t0_times(ds, history_mins, forecast_mins, sample_freq_mins):
 
     valid_t0_times = []
     for _, row in contiguous_t0_periods.iterrows():
-        start_dt = row["start_dt"]
-        end_dt = row["end_dt"]
-        valid_t0_times.append(pd.date_range(row["start_dt"], row["end_dt"], freq="5min"))
+        valid_t0_times.append(
+            pd.date_range(
+                row["start_dt"], 
+                row["end_dt"],  
+                freq=f"{sample_freq_mins}min"
+            )
+        )
 
     valid_t0_times = pd.to_datetime(np.concatenate(valid_t0_times))
 

diff --git a/tests/test_dataloader.py b/tests/test_dataloader.py
@@ -33,6 +33,21 @@ def test_find_valid_t0_times(sat_zarr_path):
     # Total                               252
 
     assert len(t0_times)==252
+
+    t0_times = find_valid_t0_times(
+        ds, 
+        history_mins=60, 
+        forecast_mins=120, 
+        sample_freq_mins=15,
+    )
+
+    # original 15 minute timesteps     288 / 3
+    # forecast length buffer      - (120 / 15) 
+    # history length buffer       -  (60 / 15) 
+    #                            ------------
+    # Total                               84
+
+    assert len(t0_times)==84    
 
 
 def test_satellite_dataset(sat_zarr_path):