copy config/dataloader/default.yaml to native_grid.yaml

Co-authored-by: Jesper Dramsch <[email protected]>
Co-authored-by: Mariana Clare <[email protected]>
Co-authored-by: Mario Santa Cruz <[email protected]>
Co-authored-by: Matthew Chantry <[email protected]>
Co-authored-by: Gabriel Moldovan <[email protected]>

src/anemoi/training/config/dataloader/native_grid.yaml

@@ -0,0 +1,58 @@
+prefetch_factor: 2
+
+num_workers:
+  training: 8
+  validation: 8
+  test: 8
+  predict: 8
+batch_size:
+  training: 2
+  validation: 4
+  test: 4
+  predict: 4
+
+# ============
+# Default effective batch_size for training is 16
+# For the o96 resolution, default per-gpu batch_size is 2 (8 gpus required)
+# The global lr is calculated as:
+# global_lr = local_lr * num_gpus_per_node * num_nodes / gpus_per_model
+# Assuming a constant effective batch_size, any change in the per_gpu batch_size
+# should come with a rescaling of the local_lr to keep a constant global_lr
+# ============
+
+# runs only N training batches [N = integer | null]
+# if null then we run through all the batches
+limit_batches:
+  training: null
+  validation: null
+  test: 20
+  predict: 20
+
+# ============
+# Dataloader definitions
+# These follow the anemoi-datasets patterns
+# You can make these as complicated for merging as you like
+# See https://anemoi-datasets.readthedocs.io
+# ============
+
+dataset: ${hardware.paths.data}/${hardware.files.dataset}
+
+training:
+  dataset: ${dataloader.dataset}
+  start: null
+  end: 2020
+  frequency: ${data.frequency}
+  drop:  []
+
+validation:
+  dataset: ${dataloader.dataset}
+  start: 2021
+  end: 2021
+  frequency: ${data.frequency}
+  drop:  []
+
+test:
+  dataset: ${dataloader.dataset}
+  start: 2022
+  end: null
+  frequency: ${data.frequency}