feat: Automatically set hyperparameters related to multi-GPU

config/config.yaml

@@ -18,7 +18,10 @@ seed: 4242
 characters_to_keep: 'abcdefghijklmnopqrstuvwxyzæøå0123456789éü '
 max_seconds_per_example: 10
 dataloader_num_workers: 8
-padding: longest  # longest/max_length/do_not_pad
+
+# Can be `longest`, `max_length` or `do_not_pad`
+# NOTE: This is automatically set to `max_length` in a multi-gpu setting
+padding: longest
 
 # This is a list of the sampling probability of each dataset, where null means that
 # each dataset will be sampled equally often

config/model/wav2vec2.yaml

@@ -18,7 +18,7 @@ mask_time_prob: 0.5
 mask_time_length: 10
 mask_feature_prob: 0.5
 mask_feature_length: 64
-layerdrop: 0.0  # NOTE: This parameter cannot be used in a multi-gpu setting!
+layerdrop: 0.1  # This will automatically be set to 0 in a multi-gpu setting
 ctc_loss_reduction: mean
 
 # Decoder hyperparameters

src/scripts/finetune_model.py

@@ -6,6 +6,7 @@
 
 import hydra
 from omegaconf import DictConfig
+import os
 
 from coral_models.finetune import finetune
 
@@ -18,6 +19,14 @@ def main(cfg: DictConfig) -> None:
         cfg (DictConfig):
             The Hydra configuration object.
     """
+    # In case we are running in a multi-GPU setting, we need to force certain
+    # hyperparameters
+    if os.getenv("WORLD_SIZE") is not None:
+        if "layerdrop" in cfg.model:
+            cfg.model.layerdrop = 0.0
+        cfg.padding = "max_length"
+
+    breakpoint()
     finetune(cfg)