Fix/small improvements (#18)

* Fix formatting for arguments description
* LoRA bias depend on bias from the config.
* LRSchedularBase class now an abstract one

src/data/downloader.py

@@ -14,7 +14,7 @@ def download(config: DictConfig, *, override_if_exists: bool = True) -> Path:
         omegaconf's dictionary with three keys: url, folder and filename
         url is from where to download the file
         folder - in which folder to put the downloaded file
-    override_if_exists: bool
+    override_if_exists : bool
         if True will download even if file with such name already exists
 
     Raises

src/model/gpt_language_model/peft/lora.py

@@ -226,6 +226,11 @@ def zero_pad(self, x: torch.Tensor) -> torch.Tensor:
         | query         | key       | value    |
         ----------------------------------------
 
+        Parameters
+        ----------
+        x : torch.Tensor
+            tensor with weights update that needs to be padded with zeros
+
         Returns
         -------
         torch.Tensor
@@ -505,7 +510,7 @@ def __init__(
             enable_lora=[True, False, True],
             fan_in_fan_out=False,
             merge_weights=True,
-            bias=False,
+            bias=bias,
         )
 
 

src/model/lr_schedulers.py

@@ -1,11 +1,11 @@
 import math
-from abc import abstractclassmethod
+from abc import ABC, abstractmethod
 from typing import Optional
 
 import torch
 
 
-class LRSchedulerBase:
+class LRSchedulerBase(ABC):
     def __init__(self, optimizer: torch.optim.Optimizer) -> None:
         """Create base class for custom learning rate schedulers.
 
@@ -17,7 +17,7 @@ def __init__(self, optimizer: torch.optim.Optimizer) -> None:
         super().__init__()
         self.optimizer = optimizer
 
-    @abstractclassmethod
+    @abstractmethod
     def _get_lr(self, iteration: int) -> float:
         pass