Update lowercase.

mlfoundations · May 19, 2024 · e53b794 · e53b794
1 parent 84e4bd4
commit e53b794
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/open_lm/datapreprocess/ray/tokenize_shuffle.py b/open_lm/datapreprocess/ray/tokenize_shuffle.py
@@ -68,7 +68,7 @@ def load_from_yaml(filename):
     # Add get_source and get_sampling_frequency methods to Sources
     def get_source_dynamic(self, key):
         for item in data["sources"]:
-            if any(marker in key.lower() for marker in item["markers"]):
+            if any(marker.lower() in key.lower() for marker in item["markers"]):
                 return Sources[item["source"]]
         return Sources.UNKNOWN