fix: 1. change relpath to abspath of dataset (#137)

* fix: 1. change relpath to abspath of dataset 2. torch set_num_threads in python3.8 on Mac system. * fix: import fingerprint_warnings from datasets
modelscope · Dec 15, 2023 · f299986 · f299986
1 parent 48c081e
commit f299986
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 10 deletions.
diff --git a/data_juicer/config/config.py b/data_juicer/config/config.py
@@ -311,6 +311,7 @@ def init_setup_from_cfg(cfg):
     :param cfg: a updated cfg
     """
 
+    cfg.export_path = os.path.abspath(cfg.export_path)
     export_path = cfg.export_path
     cfg.work_dir = os.path.dirname(export_path)
     log_dir = os.path.join(cfg.work_dir, 'log')
@@ -325,16 +326,15 @@ def init_setup_from_cfg(cfg):
 
     # check and get dataset dir
     if os.path.exists(cfg.dataset_path):
+        cfg.dataset_path = os.path.abspath(cfg.dataset_path)
         if os.path.isdir(cfg.dataset_path):
-            cfg.dataset_dir = os.path.abspath(cfg.dataset_path)
+            cfg.dataset_dir = cfg.dataset_path
         else:
-            cfg.dataset_dir = os.path.abspath(os.path.dirname(
-                cfg.dataset_path))
+            cfg.dataset_dir = os.path.dirname(cfg.dataset_path)
     else:
-        logger.error(f'Input dataset_path [{cfg.dataset_path}] is invalid. '
-                     f'Please check and retry.')
-        raise ValueError(f'Input dataset_path [{cfg.dataset_path}] is '
-                         f'invalid. Please check and retry.')
+        logger.warning(f'dataset_path [{cfg.dataset_path}] not found in local.'
+                       'Please check and retry, otherwise we will treat it '
+                       'as a remote dataset.')
 
     # whether or not to use cache management
     # disabling the cache or using checkpoint explicitly will turn off the

diff --git a/data_juicer/utils/availability_utils.py b/data_juicer/utils/availability_utils.py
@@ -1,6 +1,9 @@
+
+
 from loguru import logger
 
 UNAVAILABLE_OPERATORS = {}
+CHECK_SYSTEM_INFO_ONCE = True
 
 
 class UnavailableOperator:
@@ -52,7 +55,23 @@ def __init__(
                               f'`pip install -v -e .[{self.requires_type}]`'
 
     def __enter__(self):
-        pass
+
+        # only for python3.8 on mac
+        global CHECK_SYSTEM_INFO_ONCE
+        if CHECK_SYSTEM_INFO_ONCE:
+            import os
+            import platform
+            import sys
+            major, minor = sys.version_info[:2]
+            system = platform.system()
+            if major == 3 and minor == 8 and system == 'Darwin':
+                logger.warning(
+                    'The torch.set_num_threads function does not '
+                    'work in python3.8 version on Mac systems. We will set '
+                    'OMP_NUM_THREADS to 1 manually before importing torch')
+
+                os.environ['OMP_NUM_THREADS'] = str(1)
+                CHECK_SYSTEM_INFO_ONCE = False
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_type is ModuleNotFoundError:

diff --git a/data_juicer/utils/fingerprint_utils.py b/data_juicer/utils/fingerprint_utils.py
@@ -2,7 +2,7 @@
 
 import dill
 import xxhash
-from datasets.fingerprint import (_CACHING_ENABLED,
+from datasets.fingerprint import (_CACHING_ENABLED, fingerprint_warnings,
                                   format_kwargs_for_fingerprint,
                                   format_transform_for_fingerprint,
                                   generate_random_fingerprint,
@@ -54,7 +54,7 @@ def update_fingerprint(fingerprint, transform, transform_args):
     """
     Combining various objects to update the fingerprint.
     """
-    global fingerprint_warnings
+
     hasher = Hasher()
     hasher.update(fingerprint)
     try: