remove test code

Cathy0908 · Dec 20, 2024 · 20fc9de · 20fc9de
1 parent 3c10162
commit 20fc9de
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 6 deletions.
diff --git a/data_juicer/ops/base_op.py b/data_juicer/ops/base_op.py
@@ -143,7 +143,6 @@ def __init__(self, *args, **kwargs):
         self.history_key = kwargs.get('history_key', 'history')
 
         self.batch_size = kwargs.get('batch_size', 1000)
-        self.concurrency = kwargs.get('concurrency', 1)
 
         # whether the model can be accelerated using cuda
         _accelerator = kwargs.get('accelerator', None)

diff --git a/data_juicer/ops/mapper/image_captioning_mapper.py b/data_juicer/ops/mapper/image_captioning_mapper.py
@@ -283,11 +283,6 @@ def process_batched(self, samples, rank=None):
         :param samples:
         :return:
         """
-        import pyarrow as pa
-
-        if isinstance(samples, pa.Table):
-            samples = samples.to_pydict()
-
         # reconstruct samples from "dict of lists" to "list of dicts"
         reconstructed_samples = []
         for i in range(len(samples[self.text_key])):