fix output bug

modelscope · Nov 18, 2024 · 58e357f · 58e357f
1 parent 991e290
commit 58e357f
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/data_juicer/ops/deduplicator/ray_redis_minhash_deduplicator.py b/data_juicer/ops/deduplicator/ray_redis_minhash_deduplicator.py
@@ -302,8 +302,8 @@ def filter_with_union_find(table: pa.Table) -> pa.Table:
                                     batch_format='pyarrow').groupby(
                                         HashKeys.minhash).aggregate(
                                             UnionFn(union_find)).materialize()
-        result = dataset_with_id.map_batches(filter_with_union_find,
-                                             batch_format='pyarrow')
+        result = dataset_with_id.map_batches(
+            filter_with_union_find, batch_format='pyarrow').materialize()
         logger.info(f'Keep {result.count()} samples after MinHash dedup.')
         union_find.clean()
         return result