set availablity checking from warning to error (#297)

modelscope · Apr 18, 2024 · 1d94a67 · 1d94a67
1 parent 33f72b1
commit 1d94a67
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/data_juicer/ops/load.py b/data_juicer/ops/load.py
@@ -1,3 +1,5 @@
+import sys
+
 from loguru import logger
 
 from data_juicer.utils.availability_utils import UNAVAILABLE_OPERATORS
@@ -21,8 +23,8 @@ def load_ops(process_list, op_fusion=False):
     for process in process_list:
         op_name, args = list(process.items())[0]
         if op_name in UNAVAILABLE_OPERATORS:
-            logger.warning(UNAVAILABLE_OPERATORS[op_name].get_warning_msg())
-            continue
+            logger.error(UNAVAILABLE_OPERATORS[op_name].get_warning_msg())
+            sys.exit(UNAVAILABLE_OPERATORS[op_name].get_warning_msg())
         ops.append(OPERATORS.modules[op_name](**args))
         new_process_list.append(process)