raise error if any exception statuses at the end of a pipeline

Unstructured-IO · Jun 24, 2024 · 45a3233 · 45a3233
1 parent 139bf77
commit 45a3233
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/unstructured/ingest/v2/pipeline/pipeline.py b/unstructured/ingest/v2/pipeline/pipeline.py
@@ -30,6 +30,10 @@
 from unstructured.ingest.v2.processes.partitioner import PartitionerConfig
 
 
+class PipelineError(Exception):
+    pass
+
+
 @dataclass
 class Pipeline:
     context: ProcessorConfig
@@ -119,6 +123,8 @@ def run(self):
         finally:
             self.log_statuses()
             self.cleanup()
+            if self.context.status:
+                raise PipelineError("Pipeline did not run successfully")
 
     def clean_results(self, results: Optional[list[Union[Any, list[Any]]]]) -> Optional[list[Any]]:
         if not results: