feat: Small improvements in triggers

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- On critical error wait until the pod is killed instead of keep running
+
+### Changed
+
+- Increase time to wait without events before restarting the pod
+
 ## [1.3.9] - 2023-08-21
 
 ### Added

sekoia_automation/connector/__init__.py

@@ -30,7 +30,7 @@ class DefaultConnectorConfiguration(BaseModel):
 class Connector(Trigger):
     configuration: DefaultConnectorConfiguration
 
-    seconds_without_events = 3600
+    seconds_without_events = 3600 * 6
 
     def __init__(self, *args, **kwargs):
         executor_max_worker = kwargs.pop("executor_max_worker", 4)

sekoia_automation/trigger.py

@@ -44,6 +44,9 @@ class Trigger(ModuleItem):
     LIVENESS_PORT_FILE_NAME = "liveness_port"
     METRICS_PORT_FILE_NAME = "metrics_port"
 
+    # Time to wait for stop event to be received
+    _STOP_EVENT_WAIT = 120
+
     def __init__(self, module: Module | None = None, data_path: Path | None = None):
         super().__init__(module, data_path)
         self._configuration: dict | BaseModel | None = None
@@ -138,6 +141,10 @@ def _execute_once(self) -> None:
             self._handle_send_event_exception(ex)
         except Exception as ex:
             self._handle_trigger_exception(ex)
+        if self._critical_log_sent:
+            # Prevent the trigger from running
+            # and creating other errors until it is stopped
+            self._stop_event.wait(self._STOP_EVENT_WAIT)
 
     def execute(self) -> None:
         self._ensure_data_path_set()

tests/test_trigger.py

@@ -296,6 +296,7 @@ def run(self):
                 raise TriggerConfigurationError
 
     trigger = TestTrigger()
+    trigger._STOP_EVENT_WAIT = 0.1
     with pytest.raises(SystemExit), patch.object(
         Module, "load_config", return_value={}
     ):
@@ -322,6 +323,7 @@ def run(self):
 
     trigger = TestTrigger()
     trigger._error_count = 4
+    trigger._STOP_EVENT_WAIT = 0.1
     with pytest.raises(SystemExit), patch.object(
         Module, "load_config", return_value={}
     ):