better handling of crashes in modules' handleEvent() (smicallef#1403)

n3mas1s · Sep 24, 2021 · d35eb34 · d35eb34
1 parent 4909b97
commit d35eb34
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/sfscan.py b/sfscan.py
@@ -475,7 +475,8 @@ def waitForThreads(self):
                                 break
                             # Trigger module.finished()
                             for mod in self.__moduleInstances.values():
-                                mod.incomingEventQueue.put('FINISHED')
+                                if not mod.errorState and mod.incomingEventQueue is not None:
+                                    mod.incomingEventQueue.put('FINISHED')
                             sleep(.1)
                             while not self.threadsFinished(log_status):
                                 log_status = counter % 100 == 0
@@ -499,7 +500,7 @@ def waitForThreads(self):
                         raise AssertionError(f"{mod.__name__} requested stop")
 
                     # send it the new event if applicable
-                    if not mod.errorState:
+                    if not mod.errorState and mod.incomingEventQueue is not None:
                         watchedEvents = mod.watchedEvents()
                         if sfEvent.eventType in watchedEvents or "*" in watchedEvents:
                             mod.incomingEventQueue.put(deepcopy(sfEvent))
@@ -516,7 +517,10 @@ def threadsFinished(self, log_status=False):
         if self.eventQueue is None:
             return True
 
-        modules_waiting = {m.__name__: m.incomingEventQueue.qsize() for m in self.__moduleInstances.values()}
+        modules_waiting = {
+            m.__name__: m.incomingEventQueue.qsize() for m in
+            self.__moduleInstances.values() if m.incomingEventQueue is not None
+        }
         modules_waiting = sorted(modules_waiting.items(), key=lambda x: x[-1], reverse=True)
         modules_running = [m.__name__ for m in self.__moduleInstances.values() if m.running]
         queues_empty = [qsize == 0 for m, qsize in modules_waiting]

diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.py
@@ -405,7 +405,19 @@ def threadWorker(self):
             import traceback
             self.sf.error(f"Exception ({e.__class__.__name__}) in module {self.__name__}."
                           + traceback.format_exc())
+            # set errorState
+            self.sf.debug(f"Setting errorState for module {self.__name__}.")
             self.errorState = True
+            # clear incoming queue
+            if self.incomingEventQueue:
+                self.sf.debug(f"Emptying incomingEventQueue for module {self.__name__}.")
+                with suppress(queue.Empty):
+                    while 1:
+                        self.incomingEventQueue.get_nowait()
+                # set queue to None to prevent its use
+                # if there are leftover objects in the queue, the scan will hang.
+                self.incomingEventQueue = None
+
         finally:
             self.running = False