From d35eb342b38460931c84795ad37411bbe839812e Mon Sep 17 00:00:00 2001
From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com>
Date: Fri, 24 Sep 2021 15:45:35 -0400
Subject: [PATCH] better handling of crashes in modules' handleEvent() (#1403)

---
 sfscan.py            | 10 +++++++---
 spiderfoot/plugin.py | 12 ++++++++++++
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/sfscan.py b/sfscan.py
index 0c67007b5d..070f28d649 100644
--- a/sfscan.py
+++ b/sfscan.py
@@ -475,7 +475,8 @@ def waitForThreads(self):
                                 break
                             # Trigger module.finished()
                             for mod in self.__moduleInstances.values():
-                                mod.incomingEventQueue.put('FINISHED')
+                                if not mod.errorState and mod.incomingEventQueue is not None:
+                                    mod.incomingEventQueue.put('FINISHED')
                             sleep(.1)
                             while not self.threadsFinished(log_status):
                                 log_status = counter % 100 == 0
@@ -499,7 +500,7 @@ def waitForThreads(self):
                         raise AssertionError(f"{mod.__name__} requested stop")
 
                     # send it the new event if applicable
-                    if not mod.errorState:
+                    if not mod.errorState and mod.incomingEventQueue is not None:
                         watchedEvents = mod.watchedEvents()
                         if sfEvent.eventType in watchedEvents or "*" in watchedEvents:
                             mod.incomingEventQueue.put(deepcopy(sfEvent))
@@ -516,7 +517,10 @@ def threadsFinished(self, log_status=False):
         if self.eventQueue is None:
             return True
 
-        modules_waiting = {m.__name__: m.incomingEventQueue.qsize() for m in self.__moduleInstances.values()}
+        modules_waiting = {
+            m.__name__: m.incomingEventQueue.qsize() for m in
+            self.__moduleInstances.values() if m.incomingEventQueue is not None
+        }
         modules_waiting = sorted(modules_waiting.items(), key=lambda x: x[-1], reverse=True)
         modules_running = [m.__name__ for m in self.__moduleInstances.values() if m.running]
         queues_empty = [qsize == 0 for m, qsize in modules_waiting]
diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.py
index a4c95f79cd..294b015f25 100644
--- a/spiderfoot/plugin.py
+++ b/spiderfoot/plugin.py
@@ -405,7 +405,19 @@ def threadWorker(self):
             import traceback
             self.sf.error(f"Exception ({e.__class__.__name__}) in module {self.__name__}."
                           + traceback.format_exc())
+            # set errorState
+            self.sf.debug(f"Setting errorState for module {self.__name__}.")
             self.errorState = True
+            # clear incoming queue
+            if self.incomingEventQueue:
+                self.sf.debug(f"Emptying incomingEventQueue for module {self.__name__}.")
+                with suppress(queue.Empty):
+                    while 1:
+                        self.incomingEventQueue.get_nowait()
+                # set queue to None to prevent its use
+                # if there are leftover objects in the queue, the scan will hang.
+                self.incomingEventQueue = None
+
         finally:
             self.running = False