3.3 release preparation (#1134)

* Bump version to 3.3, add Python 3.9 testing to Travis, various module bug fixes.
smicallef · Jan 24, 2021 · 2f1d7eb · 2f1d7eb
1 parent 4ffad6a
commit 2f1d7eb
Show file tree

Hide file tree

Showing 229 changed files with 269 additions and 238 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,6 +5,7 @@ language: python
 python:
   - "3.6"
   - "3.8"
+  - "3.9"
 
 cache: pip
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-SpiderFoot 3.3-DEV
+SpiderFoot 3.3
diff --git a/dyn/HEADER.tmpl b/dyn/HEADER.tmpl
@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8">
-    <title>SpiderFoot v3.3-DEV</title>
+    <title>SpiderFoot v3.3</title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="">
     <meta name="author" content="">
@@ -64,13 +64,13 @@ class="nav-item"><a href="${docroot}/opts" class="nav-link"><i class="glyphicon
     <div class="modal-content">
 
     <div class="modal-header">
-      <h3 class="modal-title">SpiderFoot 3.3-DEV</h3>
+      <h3 class="modal-title">SpiderFoot 3.3</h3>
     </div>
 
     <div class="modal-body">
     <p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='https://twitter.com/binarypool' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p>
     <p>If you have any bugs to report, or requests for enhancements, please contact the support mailing list at <a href=mailto:[email protected]>[email protected]</a>.
-    <p>You are running version <b>3.3-DEV</b>.</p>
+    <p>You are running version <b>3.3</b>.</p>
     <p> Visit the project website at <a href='https://www.spiderfoot.net/r.php?u=aHR0cHM6Ly93d3cuc3BpZGVyZm9vdC5uZXQv&s=os_int' target=_new>https://www.spiderfoot.net/</a>.</p>
     <p> View the documentation at <a href='https://www.spiderfoot.net/documentation/' target=_new>https://www.spiderfoot.net/documentation/</a>.</p>
     <p> Check out the Github repo at <a href='https://github.com/smicallef/spiderfoot' target=_new>https://github.com/smicallef/spiderfoot</a>.</p>

diff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.py
@@ -165,7 +165,7 @@ def handleEvent(self, event):
                 ret = None
 
             if not ret:
-                self.sf.error("Unable to process empty response from archive.org: {eventData}")
+                self.sf.debug("Empty response from archive.org for {eventData}")
                 continue
 
             if len(ret['archived_snapshots']) < 1:

diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.py
@@ -438,6 +438,10 @@ def handleEvent(self, event):
 
                         try:
                             banner = prec['result']['data']['service']['banner']
+                            if '\\r\\n\\r\\n' in banner and "HTTP/" in banner:
+                                # We don't want the content after HTTP banners
+                                banner = banner.split('\\r\\n\\r\\n')[0]
+                                banner = banner.replace("\\r\\n", "\n")
                         except Exception:
                             self.sf.debug("No banner information found.")
                             continue

diff --git a/modules/sfp_pageinfo.py b/modules/sfp_pageinfo.py
@@ -85,6 +85,15 @@ def handleEvent(self, event):
             self.sf.debug("Not gathering page info for external site " + eventSource)
             return
 
+        # Ignore javascript and CSS
+        if ".css?" in eventSource or eventSource.endswith(".css"):
+            self.sf.debug("Not attempting to match CSS content.")
+            return
+
+        if ".js?" in eventSource or eventSource.endswith(".js"):
+            self.sf.debug("Not attempting to match JS content.")
+            return
+
         if eventSource in self.results:
             self.sf.debug("Already checked this page for a page type, skipping.")
             return

diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.py
@@ -161,6 +161,8 @@ def handleEvent(self, event):
         if eventName == "LINKED_URL_EXTERNAL":
             if ".amazonaws.com" in eventData:
                 b = self.sf.urlFQDN(eventData)
+                if b in self.opts['endpoints']:
+                    b += "/" + eventData.split(b + "/")[1].split("/")[0]
                 evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)
                 self.notifyListeners(evt)
             return None

diff --git a/modules/sfp_scylla.py b/modules/sfp_scylla.py
@@ -27,14 +27,14 @@ class sfp_scylla(SpiderFootPlugin):
         'useCases': ["Footprint", "Investigate", "Passive"],
         'categories': ["Leaks, Dumps and Breaches"],
         'dataSource': {
-            'website': "https://scylla.sh/",
+            'website': "https://scylla.so/",
             'model': "FREE_NOAUTH_UNLIMITED",
             'references': [
-                "https://scylla.sh/crowdsource"
+                "https://scylla.so/crowdsource"
             ],
             'favIcon': "",
             'logo': "",
-            'description': "scylla.sh has two major goals. One is to have a community-oriented database leak community "
+            'description': "scylla.so has two major goals. One is to have a community-oriented database leak community "
             "that is a useful tool for security researchers.\n"
             "The other major goal is to undercut those people that are selling databases.",
         }
@@ -76,15 +76,15 @@ def producedEvents(self):
     # Query Scylla API
     def query(self, qry, per_page=20, start=0):
         params = {
-            'q': 'Email:@' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
+            'q': 'email:@' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
             'size': str(per_page),
             'start': str(start)
         }
 
         headers = {
             'Accept': 'application/json'
         }
-        res = self.sf.fetchUrl('https://scylla.sh/search?' + urllib.parse.urlencode(params),
+        res = self.sf.fetchUrl('https://scylla.so/search?' + urllib.parse.urlencode(params),
                                headers=headers,
                                timeout=15,
                                useragent=self.opts['_useragent'],
@@ -99,7 +99,7 @@ def query(self, qry, per_page=20, start=0):
             return None
 
         if res['content'] is None:
-            self.sf.debug('No response from Scylla.sh')
+            self.sf.debug('No response from Scylla.so')
             return None
 
         try:

diff --git a/modules/sfp_shodan.py b/modules/sfp_shodan.py
@@ -98,7 +98,11 @@ def query(self, qry):
             return None
 
         try:
-            return json.loads(res['content'])
+            r = json.loads(res['content'])
+            if "error" in r:
+                self.sf.error("Error returned form SHODAN: {r['error']}")
+                return None
+            return r
         except Exception as e:
             self.sf.error(f"Error processing JSON response from SHODAN: {e}")
             return None
@@ -115,7 +119,11 @@ def searchHosts(self, qry):
             return None
 
         try:
-            return json.loads(res['content'])
+            r = json.loads(res['content'])
+            if "error" in r:
+                self.sf.error("Error returned form SHODAN: {r['error']}")
+                return None
+            return r
         except Exception as e:
             self.sf.error(f"Error processing JSON response from SHODAN: {e}")
             return None
@@ -136,7 +144,14 @@ def searchHtml(self, qry):
             return None
 
         try:
-            return json.loads(res['content'])
+            r = json.loads(res['content'])
+            if "error" in r:
+                self.sf.error("Error returned form SHODAN: {r['error']}")
+                return None
+            if r.get('total', 0) == 0:
+                self.sf.info("No SHODAN info found for {qry}")
+                return None
+            return r
         except Exception as e:
             self.sf.error(f"Error processing JSON response from SHODAN: {e}")
             return None

diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
@@ -136,7 +136,7 @@ def processUrl(self, url):
                                    self.getTarget().getNames())
 
         if not links:
-            self.sf.info(f"No links found at {url}")
+            self.sf.debug(f"No links found at {url}")
             return None
 
         # Notify modules about the links found
@@ -301,7 +301,7 @@ def handleEvent(self, event):
         if spiderTarget is None:
             return None
 
-        self.sf.info("Initiating spider of " + spiderTarget + " from " + srcModuleName)
+        self.sf.debug("Initiating spider of " + spiderTarget + " from " + srcModuleName)
 
         # Link the spidered URL to the event that triggered it
         self.urlEvents[spiderTarget] = event
@@ -368,7 +368,7 @@ def spiderFrom(self, startingPoint):
                         break
 
             nextLinks = self.cleanLinks(links)
-            self.sf.info(f"Found links: {nextLinks}")
+            self.sf.debug(f"Found links: {nextLinks}")
 
             # We've scanned through another layer of the site
             levelsTraversed += 1
@@ -379,7 +379,7 @@ def spiderFrom(self, startingPoint):
 
             # We've reached the end of our journey..
             if len(nextLinks) == 0:
-                self.sf.info("No more links found to spider, finishing..")
+                self.sf.debug("No more links found to spider, finishing..")
                 keepSpidering = False
 
             # We've been asked to stop scanning

diff --git a/sf.py b/sf.py
@@ -85,7 +85,7 @@ def main():
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.db',
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',
@@ -115,7 +115,7 @@ def main():
 
     # Legacy way to run the server
     args = None
-    p = argparse.ArgumentParser(description='SpiderFoot 3.3-DEV: Open Source Intelligence Automation.')
+    p = argparse.ArgumentParser(description='SpiderFoot 3.3: Open Source Intelligence Automation.')
     p.add_argument("-d", "--debug", action='store_true', help="Enable debug output.")
     p.add_argument("-l", metavar="IP:port", help="IP and port to listen on.")
     p.add_argument("-m", metavar="mod1,mod2,...", type=str, help="Modules to enable.")

diff --git a/sfcli.py b/sfcli.py
@@ -53,7 +53,7 @@ class bcolors:
 
 
 class SpiderFootCli(cmd.Cmd):
-    version = "3.3-DEV"
+    version = "3.3"
     pipecmd = None
     output = None
     modules = []

diff --git a/test/integration/test_sfwebui.py b/test/integration/test_sfwebui.py
@@ -22,7 +22,7 @@ def setup_server():
             '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
             '_internettlds_cache': 72,
             '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-            '__version__': '3.3-DEV',
+            '__version__': '3.3',
             '__database': 'spiderfoot.test.db',  # note: test database file
             '__modules__': None,  # List of modules. Will be set after start-up.
             '_socks1type': '',

diff --git a/test/unit/modules/test_sfp__stor_db.py b/test/unit/modules/test_sfp__stor_db.py
@@ -21,7 +21,7 @@ class TestModule_stor_db(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp__stor_stdout.py b/test/unit/modules/test_sfp__stor_stdout.py
@@ -21,7 +21,7 @@ class TestModule_stor_stdout(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_abusech.py b/test/unit/modules/test_sfp_abusech.py
@@ -21,7 +21,7 @@ class TestModuleabusech(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_abuseipdb.py b/test/unit/modules/test_sfp_abuseipdb.py
@@ -21,7 +21,7 @@ class TestModuleabuseipdb(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_accounts.py b/test/unit/modules/test_sfp_accounts.py
@@ -21,7 +21,7 @@ class TestModuleaccounts(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_adblock.py b/test/unit/modules/test_sfp_adblock.py
@@ -21,7 +21,7 @@ class TestModuleadblock(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_ahmia.py b/test/unit/modules/test_sfp_ahmia.py
@@ -21,7 +21,7 @@ class TestModuleahmia(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_alienvault.py b/test/unit/modules/test_sfp_alienvault.py
@@ -21,7 +21,7 @@ class TestModulealienvault(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_alienvaultiprep.py b/test/unit/modules/test_sfp_alienvaultiprep.py
@@ -21,7 +21,7 @@ class TestModulealienvaultiprep(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_apility.py b/test/unit/modules/test_sfp_apility.py
@@ -21,7 +21,7 @@ class TestModuleapility(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',

diff --git a/test/unit/modules/test_sfp_apple_itunes.py b/test/unit/modules/test_sfp_apple_itunes.py
@@ -21,7 +21,7 @@ class TestModuleAppleItunes(unittest.TestCase):
         '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
         '_internettlds_cache': 72,
         '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
-        '__version__': '3.3-DEV',
+        '__version__': '3.3',
         '__database': 'spiderfoot.test.db',  # note: test database file
         '__modules__': None,  # List of modules. Will be set after start-up.
         '_socks1type': '',
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ language: python @@
     python:
       - "3.6"
       - "3.8"
+      - "3.9"
     cache: pip
@@ Expand Down @@