diff --git a/modules/sfp__stor_db.py b/modules/sfp__stor_db.py index 4ba5a52b7d..582da59f4c 100644 --- a/modules/sfp__stor_db.py +++ b/modules/sfp__stor_db.py @@ -51,11 +51,10 @@ def handleEvent(self, sfEvent): if not self.opts['_store']: return - if self.opts['maxstorage'] != 0: - if len(sfEvent.data) > self.opts['maxstorage']: - self.debug("Storing an event: " + sfEvent.eventType) - self.__sfdb__.scanEventStore(self.getScanId(), sfEvent, self.opts['maxstorage']) - return + if self.opts['maxstorage'] != 0 and len(sfEvent.data) > self.opts['maxstorage']: + self.debug("Storing an event: " + sfEvent.eventType) + self.__sfdb__.scanEventStore(self.getScanId(), sfEvent, self.opts['maxstorage']) + return self.debug("Storing an event: " + sfEvent.eventType) self.__sfdb__.scanEventStore(self.getScanId(), sfEvent) diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.py index 47a196871f..49d775227f 100644 --- a/modules/sfp__stor_stdout.py +++ b/modules/sfp__stor_stdout.py @@ -85,10 +85,11 @@ def output(self, event): srcdata = srcdata[0:self.opts['_maxlength']] if self.opts['_format'] == "tab": + event_type = self.opts['_eventtypes'][event.eventType] if self.opts['_showsource']: - print(('{0:30}\t{1:45}\t{2}\t{3}'.format(event.module, self.opts['_eventtypes'][event.eventType], srcdata, data))) + print(f"{event.module.ljust(30)}\t{event_type.ljust(45)}\t{srcdata}\t{data}") else: - print(('{0:30}\t{1:45}\t{2}'.format(event.module, self.opts['_eventtypes'][event.eventType], data))) + print(f"{event.module.ljust(30)}\t{event_type.ljust(45)}\t{data}") if self.opts['_format'] == "csv": print((event.module + d + self.opts['_eventtypes'][event.eventType] + d + srcdata + d + data)) diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.py index f93229205f..195ae1a788 100644 --- a/modules/sfp_binaryedge.py +++ b/modules/sfp_binaryedge.py @@ -267,9 +267,8 @@ def handleEvent(self, event): continue if self.getTarget().matches(host, includeParents=True): - if self.opts['verify']: - if not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): - continue + if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): + continue evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event) self.notifyListeners(evt) @@ -303,10 +302,9 @@ def handleEvent(self, event): self.reportedhosts[rec] = True - if self.opts['verify']: - if not self.sf.resolveHost(rec) and not self.sf.resolveHost6(rec): - self.debug(f"Couldn't resolve {rec}, so skipping.") - continue + if self.opts['verify'] and not self.sf.resolveHost(rec) and not self.sf.resolveHost6(rec): + self.debug(f"Couldn't resolve {rec}, so skipping.") + continue e = SpiderFootEvent('INTERNET_NAME', rec, self.__name__, event) self.notifyListeners(e) diff --git a/modules/sfp_dnsdumpster.py b/modules/sfp_dnsdumpster.py index d32d674e40..cd6b3f3b99 100644 --- a/modules/sfp_dnsdumpster.py +++ b/modules/sfp_dnsdumpster.py @@ -79,8 +79,8 @@ def query(self, domain): self.error("Error obtaining CSRF tokens") self.errorState = True return ret - else: - self.debug("Successfully obtained CSRF tokens") + + self.debug("Successfully obtained CSRF tokens") # Otherwise, do the needful url = "https://dnsdumpster.com/" diff --git a/modules/sfp_greynoise.py b/modules/sfp_greynoise.py index 4967e5fb1b..1998cfe9ce 100644 --- a/modules/sfp_greynoise.py +++ b/modules/sfp_greynoise.py @@ -165,28 +165,20 @@ def handleEvent(self, event): if eventName == "NETBLOCK_OWNER": if not self.opts["netblocklookup"]: return - else: - if IPNetwork(eventData).prefixlen < self.opts["maxnetblock"]: - self.debug( - "Network size bigger than permitted: " - + str(IPNetwork(eventData).prefixlen) - + " > " - + str(self.opts["maxnetblock"]) - ) - return + + max_netblock = self.opts['maxnetblock'] + if IPNetwork(eventData).prefixlen < max_netblock: + self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") + return if eventName == "NETBLOCK_MEMBER": if not self.opts["subnetlookup"]: return - else: - if IPNetwork(eventData).prefixlen < self.opts["maxsubnet"]: - self.debug( - "Network size bigger than permitted: " - + str(IPNetwork(eventData).prefixlen) - + " > " - + str(self.opts["maxsubnet"]) - ) - return + + max_subnet = self.opts['maxsubnet'] + if IPNetwork(eventData).prefixlen < max_subnet: + self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}") + return if eventName == "IP_ADDRESS": evtType = "MALICIOUS_IPADDR" diff --git a/modules/sfp_hosting.py b/modules/sfp_hosting.py index 96587bb3dd..4733941bd7 100644 --- a/modules/sfp_hosting.py +++ b/modules/sfp_hosting.py @@ -62,11 +62,12 @@ def queryAddr(self, qaddr): data['content'] = self.sf.cacheGet("sfipcat", 48) if data['content'] is None: data = self.sf.fetchUrl(url, useragent=self.opts['_useragent']) + if data['content'] is None: self.error("Unable to fetch " + url) return None - else: - self.sf.cachePut("sfipcat", data['content']) + + self.sf.cachePut("sfipcat", data['content']) for line in data['content'].split('\n'): if "," not in line: diff --git a/modules/sfp_onyphe.py b/modules/sfp_onyphe.py index d97f1fffa5..9edf805507 100644 --- a/modules/sfp_onyphe.py +++ b/modules/sfp_onyphe.py @@ -133,7 +133,8 @@ def query(self, endpoint, ip, page=1): ) self.errorState = True return None - elif "results" not in info or info["results"] == []: + + if "results" not in info or info["results"] == []: self.info(f"No Onyphe {endpoint} data found for {ip}") return None except Exception as e: @@ -213,12 +214,9 @@ def emitDomainData(self, response, eventData, event): self.debug("Host no longer resolves to our IP.") continue - if not self.opts["cohostsamedomain"]: - if self.getTarget().matches(domain, includeParents=True): - self.debug( - "Skipping " + domain + " because it is on the same domain." - ) - continue + if not self.opts["cohostsamedomain"] and self.getTarget().matches(domain, includeParents=True): + self.debug(f"Skipping {domain} because it is on the same domain.") + continue evt = SpiderFootEvent("CO_HOSTED_SITE", domain, self.__name__, event) self.notifyListeners(evt) diff --git a/modules/sfp_securitytrails.py b/modules/sfp_securitytrails.py index fd807e3252..5f9044084b 100644 --- a/modules/sfp_securitytrails.py +++ b/modules/sfp_securitytrails.py @@ -125,6 +125,7 @@ def query(self, qry, querytype, page=1, accum=None): info = json.loads(res['content']) if querytype == "domain": return info.get('subdomains', None) + if info.get("record_count", 0) > 100: if len(info.get('records', [])) >= 100: # Avoid throttling @@ -134,12 +135,12 @@ def query(self, qry, querytype, page=1, accum=None): else: accum = info.get('records') return self.query(qry, querytype, page + 1, accum) - else: - # We are at the last page - accum.extend(info.get('records', [])) - return accum - else: - return info.get('records', []) + + # We are at the last page + accum.extend(info.get('records', [])) + return accum + + return info.get('records', []) except Exception as e: self.error("Error processing JSON response from SecurityTrails: " + str(e)) return None diff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.py index be0b02e787..57a66ef879 100644 --- a/modules/sfp_socialprofiles.py +++ b/modules/sfp_socialprofiles.py @@ -219,8 +219,8 @@ def handleEvent(self, event): self.debug("Match found: " + match) if match in instances: continue - else: - instances.append(match) + + instances.append(match) if self.checkForStop(): return @@ -242,17 +242,18 @@ def handleEvent(self, event): if pres["content"] is None: continue - else: - found = False - for kw in self.keywords: - if re.search( - r"[^a-zA-Z\-\_]" + kw + r"[^a-zA-Z\-\_]", - pres["content"], - re.IGNORECASE, - ): - found = True - if not found: - continue + + found = False + for kw in self.keywords: + if re.search( + r"[^a-zA-Z\-\_]" + kw + r"[^a-zA-Z\-\_]", + pres["content"], + re.IGNORECASE, + ): + found = True + + if not found: + continue self.info("Social Media Profile found at " + site + ": " + match) match = urllib.parse.unquote(match) diff --git a/modules/sfp_tool_nbtscan.py b/modules/sfp_tool_nbtscan.py index d94acc33a2..cfa87b6062 100644 --- a/modules/sfp_tool_nbtscan.py +++ b/modules/sfp_tool_nbtscan.py @@ -117,12 +117,12 @@ def handleEvent(self, event): if eventData in self.results: self.debug(f"Skipping {eventData} as already scanned.") return - else: - # Might be a subnet within a subnet or IP within a subnet - for addr in self.results: - if IPNetwork(eventData) in IPNetwork(addr): - self.debug(f"Skipping {eventData} as already within a scanned range.") - return + + # Might be a subnet within a subnet or IP within a subnet + for addr in self.results: + if IPNetwork(eventData) in IPNetwork(addr): + self.debug(f"Skipping {eventData} as already within a scanned range.") + return self.results[eventData] = True diff --git a/modules/sfp_tool_nmap.py b/modules/sfp_tool_nmap.py index 577e32e60f..63f0231958 100644 --- a/modules/sfp_tool_nmap.py +++ b/modules/sfp_tool_nmap.py @@ -104,12 +104,12 @@ def handleEvent(self, event): if eventData in self.results: self.debug("Skipping " + eventData + " as already scanned.") return - else: - # Might be a subnet within a subnet or IP within a subnet - for addr in self.results: - if IPNetwork(eventData) in IPNetwork(addr): - self.debug("Skipping " + eventData + " as already within a scanned range.") - return + + # Might be a subnet within a subnet or IP within a subnet + for addr in self.results: + if IPNetwork(eventData) in IPNetwork(addr): + self.debug(f"Skipping {eventData} as already within a scanned range.") + return self.results[eventData] = True diff --git a/modules/sfp_tool_onesixtyone.py b/modules/sfp_tool_onesixtyone.py index 624a8998c6..e12a97b460 100644 --- a/modules/sfp_tool_onesixtyone.py +++ b/modules/sfp_tool_onesixtyone.py @@ -66,10 +66,9 @@ def setup(self, sfc, userOpts=dict()): # Write communities to file for use later on try: _, self.communitiesFile = tempfile.mkstemp("communities") - f = open(self.communitiesFile, "w") - for community in self.opts['communities'].split(","): - f.write(community.strip() + "\n") - f.close() + with open(self.communitiesFile, "w") as f: + for community in self.opts['communities'].split(","): + f.write(community.strip() + "\n") except BaseException as e: self.error(f"Unable to write communities file ({self.communitiesFile}): {e}") self.errorState = True @@ -133,12 +132,12 @@ def handleEvent(self, event): if eventData in self.results: self.debug(f"Skipping {eventData} as already scanned.") return - else: - # Might be a subnet within a subnet or IP within a subnet - for addr in self.results: - if IPNetwork(eventData) in IPNetwork(addr): - self.debug(f"Skipping {eventData} as already within a scanned range.") - return + + # Might be a subnet within a subnet or IP within a subnet + for addr in self.results: + if IPNetwork(eventData) in IPNetwork(addr): + self.debug(f"Skipping {eventData} as already within a scanned range.") + return self.results[eventData] = True diff --git a/modules/sfp_tool_testsslsh.py b/modules/sfp_tool_testsslsh.py index 7d8d62f2aa..6b8d022293 100644 --- a/modules/sfp_tool_testsslsh.py +++ b/modules/sfp_tool_testsslsh.py @@ -133,17 +133,17 @@ def handleEvent(self, event): if eventData in self.results: self.debug(f"Skipping {eventData} as already scanned.") return - else: - if eventName != "INTERNET_NAME": - # Might be a subnet within a subnet or IP within a subnet - for addr in self.results: - try: - if IPNetwork(eventData) in IPNetwork(addr): - self.debug(f"Skipping {eventData} as already within a scanned range.") - return - except BaseException: - # self.results will also contain hostnames - continue + + if eventName != "INTERNET_NAME": + # Might be a subnet within a subnet or IP within a subnet + for addr in self.results: + try: + if IPNetwork(eventData) in IPNetwork(addr): + self.debug(f"Skipping {eventData} as already within a scanned range.") + return + except BaseException: + # self.results will also contain hostnames + continue # If we weren't passed a netblock, this will be empty if not targets: diff --git a/modules/sfp_viewdns.py b/modules/sfp_viewdns.py index bb1a22c017..9dabb2dfdc 100644 --- a/modules/sfp_viewdns.py +++ b/modules/sfp_viewdns.py @@ -240,10 +240,9 @@ def handleEvent(self, event): if self.cohostcount >= self.opts['maxcohost']: continue - if eventName in ["IP_ADDRESS", "IPV6_ADDRESS"] and self.opts['verify']: - if not self.sf.validateIP(domain, eventData): - self.debug(f"Host {domain} no longer resolves to IP address: {eventData}") - continue + if eventName in ["IP_ADDRESS", "IPV6_ADDRESS"] and self.opts['verify'] and not self.sf.validateIP(domain, eventData): + self.debug(f"Host {domain} no longer resolves to IP address: {eventData}") + continue self.cohostcount += 1 diff --git a/modules/sfp_whoxy.py b/modules/sfp_whoxy.py index ca407e0b26..dc76bc2f01 100644 --- a/modules/sfp_whoxy.py +++ b/modules/sfp_whoxy.py @@ -101,10 +101,12 @@ def query(self, qry, querytype, page=1, accum=None): try: info = json.loads(res['content']) + if info.get("status", 0) == 0: self.error("Error querying Whoxy: " + info.get("status_reason", "Unknown")) self.errorState = True return None + if info.get("total_pages", 1) > 1: if info.get("current_page") < info.get("total_pages"): if accum: @@ -112,12 +114,12 @@ def query(self, qry, querytype, page=1, accum=None): else: accum = info.get('search_result') return self.query(qry, querytype, page + 1, accum) - else: - # We are at the last page - accum.extend(info.get('search_result', [])) - return accum - else: - return info.get('search_result', []) + + # We are at the last page + accum.extend(info.get('search_result', [])) + return accum + + return info.get('search_result', []) except Exception as e: self.error("Error processing JSON response from Whoxy: " + str(e)) return None diff --git a/setup.cfg b/setup.cfg index ceeccbecb5..c04a1dc627 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,27 +4,20 @@ max-complexity = 60 docstring-convention = google ignore-decorators = property select = C,E,F,W,B,B9,DAR,DUO,R,A,S,Q0,SIM,SFS -# Note: ANN and SIM tests should be reviewed and fixed instead of ignored -extend-ignore = E501 W503 B006 E800 B950 SFS301 SF01 Q000 SIM102 SIM113 SIM114 B902 B907 I D ANN +# Note: B902, B907 and ANN should be fixed instead of ignored +extend-ignore = E501 W503 B006 B950 SFS301 SF01 Q000 B902 B907 ANN # Note: most of these should be fixed instead of ignored per-file-ignores = spiderfoot/event.py:A003 - spiderfoot/plugin.py:B902 - spiderfoot/db.py:SFS101,B902 - sf.py:SFS201,B902 - sflib.py:SFS101,SIM110,B902 - sfscan.py:SIM105,B902 - modules/*:SIM115,B902,B907,R505 - modules/sfp_alienvault.py:B902,C901 - modules/sfp_binaryedge.py:B902,C901 - modules/sfp__stor_stdout.py:SFS201 + spiderfoot/db.py:SFS101 + modules/*:SIM102,SIM113,SIM114 + modules/sfp_alienvault.py:C901 + modules/sfp_binaryedge.py:C901 modules/sfp_bitcoin.py:SFS101 - modules/sfp_socialprofiles.py:R507 - spiderfoot/correlation.py:SIM110,B902,E275 spiderfoot/__init__.py:F401 - sfcli.py:DAR,B902 - sfwebui.py:A001,A002,B902,B905,R505 - test/*:SIM117,B904,E241,ANN + sfcli.py:DAR + sfwebui.py:A001,A002,B905 + test/*:SIM117,B904,ANN docs/conf.py:A [darglint] diff --git a/sf.py b/sf.py index 54b5d9f06b..b59b6c95f4 100755 --- a/sf.py +++ b/sf.py @@ -201,7 +201,7 @@ def main() -> None: for m in sorted(sfModules.keys()): if "__" in m: continue - print(('{0:25} {1}'.format(m, sfModules[m]['descr']))) + print(f"{m.ljust(25)} {sfModules[m]['descr']}") sys.exit(0) if args.types: @@ -213,7 +213,7 @@ def main() -> None: types[r[1]] = r[0] for t in sorted(types.keys()): - print(('{0:45} {1}'.format(t, types[t]))) + print(f"{t.ljust(45)} {types[t]}") sys.exit(0) if args.l: @@ -413,12 +413,12 @@ def start_scan(sfConfig: dict, sfModules: dict, args, loggingQueue) -> None: if args.r: if delim == "\t": - headers = '{0:30}{1}{2:45}{3}{4}{5}{6}'.format("Source", delim, "Type", delim, "Source Data", delim, "Data") + headers = delim.join(["Source".ljust(30), "Type".ljust(45), "Source Data", "Data"]) else: headers = delim.join(["Source", "Type", "Source Data", "Data"]) else: if delim == "\t": - headers = '{0:30}{1}{2:45}{3}{4}'.format("Source", delim, "Type", delim, "Data") + headers = delim.join(["Source".ljust(30), "Type".ljust(45), "Data"]) else: headers = delim.join(["Source", "Type", "Data"]) diff --git a/sflib.py b/sflib.py index c8a0e98747..2c56532781 100644 --- a/sflib.py +++ b/sflib.py @@ -317,7 +317,7 @@ def configSerialize(self, opts: dict, filterSystem: bool = True): TypeError: arg type was invalid """ if not isinstance(opts, dict): - raise TypeError("opts is %s; expected dict()" % type(opts)) + raise TypeError(f"opts is {type(opts)}; expected dict()") storeopts = dict() @@ -426,7 +426,7 @@ def configUnserialize(self, opts: dict, referencePoint: dict, filterSystem: bool return returnOpts if not isinstance(referencePoint['__modules__'], dict): - raise TypeError("referencePoint['__modules__'] is %s; expected dict()" % type(referencePoint['__modules__'])) + raise TypeError(f"referencePoint['__modules__'] is {type(referencePoint['__modules__'])}; expected dict()") # Module options # A lot of mess to handle typing.. @@ -938,11 +938,7 @@ def validateIP(self, host: str, ip: str) -> bool: if not addrs: return False - for addr in addrs: - if str(addr) == ip: - return True - - return False + return any(str(addr) == ip for addr in addrs) def safeSocket(self, host: str, port: int, timeout: int) -> 'ssl.SSLSocket': """Create a safe socket that's using SOCKS/TOR if it was enabled. @@ -991,7 +987,7 @@ def sslDerToPem(self, der_cert: bytes) -> str: """ if not isinstance(der_cert, bytes): - raise TypeError("der_cert is %s; expected bytes()" % type(der_cert)) + raise TypeError(f"der_cert is {type(der_cert)}; expected bytes()") return ssl.DER_cert_to_PEM_cert(der_cert) diff --git a/sfscan.py b/sfscan.py index 702ea5790c..e2eefb07a8 100644 --- a/sfscan.py +++ b/sfscan.py @@ -388,11 +388,9 @@ def __startScan(self) -> None: psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain - if self.__targetType == 'INTERNET_NAME': - if self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): - firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, - "SpiderFoot UI", rootEvent) - psMod.notifyListeners(firstEvent) + if self.__targetType == 'INTERNET_NAME' and self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): + firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, "SpiderFoot UI", rootEvent) + psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to @@ -569,11 +567,9 @@ def threadsFinished(self, log_status: bool = False) -> bool: if not modules_running and not queues_empty: self.__sf.debug("Clearing queues for stalled/aborted modules.") for mod in self.__moduleInstances.values(): - try: + with suppress(Exception): while True: mod.incomingEventQueue.get_nowait() - except Exception: - pass if log_status: events_queued = ", ".join([f"{mod}: {qsize:,}" for mod, qsize in modules_waiting[:5] if qsize > 0]) diff --git a/sfwebui.py b/sfwebui.py index 2bf173dce9..680f8cdfc3 100644 --- a/sfwebui.py +++ b/sfwebui.py @@ -412,7 +412,7 @@ def scancorrelationsexport(self: 'SpiderFootWebUi', id: str, filetype: str = "cs cherrypy.response.headers['Pragma'] = "no-cache" return self.buildExcel(rows, headings, sheetNameIndex=0) - elif filetype.lower() == 'csv': + if filetype.lower() == 'csv': fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(headings) @@ -434,8 +434,7 @@ def scancorrelationsexport(self: 'SpiderFootWebUi', id: str, filetype: str = "cs cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue().encode('utf-8') - else: - return self.error("Invalid export filetype.") + return self.error("Invalid export filetype.") @cherrypy.expose def scaneventresultexport(self: 'SpiderFootWebUi', id: str, type: str, filetype: str = "csv", dialect: str = "excel") -> str: @@ -469,7 +468,7 @@ def scaneventresultexport(self: 'SpiderFootWebUi', id: str, type: str, filetype: return self.buildExcel(rows, ["Updated", "Type", "Module", "Source", "F/P", "Data"], sheetNameIndex=1) - elif filetype.lower() == 'csv': + if filetype.lower() == 'csv': fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"]) @@ -486,8 +485,7 @@ def scaneventresultexport(self: 'SpiderFootWebUi', id: str, type: str, filetype: cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue().encode('utf-8') - else: - return self.error("Invalid export filetype.") + return self.error("Invalid export filetype.") @cherrypy.expose def scaneventresultexportmulti(self: 'SpiderFootWebUi', ids: str, filetype: str = "csv", dialect: str = "excel") -> str: @@ -537,7 +535,7 @@ def scaneventresultexportmulti(self: 'SpiderFootWebUi', ids: str, filetype: str return self.buildExcel(rows, ["Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"], sheetNameIndex=2) - elif filetype.lower() == 'csv': + if filetype.lower() == 'csv': fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"]) @@ -559,8 +557,7 @@ def scaneventresultexportmulti(self: 'SpiderFootWebUi', ids: str, filetype: str cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue().encode('utf-8') - else: - return self.error("Invalid export filetype.") + return self.error("Invalid export filetype.") @cherrypy.expose def scansearchresultexport(self: 'SpiderFootWebUi', id: str, eventType: str = None, value: str = None, filetype: str = "csv", dialect: str = "excel") -> str: @@ -594,7 +591,7 @@ def scansearchresultexport(self: 'SpiderFootWebUi', id: str, eventType: str = No return self.buildExcel(rows, ["Updated", "Type", "Module", "Source", "F/P", "Data"], sheetNameIndex=1) - elif filetype.lower() == 'csv': + if filetype.lower() == 'csv': fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"]) @@ -608,8 +605,7 @@ def scansearchresultexport(self: 'SpiderFootWebUi', id: str, eventType: str = No cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue().encode('utf-8') - else: - return self.error("Invalid export filetype.") + return self.error("Invalid export filetype.") @cherrypy.expose def scanexportjsonmulti(self: 'SpiderFootWebUi', ids: str) -> str: @@ -1106,28 +1102,28 @@ def savesettings(self: 'SpiderFootWebUi', allopts: str, token: str, configFile: if str(token) != str(self.token): return self.error(f"Invalid token ({token})") - if configFile: # configFile seems to get set even if a file isn't uploaded - if configFile.file: + # configFile seems to get set even if a file isn't uploaded + if configFile and configFile.file: + try: contents = configFile.file.read() if isinstance(contents, bytes): contents = contents.decode('utf-8') - try: - tmp = dict() - for line in contents.split("\n"): - if "=" not in line: - continue + tmp = dict() + for line in contents.split("\n"): + if "=" not in line: + continue - opt_array = line.strip().split("=") - if len(opt_array) == 1: - opt_array[1] = "" + opt_array = line.strip().split("=") + if len(opt_array) == 1: + opt_array[1] = "" - tmp[opt_array[0]] = '='.join(opt_array[1:]) + tmp[opt_array[0]] = '='.join(opt_array[1:]) - allopts = json.dumps(tmp).encode('utf-8') - except Exception as e: - return self.error(f"Failed to parse input file. Was it generated from SpiderFoot? ({e})") + allopts = json.dumps(tmp).encode('utf-8') + except Exception as e: + return self.error(f"Failed to parse input file. Was it generated from SpiderFoot? ({e})") # Reset config to default if allopts == "RESET": diff --git a/spiderfoot/correlation.py b/spiderfoot/correlation.py index 89e5955729..8b812fb150 100644 --- a/spiderfoot/correlation.py +++ b/spiderfoot/correlation.py @@ -422,10 +422,7 @@ def event_keep(self, event: dict, field: str, patterns: str, patterntype: str) - if "." in field: key, field = field.split(".") - for subevent in event[key]: - if self.event_keep(subevent, field, patterns, patterntype): - return True - return False + return any(self.event_keep(subevent, field, patterns, patterntype) for subevent in event[key]) value = event[field] @@ -638,10 +635,9 @@ def check_event(events: list, reference: list) -> bool: except Exception: pass - if rule['match_method'] == 'exact': - if event_data in reference: - self.log.debug(f"found exact match: {event_data} in {reference}") - return True + if rule['match_method'] == 'exact' and event_data in reference: + self.log.debug(f"found exact match: {event_data} in {reference}") + return True if rule['match_method'] == 'contains': for r in reference: @@ -675,7 +671,7 @@ def check_event(events: list, reference: list) -> bool: # delete the bucket if there are no events > collection 0 if pluszerocount == 0: - del(buckets[bucket]) + del (buckets[bucket]) def analysis_first_collection_only(self, rule: dict, buckets: dict) -> None: """analysis_first_collection_only TBD @@ -699,13 +695,13 @@ def analysis_first_collection_only(self, rule: dict, buckets: dict) -> None: delete = True break if delete: - del(buckets[bucket]) + del (buckets[bucket]) # Remove buckets with collection > 0 values for bucket in list(buckets.keys()): for e in buckets[bucket]: if e['_collection'] > 0: - del(buckets[bucket]) + del (buckets[bucket]) break def analysis_outlier(self, rule: dict, buckets: dict) -> None: @@ -722,7 +718,7 @@ def analysis_outlier(self, rule: dict, buckets: dict) -> None: if len(list(countmap.keys())) == 0: for bucket in list(buckets.keys()): - del(buckets[bucket]) + del (buckets[bucket]) return total = float(sum(countmap.values())) @@ -733,7 +729,7 @@ def analysis_outlier(self, rule: dict, buckets: dict) -> None: if avgpct < rule.get('noisy_percent', 10): self.log.debug(f"Not correlating because the average percent is {avgpct} (too anomalous)") for bucket in list(buckets.keys()): - del(buckets[bucket]) + del (buckets[bucket]) return # Figure out which buckets don't contain outliers and delete them @@ -743,7 +739,7 @@ def analysis_outlier(self, rule: dict, buckets: dict) -> None: delbuckets.append(bucket) for bucket in set(delbuckets): - del(buckets[bucket]) + del (buckets[bucket]) def analysis_threshold(self, rule: dict, buckets: dict) -> None: """analysis_treshold TBD @@ -769,14 +765,14 @@ def analysis_threshold(self, rule: dict, buckets: dict) -> None: # Delete the bucket of events if it didn't meet the # analysis criteria. if bucket in buckets: - del(buckets[bucket]) + del (buckets[bucket]) continue # If we're only looking at the number of times the requested # field appears in the bucket... uniques = len(list(countmap.keys())) if uniques < rule.get('minimum', 0) or uniques > rule.get('maximum', 999999999): - del(buckets[bucket]) + del (buckets[bucket]) def analyze_field_scope(self, field: str) -> list: """Analysis field scope. diff --git a/spiderfoot/db.py b/spiderfoot/db.py index ea0478d193..b206659318 100644 --- a/spiderfoot/db.py +++ b/spiderfoot/db.py @@ -1382,9 +1382,8 @@ def scanEventStore(self, instanceId: str, sfEvent, truncateSize: int = 0) -> Non if not isinstance(sfEvent.module, str): raise TypeError(f"sfEvent.module is {type(sfEvent.module)}; expected str()") from None - if not sfEvent.module: - if sfEvent.eventType != "ROOT": - raise ValueError("sfEvent.module is empty") from None + if not sfEvent.module and sfEvent.eventType != "ROOT": + raise ValueError("sfEvent.module is empty") from None if not isinstance(sfEvent.confidence, int): raise TypeError(f"sfEvent.confidence is {type(sfEvent.confidence)}; expected int()") from None @@ -1404,9 +1403,8 @@ def scanEventStore(self, instanceId: str, sfEvent, truncateSize: int = 0) -> Non if not 0 <= sfEvent.risk <= 100: raise ValueError(f"sfEvent.risk value is {type(sfEvent.risk)}; expected 0 - 100") from None - if not isinstance(sfEvent.sourceEvent, SpiderFootEvent): - if sfEvent.eventType != "ROOT": - raise TypeError(f"sfEvent.sourceEvent is {type(sfEvent.sourceEvent)}; expected str()") from None + if not isinstance(sfEvent.sourceEvent, SpiderFootEvent) and sfEvent.eventType != "ROOT": + raise TypeError(f"sfEvent.sourceEvent is {type(sfEvent.sourceEvent)}; expected str()") from None if not isinstance(sfEvent.sourceEventHash, str): raise TypeError(f"sfEvent.sourceEventHash is {type(sfEvent.sourceEventHash)}; expected str()") from None @@ -1417,9 +1415,8 @@ def scanEventStore(self, instanceId: str, sfEvent, truncateSize: int = 0) -> Non storeData = sfEvent.data # truncate if required - if isinstance(truncateSize, int): - if truncateSize > 0: - storeData = storeData[0:truncateSize] + if isinstance(truncateSize, int) and truncateSize > 0: + storeData = storeData[0:truncateSize] # retrieve scan results qry = "INSERT INTO tbl_scan_results \ diff --git a/spiderfoot/event.py b/spiderfoot/event.py index a34fb82584..5e96d0e60e 100644 --- a/spiderfoot/event.py +++ b/spiderfoot/event.py @@ -226,9 +226,8 @@ def module(self, module: str) -> None: if not isinstance(module, str): raise TypeError(f"module is {type(module )}; expected str()") - if not module: - if self.eventType != "ROOT": - raise ValueError("module is empty") + if not module and self.eventType != "ROOT": + raise ValueError("module is empty") self._module = module @@ -296,9 +295,8 @@ def asDict(self) -> dict: 'source': '' } - if self.sourceEvent is not None: - if self.sourceEvent.data is not None: - evtDict['source'] = self.sourceEvent.data + if self.sourceEvent is not None and self.sourceEvent.data is not None: + evtDict['source'] = self.sourceEvent.data return evtDict diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.py index 382c595110..df12b24536 100644 --- a/spiderfoot/plugin.py +++ b/spiderfoot/plugin.py @@ -329,12 +329,10 @@ def notifyListeners(self, sfEvent) -> None: eventName = sfEvent.eventType eventData = sfEvent.data - if self.__outputFilter__: - # Be strict about what events to pass on, unless they are - # the ROOT event or the event type of the target. - if eventName not in ('ROOT', self.getTarget().targetType): - if eventName not in self.__outputFilter__: - return + # Be strict about what events to pass on, unless they are + # the ROOT event or the event type of the target. + if self.__outputFilter__ and eventName not in ['ROOT', self.getTarget().targetType, self.__outputFilter__]: + return storeOnly = False # Under some conditions, only store and don't notify @@ -360,10 +358,9 @@ def notifyListeners(self, sfEvent) -> None: prevEvent = sfEvent.sourceEvent while prevEvent is not None: - if prevEvent.sourceEvent is not None: - if prevEvent.sourceEvent.eventType == sfEvent.eventType and prevEvent.sourceEvent.data.lower() == eventData.lower(): - storeOnly = True - break + if prevEvent.sourceEvent is not None and prevEvent.sourceEvent.eventType == sfEvent.eventType and prevEvent.sourceEvent.data.lower() == eventData.lower(): + storeOnly = True + break prevEvent = prevEvent.sourceEvent # output to queue if applicable diff --git a/test/unit/spiderfoot/test_spiderfoothelpers.py b/test/unit/spiderfoot/test_spiderfoothelpers.py index e83bd0b79f..77b6415654 100644 --- a/test/unit/spiderfoot/test_spiderfoothelpers.py +++ b/test/unit/spiderfoot/test_spiderfoothelpers.py @@ -178,7 +178,7 @@ def test_extractLinksFromHtml_argument_data_containing_malformed_html_with_links ) def test_extractLinksFromHtml_invalid_url_should_raise_TypeError(self): - invalid_types = [None, bytes(), list(), dict()] + invalid_types = [None, bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError):