From d62fe9170d5422374107d2d8a90cfc87d024d25d Mon Sep 17 00:00:00 2001 From: bcoles Date: Sat, 11 Sep 2021 22:33:31 +1000 Subject: [PATCH] sfwebui: Add annotations and docstrings (#1350) --- setup.cfg | 3 +- sfwebui.py | 179 ++++++++++++++++-------------- test/unit/test_spiderfootwebui.py | 51 ++++++++- 3 files changed, 143 insertions(+), 90 deletions(-) diff --git a/setup.cfg b/setup.cfg index 88c7194618..695c0cdf0e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,6 @@ per-file-ignores = spiderfoot/db.py:SFS101,B902 sf.py:SFS201,B902 sflib.py:SFS101,SIM110,SIM111,B902 - sfwebui.py:A,B902 sfscan.py:SIM105,B902 modules/*:B902 modules/sfp_binaryedge.py:B902,C901 @@ -22,8 +21,10 @@ per-file-ignores = modules/sfp_reversewhois.py:R504 modules/sfp_ripe.py:SIM110,B902 modules/sfp_zetalytics.py:R504,B902 + spiderfoot/__init__.py:F401 sfcli.py:DAR,B902 + sfwebui.py:A001,A002,B902 test/*:SIM117,ANN docs/conf.py:A diff --git a/sfwebui.py b/sfwebui.py index 3f01f2df1c..3b1b0d4bad 100644 --- a/sfwebui.py +++ b/sfwebui.py @@ -52,7 +52,7 @@ class SpiderFootWebUi: token = None docroot = '' - def __init__(self, web_config, config, loggingQueue=None): + def __init__(self: 'SpiderFootWebUi', web_config: dict, config: dict, loggingQueue: 'logging.handlers.QueueListener' = None) -> None: """Initialize web server. Args: @@ -121,7 +121,7 @@ def __init__(self, web_config, config, loggingQueue=None): "tools.response_headers.headers": secure_headers.framework.cherrypy() }) - def error_page(self): + def error_page(self: 'SpiderFootWebUi') -> None: """Error page.""" cherrypy.response.status = 500 @@ -130,7 +130,7 @@ def error_page(self): else: cherrypy.response.body = b"Error" - def error_page_401(self, status, message, traceback, version): + def error_page_401(self: 'SpiderFootWebUi', status: str, message: str, traceback: str, version: str) -> str: """Unauthorized access HTTP 401 error page. Args: @@ -144,7 +144,7 @@ def error_page_401(self, status, message, traceback, version): """ return "" - def error_page_404(self, status, message, traceback, version): + def error_page_404(self: 'SpiderFootWebUi', status: str, message: str, traceback: str, version: str) -> str: """Not found error page 404. Args: @@ -159,7 +159,7 @@ def error_page_404(self, status, message, traceback, version): templ = Template(filename='spiderfoot/templates/error.tmpl', lookup=self.lookup) return templ.render(message='Not Found', docroot=self.docroot, status=status, version=__version__) - def jsonify_error(self, status, message): + def jsonify_error(self: 'SpiderFootWebUi', status: str, message: str) -> dict: """Jsonify error response. Args: @@ -167,7 +167,7 @@ def jsonify_error(self, status, message): message (str): Error message Returns: - str: HTTP response template + dict: HTTP error response template """ cherrypy.response.headers['Content-Type'] = 'application/json' cherrypy.response.status = status @@ -178,7 +178,7 @@ def jsonify_error(self, status, message): } } - def error(self, message): + def error(self: 'SpiderFootWebUi', message: str) -> None: """Show generic error page with error message. Args: @@ -190,7 +190,7 @@ def error(self, message): templ = Template(filename='spiderfoot/templates/error.tmpl', lookup=self.lookup) return templ.render(message=message, docroot=self.docroot, version=__version__) - def cleanUserInput(self, inputList): + def cleanUserInput(self: 'SpiderFootWebUi', inputList: list) -> list: """Sanitize user input, poorly. Args: @@ -219,13 +219,13 @@ def cleanUserInput(self, inputList): return ret - def searchBase(self, id=None, eventType=None, value=None): + def searchBase(self: 'SpiderFootWebUi', id: str = None, eventType: str = None, value: str = None) -> list: """Search. Args: - id: TBD - eventType: TBD - value: TBD + id (str): scan ID + eventType (str): TBD + value (str): TBD Returns: list: search results @@ -271,7 +271,17 @@ def searchBase(self, id=None, eventType=None, value=None): return retdata - def buildExcel(self, data, columnNames, sheetNameIndex=0): + def buildExcel(self: 'SpiderFootWebUi', data: list, columnNames: list, sheetNameIndex: int = 0) -> str: + """Convert supplied raw data into GEXF (Graph Exchange XML Format) format (e.g. for Gephi). + + Args: + data (list): Scan result as list + columnNames (list): column names + sheetNameIndex (int): TBD + + Returns: + str: Excel workbook + """ rowNums = dict() workbook = openpyxl.Workbook() defaultSheet = workbook.active @@ -315,7 +325,7 @@ def buildExcel(self, data, columnNames, sheetNameIndex=0): # @cherrypy.expose - def scanexportlogs(self, id, dialect="excel") -> str: + def scanexportlogs(self: 'SpiderFootWebUi', id: str, dialect: str = "excel") -> bytes: """Get scan log Args: @@ -323,7 +333,7 @@ def scanexportlogs(self, id, dialect="excel") -> str: dialect (str): CSV dialect (default: excel) Returns: - str: scan logs in CSV format + bytes: scan logs in CSV format """ dbh = SpiderFootDb(self.config) @@ -353,7 +363,7 @@ def scanexportlogs(self, id, dialect="excel") -> str: return fileobj.getvalue().encode('utf-8') @cherrypy.expose - def scaneventresultexport(self, id, type, filetype="csv", dialect="excel") -> str: + def scaneventresultexport(self: 'SpiderFootWebUi', id: str, type: str, filetype: str = "csv", dialect: str = "excel") -> str: """Get scan event result data in CSV or Excel format Args: @@ -405,7 +415,7 @@ def scaneventresultexport(self, id, type, filetype="csv", dialect="excel") -> st return self.error("Invalid export filetype.") @cherrypy.expose - def scaneventresultexportmulti(self, ids, filetype="csv", dialect="excel") -> str: + def scaneventresultexportmulti(self: 'SpiderFootWebUi', ids: str, filetype: str = "csv", dialect: str = "excel") -> str: """Get scan event result data in CSV or Excel format for multiple scans Args: @@ -478,7 +488,7 @@ def scaneventresultexportmulti(self, ids, filetype="csv", dialect="excel") -> st return self.error("Invalid export filetype.") @cherrypy.expose - def scansearchresultexport(self, id, eventType=None, value=None, filetype="csv", dialect="excel") -> str: + def scansearchresultexport(self: 'SpiderFootWebUi', id: str, eventType: str = None, value: str = None, filetype: str = "csv", dialect: str = "excel") -> str: """Get search result data in CSV or Excel format Args: @@ -527,7 +537,7 @@ def scansearchresultexport(self, id, eventType=None, value=None, filetype="csv", return self.error("Invalid export filetype.") @cherrypy.expose - def scanexportjsonmulti(self, ids) -> str: + def scanexportjsonmulti(self: 'SpiderFootWebUi', ids: str) -> str: """Get scan event result data in JSON format for multiple scans. Args: @@ -581,7 +591,7 @@ def scanexportjsonmulti(self, ids) -> str: return json.dumps(scaninfo).encode('utf-8') @cherrypy.expose - def scanviz(self, id, gexf="0"): + def scanviz(self: 'SpiderFootWebUi', id: str, gexf: str = "0") -> str: """Export entities from scan results for visualising. Args: @@ -619,7 +629,7 @@ def scanviz(self, id, gexf="0"): return SpiderFootHelpers.buildGraphGexf([root], "SpiderFoot Export", data) @cherrypy.expose - def scanvizmulti(self, ids, gexf="1") -> str: + def scanvizmulti(self: 'SpiderFootWebUi', ids: str, gexf: str = "1") -> str: """Export entities results from multiple scans in GEXF format. Args: @@ -661,7 +671,7 @@ def scanvizmulti(self, ids, gexf="1") -> str: @cherrypy.expose @cherrypy.tools.json_out() - def scanopts(self, id): + def scanopts(self: 'SpiderFootWebUi', id: str) -> str: """Return configuration used for the specified scan as JSON. Args: @@ -708,7 +718,7 @@ def scanopts(self, id): return ret @cherrypy.expose - def rerunscan(self, id): + def rerunscan(self: 'SpiderFootWebUi', id: str) -> None: """Rerun a scan. Args: @@ -768,14 +778,14 @@ def rerunscan(self, id): raise cherrypy.HTTPRedirect(f"{self.docroot}/scaninfo?id={scanId}", status=302) @cherrypy.expose - def rerunscanmulti(self, ids): + def rerunscanmulti(self: 'SpiderFootWebUi', ids: str) -> str: """Rerun scans. Args: ids (str): comma separated list of scan IDs Returns: - None + str: Scan list page HTML """ # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) @@ -823,11 +833,11 @@ def rerunscanmulti(self, ids): return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST", version=__version__) @cherrypy.expose - def newscan(self): + def newscan(self: 'SpiderFootWebUi') -> str: """Configure a new scan. Returns: - None + str: New scan page HTML """ dbh = SpiderFootDb(self.config) types = dbh.eventTypes() @@ -837,15 +847,14 @@ def newscan(self): selectedmods="", scantarget="", version=__version__) @cherrypy.expose - def clonescan(self, id): - """ - Clone an existing scan (pre-selected options in the newscan page). + def clonescan(self: 'SpiderFootWebUi', id: str) -> str: + """Clone an existing scan (pre-selected options in the newscan page). Args: id (str): scan ID to clone Returns: - None + str: New scan page HTML pre-populated with options from cloned scan. """ dbh = SpiderFootDb(self.config) types = dbh.eventTypes() @@ -876,24 +885,24 @@ def clonescan(self, id): scantarget=str(scantarget), version=__version__) @cherrypy.expose - def index(self): + def index(self: 'SpiderFootWebUi') -> str: """Show scan list page. Returns: - None + str: Scan list page HTML """ templ = Template(filename='spiderfoot/templates/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', docroot=self.docroot, version=__version__) @cherrypy.expose - def scaninfo(self, id): + def scaninfo(self: 'SpiderFootWebUi', id: str) -> str: """Information about a selected scan. Args: id (str): scan id Returns: - None + str: scan info page HTML """ dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) @@ -905,14 +914,14 @@ def scaninfo(self, id): pageid="SCANLIST") @cherrypy.expose - def opts(self, updated=None): + def opts(self: 'SpiderFootWebUi', updated: str = None) -> str: """Show module and global settings page. Args: - updated: TBD + updated (str): scan options were updated successfully Returns: - None + str: scan options page HTML """ templ = Template(filename='spiderfoot/templates/opts.tmpl', lookup=self.lookup) self.token = random.SystemRandom().randint(0, 99999999) @@ -920,14 +929,14 @@ def opts(self, updated=None): updated=updated, docroot=self.docroot) @cherrypy.expose - def optsexport(self, pattern=None): + def optsexport(self: 'SpiderFootWebUi', pattern: str = None) -> str: """Export configuration. Args: - pattern: TBD + pattern (str): TBD Returns: - None + str: Configuration settings """ sf = SpiderFoot(self.config) conf = sf.configSerialize(self.config) @@ -949,7 +958,7 @@ def optsexport(self, pattern=None): @cherrypy.expose @cherrypy.tools.json_out() - def optsraw(self): + def optsraw(self: 'SpiderFootWebUi') -> str: """Return global and module settings as json. Returns: @@ -973,7 +982,7 @@ def optsraw(self): @cherrypy.expose @cherrypy.tools.json_out() - def scandelete(self, id): + def scandelete(self: 'SpiderFootWebUi', id: str) -> str: """Delete scan(s). Args: @@ -1002,13 +1011,13 @@ def scandelete(self, id): return "" @cherrypy.expose - def savesettings(self, allopts, token, configFile=None): + def savesettings(self: 'SpiderFootWebUi', allopts: str, token: str, configFile: 'cherrypy._cpreqbody.Part' = None) -> None: """Save settings, also used to completely reset them to default. Args: allopts: TBD - token: CSRF token - configFile: TBD + token (str): CSRF token + configFile (cherrypy._cpreqbody.Part): TBD Returns: None @@ -1017,7 +1026,7 @@ def savesettings(self, allopts, token, configFile=None): HTTPRedirect: redirect to scan settings """ if str(token) != str(self.token): - return self.error(f"Invalid token ({self.token})") + return self.error(f"Invalid token ({token})") if configFile: # configFile seems to get set even if a file isn't uploaded if configFile.file: @@ -1069,12 +1078,12 @@ def savesettings(self, allopts, token, configFile=None): raise cherrypy.HTTPRedirect(f"{self.docroot}/opts?updated=1") @cherrypy.expose - def savesettingsraw(self, allopts, token): + def savesettingsraw(self: 'SpiderFootWebUi', allopts: str, token: str) -> str: """Save settings, also used to completely reset them to default. Args: allopts: TBD - token: CSRF token + token (str): CSRF token Returns: str: save success as JSON @@ -1082,7 +1091,7 @@ def savesettingsraw(self, allopts, token): cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" if str(token) != str(self.token): - return json.dumps(["ERROR", f"Invalid token ({self.token})."]).encode('utf-8') + return json.dumps(["ERROR", f"Invalid token ({token})."]).encode('utf-8') # Reset config to default if allopts == "RESET": @@ -1110,7 +1119,7 @@ def savesettingsraw(self, allopts, token): return json.dumps(["SUCCESS", ""]).encode('utf-8') - def reset_settings(self): + def reset_settings(self: 'SpiderFootWebUi') -> bool: """Reset settings to default. Returns: @@ -1126,7 +1135,7 @@ def reset_settings(self): return True @cherrypy.expose - def resultsetfp(self, id, resultids, fp): + def resultsetfp(self: 'SpiderFootWebUi', id: str, resultids: str, fp: str) -> str: """Set a bunch of results (hashes) as false positive. Args: @@ -1183,7 +1192,7 @@ def resultsetfp(self, id, resultids, fp): @cherrypy.expose @cherrypy.tools.json_out() - def eventtypes(self): + def eventtypes(self: 'SpiderFootWebUi') -> str: """List all event types. Returns: @@ -1202,7 +1211,7 @@ def eventtypes(self): @cherrypy.expose @cherrypy.tools.json_out() - def modules(self): + def modules(self: 'SpiderFootWebUi') -> str: """List all modules. Returns: @@ -1222,17 +1231,17 @@ def modules(self): @cherrypy.expose @cherrypy.tools.json_out() - def ping(self): + def ping(self: 'SpiderFootWebUi') -> list: """For the CLI to test connectivity to this server. Returns: - str: SpiderFoot version as JSON + list: SpiderFoot version as JSON """ return ["SUCCESS", __version__] @cherrypy.expose @cherrypy.tools.json_out() - def query(self, query): + def query(self: 'SpiderFootWebUi', query: str) -> str: """For the CLI to run queries against the database. Args: @@ -1258,7 +1267,7 @@ def query(self, query): return self.jsonify_error('500', str(e)) @cherrypy.expose - def startscan(self, scanname, scantarget, modulelist, typelist, usecase): + def startscan(self: 'SpiderFootWebUi', scanname: str, scantarget: str, modulelist: str, typelist: str, usecase: str) -> str: """Initiate a scan. Args: @@ -1391,7 +1400,7 @@ def startscan(self, scanname, scantarget, modulelist, typelist, usecase): @cherrypy.expose @cherrypy.tools.json_out() - def stopscan(self, id): + def stopscan(self: 'SpiderFootWebUi', id: str) -> str: """Stop a scan. Args: @@ -1433,17 +1442,17 @@ def stopscan(self, id): @cherrypy.expose @cherrypy.tools.json_out() - def scanlog(self, id, limit=None, rowId=None, reverse=None): + def scanlog(self: 'SpiderFootWebUi', id: str, limit: str = None, rowId: str = None, reverse: str = None) -> list: """Scan log data. Args: id (str): scan ID - limit: TBD - rowId: TBD - reverse: TBD + limit (str): TBD + rowId (str): TBD + reverse (str): TBD Returns: - str: JSON + list: scan log """ dbh = SpiderFootDb(self.config) retdata = [] @@ -1461,15 +1470,15 @@ def scanlog(self, id, limit=None, rowId=None, reverse=None): @cherrypy.expose @cherrypy.tools.json_out() - def scanerrors(self, id, limit=None): + def scanerrors(self: 'SpiderFootWebUi', id: str, limit: str = None) -> list: """Scan error data. Args: id (str): scan ID - limit: TBD + limit (str): limit number of results Returns: - str: scan errors as JSON + list: scan errors """ dbh = SpiderFootDb(self.config) retdata = [] @@ -1487,11 +1496,11 @@ def scanerrors(self, id, limit=None): @cherrypy.expose @cherrypy.tools.json_out() - def scanlist(self): + def scanlist(self: 'SpiderFootWebUi') -> list: """Produce a list of scans. Returns: - str: scan list as JSON + list: scan list """ dbh = SpiderFootDb(self.config) data = dbh.scanInstanceList() @@ -1516,14 +1525,14 @@ def scanlist(self): @cherrypy.expose @cherrypy.tools.json_out() - def scanstatus(self, id): + def scanstatus(self: 'SpiderFootWebUi', id: str) -> list: """Show basic information about a scan, including status and number of each event type. Args: id (str): scan ID Returns: - str: scan status as JSON + list: scan status """ dbh = SpiderFootDb(self.config) data = dbh.scanInstanceGet(id) @@ -1539,15 +1548,15 @@ def scanstatus(self, id): @cherrypy.expose @cherrypy.tools.json_out() - def scansummary(self, id, by): + def scansummary(self: 'SpiderFootWebUi', id: str, by: str) -> list: """Summary of scan results. Args: id (str): scan ID - by: TBD + by (str): filter by type Returns: - str: scan summary as JSON + list: scan summary """ retdata = [] @@ -1573,16 +1582,16 @@ def scansummary(self, id, by): @cherrypy.expose @cherrypy.tools.json_out() - def scaneventresults(self, id, eventType, filterfp=False): + def scaneventresults(self: 'SpiderFootWebUi', id: str, eventType: str, filterfp: bool = False) -> list: """Return all event results for a scan as JSON. Args: id (str): scan ID eventType (str): filter by event type - filterfp: TBD + filterfp (bool): remove false positives from search results Returns: - str: scan results as JSON + list: scan results """ retdata = [] @@ -1613,7 +1622,7 @@ def scaneventresults(self, id, eventType, filterfp=False): @cherrypy.expose @cherrypy.tools.json_out() - def scaneventresultsunique(self, id, eventType, filterfp=False): + def scaneventresultsunique(self: 'SpiderFootWebUi', id: str, eventType: str, filterfp: bool = False) -> list: """Return unique event results for a scan as JSON. Args: @@ -1622,7 +1631,7 @@ def scaneventresultsunique(self, id, eventType, filterfp=False): filterfp (bool): remove false positives from search results Returns: - str: unique search results as JSON + list: unique search results """ dbh = SpiderFootDb(self.config) retdata = [] @@ -1640,7 +1649,7 @@ def scaneventresultsunique(self, id, eventType, filterfp=False): @cherrypy.expose @cherrypy.tools.json_out() - def search(self, id=None, eventType=None, value=None): + def search(self: 'SpiderFootWebUi', id: str = None, eventType: str = None, value: str = None) -> list: """Search scans. Args: @@ -1649,7 +1658,7 @@ def search(self, id=None, eventType=None, value=None): value (str): filter search results by event value Returns: - str: search results as JSON + list: search results """ try: return self.searchBase(id, eventType, value) @@ -1658,14 +1667,14 @@ def search(self, id=None, eventType=None, value=None): @cherrypy.expose @cherrypy.tools.json_out() - def scanhistory(self, id): + def scanhistory(self: 'SpiderFootWebUi', id: str) -> list: """Historical data for a scan. Args: id (str): scan ID Returns: - str: scan history as JSON + list: scan history """ if not id: return self.jsonify_error('404', "No scan specified") @@ -1679,7 +1688,7 @@ def scanhistory(self, id): @cherrypy.expose @cherrypy.tools.json_out() - def scanelementtypediscovery(self, id, eventType): + def scanelementtypediscovery(self: 'SpiderFootWebUi', id: str, eventType: str) -> dict: """Scan element type discovery. Args: @@ -1687,7 +1696,7 @@ def scanelementtypediscovery(self, id, eventType): eventType (str): filter by event type Returns: - str: JSON + dict """ dbh = SpiderFootDb(self.config) pc = dict() diff --git a/test/unit/test_spiderfootwebui.py b/test/unit/test_spiderfootwebui.py index 00fb507c4d..e433d0f463 100644 --- a/test/unit/test_spiderfootwebui.py +++ b/test/unit/test_spiderfootwebui.py @@ -11,15 +11,25 @@ class TestSpiderFootWebUi(unittest.TestCase): Test SpiderFootWebUi """ - def test_init_no_options_should_raise(self): + def test_init_config_invalid_type_should_raise_TypeError(self): """ - Test __init__(self, config) + Test __init__(self, config, web_config) + """ + opts = self.default_options + opts['__modules__'] = dict() + + with self.assertRaises(TypeError): + SpiderFootWebUi(None, opts) + + def test_init_no_web_config_should_raise(self): + """ + Test __init__(self, config, web_config) """ with self.assertRaises(TypeError): - SpiderFootWebUi(None, None) + SpiderFootWebUi(self.web_default_options, None) with self.assertRaises(ValueError): - SpiderFootWebUi(dict(), dict()) + SpiderFootWebUi(self.web_default_options, dict()) def test_init(self): """ @@ -41,6 +51,17 @@ def test_error_page(self): sfwebui = SpiderFootWebUi(self.web_default_options, opts) sfwebui.error_page() + def test_error_page_401(self): + """ + Test error_page(self) + """ + opts = self.default_options + opts['__modules__'] = dict() + + sfwebui = SpiderFootWebUi(self.web_default_options, opts) + error_page_401 = sfwebui.error_page_401(None, None, None, None) + self.assertIsInstance(error_page_401, str) + def test_error_page_404(self): """ Test error_page_404(self, status, message, traceback, version) @@ -128,6 +149,28 @@ def test_scan_search_result_export(self): search_results = sfwebui.scansearchresultexport("", None, None, "excel") self.assertIsInstance(search_results, bytes) + def test_scan_export_logs_invalid_scan_id_should_return_string(self): + """ + Test scanexportlogs(self: 'SpiderFootWebUi', id: str, dialect: str = "excel") -> str + """ + opts = self.default_options + opts['__modules__'] = dict() + sfwebui = SpiderFootWebUi(self.web_default_options, opts) + logs = sfwebui.scanexportlogs(None, "excel") + self.assertIsInstance(logs, str) + self.assertIn("Scan ID not found.", logs) + + @unittest.skip("todo") + def test_scan_export_logs_should_return_bytes(self): + """ + Test scanexportlogs(self: 'SpiderFootWebUi', id: str, dialect: str = "excel") -> str + """ + opts = self.default_options + opts['__modules__'] = dict() + sfwebui = SpiderFootWebUi(self.web_default_options, opts) + logs = sfwebui.scanexportlogs("scan id", "excel") + self.assertIsInstance(logs, bytes) + @unittest.skip("todo") def test_scan_export_json_multi(self): """