From dccf21fc837592c0319b3df5bafc376a6a773088 Mon Sep 17 00:00:00 2001 From: Pierre Lalet Date: Wed, 15 Sep 2021 22:33:40 +0200 Subject: [PATCH] Add IVRE module --- modules/sfp_ivre.py | 344 ++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 345 insertions(+) create mode 100644 modules/sfp_ivre.py diff --git a/modules/sfp_ivre.py b/modules/sfp_ivre.py new file mode 100644 index 00000000000..02f4098ae00 --- /dev/null +++ b/modules/sfp_ivre.py @@ -0,0 +1,344 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------- +# Name: sfp_ivre +# Purpose: Query an IVRE instance +# +# Author: Pierre Lalet +# +# Created: 2021-09-14 +# Copyright: (c) Pierre Lalet +# Licence: GPL +# ------------------------------------------------------------------------------- + +import hashlib +import json + +from netaddr import IPNetwork +from ivre.db import db, DBPassive +from ivre import config, utils + +from spiderfoot import SpiderFootEvent, SpiderFootPlugin + + +def flt_from_asnum(dbase, asnum): + return dbase.searchasnum(int(asnum)) + + +def flt_from_ip(dbase, addr): + yield dbase.searchhost(addr) + + +def flt_from_net(dbase, net): + yield dbase.searchnet(net) + + +def flt_from_port(dbase, proto, port): + try: + yield dbase.searchport(port, protocol=proto) + except ValueError: + yield dbase.searchnonexistent() + + +def flt_from_cert(dbase, cert): + yield dbase.searchcert( + sha256=hashlib.new("sha256", utils.decode_b64(cert.encode())).hexdigest() + ) + + +def flt_from_fqdn(dbase, name): + if isinstance(dbase, DBPassive): + yield dbase.searchdns(name=name) + yield dbase.searchdns(name=name, reverse=True) + else: + yield dbase.searchhostname(name) + + +EVENTS_FILTERS = { + "BGP_AS_MEMBER": flt_from_asnum, + "IP_ADDRESS": flt_from_ip, + "IPV6_ADDRESS": flt_from_ip, + "NETBLOCK_MEMBER": flt_from_net, + # we probably don't want to get all hosts with port 80 open... + # "TCP_PORT_OPEN": lambda dbase, port: flt_from_port(dbase, "tcp", port), + # "UDP_PORT_OPEN": lambda dbase, port: flt_from_port(dbase, "udp", port), + "SSL_CERTIFICATE_RAW": flt_from_cert, + "INTERNET_NAME": flt_from_fqdn, +} + + +class sfp_ivre(SpiderFootPlugin): + + meta = { + "name": "IVRE", + "summary": "Obtain information from an IVRE instance.", + # This module could be flagged as "tool", but it does not run + # an external tool to collect data, it relies on data already + # collected in IVRE. + "flags": [], + "useCases": ["Passive", "Footprint", "Investigate"], + "categories": ["Crawling and Scanning", "DNS", "Passive DNS"], + "dataSource": { + "website": "https://ivre.rocks/", + "model": "PRIVATE_ONLY", + "references": ["https://ivre.rocks/"], + "favIcon": "https://ivre.rocks/favicon.ico", + "logo": "https://ivre.rocks/logo-big.png", + "description": "Unlike other modules, IVRE is not a service but " + "an open-source software you need to run. Install an IVRE instance, " + "configure it, and add your own data (scan results, passive intelligence).
" + "Important: set the use_passive, use_scans " + "and/or use_data settings to True to enable the module.
" + "IVRE has several use-cases, such as running your own Shodan-like service " + "(based on powerful open-source tools such as Masscan, Nmap, ZGrab2, ZDns), " + "passively gather intelligence from network traffic (including running a " + "Passive DNS service, collecting and analyzing X509 certificates, HTTP " + "headers, TCP banners, etc.), analyzing scanners hits against simple " + "honeypots, etc.
You may want to read " + 'IVRE use ' + "cases.
" + }, + } + + opts = { + "check_asmembers_bool": False, + "check_netmembers_bool": False, + "check_asmembers_max": 4096, + "check_netmembers_max": 24, + "use_data": False, + "use_passive": False, + "use_scans": False, + } + + optdescs = { + "check_asmembers_bool": "Check hosts from AS numbers?", + "check_netmembers_bool": "Check hosts from netblocks?", + "check_asmembers_max": "If looking up AS members, the maximum number of addresses in an AS to look up all IPs within", + "check_netmembers_max": "If looking up netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", + "use_data": "Use data from the data purpose (MaxMind)", + "use_passive": "Use data from the passive purpose", + "use_scans": "Use data from the nmap (scans) purpose", + } + + # Tracking results can be helpful to avoid reporting/processing duplicates + results = None + + # Tracking the error state of the module can be useful to detect when a third party + # has failed and you don't wish to process any more events. + errorState = False + + def setup(self, sfc, userOpts=None): + self.sf = sfc + self.results = self.tempStorage() + if userOpts is None: + userOpts = {} + + for opt in list(userOpts): + self.opts[opt] = userOpts[opt] + + self.db = db + + + def watchedEvents(self): + return list(EVENTS_FILTERS) + # return [ + # "BGP_AS_MEMBER", + # "NETBLOCK_MEMBER", + # "IP_ADDRESS", + # "IPV6_ADDRESS", + # "DOMAIN_NAME", + # "DOMAIN_NAME_PARENT", + # "INTERNET_NAME", + # "TCP_PORT_OPEN", + # "TCP_PORT_OPEN_BANNER", + # "WEBSERVER_BANNER", + # "WEBSERVER_HTTPHEADERS", + # "UDP_PORT_OPEN", + # "UDP_PORT_OPEN_INFO", + # "SSL_CERTIFICATE_RAW", + # ] + + def producedEvents(self): + return [ + "BGP_AS_MEMBER", + "NETBLOCK_MEMBER", + "IP_ADDRESS", + "IPV6_ADDRESS", + "DOMAIN_NAME", + "DOMAIN_NAME_PARENT", + "INTERNET_NAME", + "TCP_PORT_OPEN", + "TCP_PORT_OPEN_BANNER", + "WEBSERVER_BANNER", + "WEBSERVER_HTTPHEADERS", + "UDP_PORT_OPEN", + "UDP_PORT_OPEN_INFO", + "SSL_CERTIFICATE_RAW", + "GEOINFO", + "COUNTRY_NAME", + "SOFTWARE_USED", + "USERNAME", + ] + + _GEOINFO_KEYS = [ + ["city"], + ["region_name", "region_code"], + ["postal_code"], + ["country_code", "country_name"], + ] + + def handle_data_record(self, event, rec): + if "country_name" in rec: + yield SpiderFootEvent( + "COUNTRY_NAME", rec["country_name"], self.__name__, event + ) + elif "country_code" in rec: + yield SpiderFootEvent( + "COUNTRY_NAME", rec["country_code"], self.__name__, event + ) + if "as_num" in rec: + yield SpiderFootEvent( + "BGP_AS_MEMBER", str(rec["as_num"]), self.__name__, event + ) + if any(key in rec for keys in self._GEOINFO_KEYS for key in keys): + location = [] + for keys in _GEOINFO_KEYS: + for key in keys: + if key in rec: + location.append(rec[key]) + break + yield SpiderFootEvent("GEOINFO", ", ".join(location), self.__name__, event) + + def handle_passive_record(self, event, rec): + if "addr" in rec: + if ":" in rec["addr"]: + yield SpiderFootEvent("IPV6_ADDRESS", rec["addr"], self.__name__, event) + else: + yield SpiderFootEvent("IP_ADDRESS", rec["addr"], self.__name__, event) + if rec["recontype"] == "DNS_ANSWER": + names = [rec["value"]] + if "targetval" in rec: + names.append(rec["targetval"]) + for name in names: + newevt = SpiderFootEvent("INTERNET_NAME", name, self.__name__, event) + yield newevt + if "." not in name: + continue + domain = name.split(".", 1)[1] + yield SpiderFootEvent("DOMAIN_NAME", domain, self.__name__, newevt) + return + if rec["recontype"] == "SSL_SERVER": + if rec["source"] == "cert": + yield SpiderFootEvent( + "SSL_CERTIFICATE_RAW", + utils.encode_b64(rec["value"]).decode(), + self.__name__, + event, + ) + return + if rec["recontype"] == "OPEN_PORT": + yield SpiderFootEvent( + "%s_PORT_OPEN" % rec["source"], + "%(addr)s:%(port)d" % rec, + self.__name__, + event, + ) + return + if rec["recontype"] == "TCP_SERVER_BANNER": + yield SpiderFootEvent( + "TCP_PORT_OPEN_BANNER", + rec["value"], + self.__name__, + event, + ) + return + + def handle_nmap_record(self, event, rec): + for hname in rec.get("hostnames", []): + yield SpiderFootEvent("INTERNET_NAME", hname["name"], self.__name__, event) + for port in rec.get("ports", []): + if port["port"] != -1: + if port["protocol"] in {"tcp", "udp"}: + yield SpiderFootEvent( + "%s_PORT_OPEN" % port["protocol"].upper(), + f"%{rec['addr']}:{port['port']}", + self.__name__, + event, + ) + for script in port.get("scripts", []): + if script["id"] == "banner" and port["protocol"] == "tcp": + yield SpiderFootEvent( + "TCP_PORT_OPEN_BANNER", + script["output"], + self.__name__, + event, + ) + continue + if script["id"] == "ssl-cert": + for cert in script.get("ssl-cert", []): + if "pem" not in cert: + continue + yield SpiderFootEvent( + "SSL_CERTIFICATE_RAW", + "".join(cert["pem"].splitlines()[1:-1]), + self.__name__, + event, + ) + + # Handle events sent to this module + def handleEvent(self, event): + eventName = event.eventType + srcModuleName = event.module + eventData = event.data + + # Once we are in this state, return immediately. + if self.errorState: + return + + self.sf.debug(f"Received event, {eventName}, from {srcModuleName}") + + # Don't look up stuff twice + if eventData in self.results: + self.sf.debug(f"Skipping {eventData}, already checked.") + return + + self.results[eventData] = True + + # Data + if self.opts["use_data"] and eventName in {"IP_ADDRESS", "IPV6_ADDRESS"}: + self.handle_data_record(event, self.db.data.infos_byip(eventData) or {}) + + if eventName == "NETBLOCK_OWNER": + if not self.opts["check_netmembers_bool"]: + self.sf.debug("NETBLOCK_OWNER lookups disabled") + return + max_netblock = self.opts["check_netmembers_max"] + net_size = IPNetwork(eventData).prefixlen + if net_size < max_netblock: + self.sf.debug( + f"Network size {net_size} bigger than permitted: {max_netblock}" + ) + return + + if eventName not in EVENTS_FILTERS: + self.sf.debug(f"Event {eventName} not handled") + return + + find_flt = EVENTS_FILTERS[eventName] + + for dbase, run, handler in [ + (self.db.passive, self.opts["use_passive"], self.handle_passive_record), + (self.db.nmap, self.opts["use_scans"], self.handle_nmap_record), + ]: + + if self.checkForStop(): + return + + if not run: + continue + + for flt in find_flt(dbase, eventData): + for rec in dbase.get(flt): + self.sf.debug(f"Record! {rec}") + for evt in handler(event, rec): + if evt.data != eventData: + self.notifyListeners(evt) diff --git a/requirements.txt b/requirements.txt index a8f12f360ae..9b91c62a231 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ pysocks>=1.7.1,<2 requests>=2.26.0,<3 ipwhois>=1.1.0,<1.2.0 ipaddr>=2.2.0,<3 +ivre>=0.9.16 phonenumbers>=8.12.28,<9 pygexf>=0.2.2,<0.3 PyPDF2>=1.26.0,<2