PcapXray 2.7 (#47)

* 🚧 mac crash fix for interactive graph

* 🚧 moving module calls and init as tweaks to increase perf

* ➕ (dont hide any traffic) show unknown or other clear text protocols

* graph adjustments + button option

* fix icmp identifier

* 👁️ add covert tunnel detect - starting with icmp

* more filter for covert algo

* add dns covert exfiltration filter - init

* separate multicast traffic

* ❓ ctf solving change -- alter lan schema and cover covert cases

* cleanup lan key logic, persist other changes

* some checks to record different layers ( linux cooked capture )

* handle ipv6 for multicast logic

* 🎮 add more test

Source/Module/communication_details_fetch.py

@@ -18,7 +18,7 @@ def __init__(self, option):
                 if option == "whois":
                     memory.destination_hosts[host]["domain_name"] = self.whois_info_fetch(host)
                 else:
-                    memory.destination_hosts[host]["domain_name"] = self.dns(host)
+                    memory.destination_hosts[host]["domain_name"] = trafficDetailsFetch.dns(host)
 
     def whois_info_fetch(self, ip):
         try:
@@ -27,13 +27,26 @@ def whois_info_fetch(self, ip):
            whois_info = "NoWhoIsInfo"
         return whois_info
 
-    def dns(self, ip):
+    @staticmethod
+    def dns(ip):
         try:
             dns_info = socket.gethostbyaddr(ip)[0]
         except:
             dns_info = "NotResolvable"
         return dns_info
 
+    @staticmethod
+    def is_multicast(ip):
+        if ":" in ip:
+            groups = ip.split(":")
+            if "FF0" in groups[0].upper():
+                return True
+        else:
+            octets = ip.split(".")
+            if int(octets[0]) >= 224:
+                return True
+        return False
+
 def main():
     capture = pcap_reader.PcapEngine('examples/test.pcap', "scapy")
     details = trafficDetailsFetch("sock")

Source/Module/device_details_fetch.py

@@ -20,17 +20,18 @@ def __init__(self, option="ieee"):
         self.target_oui_database = option
 
     def fetch_info(self):
-        for mac in memory.lan_hosts:
+        for host in memory.lan_hosts:
+            mac = host.split("/")[0]
             if self.target_oui_database == "api":
-                memory.lan_hosts[mac]["device_vendor"] = self.oui_identification_via_api(mac)
+                memory.lan_hosts[host]["device_vendor"] = self.oui_identification_via_api(mac)
             else:
-                memory.lan_hosts[mac]["device_vendor"], memory.lan_hosts[mac]["vendor_address"] = self.oui_identification_via_ieee(mac)
+                memory.lan_hosts[host]["device_vendor"], memory.lan_hosts[host]["vendor_address"] = self.oui_identification_via_ieee(mac)
             mac_san = mac.replace(":",".")
-            if ":" in memory.lan_hosts[mac]["ip"]:
-                ip_san = memory.lan_hosts[mac]["ip"].replace(":",".")
+            if ":" in memory.lan_hosts[host]["ip"]:
+                ip_san = memory.lan_hosts[host]["ip"].replace(":",".")
             else:
-                ip_san = memory.lan_hosts[mac]["ip"]
-            memory.lan_hosts[mac]["node"] = ip_san+"\n"+mac_san+"\n"+memory.lan_hosts[mac]['device_vendor']
+                ip_san = memory.lan_hosts[host]["ip"]
+            memory.lan_hosts[host]["node"] = ip_san+"\n"+mac_san+"\n"+memory.lan_hosts[host]['device_vendor']
 
     def oui_identification_via_api(self, mac):
         url = "http://macvendors.co/api/" + mac

Source/Module/interactive_gui.py

@@ -75,17 +75,27 @@ def gimmick_initialize(window, map):
             assert cef.__version__ >= "55.3", "CEF Python v55.3+ required to run this"
             sys.excepthook = cef.ExceptHook  # To shutdown all CEF processes on error
 
-            FourthFrame = ttk.Frame(window,  width=500, height=500, padding="10 10 10 10",relief= GROOVE)
-            FourthFrame.grid(column=50, row=10, sticky=(N, W, E, S), columnspan=200, rowspan=200, padx=5, pady=5)
-
-            browser_frame = BrowserFrame(FourthFrame)
-            browser_frame.grid(row=0, column=0,sticky=(N, W, E, S),columnspan=100, rowspan=100, padx=5, pady=5)
-
-            FourthFrame.columnconfigure(50, weight=1)
-            FourthFrame.rowconfigure(10, weight=1)
-            browser_frame.columnconfigure(0, weight=1)
-            browser_frame.rowconfigure(0, weight=1)
-
+            if not MAC:
+                FourthFrame = ttk.Frame(window,  width=500, height=500, padding="10 10 10 10",relief= GROOVE)
+                FourthFrame.grid(column=50, row=10, sticky=(N, W, E, S), columnspan=200, rowspan=200, padx=5, pady=5)
+
+                browser_frame = BrowserFrame(FourthFrame)
+                browser_frame.grid(row=0, column=0,sticky=(N, W, E, S),columnspan=100, rowspan=100, padx=5, pady=5)
+
+                FourthFrame.columnconfigure(50, weight=1)
+                FourthFrame.rowconfigure(10, weight=1)
+                browser_frame.columnconfigure(0, weight=1)
+                browser_frame.rowconfigure(0, weight=1)
+            else:
+                print("Interative graph with CEF and Tkinter is not supported on MAC. Launching Browser for InteractiveMagic!")
+                FourthFrame = ttk.Frame(window,  width=500, height=500, padding="10 10 10 10",relief= GROOVE)
+                FourthFrame.grid(column=50, row=10, sticky=(N, W, E, S), columnspan=200, rowspan=200, padx=5, pady=5)
+                mac_bug_label = ttk.Label(FourthFrame, text="Interactive Graph will launch on your browser", style="BW.TLabel")
+                mac_bug_label.grid(column=10, row=10,sticky="W")
+                FourthFrame.columnconfigure(50, weight=1)
+                FourthFrame.rowconfigure(10, weight=1)
+                import webbrowser
+                webbrowser.open(interactive_map)
             window.update()
         else:
             if FourthFrame:
@@ -124,7 +134,8 @@ def get_window_handle(self):
         if self.winfo_id() > 0 and not MAC:
             return self.winfo_id()
         elif MAC:
-            raise Exception("Couldn't obtain window handle")
+            # TODO: Handle MAC case properly the solution below from upstream crashes MAC env
+            """
             # CEF crashes in mac so temp disable
             # * CreateBrowserSync calling window handle crashes with segmentation fault 11
             # * https://github.com/cztomczak/cefpython/issues/309
@@ -133,7 +144,7 @@ def get_window_handle(self):
             # PyObjC package. If you change structure of windows then you
             # need to do modifications here as well.
             # noinspection PyUnresolvedReferences
-            """
+            
             try:
                 from AppKit import NSApp
                 # noinspection PyUnresolvedReferences
@@ -146,6 +157,10 @@ def get_window_handle(self):
             except:
                 raise Exception("Couldn't obtain window handle")
             """
+            print("Mac environment: Couldn't obtain window handle")
+            # TODO: remove this once the mac issue for CEF is resolved
+            import webbrowser
+            webbrowser.open(interactive_map)
         else:
             raise Exception("Couldn't obtain window handle")
 

Source/Module/malicious_traffic_identifier.py

@@ -3,6 +3,7 @@
 
 # Custom Module Import
 import pcap_reader
+import communication_details_fetch
 
 # Library Import
 
@@ -19,12 +20,48 @@ def __init__(self):
     def malicious_traffic_detection(self, src, dst, port):
         very_well_known_ports = [443] # used to differentiate possible mal vs serious mal
         well_known_ports = [20, 21, 22, 23, 25, 53, 69, 80, 161, 179, 389, 443]
-        if (dst in memory.destination_hosts and memory.destination_hosts[dst]["domain_name"] == "NotResolvable") or port not in well_known_ports:
-            return 1
-        else:
-            return 0
+        # Currently whitelist all the ports
+        if not communication_details_fetch.trafficDetailsFetch.is_multicast(src) and not communication_details_fetch.trafficDetailsFetch.is_multicast(dst):
+            if (dst in memory.destination_hosts and memory.destination_hosts[dst]["domain_name"] == "NotResolvable") or port > 1024:
+                return 1
+        return 0
 
     # TODO: Covert communication module --> Add here
+    # * Only add scapy first
+
+    # Covert Detection Algorithm
+    @staticmethod
+    def covert_traffic_detection(packet):
+        # covert ICMP - icmp tunneling
+        tunnelled_protocols = ["DNS", "HTTP"]
+
+        # TODO: this does not handle ipv6 --> so check before calling this function
+        #if "IP" in packet:
+        #    if communication_details_fetch.trafficDetailsFetch.is_multicast(packet["IP"].src) or communication_details_fetch.trafficDetailsFetch.is_multicast(packet["IP"].dst):
+        #        return 0
+
+        if "ICMP" in packet:
+            if "TCP in ICMP" in packet or "UDP in ICMP" in packet or "DNS" in packet:
+                #print(packet.show())
+                return 1
+            elif "padding" in packet:
+                return 1
+            elif filter(lambda x: x in str(packet["ICMP"].payload), tunnelled_protocols):
+                return 1
+        elif "DNS" in packet:
+            #print(packet["DNS"].qd.qname)
+            try:
+                if communication_details_fetch.trafficDetailsFetch.dns(packet["DNS"].qd.qname.strip()) == "NotResolvable":
+                    return 1
+                elif len(filter(str.isdigit, str(packet["DNS"].qd.qname).strip())) > 8:
+                    return 1
+            except:
+                pass
+        return 0
+
+    # Covert payload prediction algorithm
+    ##@staticmethod
+    ##def covert_payload_prediction(session):
 
 def main():
     cap = pcap_reader.PcapEngine('examples/torExample.pcap', "scapy")

Source/Module/memory.py

@@ -3,6 +3,14 @@
 
 global packet_db
 packet_db = {}
+
+# Schema
+# Key for access is MAC Address/IP Address
+# * Initially had mac address (assuming unique) as the only key
+#   - CTF problems sometime cause a scenario of same mac with different IP address so segregated this
+# * Otherwise each key holds
+#   - Mac Vendor
+#   - Ip address
 global lan_hosts
 lan_hosts = {}
 global destination_hosts

Source/Module/pcap_reader.py

@@ -7,6 +7,8 @@
 from netaddr import IPAddress
 import threading
 import base64
+import malicious_traffic_identifier
+import communication_details_fetch
 
 class PcapEngine():
     """
@@ -151,11 +153,16 @@ def analyse_packet_data(self):
                             source_private_ip = key1
 
                         # IntraNetwork Hosts list 
-                        # * When both are private they are LAN hosts
-                        if packet[eth_layer].src not in memory.lan_hosts:
-                            memory.lan_hosts[packet[eth_layer].src] = {"ip": packet[IP].src}
-                        if packet[eth_layer].dst not in memory.lan_hosts:
-                            memory.lan_hosts[packet[eth_layer].dst] = {"ip": packet[IP].dst}
+                        # * When both are private they are LAN host
+                        # TODO: this assumes a unique mac address per LAN, investigate if we need to account duplicate MAC
+                        # * This requirement occurred when working with CTF with fake MAC like 00:00:00:00:00:00
+                        if eth_layer in packet:
+                            lan_key_src = packet[eth_layer].src
+                            lan_key_dst = packet[eth_layer].dst
+                            if lan_key_src not in memory.lan_hosts:
+                                memory.lan_hosts[lan_key_src] = {"ip": packet[IP].src}
+                            if lan_key_dst not in memory.lan_hosts:
+                                memory.lan_hosts[lan_key_dst] = {"ip": packet[IP].dst}
 
                     elif private_source: # Internetwork packet
 
@@ -164,10 +171,12 @@ def analyse_packet_data(self):
                         source_private_ip = key
 
                         # IntraNetwork vs InterNetwork Hosts list
-                        if packet[eth_layer].src not in memory.lan_hosts:
-                            memory.lan_hosts[packet[eth_layer].src] = {"ip": packet[IP].src}
-                        if packet[IP].dst not in memory.destination_hosts:
-                            memory.destination_hosts[packet[IP].dst] = {"mac": packet[eth_layer].dst}
+                        if eth_layer in packet:
+                            lan_key_src = packet[eth_layer].src
+                            if lan_key_src not in memory.lan_hosts:
+                                memory.lan_hosts[lan_key_src] = {"ip": packet[IP].src}
+                            if packet[IP].dst not in memory.destination_hosts:
+                                memory.destination_hosts[packet[IP].dst] = {"mac": packet[eth_layer].dst}
 
                     elif private_destination: # Internetwork packet
 
@@ -176,10 +185,12 @@ def analyse_packet_data(self):
                         source_private_ip = key
 
                         # IntraNetwork vs InterNetwork Hosts list
-                        if packet[eth_layer].dst not in memory.lan_hosts:
-                            memory.lan_hosts[packet[eth_layer].dst] = {"ip": packet[IP].dst}
-                        if packet[IP].src not in memory.destination_hosts:
-                            memory.destination_hosts[packet[IP].src] = {"mac": packet[eth_layer].src}
+                        if eth_layer in packet:
+                            lan_key_dst = packet[eth_layer].dst
+                            if lan_key_dst not in memory.lan_hosts:
+                                memory.lan_hosts[lan_key_dst] = {"ip": packet[IP].dst}
+                            if packet[IP].src not in memory.destination_hosts:
+                                memory.destination_hosts[packet[IP].src] = {"mac": packet[eth_layer].src}
 
                     else: # public ip communication if no match
 
@@ -197,10 +208,11 @@ def analyse_packet_data(self):
 
                         # IntraNetwork Hosts list 
                         # * When both are private they are LAN hosts
-                        if packet[IP].src not in memory.destination_hosts:
-                            memory.destination_hosts[packet[IP].src] = {"mac": packet[eth_layer].src}
-                        if packet[IP].dst not in memory.destination_hosts:
-                            memory.destination_hosts[packet[IP].dst] = {"mac": packet[eth_layer].dst}
+                        if eth_layer in packet:
+                            if packet[IP].src not in memory.destination_hosts:
+                                memory.destination_hosts[packet[IP].src] = {"mac": packet[eth_layer].src}
+                            if packet[IP].dst not in memory.destination_hosts:
+                                memory.destination_hosts[packet[IP].dst] = {"mac": packet[eth_layer].dst}
 
                 elif "ICMP" in packet:
 
@@ -213,7 +225,7 @@ def analyse_packet_data(self):
                         source_private_ip = key2
                     else:
                         source_private_ip = key1
-                    source_private_ip = key
+                    #source_private_ip = key
 
                 # Fill packetDB with generated key
 
@@ -225,18 +237,32 @@ def analyse_packet_data(self):
 
                         # Ethernet Layer ( Mac address )
                         if "Ethernet" not in memory.packet_db[source_private_ip]:
-                            memory.packet_db[source_private_ip]["Ethernet"] = {}
+                            memory.packet_db[source_private_ip]["Ethernet"] = {"src":"", "dst":""}
 
                         # Record Payloads 
-                        if "Payload" not in memory.packet_db:
+                        if "Payload" not in memory.packet_db[source_private_ip]:
                             # Record unidirectional + bidirectional separate
                             memory.packet_db[source_private_ip]["Payload"] = {"forward":[],"reverse":[]}
 
+                        # Covert Communication Identifier
+                        if "covert" not in memory.packet_db[source_private_ip]:
+                            memory.packet_db[source_private_ip]["covert"] = False
+
+                    src, dst, port = source_private_ip.split("/")
+                    if memory.packet_db[source_private_ip]["covert"] == False:
+                        if not communication_details_fetch.trafficDetailsFetch.is_multicast(src) and not communication_details_fetch.trafficDetailsFetch.is_multicast(dst):
+                            if malicious_traffic_identifier.maliciousTrafficIdentifier.covert_traffic_detection(packet) == 1:
+                                memory.packet_db[source_private_ip]["covert"] = True
+
+                    # Temperory Stub
+                    # TODO: remove these pcap engine checks (confusing?), this is a temp block to develop/add support
+                    # * Once proper building is done this would be removed
                     if self.engine == "pyshark":
 
                         # Ethernet Layer
-                        memory.packet_db[source_private_ip]["Ethernet"]["src"] = packet["ETH"].src
-                        memory.packet_db[source_private_ip]["Ethernet"]["dst"] = packet["ETH"].dst
+                        if eth_layer in packet:
+                            memory.packet_db[source_private_ip]["Ethernet"]["src"] = packet["ETH"].src
+                            memory.packet_db[source_private_ip]["Ethernet"]["dst"] = packet["ETH"].dst
 
                         # <TODO>: Payload recording for pyshark
                         # Refer https://github.com/KimiNewt/pyshark/issues/264
@@ -246,12 +272,14 @@ def analyse_packet_data(self):
 
                         # Ethernet layer: store respect mac for the IP
                         if private_source:
-                            memory.packet_db[source_private_ip]["Ethernet"]["src"] = packet["Ether"].src
-                            memory.packet_db[source_private_ip]["Ethernet"]["dst"] = packet["Ether"].dst
+                            if eth_layer in packet:
+                                memory.packet_db[source_private_ip]["Ethernet"]["src"] = packet["Ether"].src
+                                memory.packet_db[source_private_ip]["Ethernet"]["dst"] = packet["Ether"].dst
                             payload = "forward"
                         else:
-                            memory.packet_db[source_private_ip]["Ethernet"]["src"] = packet["Ether"].dst
-                            memory.packet_db[source_private_ip]["Ethernet"]["dst"] = packet["Ether"].src
+                            if eth_layer in packet:
+                                memory.packet_db[source_private_ip]["Ethernet"]["src"] = packet["Ether"].dst
+                                memory.packet_db[source_private_ip]["Ethernet"]["dst"] = packet["Ether"].src
                             payload = "reverse"
 
                         # Payload 
@@ -273,7 +301,7 @@ def main():
     """
     Module Driver
     """
-    pcapfile = PcapEngine(sys.path[0]+'/examples/torExample.pcap', "scapy")
+    pcapfile = PcapEngine(sys.path[0]+'/examples/biz.pcap', "scapy")
     print(memory.packet_db.keys())
     ports = []
 
@@ -284,15 +312,12 @@ def main():
             ip, port = key.split("/")[0], int(key.split("/")[-1])
             if ip == "10.187.195.95":
                 ports.append(port)
-
-
     print(sorted(list(set(ports))))
     print(memory.lan_hosts)
     print(memory.destination_hosts)
     #print(memory.packet_db["TCP 192.168.0.26:64707 > 172.217.12.174:443"].summary())
     #print(memory.packet_db["TCP 172.217.12.174:443 > 192.168.0.26:64707"].summary())
     #memory.packet_db.conversations(type="jpg", target="> test.jpg")
-
 #main()
 
 # Sort payload by time...