Merge pull request #696 from CybercentreCanada/update/phishing-url-re…

…vamp

Put URLs POSTed to (phishing) in a separate result section

jsjaws.py

@@ -3297,7 +3297,7 @@ def _extract_urls(self, request: ServiceRequest) -> None:
         urls_result_section.set_column_order(["url", "method", "request_body"])
         urls_rows: List[TableRow] = []
         items_seen: Set[str] = set()
-        post_seen = False
+        posts_seen: List[str] = []
 
         if path.exists(self.malware_jail_urls_json_path):
             with open(self.malware_jail_urls_json_path, "r") as f:
@@ -3309,7 +3309,7 @@ def _extract_urls(self, request: ServiceRequest) -> None:
                     if not add_tag(urls_result_section, "network.dynamic.uri", item["url"], self.safelist):
                         continue
                     if item.get("method", "").lower() == "post":
-                        post_seen = True
+                        posts_seen.append(item["url"])
                         params = {"method": "POST", "headers": item.get("headers", {})}
                         if isinstance(item.get("request_body"), dict):
                             params["json"] = item.get("request_body", None)
@@ -3340,7 +3340,7 @@ def _extract_urls(self, request: ServiceRequest) -> None:
                             continue
                         item = {"url": value["url"], "method": value["method"], "request_headers": value["headers"]}
                         if item.get("method", "").lower() == "post":
-                            post_seen = True
+                            posts_seen.append(value["url"])
                             params = {"method": "POST", "headers": item.get("headers", {})}
                             if isinstance(item.get("request_body"), dict):
                                 params["json"] = item.get("request_body", None)
@@ -3370,9 +3370,6 @@ def _extract_urls(self, request: ServiceRequest) -> None:
             if self.split_reverse_join:
                 urls_result_section.heuristic.add_signature_id("split_reverse_join_url", 500)
 
-            if self.is_phishing and post_seen:
-                urls_result_section.heuristic.add_signature_id("is_phishing_url", 500)
-
             if self.weird_base64_value_set:
                 urls_result_section.heuristic.add_signature_id("weird_base64_value_set_url", 500)
 
@@ -3382,6 +3379,17 @@ def _extract_urls(self, request: ServiceRequest) -> None:
             if self.url_used_for_suspicious_exec:
                 urls_result_section.heuristic.add_signature_id("url_used_for_suspicious_exec", 500)
 
+            if self.is_phishing and posts_seen:
+                phishing_post_urls_result_section = ResultTextSection(
+                    "URLs used for POSTs, found in a file containing suspicious phishing characteristics",
+                    parent=urls_result_section,
+                )
+                phishing_post_urls_result_section.set_heuristic(1)
+                for post_seen in posts_seen:
+                    phishing_post_urls_result_section.add_line(f"\t-\t{post_seen}")
+                    add_tag(phishing_post_urls_result_section, "network.dynamic.uri", post_seen, self.safelist)
+                phishing_post_urls_result_section.heuristic.add_signature_id("is_phishing_url", 500)
+
             request.result.add_section(urls_result_section)
 
     def _extract_supplementary(self, output: List[str]) -> None:

tests/results/1216133eaa23ba3c9e89dc6828efdc9c8ef05a04d366b3cf690866f52fd1c989/result.json

@@ -1,7 +1,7 @@
 {
   "extra": {
     "drop_file": false,
-    "score": 911,
+    "score": 912,
     "sections": [
       {
         "auto_collapse": false,
@@ -246,6 +246,40 @@
         "body_format": "TABLE",
         "classification": "TLP:C",
         "depth": 0,
+        "heuristic": {
+          "attack_ids": [],
+          "frequency": 1,
+          "heur_id": 1,
+          "score": 1,
+          "score_map": {},
+          "signatures": {}
+        },
+        "promote_to": null,
+        "tags": {
+          "network": {
+            "dynamic": {
+              "domain": [
+                "badwebsiteurl.com"
+              ],
+              "uri": [
+                "https://badwebsiteurl.com/postrcv/bad.php"
+              ],
+              "uri_path": [
+                "/postrcv/bad.php"
+              ]
+            }
+          }
+        },
+        "title_text": "URLs",
+        "zeroize_on_tag_safe": false
+      },
+      {
+        "auto_collapse": false,
+        "body": "\t-\thttps://badwebsiteurl.com/postrcv/bad.php",
+        "body_config": {},
+        "body_format": "TEXT",
+        "classification": "TLP:C",
+        "depth": 1,
         "heuristic": {
           "attack_ids": [],
           "frequency": 1,
@@ -274,7 +308,7 @@
             }
           }
         },
-        "title_text": "URLs",
+        "title_text": "URLs used for POSTs, found in a file containing suspicious phishing characteristics",
         "zeroize_on_tag_safe": false
       }
     ]
@@ -299,6 +333,11 @@
   },
   "results": {
     "heuristics": [
+      {
+        "attack_ids": [],
+        "heur_id": 1,
+        "signatures": []
+      },
       {
         "attack_ids": [],
         "heur_id": 1,
@@ -350,6 +389,11 @@
     ],
     "tags": {
       "network.dynamic.domain": [
+        {
+          "heur_id": 1,
+          "signatures": [],
+          "value": "badwebsiteurl.com"
+        },
         {
           "heur_id": 1,
           "signatures": [
@@ -359,6 +403,11 @@
         }
       ],
       "network.dynamic.uri": [
+        {
+          "heur_id": 1,
+          "signatures": [],
+          "value": "https://badwebsiteurl.com/postrcv/bad.php"
+        },
         {
           "heur_id": 1,
           "signatures": [
@@ -368,6 +417,11 @@
         }
       ],
       "network.dynamic.uri_path": [
+        {
+          "heur_id": 1,
+          "signatures": [],
+          "value": "/postrcv/bad.php"
+        },
         {
           "heur_id": 1,
           "signatures": [