From b38fa944173a3641ee89ad24c0d04b8d1d2e4a76 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Thu, 7 Mar 2024 13:58:49 +0100 Subject: [PATCH 01/13] Improve parser to handle X-Forwarder-For values --- .../paloalto-ngfw/ingest/parser.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index 188b0f305..e3f497b2a 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -320,7 +320,7 @@ pipeline: - URLID - UserAgent - FileType - - Xff + - xff - Referer - Sender - Subject @@ -640,6 +640,11 @@ stages: - set: source.nat.ip: "{{parsed_event.message.sourceTranslatedAddress}}" filter: "{{parsed_event.message.sourceTranslatedAddress | is_ipaddress}}" + + - set: + network.forwarded_ip: "{{parsed_event.message.XFFAddress or parsed_event.message.xff}}" + filter: "{{parsed_event.message.XFFAddress | is_ipaddress or parsed_event.message.xff | is_ipaddress}}" + - set: source.geo.country_iso_code: "{{parsed_event.message.SourceRegion or parsed_event.message.SourceLocation}}" filter: "{{parsed_event.message.SourceLocation | length == 2 or parsed_event.message.PanOSSourceLocation | length == 2}}" @@ -654,11 +659,11 @@ stages: source.nat.port: "{{parsed_event.message.NATSourcePort or parsed_event.message.sourceTranslatedPort}}" source.packets: "{{parsed_event.message.PanOSPacketsSent or parsed_event.message.pkts_sent}}" source.port: "{{parsed_event.message.SourcePort or parsed_event.message.spt}}" - source.user.name: "{{parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_event.message.SourceUser}}" + source.user.name: "{{parsed_event.message.suser or parsed_event.message.PanOSSourceUserName}}" user_agent.name: "{{parsed_event.message.UserAgent}}" user_agent.os.name: "{{parsed_event.message.ClientOS}}" user_agent.os.version: "{{parsed_event.message.ClientOSVersion}}" - user.name: "{{parsed_event.message.SourceUser or parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}" + user.name: "{{parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}" paloalto: >- { {% set ns = namespace(first_iteration=True) %} @@ -676,6 +681,10 @@ stages: paloalto.connection.method: "{{parsed_event.message.ConnectionMethod or parsed_event.message.PanOSConnectionMethod}}" paloalto.endpoint.serial_number: "{{parsed_event.message.EndpointSerialNumber or parsed_event.message.PanOSEndpointSerialNumber}}" paloalto.threat.id: "{{parsed_event.message.ThreatID or parsed_event.message.PanOSThreatID}}" + - set: + source.user.name: "{{parsed_event.message.SourceUser}}" + user.name: "{{parsed_event.message.SourceUser}}" + filter: '{{parsed_event.message.SourceUser.startswith("x-fwd-for") == False}}' set_category_fields: actions: From fcbfb60f0e8829c9bdbc1060c8783e5a298881cd Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Thu, 7 Mar 2024 16:06:41 +0100 Subject: [PATCH 02/13] Prettier --- Palo Alto Networks/paloalto-ngfw/ingest/parser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index a2a764f77..a2afd17bc 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -684,7 +684,7 @@ stages: - set: source.user.name: "{{parsed_event.message.SourceUser}}" user.name: "{{parsed_event.message.SourceUser}}" - filter: '{{parsed_event.message.SourceUser.startswith("x-fwd-for") == False}}' + filter: '{{parsed_event.message.SourceUser.startswith("x-fwd-for") == False}}' set_category_fields: actions: From dee991b9c43624f8c0c0c610885b7b6008e93efa Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 10:24:48 +0100 Subject: [PATCH 03/13] Try user.name split --- .../paloalto-ngfw/_meta/fields.yml | 5 +++++ .../paloalto-ngfw/ingest/parser.yml | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml b/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml index 51bb74a01..1da136b9e 100644 --- a/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml +++ b/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml @@ -277,3 +277,8 @@ paloalto.threat.id: description: The identifier of the threat name: paloalto.threat.id type: keyword + +user.name_tmp: + description: temp user.name field + name: user.name_tmp + type: keyword \ No newline at end of file diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index a2afd17bc..03811d78f 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -465,6 +465,7 @@ pipeline: filter: '{{parsed_event.message.get("EventDescription") != None}}' - name: set_extracted_fields + - name: set_finalize_user_name - name: set_category_fields - name: set_ecs_deviceOutboundInterface filter: '{{parsed_event.message.get("deviceOutboundInterface") != None}}' @@ -663,7 +664,7 @@ stages: user_agent.name: "{{parsed_event.message.UserAgent}}" user_agent.os.name: "{{parsed_event.message.ClientOS}}" user_agent.os.version: "{{parsed_event.message.ClientOSVersion}}" - user.name: "{{parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}" + user.name_tmp: "{{parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}" paloalto: >- { {% set ns = namespace(first_iteration=True) %} @@ -683,9 +684,20 @@ stages: paloalto.threat.id: "{{parsed_event.message.ThreatID or parsed_event.message.PanOSThreatID}}" - set: source.user.name: "{{parsed_event.message.SourceUser}}" - user.name: "{{parsed_event.message.SourceUser}}" + user.name_tmp: "{{parsed_event.message.SourceUser}}" filter: '{{parsed_event.message.SourceUser.startswith("x-fwd-for") == False}}' + + set_finalize_user_name: + actions: + - set: + user.domain: '{{user.name_tmp.split("\\\\") | first}}' + user.name: '{{user.name_tmp.split("\\\\") | last}}' + filter: '{{user.get("name_tmp") != None and "\\\\" in user.name_tmp}}' + - set: + user.name: '{{user.name_tmp}}' + filter: '{{user.get("name_tmp") != None and and "\\\\" not in user.name_tmp}}' + set_category_fields: actions: - set: From 087347ecf388f59d762213ba9de6d525ae8c0d53 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 10:36:39 +0100 Subject: [PATCH 04/13] Try user.name split --- Palo Alto Networks/paloalto-ngfw/ingest/parser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index 03811d78f..e8f4d9a16 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -696,7 +696,7 @@ stages: filter: '{{user.get("name_tmp") != None and "\\\\" in user.name_tmp}}' - set: user.name: '{{user.name_tmp}}' - filter: '{{user.get("name_tmp") != None and and "\\\\" not in user.name_tmp}}' + filter: '{{user.get("name_tmp") != None}}' set_category_fields: actions: From 9752444f1496b11f428f1ad50d413459efc2960d Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 15:11:55 +0100 Subject: [PATCH 05/13] Split user name and domains --- .../paloalto-ngfw/_meta/fields.yml | 5 - .../paloalto-ngfw/ingest/parser.yml | 20 ++-- .../paloalto-ngfw/tests/auth_cef.json | 5 +- .../paloalto-ngfw/tests/decryption_cef.json | 11 ++- .../paloalto-ngfw/tests/fix_bug_with_int.json | 8 +- .../tests/globalprotect_cef.json | 8 +- .../tests/globalprotect_csv_2.json | 8 +- .../tests/test_globalprotect.json | 8 +- .../paloalto-ngfw/tests/test_userid.json | 5 +- .../paloalto-ngfw/tests/threat-url-xff.json | 99 +++++++++++++++++++ 10 files changed, 144 insertions(+), 33 deletions(-) create mode 100644 Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json diff --git a/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml b/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml index 1da136b9e..51bb74a01 100644 --- a/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml +++ b/Palo Alto Networks/paloalto-ngfw/_meta/fields.yml @@ -277,8 +277,3 @@ paloalto.threat.id: description: The identifier of the threat name: paloalto.threat.id type: keyword - -user.name_tmp: - description: temp user.name field - name: user.name_tmp - type: keyword \ No newline at end of file diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index e8f4d9a16..685d865b6 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -664,7 +664,7 @@ stages: user_agent.name: "{{parsed_event.message.UserAgent}}" user_agent.os.name: "{{parsed_event.message.ClientOS}}" user_agent.os.version: "{{parsed_event.message.ClientOSVersion}}" - user.name_tmp: "{{parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}" + user.name: "{{parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}" paloalto: >- { {% set ns = namespace(first_iteration=True) %} @@ -684,19 +684,23 @@ stages: paloalto.threat.id: "{{parsed_event.message.ThreatID or parsed_event.message.PanOSThreatID}}" - set: source.user.name: "{{parsed_event.message.SourceUser}}" - user.name_tmp: "{{parsed_event.message.SourceUser}}" + user.name: "{{parsed_event.message.SourceUser}}" filter: '{{parsed_event.message.SourceUser.startswith("x-fwd-for") == False}}' - set_finalize_user_name: actions: - set: - user.domain: '{{user.name_tmp.split("\\\\") | first}}' - user.name: '{{user.name_tmp.split("\\\\") | last}}' - filter: '{{user.get("name_tmp") != None and "\\\\" in user.name_tmp}}' + user.domain: '{{final.user.name.split("\\") | first}}' + user.name: '{{final.user.name.split("\\") | last}}' + filter: '{{final.user.name != null and "\\" in final.user.name}}' + - set: + source.user.domain: '{{final.source.user.name.split("\\") | first}}' + source.user.name: '{{final.source.user.name.split("\\") | last}}' + filter: '{{final.source.user.name != null and "\\" in final.source.user.name}}' - set: - user.name: '{{user.name_tmp}}' - filter: '{{user.get("name_tmp") != None}}' + destination.user.domain: '{{final.destination.user.name.split("\\") | first}}' + destination.user.name: '{{final.destination.user.name.split("\\") | last}}' + filter: '{{final.destination.user.name != null and "\\" in final.destination.user.name}}' set_category_fields: actions: diff --git a/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json index 410789f29..d2def6a93 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json @@ -26,7 +26,8 @@ "@timestamp": "2021-02-28T18:20:54Z", "destination": { "user": { - "name": "paloaltonetwork\\\\xxxxx" + "domain": "paloaltonetwork", + "name": "xxxxx" } }, "host": { @@ -72,7 +73,7 @@ "xxxxx" ], "user": [ - "paloaltonetwork\\\\xxxxx" + "xxxxx" ] } } diff --git a/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json index 49daa3726..5797073e8 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json @@ -34,7 +34,8 @@ }, "port": 20122, "user": { - "name": "paloaltonetwork\\\\\\\\xxxxx" + "domain": "paloaltonetwork", + "name": "xxxxx" } }, "log": { @@ -72,7 +73,7 @@ "1.1.1.1" ], "user": [ - "paloaltonetwork\\\\\\\\xxxxx" + "xxxxx" ] }, "rule": { @@ -88,11 +89,13 @@ }, "port": 16524, "user": { - "name": "paloaltonetwork\\\\\\\\xxxxx" + "domain": "paloaltonetwork", + "name": "xxxxx" } }, "user": { - "name": "paloaltonetwork\\\\\\\\xxxxx" + "domain": "paloaltonetwork", + "name": "xxxxx" } } } \ No newline at end of file diff --git a/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json b/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json index 1c33679f4..a1945cb5f 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json @@ -70,7 +70,7 @@ "5.6.7.8" ], "user": [ - "domain\\pusername", + "pusername", "userdest" ] }, @@ -89,11 +89,13 @@ "packets": 6, "port": 51413, "user": { - "name": "domain\\pusername" + "domain": "domain", + "name": "pusername" } }, "user": { - "name": "domain\\pusername" + "domain": "domain", + "name": "pusername" } } } \ No newline at end of file diff --git a/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json index d17e95f70..c4a3737b1 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json @@ -59,16 +59,18 @@ "machine_name2" ], "user": [ - "xxxxx\\\\\\\\xxxxx" + "xxxxx" ] }, "source": { "user": { - "name": "xxxxx\\\\\\\\xxxxx" + "domain": "xxxxx", + "name": "xxxxx" } }, "user": { - "name": "xxxxx\\\\\\\\xxxxx" + "domain": "xxxxx", + "name": "xxxxx" } } } \ No newline at end of file diff --git a/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_csv_2.json b/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_csv_2.json index 95aaa5900..d5fde0805 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_csv_2.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_csv_2.json @@ -53,7 +53,7 @@ "88.120.236.74" ], "user": [ - "example.org\\\\test" + "test" ] }, "source": { @@ -63,11 +63,13 @@ }, "ip": "88.120.236.74", "user": { - "name": "example.org\\\\test" + "domain": "example.org", + "name": "test" } }, "user": { - "name": "example.org\\\\test" + "domain": "example.org", + "name": "test" }, "user_agent": { "os": { diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_globalprotect.json b/Palo Alto Networks/paloalto-ngfw/tests/test_globalprotect.json index ce0827353..8f91c490c 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/test_globalprotect.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/test_globalprotect.json @@ -56,7 +56,7 @@ "1.2.3.4" ], "user": [ - "test.fr\\JDOE" + "JDOE" ] }, "source": { @@ -66,11 +66,13 @@ }, "ip": "1.2.3.4", "user": { - "name": "test.fr\\JDOE" + "domain": "test.fr", + "name": "JDOE" } }, "user": { - "name": "test.fr\\JDOE" + "domain": "test.fr", + "name": "JDOE" }, "user_agent": { "os": { diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_userid.json b/Palo Alto Networks/paloalto-ngfw/tests/test_userid.json index e0d997f7a..d335b8a48 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/test_userid.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/test_userid.json @@ -53,7 +53,7 @@ "1.2.3.4" ], "user": [ - "test.fr\\JDOE" + "JDOE" ] }, "source": { @@ -62,7 +62,8 @@ "port": 0 }, "user": { - "name": "test.fr\\JDOE" + "domain": "test.fr", + "name": "JDOE" } } } \ No newline at end of file diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json new file mode 100644 index 000000000..e3057dbd9 --- /dev/null +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json @@ -0,0 +1,99 @@ +{ + "input": { + "sekoiaio": { + "intake": { + "dialect": "Palo Alto NGFW", + "dialect_uuid": "903ec1b8-f206-4ba5-8563-db21da09cafd" + } + }, + "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic" + }, + "expected": { + "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic", + "event": { + "category": [ + "network" + ], + "dataset": "threat", + "kind": "event", + "outcome": "success", + "reason": "(9999)", + "type": [ + "info" + ] + }, + "@timestamp": "2024-03-12T14:02:32.650000Z", + "action": { + "name": "alert", + "outcome": "success", + "type": "url" + }, + "destination": { + "address": "192.168.0.1", + "ip": "192.168.0.1", + "nat": { + "ip": "0.0.0.0", + "port": 0 + }, + "port": 80 + }, + "file": { + "name": "www.sekoia.io", + "path": "www.sekoia.io" + }, + "host": { + "name": "FW" + }, + "http": { + "request": { + "method": "get" + } + }, + "log": { + "hostname": "FW", + "level": "informational", + "logger": "threat" + }, + "network": { + "application": "web-browsing", + "forwarded_ip": "11.22.33.44", + "transport": "tcp" + }, + "observer": { + "product": "PAN-OS", + "serial_number": "016401004874" + }, + "paloalto": { + "DGHierarchyLevel1": "0", + "DGHierarchyLevel2": "0", + "DGHierarchyLevel3": "0", + "DGHierarchyLevel4": "0", + "Threat_ContentType": "url", + "VirtualLocation": "vsys", + "VirtualSystemName": "VSYS" + }, + "related": { + "ip": [ + "0.0.0.0", + "10.0.0.2", + "192.168.0.1" + ] + }, + "rule": { + "name": "rule-internet", + "uuid": "ea3431a2-6869-4d9f-ad41-1858d80b406c" + }, + "source": { + "address": "10.0.0.2", + "ip": "10.0.0.2", + "nat": { + "ip": "0.0.0.0", + "port": 0 + }, + "port": 49802 + }, + "user_agent": { + "name": "Mozilla/4.0 (compatible; ms-office; MSOffice 16)" + } + } +} \ No newline at end of file From 6e5c81c5cf510da70c5c1f1bb06fbbfcf635e587 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 16:58:19 +0100 Subject: [PATCH 06/13] Add subtype for CEF logs and set url fields when subtype is url --- .../paloalto-ngfw/ingest/parser.yml | 21 ++++++++++++++++--- .../paloalto-ngfw/tests/auth_cef.json | 3 +++ .../paloalto-ngfw/tests/decryption_cef.json | 3 +++ .../paloalto-ngfw/tests/file_cef.json | 3 +++ .../tests/globalprotect_cef.json | 4 ++++ .../paloalto-ngfw/tests/iptag_cef.json | 3 +++ .../paloalto-ngfw/tests/threat-url-xff.json | 2 +- .../paloalto-ngfw/tests/threat_cef.json | 3 +++ .../paloalto-ngfw/tests/traffic1_csv.json | 3 +++ .../paloalto-ngfw/tests/traffic2_csv.json | 3 +++ .../paloalto-ngfw/tests/traffic_cef.json | 3 +++ .../paloalto-ngfw/tests/url_cef.json | 3 +++ .../paloalto-ngfw/tests/userid_cef.json | 3 +++ 13 files changed, 53 insertions(+), 4 deletions(-) diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index 685d865b6..a46246371 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -508,7 +508,7 @@ stages: "@timestamp": "{{parsed_timestamp.datetime}}" event.start: "{{parsed_start.datetime}}" action.name: "{{parsed_event.message.Action or parsed_description.message.action}}" - action.type: "{{parsed_event.message.Subtype}}" + action.type: "{{parsed_event.message.Subtype|lower or parsed_event.message.Name|lower}}" container.id: "{{parsed_event.message.ContainerID}}" container.name: "{{parsed_event.message.ContainerName}}" destination.address: "{{parsed_event.message.DestinationAddress or parsed_event.message.dst or parsed_description.message.dst_addr}}" @@ -562,8 +562,6 @@ stages: event.kind: "event" event.reason: "{{parsed_event.message.reason or parsed_event.message.Threat_ContentName or parsed_event.message.EventDescription or parsed_event.message.PanOSConnectionError}}" event.module: "{{parsed_description.message.module}}" - file.path: "{{parsed_event.message.URLFilename}}" - file.name: "{{parsed_event.message.FileName or parsed_event.message.URLFilename or parsed_description.message.filename}}" host.hostname: "{{parsed_event.message.dvchost or parsed_event.message.PanOSEndpointDeviceName}}" host.name: "{{parsed_event.message.dvchost or parsed_event.message.PanOSEndpointDeviceName or parsed_event.message.LogSourceName or parsed_event.message.MachineName or parsed_event.message.shost}}" host.id: "{{parsed_event.message.deviceExternalId}}" @@ -594,6 +592,23 @@ stages: rule.uuid: "{{parsed_event.message.PanOSRuleUUID or parsed_event.message.RuleUUID}}" source.bytes: "{{parsed_event.message.BytesSent or parsed_event.message.in}}" + - set: + file.path: "{{parsed_event.message.URLFilename}}" + file.name: "{{parsed_event.message.FileName or parsed_event.message.URLFilename or parsed_description.message.filename}}" + filter: "{{final.action.type != 'url'}}" + + - set: + url.original: '{{parsed_event.message.FileName or parsed_event.message.URLFilename}}' + url.domain: '{{final.url.original.split("/")[0].split(":")[0]}}' + url.port: '{{final.url.original.split("/")[0].split(":")[1]}}' + url.path: '{{final.url.original.split("?")[0].split("/")[1:] | join("/")}}' + url.query: '{{final.url.original.split("?")[1]}}' + filter: "{{final.action.type == 'url'}}" + + - delete: + - url.original + filter: "{{final.action.type == 'url'}}" + - set: source.ip: "{{parsed_event.message.PublicIP}}" filter: "{{parsed_event.message.PublicIP | is_ipaddress}}" diff --git a/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json index d2def6a93..a64b5708d 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json @@ -24,6 +24,9 @@ ] }, "@timestamp": "2021-02-28T18:20:54Z", + "action": { + "type": "radius" + }, "destination": { "user": { "domain": "paloaltonetwork", diff --git a/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json index 5797073e8..8ac1b4db9 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json @@ -25,6 +25,9 @@ ] }, "@timestamp": "2021-03-01T20:35:54Z", + "action": { + "type": "end" + }, "destination": { "address": "1.1.1.1", "ip": "1.1.1.1", diff --git a/Palo Alto Networks/paloalto-ngfw/tests/file_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/file_cef.json index 3a8301947..66940e42f 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/file_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/file_cef.json @@ -25,6 +25,9 @@ ] }, "@timestamp": "2021-03-01T21:06:06Z", + "action": { + "type": "file" + }, "destination": { "address": "1.1.1.1", "ip": "1.1.1.1", diff --git a/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json index c4a3737b1..bd03a678e 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json @@ -25,6 +25,10 @@ ] }, "@timestamp": "2021-03-01T20:35:54Z", + "action": { + "name": "satellite-gateway-update-route", + "type": "globalprotect" + }, "host": { "hostname": "machine_name2", "name": "machine_name2", diff --git a/Palo Alto Networks/paloalto-ngfw/tests/iptag_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/iptag_cef.json index 7aa80697f..7366d48c0 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/iptag_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/iptag_cef.json @@ -24,6 +24,9 @@ ] }, "@timestamp": "2021-03-01T21:20:13Z", + "action": { + "type": "iptag" + }, "destination": { "address": "1.1.1.1", "ip": "1.1.1.1" diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json index e3057dbd9..c90839c3b 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json @@ -6,7 +6,7 @@ "dialect_uuid": "903ec1b8-f206-4ba5-8563-db21da09cafd" } }, - "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic" + "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io:443/catalog/integrations?query=this\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic" }, "expected": { "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic", diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/threat_cef.json index a7cb0b207..c1c4875f4 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/threat_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat_cef.json @@ -25,6 +25,9 @@ ] }, "@timestamp": "2021-03-01T20:48:21Z", + "action": { + "type": "spyware" + }, "destination": { "geo": { "country_iso_code": "BR" diff --git a/Palo Alto Networks/paloalto-ngfw/tests/traffic1_csv.json b/Palo Alto Networks/paloalto-ngfw/tests/traffic1_csv.json index ef35c1025..7ccca5b22 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/traffic1_csv.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/traffic1_csv.json @@ -27,6 +27,9 @@ ] }, "@timestamp": "2022-07-31T12:46:24Z", + "action": { + "type": "end" + }, "destination": { "address": "5.6.7.8", "bytes": 5651, diff --git a/Palo Alto Networks/paloalto-ngfw/tests/traffic2_csv.json b/Palo Alto Networks/paloalto-ngfw/tests/traffic2_csv.json index 0d5f80841..ec742e799 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/traffic2_csv.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/traffic2_csv.json @@ -27,6 +27,9 @@ ] }, "@timestamp": "2022-08-02T06:42:20Z", + "action": { + "type": "end" + }, "destination": { "address": "1.1.1.1", "bytes": 2755, diff --git a/Palo Alto Networks/paloalto-ngfw/tests/traffic_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/traffic_cef.json index a5d0c867e..8efb39cb8 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/traffic_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/traffic_cef.json @@ -27,6 +27,9 @@ ] }, "@timestamp": "2021-02-27T20:16:21Z", + "action": { + "type": "end" + }, "destination": { "address": "1.1.1.1", "bytes": 400448, diff --git a/Palo Alto Networks/paloalto-ngfw/tests/url_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/url_cef.json index 5c159fe03..7eb566069 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/url_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/url_cef.json @@ -25,6 +25,9 @@ ] }, "@timestamp": "2021-03-01T20:48:21Z", + "action": { + "type": "url" + }, "destination": { "address": "1.1.1.1", "geo": { diff --git a/Palo Alto Networks/paloalto-ngfw/tests/userid_cef.json b/Palo Alto Networks/paloalto-ngfw/tests/userid_cef.json index 603a18b20..9fc04d91b 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/userid_cef.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/userid_cef.json @@ -24,6 +24,9 @@ ] }, "@timestamp": "2021-03-01T21:06:02Z", + "action": { + "type": "logout" + }, "destination": { "address": "1.1.1.1", "ip": "1.1.1.1", From a7fdf850f13722d0700e4d54d5c5fd9b16725fd8 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 17:00:33 +0100 Subject: [PATCH 07/13] Add subtype for CEF logs and set url fields when subtype is url --- .../paloalto-ngfw/tests/test_threat.json | 13 +++++++++---- .../paloalto-ngfw/tests/threat-url-xff.json | 18 +++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json b/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json index 05f0c1aae..28a33cdb2 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json @@ -37,10 +37,6 @@ }, "port": 2222 }, - "file": { - "name": "test.fr:9999/", - "path": "test.fr:9999/" - }, "host": { "name": "TEST-01" }, @@ -66,6 +62,9 @@ "VirtualLocation": "vsys1" }, "related": { + "hosts": [ + "test.fr" + ], "ip": [ "0.0.0.0", "1.2.3.4", @@ -88,6 +87,12 @@ "port": 58444 }, "port": 58444 + }, + "url": { + "domain": "test.fr", + "port": 9999, + "registered_domain": "test.fr", + "top_level_domain": "fr" } } } \ No newline at end of file diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json index c90839c3b..d53576852 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json @@ -9,7 +9,7 @@ "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io:443/catalog/integrations?query=this\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic" }, "expected": { - "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic", + "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io:443/catalog/integrations?query=this\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic", "event": { "category": [ "network" @@ -37,10 +37,6 @@ }, "port": 80 }, - "file": { - "name": "www.sekoia.io", - "path": "www.sekoia.io" - }, "host": { "name": "FW" }, @@ -73,6 +69,9 @@ "VirtualSystemName": "VSYS" }, "related": { + "hosts": [ + "www.sekoia.io" + ], "ip": [ "0.0.0.0", "10.0.0.2", @@ -92,6 +91,15 @@ }, "port": 49802 }, + "url": { + "domain": "www.sekoia.io", + "path": "catalog/integrations", + "port": 443, + "query": "query=this", + "registered_domain": "sekoia.io", + "subdomain": "www", + "top_level_domain": "io" + }, "user_agent": { "name": "Mozilla/4.0 (compatible; ms-office; MSOffice 16)" } From 61acbf31fe4ab7c45a62698df818b12667ee8f8a Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 17:05:21 +0100 Subject: [PATCH 08/13] Fix Threat smart description --- .../paloalto-ngfw/_meta/smart-descriptions.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json b/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json index 3cab0b2d4..53f7f5904 100644 --- a/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json +++ b/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json @@ -139,6 +139,12 @@ { "field": "log.logger", "value": "threat" + }, + { + "field": "source.ip" + }, + { + "field": "destination.ip" } ], "relationships": [ From e8dbb05caf418eb092b20f7596426773e02d7fab Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 17:29:17 +0100 Subject: [PATCH 09/13] Prettier & linting --- Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json index d53576852..c0fccc875 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json @@ -1,12 +1,12 @@ { "input": { + "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io:443/catalog/integrations?query=this\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic", "sekoiaio": { "intake": { "dialect": "Palo Alto NGFW", "dialect_uuid": "903ec1b8-f206-4ba5-8563-db21da09cafd" } - }, - "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io:443/catalog/integrations?query=this\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic" + } }, "expected": { "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io:443/catalog/integrations?query=this\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic", From e60641c60e9378f385e907cedee5ed29c73eab9e Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 12 Mar 2024 17:33:03 +0100 Subject: [PATCH 10/13] Prettier & linting --- Palo Alto Networks/paloalto-ngfw/ingest/parser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index a46246371..3deb66019 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -598,7 +598,7 @@ stages: filter: "{{final.action.type != 'url'}}" - set: - url.original: '{{parsed_event.message.FileName or parsed_event.message.URLFilename}}' + url.original: "{{parsed_event.message.FileName or parsed_event.message.URLFilename}}" url.domain: '{{final.url.original.split("/")[0].split(":")[0]}}' url.port: '{{final.url.original.split("/")[0].split(":")[1]}}' url.path: '{{final.url.original.split("?")[0].split("/")[1:] | join("/")}}' From 8e297b2676284d3ed3ba742094801f31ff28eb47 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Wed, 13 Mar 2024 08:18:35 +0100 Subject: [PATCH 11/13] Replace parsed_event.message.Threat_Category by parsed_event.message.Action in event.action field --- Palo Alto Networks/paloalto-ngfw/ingest/parser.yml | 2 +- Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/fix_bug_without_int.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/icmp_allow_csv.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/tcp_allow_csv.json | 1 + .../paloalto-ngfw/tests/test_file_alert_json.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/test_threat.json | 1 + .../paloalto-ngfw/tests/test_traffic_event_1_json.json | 1 + .../paloalto-ngfw/tests/test_traffic_event_2_json.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/threat_csv.json | 2 +- .../paloalto-ngfw/tests/traffic_with_resotimestamp.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/udp_deny_csv.json | 1 + Palo Alto Networks/paloalto-ngfw/tests/wildfire1_json.json | 1 + 14 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml index 3deb66019..3918356a8 100644 --- a/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml +++ b/Palo Alto Networks/paloalto-ngfw/ingest/parser.yml @@ -556,7 +556,7 @@ stages: email.from.address: "{{parsed_event.message.Sender}}" email.subject: "{{parsed_event.message.Subject}}" email.to.address: "{{parsed_event.message.Recipient}}" - event.action: "{{parsed_event.message.act or parsed_event.message.Threat_Category or parsed_description.message.action}}" + event.action: "{{parsed_event.message.act or parsed_event.message.Action or parsed_description.message.action}}" event.timezone: "{{parsed_event.message.dtz}}" event.dataset: "{{parsed_event.message.DeviceEventClassID|lower or parsed_event.message.Type|lower or parsed_event.message.LogType|lower}}" event.kind: "event" diff --git a/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json b/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json index a1945cb5f..2014b1d1f 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json @@ -11,6 +11,7 @@ "expected": { "message": "1,2023/06/16 10:41:44,001701003551,TRAFFIC,end,2305,2023/06/16 10:41:44,1.2.3.4,5.6.7.8,0.0.0.0,0.0.0.0,GEN_WINLOG_Users,domain\\pusername,userdest,windows-remote-management,vsys1,PDT_STD,INFRA_ADM,aaa.111,aaa.111,Syslog_Test,2023/06/16 10:41:44,234981,1,51413,5985,0,0,15,tcp,allow,2346,1974,372,9,90,16,30,0,69678105127,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,3,tcp-fin,0,0,0,0,,FWPA01,from-policy,,,0,,0,,N/A,0,0,0,0,5e7eca5b-f585-4633-bbd4-9ed431f7f95b,0,0,,,,,,,", "event": { + "action": "allow", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_without_int.json b/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_without_int.json index 35d10817d..80236dd78 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_without_int.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/fix_bug_without_int.json @@ -11,6 +11,7 @@ "expected": { "message": "1,2023/06/16 10:41:44,001701003551,TRAFFIC,end,2305,2023/06/16 10:41:44,1.2.3.4,5.6.7.8,0.0.0.0,0.0.0.0,GEN_WINLOG_Users,domainusername,destuser,windows-remote-management,vsys1,PDT_STD,INFRA_ADM,aaa.111,aaa.111,Syslog_Test,2023/06/16 10:41:44,234981,1,51413,5985,0,0,0x1c,tcp,allow,2346,1974,372,9,2023/06/16 10:41:26,16,not-resolved,0,69678105127,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,3,tcp-fin,0,0,0,0,,FWPA01,from-policy,,,0,,0,,N/A,0,0,0,0,5e7eca5b-f585-4633-bbd4-9ed431f7f95b,0,0,,,,,,,", "event": { + "action": "allow", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/icmp_allow_csv.json b/Palo Alto Networks/paloalto-ngfw/tests/icmp_allow_csv.json index d82939215..658dbc4ce 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/icmp_allow_csv.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/icmp_allow_csv.json @@ -11,6 +11,7 @@ "expected": { "message": "<14>Sep 16 10:00:02 PP 1,9/16/19 10:00,1801017000,TRAFFIC,start,2049,9/16/19 10:00,1.2.3.4,4.3.2.1,1.2.3.4,10.0.1.2,PING,,,ping,vsys,AAAAA,Zone1,ethernet1/1,ae2.11,Secure,9/16/19 10:00,24100,3,0,0,0,0,0x500000,icmp,allow,222,222,0,3,9/16/19 10:00,0,any,0,50660388939,0x0,Spain,France,0,3,0,n/a,0,0,0,0,,PA,from-policy,,,0,,0,,N/A,0,0,0,0", "event": { + "action": "allow", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/tcp_allow_csv.json b/Palo Alto Networks/paloalto-ngfw/tests/tcp_allow_csv.json index f317fbfc4..9d58d1a5e 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/tcp_allow_csv.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/tcp_allow_csv.json @@ -11,6 +11,7 @@ "expected": { "message": "<14>Sep 16 10:00:02 PA-1 1,9/16/19 10:00,1801016000,TRAFFIC,start,2049,9/16/19 10:00,1.2.3.4,4.3.2.1,0.0.0.0,0.0.0.0,proxy1,,,web-browsing,vsys1234,v10213,zone1,a.1,b.2,Secure,9/16/19 10:00,60000,1,61000,80,0,0,0x0,tcp,allow,800,700,70,2,9/16/19 10:00,0,any,0,50660381839,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,2,1,n/a,0,0,0,0,,PP,from-policy,,,0,,0,,N/A,0,0,0,0", "event": { + "action": "allow", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_file_alert_json.json b/Palo Alto Networks/paloalto-ngfw/tests/test_file_alert_json.json index 17a3787c8..bd2f0569f 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/test_file_alert_json.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/test_file_alert_json.json @@ -11,6 +11,7 @@ "expected": { "message": "{\"TimeReceived\": \"2024-02-06T18:17:09.000000Z\", \"DeviceSN\": \"no-serial\", \"LogType\": \"THREAT\", \"Subtype\": \"file\", \"SubType\": \"file\", \"ConfigVersion\": \"10.2\", \"TimeGenerated\": \"2024-02-06T18:17:02.000000Z\", \"SourceAddress\": \"1.2.3.4\", \"DestinationAddress\": \"5.6.7.8\", \"NATSource\": \"9.10.11.12\", \"NATDestination\": \"5.6.7.8\", \"Rule\": \"Global_Outbound_internet_access\", \"SourceUser\": \"john.doe@example.com\", \"DestinationUser\": null, \"Application\": \"web-browsing\", \"VirtualLocation\": \"vsys1\", \"FromZone\": \"trust\", \"ToZone\": \"untrust\", \"InboundInterface\": \"tunnel.1\", \"OutboundInterface\": \"ethernet1/1\", \"LogSetting\": \"default\", \"SessionID\": 1450762, \"RepeatCount\": 1, \"SourcePort\": 53514, \"DestinationPort\": 80, \"NATSourcePort\": 22444, \"NATDestinationPort\": 80, \"Protocol\": \"tcp\", \"Action\": \"alert\", \"FileName\": \"some_file_name\", \"URLCategory\": \"computer-and-internet-info\", \"VendorSeverity\": \"Low\", \"DirectionOfAttack\": \"server to client\", \"SequenceNo\": 7292474944208657622, \"SourceLocation\": \"Prisma-Mobile-Users-EMEA\", \"DestinationLocation\": \"US\", \"PacketID\": 0, \"FileHash\": null, \"ReportID\": 0, \"DGHierarchyLevel1\": 463, \"DGHierarchyLevel2\": 467, \"DGHierarchyLevel3\": 0, \"DGHierarchyLevel4\": 0, \"VirtualSystemName\": \"\", \"DeviceName\": \"GP cloud service\", \"SourceUUID\": null, \"DestinationUUID\": null, \"IMSI\": 0, \"IMEI\": null, \"ParentSessionID\": 0, \"ParentStartTime\": \"1970-01-01T00:00:00.000000Z\", \"Tunnel\": \"N/A\", \"ContentVersion\": \"577053022\", \"SigFlags\": 0, \"RuleUUID\": \"c38e111b-43fc-4de4-a17c-c372af557193\", \"HTTP2Connection\": 0, \"DynamicUserGroup\": null, \"X-Forwarded-ForIP\": null, \"SourceDeviceCategory\": null, \"SourceDeviceProfile\": null, \"SourceDeviceModel\": null, \"SourceDeviceVendor\": null, \"SourceDeviceOSFamily\": null, \"SourceDeviceOSVersion\": null, \"SourceDeviceHost\": null, \"SourceDeviceMac\": null, \"DestinationDeviceCategory\": null, \"DestinationDeviceProfile\": null, \"DestinationDeviceModel\": null, \"DestinationDeviceVendor\": null, \"DestinationDeviceOSFamily\": null, \"DestinationDeviceOSVersion\": null, \"DestinationDeviceHost\": null, \"DestinationDeviceMac\": null, \"ContainerID\": null, \"ContainerNameSpace\": null, \"ContainerName\": null, \"SourceEDL\": null, \"DestinationEDL\": null, \"HostID\": null, \"EndpointSerialNumber\": null, \"DomainEDL\": null, \"SourceDynamicAddressGroup\": null, \"DestinationDynamicAddressGroup\": null, \"PartialHash\": 0, \"TimeGeneratedHighResolution\": \"2024-02-06T18:17:02.077000Z\", \"ReasonForDataFilteringAction\": null, \"Justification\": null, \"NSSAINetworkSliceType\": null}", "event": { + "action": "alert", "category": [ "file" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json b/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json index 28a33cdb2..1375dbbad 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/test_threat.json @@ -11,6 +11,7 @@ "expected": { "message": "1,2024/01/12 11:21:15,016201000000,THREAT,url,2562,2024/01/12 11:21:15,1.2.3.4,5.6.7.8,9.10.11.12,0.0.0.0,SAAS vers log,,,ssl,vsys1,Outside,test-Externe,a11.30,a11.25,Panorama,2024/01/12 11:21:15,200000,1,58444,2222,58444,2222,0x50b444,tcp,alert,\"test.fr:9999/\",(9999),test,informational,client-to-server,55555555555555555555,0x8000000000000000,US,France,,,0,,,0,,,,,,,,0,0,0,0,0,,TEST-01,,,,,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"test,low-risk\",96eeeef8-bd9c-4145,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-01-12T11:21:15.190+01:00,,,,encrypted-tunnel,networking,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,ssl,no,no,", "event": { + "action": "alert", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_1_json.json b/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_1_json.json index 83e5d5569..e9ef7bddb 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_1_json.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_1_json.json @@ -11,6 +11,7 @@ "expected": { "message": "{\"Action\": \"allow\",\"ActionSource\": \"from-policy\",\"Application\": \"incomplete\",\"Bytes\": 74,\"BytesReceived\": 0,\"BytesSent\": 74,\"ChunksReceived\": 0,\"ChunksSent\": 0,\"ChunksTotal\": 0,\"ConfigVersion\": \"10.1\",\"ContainerID\": null,\"ContainerName\": null,\"ContainerNameSpace\": null,\"DGHierarchyLevel1\": 0,\"DGHierarchyLevel2\": 0,\"DGHierarchyLevel3\": 0,\"DGHierarchyLevel4\": 0,\"DestinationAddress\": \"5.6.7.8\",\"DestinationDeviceCategory\": null,\"DestinationDeviceHost\": null,\"DestinationDeviceMac\": null,\"DestinationDeviceModel\": null,\"DestinationDeviceOSFamily\": null,\"DestinationDeviceOSVersion\": null,\"DestinationDeviceProfile\": null,\"DestinationDeviceVendor\": null,\"DestinationDynamicAddressGroup\": null,\"DestinationEDL\": null,\"DestinationLocation\": \"US\",\"DestinationPort\": 443,\"DestinationUUID\": null,\"DestinationUser\": null,\"DeviceName\": \"PA-VM\",\"DeviceSN\": \"007954000351998\",\"DynamicUserGroupName\": null,\"EndpointAssociationID\": 0,\"EndpointSerialNumber\": null,\"FromZone\": \"untrusted\",\"GPHostID\": null,\"HASessionOwner\": null,\"HTTP2Connection\": 0,\"IMEI\": null,\"IMSI\": 0,\"InboundInterface\": \"ethernet1/1\",\"LinkChangeCount\": 0,\"LinkSwitches\": null,\"LogSetting\": \"default\",\"LogType\": \"TRAFFIC\",\"NATDestination\": \"\",\"NATDestinationPort\": 0,\"NATSource\": \"\",\"NATSourcePort\": 0,\"NSSAINetworkSliceDifferentiator\": null,\"NSSAINetworkSliceType\": null,\"OutboundInterface\": \"ethernet1/1\",\"PacketsReceived\": 0,\"PacketsSent\": 1,\"PacketsTotal\": 1,\"ParentSessionID\": 0,\"ParentStarttime\": \"1970-01-01T00:00:00.000000Z\",\"Protocol\": \"tcp\",\"RepeatCount\": 1,\"Rule\": \"intrazone-default\",\"RuleUUID\": \"f903db52-4b89-4610-b908-67be412704f0\",\"SDWANCluster\": null,\"SDWANClusterType\": null,\"SDWANDeviceType\": null,\"SDWANPolicyName\": null,\"SDWANSite\": null,\"SequenceNo\": 7195838274152187101,\"SessionDuration\": 0,\"SessionEndReason\": \"aged-out\",\"SessionID\": 17635,\"SessionStartTime\": \"2023-02-03T16:46:00.000000Z\",\"SourceAddress\": \"1.2.3.4\",\"SourceDeviceCategory\": null,\"SourceDeviceHost\": null,\"SourceDeviceMac\": null,\"SourceDeviceModel\": null,\"SourceDeviceOSFamily\": null,\"SourceDeviceOSVersion\": null,\"SourceDeviceProfile\": null,\"SourceDeviceVendor\": null,\"SourceDynamicAddressGroup\": null,\"SourceEDL\": null,\"SourceLocation\": \"1.2.0.0-1.2.255.255\",\"SourcePort\": 59087,\"SourceUUID\": null,\"SourceUser\": null,\"Subtype\": \"end\",\"TimeGenerated\": \"2023-02-03T16:46:07.000000Z\",\"TimeGeneratedHighResolution\": \"2023-02-03T16:46:07.584000Z\",\"TimeReceived\": \"2023-02-03T16:46:14.000000Z\",\"ToZone\": \"untrusted\",\"Tunnel\": \"N/A\",\"URLCategory\": \"any\",\"VirtualLocation\": \"vsys1\",\"VirtualSystemName\": \"\",\"X-Forwarded-ForIP\": null}", "event": { + "action": "allow", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_2_json.json b/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_2_json.json index 0d46d8854..38d770ef0 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_2_json.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/test_traffic_event_2_json.json @@ -11,6 +11,7 @@ "expected": { "message": "{\"Action\": \"allow\",\"ActionSource\": \"from-policy\",\"Application\": \"incomplete\",\"Bytes\": 74,\"BytesReceived\": 0,\"BytesSent\": 74,\"ChunksReceived\": 0,\"ChunksSent\": 0,\"ChunksTotal\": 0,\"ConfigVersion\": \"10.1\",\"ContainerID\": null,\"ContainerName\": null,\"ContainerNameSpace\": null,\"DGHierarchyLevel1\": 0,\"DGHierarchyLevel2\": 0,\"DGHierarchyLevel3\": 0,\"DGHierarchyLevel4\": 0,\"DestinationAddress\": \"5.6.7.8\",\"DestinationDeviceCategory\": null,\"DestinationDeviceHost\": null,\"DestinationDeviceMac\": null,\"DestinationDeviceModel\": null,\"DestinationDeviceOSFamily\": null,\"DestinationDeviceOSVersion\": null,\"DestinationDeviceProfile\": null,\"DestinationDeviceVendor\": null,\"DestinationDynamicAddressGroup\": null,\"DestinationEDL\": null,\"DestinationLocation\": \"US\",\"DestinationPort\": 443,\"DestinationUUID\": null,\"DestinationUser\": null,\"DeviceName\": \"PA-VM\",\"DeviceSN\": \"007954000351998\",\"DynamicUserGroupName\": null,\"EndpointAssociationID\": 0,\"EndpointSerialNumber\": null,\"FromZone\": \"untrusted\",\"GPHostID\": null,\"HASessionOwner\": null,\"HTTP2Connection\": 0,\"IMEI\": null,\"IMSI\": 0,\"InboundInterface\": \"ethernet1/1\",\"LinkChangeCount\": 0,\"LinkSwitches\": null,\"LogSetting\": \"default\",\"LogType\": \"TRAFFIC\",\"NATDestination\": \"\",\"NATDestinationPort\": 0,\"NATSource\": \"\",\"NATSourcePort\": 0,\"NSSAINetworkSliceDifferentiator\": null,\"NSSAINetworkSliceType\": null,\"OutboundInterface\": \"ethernet1/1\",\"PacketsReceived\": 0,\"PacketsSent\": 1,\"PacketsTotal\": 1,\"ParentSessionID\": 0,\"ParentStarttime\": \"1970-01-01T00:00:00.000000Z\",\"Protocol\": \"tcp\",\"RepeatCount\": 1,\"Rule\": \"intrazone-default\",\"RuleUUID\": \"f903db52-4b89-4610-b908-67be412704f0\",\"SDWANCluster\": null,\"SDWANClusterType\": null,\"SDWANDeviceType\": null,\"SDWANPolicyName\": null,\"SDWANSite\": null,\"SequenceNo\": 7195838274152187100,\"SessionDuration\": 0,\"SessionEndReason\": \"aged-out\",\"SessionID\": 17634,\"SessionStartTime\": \"2023-02-03T16:45:44.000000Z\",\"SourceAddress\": \"1.2.3.4\",\"SourceDeviceCategory\": null,\"SourceDeviceHost\": null,\"SourceDeviceMac\": null,\"SourceDeviceModel\": null,\"SourceDeviceOSFamily\": null,\"SourceDeviceOSVersion\": null,\"SourceDeviceProfile\": null,\"SourceDeviceVendor\": null,\"SourceDynamicAddressGroup\": null,\"SourceEDL\": null,\"SourceLocation\": \"1.2.0.0-1.2.255.255\",\"SourcePort\": 59087,\"SourceUUID\": null,\"SourceUser\": null,\"Subtype\": \"end\",\"TimeGenerated\": \"2023-02-03T16:45:52.000000Z\",\"TimeGeneratedHighResolution\": \"2023-02-03T16:45:52.582000Z\",\"TimeReceived\": \"2023-02-03T16:45:56.000000Z\",\"ToZone\": \"untrusted\",\"Tunnel\": \"N/A\",\"URLCategory\": \"any\",\"VirtualLocation\": \"vsys1\",\"VirtualSystemName\": \"\",\"X-Forwarded-ForIP\": null}", "event": { + "action": "allow", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json index c0fccc875..5b4674816 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json @@ -11,6 +11,7 @@ "expected": { "message": "1,2024/03/12 15:02:32,016401004874,THREAT,url,2816,2024/03/12 15:02:32,10.0.0.2,192.168.0.1,0.0.0.0,0.0.0.0,rule-internet,x-fwd-for: 11.22.33.44,,web-browsing,vsys,env,zone2,a1.1,aec.2,default,2024/03/12 15:02:32,1384927,1,49802,80,0,0,0x8b000,tcp,alert,\"www.sekoia.io:443/catalog/integrations?query=this\",(9999),any,informational,client-to-server,7324220311048193508,0x0,10.0.0.0-10.255.255.255,192.168.0.0-192.168.255.255,,,0,,,10,Mozilla/4.0 (compatible; ms-office; MSOffice 16),,\"11.22.33.44\",,,,,0,0,0,0,0,VSYS,FW,,,,get,0,,0,,N/A,N/A,AppThreat-0-0,0x0,0,4294967295,,\"Sekoia,cybertech,low-risk\",ea3431a2-6869-4d9f-ad41-1858d80b406c,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-03-12T15:02:32.650+01:00,,,,internet-utility,general-internet,browser-based,4,\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\",,web-browsing,no,no,,,NonProxyTraffic", "event": { + "action": "alert", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat_csv.json b/Palo Alto Networks/paloalto-ngfw/tests/threat_csv.json index 5c152d751..7281cc3af 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/threat_csv.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat_csv.json @@ -11,7 +11,7 @@ "expected": { "message": "1,2021/08/31 14:00:02,001701000000,THREAT,vulnerability,2049,2021/08/31 14:00:02,10.0.0.2,10.2.0.1,0.0.0.0,0.0.0.0,abcd,,,web-browsing,vsys,env,zone2,a1.1,aec.2,podl,2021/08/31 14:00:02,279429,2,12345,80,0,0,0x2000,tcp,alert,\"EXAMPLE.PDF\",PDF Exploit Evasion Found(34805),any,informational,server-to-client,1320000,0x2000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,,0,,,1,,,,,,,,0,0,0,0,0,,FW,,,,,0,,0,,N/A,code-execution,AppThreat-0000-1111,0x0,0,422342342,", "event": { - "action": "code-execution", + "action": "alert", "category": [ "vulnerability" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/traffic_with_resotimestamp.json b/Palo Alto Networks/paloalto-ngfw/tests/traffic_with_resotimestamp.json index 6bd682f66..101ad2d71 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/traffic_with_resotimestamp.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/traffic_with_resotimestamp.json @@ -11,6 +11,7 @@ "expected": { "message": "1,2024/01/03 13:15:29,026701002040,TRAFFIC,end,2816,2024/01/03 13:15:29,1.2.3.4,5.6.7.8,0.0.0.0,0.0.0.0,MyRule,,,ssl,vsys1,Z_DMZ_PROXY,Z_INTERCO_WAN,ethernet1/22.301,ethernet1/3.104,Log Profile,2024/01/03 13:15:29,219781,1,60975,443,0,0,0x41c,tcp,allow,5773,758,5015,14,2024/01/03 13:15:14,0,not-resolved,,7312415129244589397,0x0,10.0.0.0-10.255.255.255,United States,,7,7,tcp-fin,0,0,0,0,,PA2314-CD,from-policy,,,0,,0,,N/A,0,0,0,0,0bbe5a53-f498-4cc2-a170-ced134f4824c,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2024-01-03T13:15:30.547+01:00,,,encrypted-tunnel,networking,browser-based,4,\\\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\\\",,ssl,no,no,0,NonProxyTraffic,", "event": { + "action": "allow", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/udp_deny_csv.json b/Palo Alto Networks/paloalto-ngfw/tests/udp_deny_csv.json index 177a6614c..023e77db3 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/udp_deny_csv.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/udp_deny_csv.json @@ -11,6 +11,7 @@ "expected": { "message": "<14>Sep 16 10:00:00 PA 1,9/16/19 10:00,1801017000,TRAFFIC,deny,2049,9/16/19 10:00,10.0.0.2,1.2.3.4,5.4.4.3,5.4.3.2,DENYALL,,,protection,vsys1,DNS,AAAAA,ae2.503,ethernet1/1,Secure,9/16/19 10:00,11111,1,130000,53,6379,53,0x400000,udp,reset-both,284,284,0,1,9/16/19 10:00,0,any,0,50660381851,0x0,10.0.0.0-10.255.255.255,Spain,0,1,0,policy-deny,0,0,0,0,,PA-1,from-application,,,0,,0,,N/A,0,0,0,0", "event": { + "action": "reset-both", "category": [ "network" ], diff --git a/Palo Alto Networks/paloalto-ngfw/tests/wildfire1_json.json b/Palo Alto Networks/paloalto-ngfw/tests/wildfire1_json.json index e433cc12d..5174efeb7 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/wildfire1_json.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/wildfire1_json.json @@ -11,6 +11,7 @@ "expected": { "message": "{\"TimeReceived\":\"2023-05-30T06:54:42.000000Z\",\"DeviceSN\":\"111111111111\",\"LogType\":\"THREAT\",\"Subtype\":\"wildfire\",\"ConfigVersion\":\"10.1\",\"TimeGenerated\":\"2023-05-30T06:52:13.000000Z\",\"SourceAddress\":\"1.2.3.4\",\"DestinationAddress\":\"5.6.7.8\",\"NATSource\":\"4.3.2.1\",\"NATDestination\":\"8.7.6.5\",\"Rule\":\"Normal Internet Access browser\",\"SourceUser\":\"john.doe@example.org\",\"DestinationUser\":null,\"Application\":\"web-browsing\",\"VirtualLocation\":\"vsys1\",\"FromZone\":\"Trust\",\"ToZone\":\"Untrust\",\"InboundInterface\":\"ethernet1/20\",\"OutboundInterface\":\"ethernet1/1\",\"LogSetting\":\"Panorama_CDL\",\"SessionID\":444444,\"RepeatCount\":1,\"SourcePort\":55555,\"DestinationPort\":80,\"NATSourcePort\":40114,\"NATDestinationPort\":80,\"Protocol\":\"tcp\",\"Action\":\"block\",\"FileName\":\"mp3.exe\",\"ThreatID\":\"Windows Executable (EXE)(52020)\",\"VendorSeverity\":\"Informational\",\"DirectionOfAttack\":\"server to client\",\"SequenceNo\":7117268851537282868,\"SourceLocation\":\"10.0.0.0-10.255.255.255\",\"DestinationLocation\":\"CN\",\"PacketID\":0,\"FileHash\":\"adc83b19e793491b1c6ea0fd8b46cd9f32e592fc\",\"ApplianceOrCloud\":\"wildfire.paloaltonetworks.com\\u0000\",\"URLCounter\":1,\"FileType\":\"pe\",\"SenderEmail\":null,\"EmailSubject\":null,\"RecipientEmail\":null,\"ReportID\":33333333333,\"DGHierarchyLevel1\":997,\"DGHierarchyLevel2\":738,\"DGHierarchyLevel3\":0,\"DGHierarchyLevel4\":0,\"VirtualSystemName\":\"\",\"DeviceName\":\"MyDevice\",\"SourceUUID\":null,\"DestinationUUID\":null,\"IMSI\":0,\"IMEI\":null,\"ParentSessionID\":0,\"ParentStarttime\":\"1970-01-01T00:00:00.000000Z\",\"Tunnel\":\"N/A\",\"ThreatCategory\":\"unknown\",\"ContentVersion\":\"0\",\"SigFlags\":\"0x0\",\"RuleUUID\":\"50afdf91-0d37-4729-8052-1382912d9895\",\"HTTP2Connection\":0,\"DynamicUserGroupName\":null,\"X-Forwarded-ForIP\":null,\"SourceDeviceCategory\":null,\"SourceDeviceProfile\":null,\"SourceDeviceModel\":null,\"SourceDeviceVendor\":null,\"SourceDeviceOSFamily\":null,\"SourceDeviceOSVersion\":null,\"SourceDeviceHost\":null,\"SourceDeviceMac\":null,\"DestinationDeviceCategory\":null,\"DestinationDeviceProfile\":null,\"DestinationDeviceModel\":null,\"DestinationDeviceVendor\":null,\"DestinationDeviceOSFamily\":null,\"DestinationDeviceOSVersion\":null,\"DestinationDeviceHost\":null,\"DestinationDeviceMac\":null,\"ContainerID\":null,\"ContainerNameSpace\":null,\"ContainerName\":null,\"SourceEDL\":null,\"DestinationEDL\":null,\"HostID\":null,\"EndpointSerialNumber\":\"xxxxxxxxxxx\",\"DomainEDL\":null,\"SourceDynamicAddressGroup\":null,\"DestinationDynamicAddressGroup\":null,\"PartialHash\":0,\"TimeGeneratedHighResolution\":\"2023-05-30T06:52:14.052000Z\",\"NSSAINetworkSliceType\":null}\n", "event": { + "action": "block", "category": [ "malware" ], From 98a972247a44c7e3472e04c5c5e85ea209b80231 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Tue, 4 Jun 2024 14:21:55 +0200 Subject: [PATCH 12/13] fix(PaloAlto): fix after merge --- Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json | 1 - 1 file changed, 1 deletion(-) diff --git a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json index 5b4674816..478fe08be 100644 --- a/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json +++ b/Palo Alto Networks/paloalto-ngfw/tests/threat-url-xff.json @@ -16,7 +16,6 @@ "network" ], "dataset": "threat", - "kind": "event", "outcome": "success", "reason": "(9999)", "type": [ From 32583cabac042dd7eba9a3db5152b5a613f25ff5 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Tue, 4 Jun 2024 15:04:44 +0200 Subject: [PATCH 13/13] fix(PaloAlto): add a missing smart-description --- .../_meta/smart-descriptions.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json b/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json index 53f7f5904..1b89c020f 100644 --- a/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json +++ b/Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json @@ -155,6 +155,28 @@ } ] }, + { + "value": "{event.action} threat between {source.nat.ip} and {destination.nat.ip}", + "conditions": [ + { + "field": "log.logger", + "value": "threat" + }, + { + "field": "source.nat.ip" + }, + { + "field": "destination.nat.ip" + } + ], + "relationships": [ + { + "source": "source.ip", + "target": "destination.ip", + "type": "connected to" + } + ] + }, { "value": "{log.logger} type {paloalto.PanOSThreatID} detected from {source.ip} user {user.name} to {destination.ip}:{destination.port} targeting host {host.name}", "conditions": [