Skip to content

Commit

Permalink
Merge pull request SEKOIA-IO#1136 from SEKOIA-IO/feat/improve_postfix…
Browse files Browse the repository at this point in the history
…_parser

Improve Postfix parser
  • Loading branch information
squioc authored Jul 8, 2024
2 parents b16caa0 + 10c8931 commit 20830a2
Show file tree
Hide file tree
Showing 137 changed files with 5,763 additions and 18 deletions.
16 changes: 10 additions & 6 deletions Postfix/postfix/ingest/parser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@ name: Postfix
ignored_values: ["-"]
pipeline:
- name: event
filter: "{{original.log.syslog.get('appname') != None}}"
external:
name: grok.case_match
properties:
input_field: "{{original.message}}"
output_field: message
pattern_key: "{{original.log.syslog.appname}}"
pattern_key: "{{original.get('log', {}).get('syslog', {}).get('appname', 'default')}}"
patterns:
"postfix/smtp": "%{POSTFIX_SMTP}"
"postfix/smtpd": "%{POSTFIX_SMTPD}"
Expand All @@ -20,14 +19,15 @@ pipeline:
"postfix-nospam/cleanup": "%{POSTFIX_CLEANUP}"
"postfix-nospam/smtp": "%{POSTFIX_SMTP}"
"postfix-nospam/qmgr": "%{POSTFIX_QMGR}"
"postfix/pipe": "%{POSTFIX_DISCARD}"
"postfix/pipe": "%{POSTFIX_LOCAL}"
"spamd": "%{SPAMD}"
"postgrey": "%{POSTFIX_POSTGREY}"
"postfix/local": "%{POSTFIX_LOCAL}"
"postfix/error": "%{POSTFIX_DISCARD}"
"postfix/anvil": "%{POSTFIX_ANVIL}"
"postfix/pickup": "%{POSTFIX_PICKUP}"
"postfix": "%{POSTFIX}"
"default": "%{POSTFIX_POSTGREY}|%{POSTFIX}|%{SPAMD}|%{POSTFIX_POLCY_SPF}|%{POSTFIX_CLEANUP}|%{POSTFIX_QMGR}|%{POSTFIX_SMTPD}"

custom_patterns:
EMAILLOCALPART: "[a-zA-Z0-9_.+-=:]+"
Expand All @@ -37,7 +37,7 @@ pipeline:
GREEDYDATA_NO_TREMMA: '[^\"]*'
NOQUEUE_FALLBACK: 'NOQUEUE: %{DATA:action_name}: %{DATA:action_type} from %{HOSTNAME:source_domain}?\[%{IP:source_address}\]:%{GREEDYDATA}from=<%{EMAILADDRESS:postfix_from}> to=<%{EMAILADDRESS:postfix_to}>%{GREEDYDATA}'
POSTFIX_POSTGREY: "%{POSTFIX_PASS}|%{POSTFIX_WHITELISTED}"
POSTFIX: "%{POSTFIX_PROXY}|%{POSTFIX_NOQUEUE}|%{POSTFIX_TLS}|%{POSTFIX_PIPE}|%{POSTFIX_POSTSCREEN}|%{POSTFIX_DNSBLOG}|%{POSTFIX_ANVIL}|%{POSTFIX_SMTP}|%{POSTFIX_DISCARD}|%{POSTFIX_LMTP}|%{POSTFIX_PICKUP}|%{POSTFIX_TLSPROXY}|%{POSTFIX_MASTER}|%{POSTFIX_BOUNCE}|%{POSTFIX_SENDMAIL}|%{POSTFIX_POSTDROP}|%{POSTFIX_SCACHE}|%{POSTFIX_TRIVIAL_REWRITE}|%{POSTFIX_TLSMGR}|%{POSTFIX_LOCAL}|%{POSTFIX_VIRTUAL}|%{POSTFIX_ERROR}|%{POSTFIX_POSTSUPER}|%{POSTFIX_PASS}|%{POSTFIX_WARNING_FILENAME}|%{POSTFIX_START}"
POSTFIX: "%{POSTFIX_PROXY}|%{POSTFIX_NOQUEUE}|%{POSTFIX_TLS}|%{POSTFIX_SMTP}|%{POSTFIX_PIPE}|%{POSTFIX_POSTSCREEN}|%{POSTFIX_DNSBLOG}|%{POSTFIX_ANVIL}|%{POSTFIX_DISCARD}|%{POSTFIX_LMTP}|%{POSTFIX_PICKUP}|%{POSTFIX_TLSPROXY}|%{POSTFIX_MASTER}|%{POSTFIX_BOUNCE}|%{POSTFIX_SENDMAIL}|%{POSTFIX_POSTDROP}|%{POSTFIX_SCACHE}|%{POSTFIX_TRIVIAL_REWRITE}|%{POSTFIX_TLSMGR}|%{POSTFIX_LOCAL}|%{POSTFIX_VIRTUAL}|%{POSTFIX_ERROR}|%{POSTFIX_POSTSUPER}|%{POSTFIX_PASS}|%{POSTFIX_WARNING_FILENAME}|%{POSTFIX_START}"
POSTFIX_TLS: "%{POSTFIX_QUEUEID}: %{POSTFIX_TLSUSED}"
POSTFIX_NOQUEUE: "NOQUEUE: client=%{POSTFIX_DESTINATION_INFO}"
POSTFIX_PROXY: "proxy-%{POSTFIX_ACTION:action_name}: (%{POSTFIX_SMTP_STAGE:action_type}): %{POSTFIX_PROXY_MESSAGE:postfix_proxy_message};( from=<(%{EMAILADDRESS:postfix_from})?>)?( to=<(%{EMAILADDRESS:postfix_to})?>)?(%{DATA} helo=<%{DATA:source_domain}>)?%{GREEDYDATA}"
Expand Down Expand Up @@ -65,7 +65,7 @@ pipeline:
POSTFIX_KEYVALUE: "%{POSTFIX_QUEUEID}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}"
POSTFIX_KEYVALUE_DATA: '([\w-]+=[^;]*)'
POSTFIX_LMTP: "%{POSTFIX_SMTP}" ###
POSTFIX_LOCAL: "%{POSTFIX_KEYVALUE}|%{POSTFIX_WARNING}"
POSTFIX_LOCAL: "%{POSTFIX_SMTP_RELAY}|%{POSTFIX_KEYVALUE}|%{POSTFIX_WARNING}"
POSTFIX_LOSTCONN: "(Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1)"
POSTFIX_LOSTCONN_REASONS: "(receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once)"
POSTFIX_MASTER: "%{POSTFIX_MASTER_START}|%{POSTFIX_MASTER_EXIT}|%{POSTFIX_WARNING}"
Expand Down Expand Up @@ -353,7 +353,11 @@ stages:
- set:
network.protocol: "smtp"
filter: "{{event.message.POSTFIX_SMTPD_CONNECT != None and event.message.tmp_ESMTP != None}}"
filter: "{{event.message.POSTFIX_SMTPD_CONNECT != null and event.message.tmp_ESMTP != null}}"
- set:
network.protocol: "smtp"
filter: "{{original.log.syslog.appname == 'postfix'}}"
- set:
action.type: "{{event.message.action_type}}"
Expand Down
11 changes: 5 additions & 6 deletions Postfix/postfix/tests/local3.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,14 @@
"action": {
"name": "sent",
"outcome": "success",
"outcome_reason": "success",
"target": "network-traffic"
},
"destination": {
"address": "example.org",
"domain": "example.org",
"address": "127.0.0.1",
"domain": "127.0.0.1",
"ip": "127.0.0.1",
"port": 10025,
"registered_domain": "example.org",
"top_level_domain": "org"
"port": 10025
},
"email": {
"to": {
Expand All @@ -48,7 +47,7 @@
},
"related": {
"hosts": [
"example.org"
"127.0.0.1"
],
"ip": [
"127.0.0.1"
Expand Down
11 changes: 5 additions & 6 deletions Postfix/postfix/tests/pipe3.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,14 @@
"action": {
"name": "sent",
"outcome": "success",
"outcome_reason": "success",
"target": "network-traffic"
},
"destination": {
"address": "example.org",
"domain": "example.org",
"address": "127.0.0.1",
"domain": "127.0.0.1",
"ip": "127.0.0.1",
"port": 10025,
"registered_domain": "example.org",
"top_level_domain": "org"
"port": 10025
},
"email": {
"to": {
Expand All @@ -48,7 +47,7 @@
},
"related": {
"hosts": [
"example.org"
"127.0.0.1"
],
"ip": [
"127.0.0.1"
Expand Down
28 changes: 28 additions & 0 deletions Postfix/postfix/tests/wo_appname_anvil.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"input": {
"message": "statistics: max connection count 10 for (smtp:1.2.3.4) at Sep 11 10:47:30"
},
"expected": {
"message": "statistics: max connection count 10 for (smtp:1.2.3.4) at Sep 11 10:47:30",
"event": {
"category": [
"email"
],
"type": [
"info"
]
},
"network": {
"protocol": "smtp"
},
"related": {
"ip": [
"1.2.3.4"
]
},
"source": {
"address": "1.2.3.4",
"ip": "1.2.3.4"
}
}
}
43 changes: 43 additions & 0 deletions Postfix/postfix/tests/wo_appname_appname_postfix_error.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
{
"input": {
"message": "2298F5F619: to=<[email protected]>, relay=none, delay=89758, delays=89758/0.02/0/0, dsn=4.4.1, status=deferred (delivery temporarily suspended: connect to mail.corp.com[1.1.1.1]:25: Connection timed out) 215"
},
"expected": {
"message": "2298F5F619: to=<[email protected]>, relay=none, delay=89758, delays=89758/0.02/0/0, dsn=4.4.1, status=deferred (delivery temporarily suspended: connect to mail.corp.com[1.1.1.1]:25: Connection timed out) 215",
"event": {
"category": [
"email"
],
"outcome": "success",
"type": [
"info"
]
},
"action": {
"name": "deferred",
"outcome": "success",
"target": "network-traffic"
},
"destination": {
"address": "corp.com",
"domain": "corp.com",
"registered_domain": "corp.com",
"top_level_domain": "com"
},
"email": {
"to": {
"address": [
"[email protected]"
]
}
},
"network": {
"protocol": "smtp"
},
"related": {
"hosts": [
"corp.com"
]
}
}
}
44 changes: 44 additions & 0 deletions Postfix/postfix/tests/wo_appname_appname_postfix_local.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"input": {
"message": "11FDF5F62A: to=<[email protected]>, relay=local, delay=80181, delays=80181/0.02/0/0, dsn=4.0.0, status=deferred (user lookup error)"
},
"expected": {
"message": "11FDF5F62A: to=<[email protected]>, relay=local, delay=80181, delays=80181/0.02/0/0, dsn=4.0.0, status=deferred (user lookup error)",
"event": {
"category": [
"email"
],
"outcome": "success",
"type": [
"info"
]
},
"action": {
"name": "deferred",
"outcome": "success",
"target": "network-traffic"
},
"destination": {
"address": "sub.corp.com",
"domain": "sub.corp.com",
"registered_domain": "corp.com",
"subdomain": "sub",
"top_level_domain": "com"
},
"email": {
"to": {
"address": [
"[email protected]"
]
}
},
"network": {
"protocol": "smtp"
},
"related": {
"hosts": [
"sub.corp.com"
]
}
}
}
43 changes: 43 additions & 0 deletions Postfix/postfix/tests/wo_appname_bounced.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
{
"input": {
"message": "3D770111AF50: to=<[email protected]>, relay=none, delay=1.2, delays=1.1/0/0.03/0, dsn=5.4.4, status=bounced (Host or domain name not found. Name service error for name=corp.com type=AAAA: Host not found)"
},
"expected": {
"message": "3D770111AF50: to=<[email protected]>, relay=none, delay=1.2, delays=1.1/0/0.03/0, dsn=5.4.4, status=bounced (Host or domain name not found. Name service error for name=corp.com type=AAAA: Host not found)",
"event": {
"category": [
"email"
],
"outcome": "success",
"type": [
"info"
]
},
"action": {
"name": "bounced",
"outcome": "success",
"target": "network-traffic"
},
"destination": {
"address": "corp.com",
"domain": "corp.com",
"registered_domain": "corp.com",
"top_level_domain": "com"
},
"email": {
"to": {
"address": [
"[email protected]"
]
}
},
"network": {
"protocol": "smtp"
},
"related": {
"hosts": [
"corp.com"
]
}
}
}
55 changes: 55 additions & 0 deletions Postfix/postfix/tests/wo_appname_cleanup.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{
"input": {
"message": "77EFFC0015: warning: header Content-Disposition: inline; filename=\"image003.jpg\"; size=26055;??creation-date=\"Thu, 12 Sep 2019 12:39:01 GMT\";??modification-date=\"Thu, 12 Sep 2019 12:40:01 GMT\" from mail.outbound.protection.outlook.com[1.1.1.1]; from=<> to=<[email protected]> proto=ESMTP helo=<NAM03.outbound.protection.outlook.com>"
},
"expected": {
"message": "77EFFC0015: warning: header Content-Disposition: inline; filename=\"image003.jpg\"; size=26055;??creation-date=\"Thu, 12 Sep 2019 12:39:01 GMT\";??modification-date=\"Thu, 12 Sep 2019 12:40:01 GMT\" from mail.outbound.protection.outlook.com[1.1.1.1]; from=<> to=<[email protected]> proto=ESMTP helo=<NAM03.outbound.protection.outlook.com>",
"event": {
"category": [
"email"
],
"type": [
"info"
]
},
"destination": {
"address": "exemple.com",
"domain": "exemple.com",
"registered_domain": "exemple.com",
"top_level_domain": "com"
},
"email": {
"to": {
"address": [
"[email protected]"
]
}
},
"file": {
"created": "2019-09-12T12:39:01Z",
"ctime": "2019-09-12T12:40:01Z",
"name": "image003.jpg",
"size": 26055
},
"network": {
"protocol": "smtp"
},
"related": {
"hosts": [
"exemple.com",
"mail.outbound.protection.outlook.com"
],
"ip": [
"1.1.1.1"
]
},
"source": {
"address": "mail.outbound.protection.outlook.com",
"domain": "mail.outbound.protection.outlook.com",
"ip": "1.1.1.1",
"registered_domain": "outlook.com",
"subdomain": "mail.outbound.protection",
"top_level_domain": "com"
}
}
}
54 changes: 54 additions & 0 deletions Postfix/postfix/tests/wo_appname_cleanup2.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
"input": {
"message": "3D770111AF50: warning: header Subject: Manquants LASTNAME GB Nouvelle version from unknown[10.1.1.1]; from=<[email protected]> to=<[email protected]> proto=ESMTP helo=<SUBDOMAIN.CORP.COM>"
},
"expected": {
"message": "3D770111AF50: warning: header Subject: Manquants LASTNAME GB Nouvelle version from unknown[10.1.1.1]; from=<[email protected]> to=<[email protected]> proto=ESMTP helo=<SUBDOMAIN.CORP.COM>",
"event": {
"category": [
"email"
],
"type": [
"info"
]
},
"destination": {
"address": "corp.com",
"domain": "corp.com",
"registered_domain": "corp.com",
"top_level_domain": "com"
},
"email": {
"from": {
"address": [
"[email protected]"
]
},
"to": {
"address": [
"[email protected]"
]
}
},
"network": {
"protocol": "smtp"
},
"related": {
"hosts": [
"SUBDOMAIN.CORP.COM",
"corp.com"
],
"ip": [
"10.1.1.1"
]
},
"source": {
"address": "SUBDOMAIN.CORP.COM",
"domain": "SUBDOMAIN.CORP.COM",
"ip": "10.1.1.1",
"registered_domain": "CORP.COM",
"subdomain": "SUBDOMAIN",
"top_level_domain": "COM"
}
}
}
Loading

0 comments on commit 20830a2

Please sign in to comment.