Merge pull request #184 from whyscream/dpslavov/main

Various pattern additions

50-filter-postfix.conf

@@ -175,6 +175,13 @@ filter {
             tag_on_failure => [ "_grok_postfix_script_nomatch" ]
             add_tag        => [ "_grok_postfix_success" ]
         }
+    } else if [program] =~ /^postfix.*\/verify$/ {
+        grok {
+            patterns_dir   => "/etc/logstash/patterns.d"
+            match          => [ "message", "^%{POSTFIX_VERIFY}$" ]
+            tag_on_failure => [ "_grok_postfix_verify_nomatch" ]
+            add_tag        => [ "_grok_postfix_success" ]
+        }
     } else if [program] =~ /^postfix.*/ {
         mutate {
             add_tag => [ "_grok_postfix_program_nomatch" ]
@@ -237,6 +244,8 @@ filter {
             "postfix_client_port", "integer",
             "postfix_cmd_auth", "integer",
             "postfix_cmd_auth_accepted", "integer",
+            "postfix_cmd_bdat", "integer",
+            "postfix_cmd_bdat_accepted", "integer",
             "postfix_cmd_count", "integer",
             "postfix_cmd_count_accepted", "integer",
             "postfix_cmd_data", "integer",
@@ -247,6 +256,8 @@ filter {
             "postfix_cmd_helo_accepted", "integer",
             "postfix_cmd_mail", "integer",
             "postfix_cmd_mail_accepted", "integer",
+            "postfix_cmd_noop", "integer",
+            "postfix_cmd_noop_accepted", "integer",
             "postfix_cmd_quit", "integer",
             "postfix_cmd_quit_accepted", "integer",
             "postfix_cmd_rcpt", "integer",
@@ -266,6 +277,8 @@ filter {
             "postfix_size", "integer",
             "postfix_status_code", "integer",
             "postfix_termination_signal", "integer",
+            "postfix_verify_cache_dropped", "integer",
+            "postfix_verify_cache_retained", "integer",
 
             # list of float fields
             "postfix_delay", "float",

postfix.grok

@@ -1,10 +1,18 @@
 # Version: 1.0.0
 
+# helper patterns
+GREEDYDATA_NO_COLON [^:]*
+GREEDYDATA_NO_SEMICOLON [^;]*
+GREEDYDATA_NO_BRACKET [^<>]*
+STATUS_WORD [\w-]*
+IP_UNKNOWN unknown
+
 # common postfix patterns
 POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}|NOQUEUE)
-POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[%{IP:postfix_client_ip}\](:%{INT:postfix_client_port})?
+
+POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\](:%{INT:postfix_client_port})?
 POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service}
-POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.)
+POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|BDAT|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.)
 POSTFIX_ACTION (accept|defer|discard|filter|header-redirect|reject|reject_warning)
 POSTFIX_STATUS_CODE \d{3}
 POSTFIX_STATUS_CODE_ENHANCED \d\.\d+\.\d+
@@ -15,21 +23,17 @@ POSTFIX_TIME_UNIT %{NUMBER}[smhd]
 POSTFIX_KEYVALUE_DATA [\w-]+=[^;]*
 POSTFIX_KEYVALUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
 POSTFIX_WARNING_LEVEL (warning|fatal|info)
+POSTFIX_VERIFY_CLEANUP_TYPE (full|partial)
 
 POSTFIX_TLSCONN (Anonymous|Trusted|Untrusted|Verified) TLS connection established (to %{POSTFIX_RELAY_INFO}|from %{POSTFIX_CLIENT_INFO}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\)
 POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY_INFO}: %{GREEDYDATA:postfix_tls_error}
 
 POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission}
-POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1)
-POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once)
+POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1|Address not available|Operation timed out|Address not available|Operation timed out)
+POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once|sending %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})
 POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.*
-POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count}
+POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(bdat=(%{INT:postfix_cmd_bdat_accepted}/)?%{INT:postfix_cmd_bdat} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(noop=(%{INT:postfix_cmd_noop_accepted}/)?%{INT:postfix_cmd_noop} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count}
 
-# helper patterns
-GREEDYDATA_NO_COLON [^:]*
-GREEDYDATA_NO_SEMICOLON [^;]*
-GREEDYDATA_NO_BRACKET [^<>]*
-STATUS_WORD [\w-]*
 
 # warning patterns
 POSTFIX_WARNING_WITH_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT_INFO}: )?%{GREEDYDATA:postfix_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data}
@@ -87,9 +91,9 @@ POSTFIX_DNSBLOG_LISTING addr %{IP:postfix_client_ip} listed by domain %{HOSTNAME
 POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT_INFO}
 
 # anvil patterns
-POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
+POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_service}:(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
 POSTFIX_ANVIL_CONN_CACHE statistics: max cache size %{NUMBER:postfix_anvil_cache_size} at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
-POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
+POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_service}:(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
 
 # smtp patterns
 POSTFIX_SMTP_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_smtp_response}\))?
@@ -98,6 +102,7 @@ POSTFIX_SMTP_SSLCONNERR SSL_connect error to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LO
 POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY_INFO}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})?
 POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY_INFO} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})?
 POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\)
+POSTFIX_SMTP_RELAYREFUSED %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} refused to talk to me: %{GREEDYDATA:postfix_smtp_response}
 POSTFIX_SMTP_SSLAUTHERR %{POSTFIX_QUEUEID:postfix_queueid}: SASL authentication failed; server %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response}
 POSTFIX_SMTP_UTF8 host %{POSTFIX_RELAY_INFO} offers SMTPUTF8 support, but not 8BITMIME
 POSTFIX_SMTP_PIX %{POSTFIX_QUEUEID:postfix_queueid}: enabling PIX workarounds: %{DATA:postfix_pix_workaround} for %{POSTFIX_RELAY_INFO}
@@ -114,6 +119,9 @@ POSTFIX_SCACHE_LOOKUPS statistics: (address|domain) lookup hits=%{INT:postfix_sc
 POSTFIX_SCACHE_SIMULTANEOUS statistics: max simultaneous domains=%{INT:postfix_scache_domains} addresses=%{INT:postfix_scache_addresses} connection=%{INT:postfix_scache_connection}
 POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_scache_timestamp}
 
+# verify patterns
+POSTFIX_VERIFY_CACHE cache %{DATA} %{POSTFIX_VERIFY_CLEANUP_TYPE:postfix_verify_cleanup_type} cleanup: retained=%{INT:postfix_verify_cache_retained} dropped=%{INT:postfix_verify_cache_dropped} entries
+
 # aggregate all patterns
 POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE}
 POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MESSAGEID}|%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_CLEANUP_PREPEND}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}
@@ -122,7 +130,7 @@ POSTFIX_PIPE %{POSTFIX_PIPE_ANY}
 POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING}
 POSTFIX_DNSBLOG %{POSTFIX_DNSBLOG_LISTING}|%{POSTFIX_WARNING}
 POSTFIX_ANVIL %{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT}
-POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_SSLAUTHERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX}
+POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_SSLAUTHERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_SMTP_RELAYREFUSED}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX}
 POSTFIX_DISCARD %{POSTFIX_DISCARD_ANY}|%{POSTFIX_WARNING}
 POSTFIX_LMTP %{POSTFIX_SMTP}
 POSTFIX_PICKUP %{POSTFIX_KEYVALUE}
@@ -140,3 +148,4 @@ POSTFIX_ERROR %{POSTFIX_ERROR_ANY}
 POSTFIX_POSTSUPER %{POSTFIX_POSTSUPER_ACTION}|%{POSTFIX_POSTSUPER_SUMMARY}
 POSTFIX_POSTMAP %{POSTFIX_WARNING}
 POSTFIX_SCRIPT %{POSTFIX_WARNING}
+POSTFIX_VERIFY %{POSTFIX_VERIFY_CACHE}

test/anvil_0008.yaml

@@ -0,0 +1,7 @@
+pattern: ^%{POSTFIX_ANVIL}$
+data: "statistics: max connection count 1 for (smtp:unknown) at Sep 7 07:14:19"
+results:
+    postfix_anvil_conn_count: 1
+    postfix_service: smtp
+    postfix_client_ip_unknown: unknown
+    postfix_anvil_timestamp: Sep 7 07:14:19

test/anvil_0009.yaml

@@ -0,0 +1,8 @@
+pattern: ^%{POSTFIX_ANVIL}$
+data: "statistics: max connection rate 1/60s for (smtp:unknown) at Sep 7 07:14:19"
+results:
+    postfix_anvil_conn_rate: 1
+    postfix_anvil_conn_period: 60s
+    postfix_service: smtp
+    postfix_client_ip_unknown: unknown
+    postfix_anvil_timestamp: Sep 7 07:14:19

test/smtp_0006.yaml

@@ -2,5 +2,7 @@ pattern: ^%{POSTFIX_SMTP}$
 data: "B99FE3D: lost connection with mx3.hotmail.com[65.55.37.72] while receiving the initial server greeting"
 results:
     postfix_queueid: B99FE3D
+    postfix_smtp_lostconn_data: lost connection
+    postfix_smtp_lostconn_reason: receiving the initial server greeting
     postfix_relay_hostname: mx3.hotmail.com
     postfix_relay_ip: 65.55.37.72

test/smtp_0033.yaml

@@ -0,0 +1,8 @@
+pattern: ^%{POSTFIX_SMTP}$
+data: "745137AA: lost connection with hotmail-com.olc.protection.outlook.com[104.47.55.161] while sending RCPT TO"
+results:
+    postfix_queueid: 745137AA
+    postfix_smtp_lostconn_data: lost connection
+    postfix_smtp_lostconn_reason: sending RCPT TO
+    postfix_relay_hostname: hotmail-com.olc.protection.outlook.com
+    postfix_relay_ip: 104.47.55.161

test/smtp_0034.yaml

@@ -0,0 +1,7 @@
+pattern: ^%{POSTFIX_SMTP}$
+data: "40197AA: host mg2.egov.bg[213.91.191.86] refused to talk to me: 450 4.3.2 try again later"
+results:
+    postfix_queueid: 40197AA
+    postfix_relay_hostname: mg2.egov.bg
+    postfix_relay_ip: 213.91.191.86
+    postfix_smtp_response: 450 4.3.2 try again later

test/smtpd_0028.yaml

@@ -4,4 +4,3 @@ results:
     postfix_client_hostname: unknown
     postfix_client_ip: 72.10.165.66
     postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 data=0/1 quit=1 commands=5/7
-

test/smtpd_0032.yaml

@@ -0,0 +1,7 @@
+pattern: ^%{POSTFIX_SMTPD}$
+data: "disconnect from unknown[unknown] commands=0/0"
+results:
+    postfix_client_hostname: unknown
+    postfix_client_ip_unknown: unknown
+    postfix_command_counter_data: commands=0/0
+

test/smtpd_0033.yaml

@@ -0,0 +1,7 @@
+pattern: ^%{POSTFIX_SMTPD}$
+data: "lost connection after BDAT from example.net[158.247.23.50]"
+results:
+    postfix_smtpd_lostconn_data: lost connection
+    postfix_smtp_stage: BDAT
+    postfix_client_hostname: example.net
+    postfix_client_ip: 158.247.23.50

test/smtpd_0034.yaml

@@ -0,0 +1,6 @@
+pattern: ^%{POSTFIX_SMTPD}$
+data: "disconnect from example.net[158.247.23.50] ehlo=2 starttls=1 mail=1 rcpt=0/1 bdat=0/1 commands=4/6"
+results:
+    postfix_client_hostname: example.net
+    postfix_client_ip: 158.247.23.50
+    postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 bdat=0/1 commands=4/6

test/smtpd_0035.yaml

@@ -0,0 +1,6 @@
+pattern: ^%{POSTFIX_SMTPD}$
+data: "disconnect from unknown[93.188.162.137] ehlo=2 starttls=1 mail=1 rcpt=0/1 rset=1 noop=1 quit=1 commands=7/8"
+results:
+    postfix_client_hostname: unknown
+    postfix_client_ip: 93.188.162.137
+    postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 rset=1 noop=1 quit=1 commands=7/8

test/verify_0001.yaml

@@ -0,0 +1,6 @@
+pattern: ^%{POSTFIX_VERIFY}$
+data: "cache lmdb:/var/lib/postfix/verify_cache full cleanup: retained=724 dropped=6 entries"
+results:
+    postfix_verify_cleanup_type: full
+    postfix_verify_cache_retained: 724
+    postfix_verify_cache_dropped: 6

test/verify_0002.yaml

@@ -0,0 +1,6 @@
+pattern: ^%{POSTFIX_VERIFY}$
+data: "cache ??????????????????????????????????? partial cleanup: retained=1 dropped=0 entries"
+results:
+    postfix_verify_cleanup_type: partial
+    postfix_verify_cache_retained: 1
+    postfix_verify_cache_dropped: 0