From dfe1970afb17eb13373cd77741bf3371297390b6 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Mon, 2 Aug 2021 15:41:53 +0000 Subject: [PATCH 1/3] refactor: refactor message inspectors into own classes --- lib/postal.rb | 3 + lib/postal/message_db/message.rb | 18 +-- lib/postal/message_inspection.rb | 135 +++--------------- lib/postal/message_inspector.rb | 38 +++++ lib/postal/message_inspectors.rb | 9 ++ lib/postal/message_inspectors/clamav.rb | 45 ++++++ .../message_inspectors/spam_assassin.rb | 54 +++++++ lib/postal/spam_check.rb | 2 +- 8 files changed, 178 insertions(+), 126 deletions(-) create mode 100644 lib/postal/message_inspector.rb create mode 100644 lib/postal/message_inspectors.rb create mode 100644 lib/postal/message_inspectors/clamav.rb create mode 100644 lib/postal/message_inspectors/spam_assassin.rb diff --git a/lib/postal.rb b/lib/postal.rb index 6186cccc..4b96e6e7 100644 --- a/lib/postal.rb +++ b/lib/postal.rb @@ -15,6 +15,8 @@ module Postal autoload :Job autoload :MessageDB autoload :MessageInspection + autoload :MessageInspector + autoload :MessageInspectors autoload :MessageParser autoload :MessageRequeuer autoload :MXLookup @@ -37,6 +39,7 @@ def self.eager_load! super Postal::MessageDB.eager_load! Postal::SMTPServer.eager_load! + Postal::MessageInspectors.eager_load! end end diff --git a/lib/postal/message_db/message.rb b/lib/postal/message_db/message.rb index c288f836..7a7587aa 100644 --- a/lib/postal/message_db/message.rb +++ b/lib/postal/message_db/message.rb @@ -498,14 +498,16 @@ def rcpt_to_return_path? # Inspect this message # def inspect_message - if result = MessageInspection.new(self.raw_message, self.scope&.to_sym) - # Update the messages table with the results of our inspection - update(:inspected => 1, :spam_score => result.filtered_spam_score, :threat => result.threat?, :threat_details => result.threat_message) - # Add any spam details into the spam checks database - self.database.insert_multi(:spam_checks, [:message_id, :code, :score, :description], result.filtered_spam_checks.map { |d| [self.id, d.code, d.score, d.description]}) - # Return the result - result - end + result = MessageInspection.scan(self, self.scope&.to_sym) + + # Update the messages table with the results of our inspection + update(:inspected => 1, :spam_score => result.spam_score, :threat => result.threat?, :threat_details => result.threat_message) + + # Add any spam details into the spam checks database + self.database.insert_multi(:spam_checks, [:message_id, :code, :score, :description], result.spam_checks.map { |d| [self.id, d.code, d.score, d.description] }) + + # Return the result + result end # diff --git a/lib/postal/message_inspection.rb b/lib/postal/message_inspection.rb index f7f560ff..4096cd72 100644 --- a/lib/postal/message_inspection.rb +++ b/lib/postal/message_inspection.rb @@ -1,140 +1,41 @@ -require 'timeout' -require 'socket' -require 'json' - module Postal class MessageInspection - SPAM_EXCLUSIONS = { - :outgoing => ['NO_RECEIVED', 'NO_RELAYS', 'ALL_TRUSTED', 'FREEMAIL_FORGED_REPLYTO', 'RDNS_DYNAMIC', 'CK_HELO_GENERIC', /^SPF\_/, /^HELO\_/, /DKIM_/, /^RCVD_IN_/], - :incoming => [] - } + attr_reader :message + attr_reader :scope + attr_reader :spam_checks + attr_accessor :threat + attr_accessor :threat_message - def initialize(message, scope = :incoming) + def initialize(message, scope) @message = message @scope = scope - @threat = false - @spam_score = 0.0 @spam_checks = [] - - if Postal.config.spamd.enabled? - scan_for_spam - end - - if Postal.config.clamav.enabled? - scan_for_threats - end + @threat = false end def spam_score - @spam_score - end - - def spam_checks - @spam_checks - end + return 0 if @spam_checks.empty? - def filtered_spam_checks - @filtered_spam_checks ||= @spam_checks.reject do |check| - SPAM_EXCLUSIONS[@scope].any? do |item| - item == check.code || (item.is_a?(Regexp) && item =~ check.code) - end - end - end - - def filtered_spam_score - filtered_spam_checks.inject(0.0) do |total, check| - total += check.score || 0.0 - end.round(2) + @spam_checks.sum(&:score) end def threat? - @threat - end - - def threat_message - @threat_message + @threat == true end - private - - def scan_for_spam - data = nil - Timeout.timeout(15) do - tcp_socket = TCPSocket.new(Postal.config.spamd.host, Postal.config.spamd.port) - tcp_socket.write("REPORT SPAMC/1.2\r\n") - tcp_socket.write("Content-length: #{@message.bytesize}\r\n") - tcp_socket.write("\r\n") - tcp_socket.write(@message) - tcp_socket.close_write - data = tcp_socket.read - end - - spam_checks = [] - total = 0.0 - rules = data ? data.split(/^---(.*)\r?\n/).last.split(/\r?\n/) : [] - while line = rules.shift - if line =~ /\A([\- ]?[\d\.]+)\s+(\w+)\s+(.*)/ - total += $1.to_f - spam_checks << SPAMCheck.new($2, $1.to_f, $3) - else - spam_checks.last.description << " " + line.strip - end + def scan + MessageInspector.inspectors.each do |inspector| + inspector.inspect_message(self) end - - @spam_score = total.round(1) - @spam_checks = spam_checks - - rescue Timeout::Error - @spam_checks = [SPAMCheck.new("TIMEOUT", 0, "Timed out when scanning for spam")] - rescue => e - logger.error "Error talking to spamd: #{e.class} (#{e.message})" - logger.error e.backtrace[0,5] - @spam_checks = [SPAMCheck.new("ERROR", 0, "Error when scanning for spam")] - ensure - tcp_socket.close rescue nil end - def scan_for_threats - @threat = false - - data = nil - Timeout.timeout(10) do - tcp_socket = TCPSocket.new(Postal.config.clamav.host, Postal.config.clamav.port) - tcp_socket.write("zINSTREAM\0") - tcp_socket.write([@message.bytesize].pack("N")) - tcp_socket.write(@message) - tcp_socket.write([0].pack("N")) - tcp_socket.close_write - data = tcp_socket.read - end - - if data && data =~ /\Astream\:\s+(.*?)[\s\0]+?/ - if $1.upcase == 'OK' - @threat = false - @threat_message = "No threats found" - else - @threat = true - @threat_message = $1 - end - else - @threat = false - @threat_message = "Could not scan message" + class << self + def scan(message, scope) + inspection = new(message, scope) + inspection.scan + inspection end - rescue Timeout::Error - @threat = false - @threat_message = "Timed out scanning for threats" - rescue => e - logger.error "Error talking to clamav: #{e.class} (#{e.message})" - logger.error e.backtrace[0,5] - @threat = false - @threat_message = "Error when scanning for threats" - ensure - tcp_socket.close rescue nil - end - - def logger - Postal.logger_for(:message_inspection) end end diff --git a/lib/postal/message_inspector.rb b/lib/postal/message_inspector.rb new file mode 100644 index 00000000..25da0d5e --- /dev/null +++ b/lib/postal/message_inspector.rb @@ -0,0 +1,38 @@ +module Postal + class MessageInspector + + def initialize(config) + @config = config + end + + # Inspect a message and update the inspection with the results + # as appropriate. + def inspect_message(message, scope, inspection) + end + + private + + def logger + Postal.logger_for(:message_inspection) + end + + class << self + # Return an array of all inspectors that are available for this + # installation. + def inspectors + Array.new.tap do |inspectors| + + if Postal.config.spamd&.enabled + inspectors << MessageInspectors::SpamAssassin.new(Postal.config.spamd) + end + + if Postal.config.clamav&.enabled + inspectors << MessageInspectors::Clamav.new(Postal.config.clamav) + end + + end + end + end + + end +end diff --git a/lib/postal/message_inspectors.rb b/lib/postal/message_inspectors.rb new file mode 100644 index 00000000..916a3a53 --- /dev/null +++ b/lib/postal/message_inspectors.rb @@ -0,0 +1,9 @@ +module Postal + module MessageInspectors + extend ActiveSupport::Autoload + eager_autoload do + autoload :Clamav + autoload :SpamAssassin + end + end +end diff --git a/lib/postal/message_inspectors/clamav.rb b/lib/postal/message_inspectors/clamav.rb new file mode 100644 index 00000000..dbbda7d2 --- /dev/null +++ b/lib/postal/message_inspectors/clamav.rb @@ -0,0 +1,45 @@ +module Postal + module MessageInspectors + class Clamav < MessageInspector + + def inspect_message(inspection) + raw_message = inspection.message.raw_message + + data = nil + Timeout.timeout(10) do + tcp_socket = TCPSocket.new(@config.host, @config.port) + tcp_socket.write("zINSTREAM\0") + tcp_socket.write([raw_message.bytesize].pack("N")) + tcp_socket.write(raw_message) + tcp_socket.write([0].pack("N")) + tcp_socket.close_write + data = tcp_socket.read + end + + if data && data =~ /\Astream\:\s+(.*?)[\s\0]+?/ + if $1.upcase == 'OK' + inspection.threat = false + inspection.threat_message = "No threats found" + else + inspection.threat = true + inspection.threat_message = $1 + end + else + inspection.threat = false + inspection.threat_message = "Could not scan message" + end + rescue Timeout::Error + inspection.threat = false + inspection.threat_message = "Timed out scanning for threats" + rescue => e + logger.error "Error talking to clamav: #{e.class} (#{e.message})" + logger.error e.backtrace[0,5] + inspection.threat = false + inspection.threat_message = "Error when scanning for threats" + ensure + tcp_socket.close rescue nil + end + + end + end +end diff --git a/lib/postal/message_inspectors/spam_assassin.rb b/lib/postal/message_inspectors/spam_assassin.rb new file mode 100644 index 00000000..e84a136e --- /dev/null +++ b/lib/postal/message_inspectors/spam_assassin.rb @@ -0,0 +1,54 @@ +module Postal + module MessageInspectors + class SpamAssassin < MessageInspector + + EXCLUSIONS = { + :outgoing => ['NO_RECEIVED', 'NO_RELAYS', 'ALL_TRUSTED', 'FREEMAIL_FORGED_REPLYTO', 'RDNS_DYNAMIC', 'CK_HELO_GENERIC', /^SPF\_/, /^HELO\_/, /DKIM_/, /^RCVD_IN_/], + :incoming => [] + } + + def inspect_message(inspection) + data = nil + raw_message = inspection.message.raw_message + Timeout.timeout(15) do + tcp_socket = TCPSocket.new(@config.host, @config.port) + tcp_socket.write("REPORT SPAMC/1.2\r\n") + tcp_socket.write("Content-length: #{raw_message.bytesize}\r\n") + tcp_socket.write("\r\n") + tcp_socket.write(raw_message) + tcp_socket.close_write + data = tcp_socket.read + end + + spam_checks = [] + total = 0.0 + rules = data ? data.split(/^---(.*)\r?\n/).last.split(/\r?\n/) : [] + while line = rules.shift + if line =~ /\A([\- ]?[\d\.]+)\s+(\w+)\s+(.*)/ + total += $1.to_f + spam_checks << SpamCheck.new($2, $1.to_f, $3) + else + spam_checks.last.description << " " + line.strip + end + end + + checks = spam_checks.reject { |s| EXCLUSIONS[inspection.scope].include?(s.code) } + checks.each do |check| + inspection.spam_checks << check + end + + rescue Timeout::Error + inspection.spam_checks << SpamCheck.new("TIMEOUT", 0, "Timed out when scanning for spam") + + rescue => e + logger.error "Error talking to spamd: #{e.class} (#{e.message})" + logger.error e.backtrace[0,5] + inspection.spam_checks << SpamCheck.new("ERROR", 0, "Error when scanning for spam") + + ensure + tcp_socket.close rescue nil + end + + end + end +end diff --git a/lib/postal/spam_check.rb b/lib/postal/spam_check.rb index ea0e48a0..799531b9 100644 --- a/lib/postal/spam_check.rb +++ b/lib/postal/spam_check.rb @@ -1,5 +1,5 @@ module Postal - class SPAMCheck + class SpamCheck attr_reader :code, :score, :description From 724325a1b97d61ef1e134240e4f70aaad39dbf98 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Mon, 2 Aug 2021 16:17:18 +0000 Subject: [PATCH 2/3] feat: support for configuring the default spam threshold values for new servers --- app/models/server.rb | 4 ++-- config/postal.defaults.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/server.rb b/app/models/server.rb index 53c46497..c2b73195 100644 --- a/app/models/server.rb +++ b/app/models/server.rb @@ -69,8 +69,8 @@ class Server < ApplicationRecord default_value :raw_message_retention_days, -> { 30 } default_value :raw_message_retention_size, -> { 2048 } default_value :message_retention_days, -> { 60 } - default_value :spam_threshold, -> { 5.0 } - default_value :spam_failure_threshold, -> { 20.0 } + default_value :spam_threshold, -> { Postal.config.general.default_spam_threshold } + default_value :spam_failure_threshold, -> { Postal.config.general.default_spam_failure_threshold } validates :name, :presence => true, :uniqueness => {:scope => :organization_id} validates :mode, :inclusion => {:in => MODES} diff --git a/config/postal.defaults.yml b/config/postal.defaults.yml index 7d237c69..5b30dfbc 100644 --- a/config/postal.defaults.yml +++ b/config/postal.defaults.yml @@ -15,6 +15,8 @@ general: maximum_hold_expiry_days: 7 suppression_list_removal_delay: 30 use_local_ns_for_domains: false + default_spam_threshold: 5.0 + default_spam_failure_threshold: 20.0 web_server: bind_address: 127.0.0.1 From a1277baba56ea6d6b4da4bba87b00cd3dbf0305e Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Mon, 2 Aug 2021 16:18:07 +0000 Subject: [PATCH 3/3] feat: support for using rspamd for spam filtering --- config/postal.defaults.yml | 8 +++ lib/postal/message_inspector.rb | 4 +- lib/postal/message_inspectors.rb | 1 + lib/postal/message_inspectors/rspamd.rb | 74 +++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 lib/postal/message_inspectors/rspamd.rb diff --git a/config/postal.defaults.yml b/config/postal.defaults.yml index 5b30dfbc..a69fe345 100644 --- a/config/postal.defaults.yml +++ b/config/postal.defaults.yml @@ -102,6 +102,14 @@ rails: environment: production secret_key: +rspamd: + enabled: false + host: 127.0.0.1 + port: 11334 + ssl: false + password: null + flags: null + spamd: enabled: false host: 127.0.0.1 diff --git a/lib/postal/message_inspector.rb b/lib/postal/message_inspector.rb index 25da0d5e..10224a36 100644 --- a/lib/postal/message_inspector.rb +++ b/lib/postal/message_inspector.rb @@ -22,7 +22,9 @@ class << self def inspectors Array.new.tap do |inspectors| - if Postal.config.spamd&.enabled + if Postal.config.rspamd&.enabled + inspectors << MessageInspectors::Rspamd.new(Postal.config.rspamd) + elsif Postal.config.spamd&.enabled inspectors << MessageInspectors::SpamAssassin.new(Postal.config.spamd) end diff --git a/lib/postal/message_inspectors.rb b/lib/postal/message_inspectors.rb index 916a3a53..33b7565c 100644 --- a/lib/postal/message_inspectors.rb +++ b/lib/postal/message_inspectors.rb @@ -3,6 +3,7 @@ module MessageInspectors extend ActiveSupport::Autoload eager_autoload do autoload :Clamav + autoload :Rspamd autoload :SpamAssassin end end diff --git a/lib/postal/message_inspectors/rspamd.rb b/lib/postal/message_inspectors/rspamd.rb new file mode 100644 index 00000000..914a7806 --- /dev/null +++ b/lib/postal/message_inspectors/rspamd.rb @@ -0,0 +1,74 @@ +require 'net/http' + +module Postal + module MessageInspectors + class Rspamd < MessageInspector + + class Error < StandardError + end + + def inspect_message(inspection) + response = request(inspection.message, inspection.scope) + response = JSON.parse(response.body) + return unless response['symbols'].is_a?(Hash) + + response['symbols'].values.each do |symbol| + next if symbol['description'].blank? + + inspection.spam_checks << SpamCheck.new(symbol['name'], symbol['score'], symbol['description']) + end + rescue Error => e + inspection.spam_checks << SpamCheck.new("ERROR", 0, e.message) + end + + private + + def request(message, scope) + http = Net::HTTP.new(@config.host, @config.port) + http.use_ssl = true if @config.ssl + http.read_timeout = 10 + http.open_timeout = 10 + + raw_message = message.raw_message + + request = Net::HTTP::Post.new('/checkv2') + request.body = raw_message + request['Content-Length'] = raw_message.bytesize.to_s + request['Password'] = @config.password if @config.password + request['Flags'] = @config.flags if @config.flags + request['User-Agent'] = 'Postal' + request['Deliver-To'] = message.rcpt_to + request['From'] = message.mail_from + request['Rcpt'] = message.rcpt_to + request['Queue-Id'] = message.token + + if scope == :outgoing + request['User'] = '' + # We don't actually know the IP but an empty input here will + # still trigger rspamd to treat this as an outbound email + # and disable certain checks. + # https://rspamd.com/doc/tutorials/scanning_outbound.html + request['Ip'] = '' + end + + response = nil + begin + response = http.request(request) + rescue Exception => e + logger.error "Error talking to rspamd: #{e.class} (#{e.message})" + logger.error e.backtrace[0,5] + + raise Error, "Error when scanning with rspamd (#{e.class})" + end + + unless response.is_a?(Net::HTTPOK) + logger.info "Got #{response.code} status from rspamd, wanted 200" + raise Error, "Error when scanning with rspamd (got #{response.code})" + end + + response + end + + end + end +end