From 0fe71e88eb5c98acbf4694f7cbfa69f3e9980b81 Mon Sep 17 00:00:00 2001 From: Malcolm Baig Date: Wed, 29 May 2024 13:08:42 +0000 Subject: [PATCH] Remove Logstash integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’ve implemented log shipping at the platform level and this custom code to send our logs to Logstash is no longer required. --- Gemfile | 6 +- Gemfile.lock | 4 - config/environments/development.rb | 4 + config/environments/production.rb | 5 + config/initializers/logstash.rb | 8 -- config/initializers/semantic_logger.rb | 54 ++------- .../initializers/log_stash_formatter_spec.rb | 111 ------------------ 7 files changed, 20 insertions(+), 172 deletions(-) delete mode 100644 config/initializers/logstash.rb delete mode 100644 spec/config/initializers/log_stash_formatter_spec.rb diff --git a/Gemfile b/Gemfile index c486f7d3..b82e747d 100644 --- a/Gemfile +++ b/Gemfile @@ -59,8 +59,6 @@ gem "clockwork" # Generate JSON Web Tokens gem "jwt" -gem "logstash-logger" -gem "rails_semantic_logger" gem 'activerecord-session_store' @@ -95,3 +93,7 @@ end group :development, :test, :review do gem "factory_bot_rails" end + +group :development, :production do + gem "rails_semantic_logger" +end diff --git a/Gemfile.lock b/Gemfile.lock index fbac7239..90b0620d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -251,9 +251,6 @@ GEM language_server-protocol (3.17.0.3) launchy (2.5.2) addressable (~> 2.8) - logstash-event (1.2.02) - logstash-logger (0.26.1) - logstash-event (~> 1.2) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -602,7 +599,6 @@ DEPENDENCIES jsbundling-rails jwt launchy - logstash-logger okcomputer omniauth-oauth2 (~> 1.8) omniauth-rails_csrf_protection diff --git a/config/environments/development.rb b/config/environments/development.rb index d6da02b7..a0fdbdf0 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -59,4 +59,8 @@ # Uncomment if you wish to allow Action Cable access from any origin. # config.action_cable.disable_request_forgery_protection = true + # + config.log_level = :debug # Or :info + config.log_format = :color # Console colorised non-json output + config.semantic_logger.backtrace_level = :debug # Show file and line number (expensive: not for production) end diff --git a/config/environments/production.rb b/config/environments/production.rb index 645a354e..87484111 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -80,4 +80,9 @@ # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false + + config.log_level = :info # Or :warn, or :error + config.log_format = :json # For parsing in Logit + config.rails_semantic_logger.add_file_appender = false # Don't log to file + config.active_record.logger = nil # Don't log SQL end diff --git a/config/initializers/logstash.rb b/config/initializers/logstash.rb deleted file mode 100644 index a1be7fcd..00000000 --- a/config/initializers/logstash.rb +++ /dev/null @@ -1,8 +0,0 @@ -LogStashLogger.configure do |config| - config.customize_event do |event| - event["environment"] = Rails.env - event["host"] = ENV["HOSTING_DOMAIN"] - event["hosting_environment"] = ENV["HOSTING_ENVIRONMENT_NAME"] - event["type"] = "rails" - end -end diff --git a/config/initializers/semantic_logger.rb b/config/initializers/semantic_logger.rb index 8613ddb3..8b7aa1b6 100644 --- a/config/initializers/semantic_logger.rb +++ b/config/initializers/semantic_logger.rb @@ -1,50 +1,10 @@ -class LogStashFormatter < SemanticLogger::Formatters::Raw - def call(log, logger) - super(log, logger) +return unless defined?(SemanticLogger) - format_exception - format_stacktrace - - hash.to_json - end - - def format_exception - exception_message = hash.dig(:exception, :message) - return if exception_message.blank? - - hash[:message] = "Exception occured: #{exception_message}" - end - - def format_stacktrace - stack_trace = hash.dig(:exception, :stack_trace) - return if stack_trace.blank? - - hash[:stacktrace] = stack_trace.first(3) - hash[:exception].delete(:stack_trace) - end +Rails.application.configure do + config.semantic_logger.application = "" # This is added by logstash from its tags + config.log_tags = [:request_id] # Prepend all log lines with the following tags end -if ENV["LOGSTASH_HOST"] && ENV["LOGSTASH_PORT"] - warn("logstash configured, sending logs there") - - # For some reason logstash / elasticsearch drops events where the payload - # is a hash. These are more conveniently accessed at the top level of the - # event, anyway, so we move it there. - customize_event = ->(event) do - if event["payload"].present? - event.append(event["payload"]) - event["payload"] = nil - end - end - - logger = - LogStashLogger.new( - { - host: ENV["LOGSTASH_HOST"], - port: ENV["LOGSTASH_PORT"], - ssl_enable: true, - type: :tcp - }.merge(customize_event:) - ) - SemanticLogger.add_appender(logger:, level: :info, formatter: LogStashFormatter.new) -end +SemanticLogger.add_appender(io: $stdout, level: Rails.application.config.log_level, +formatter: Rails.application.config.log_format) +Rails.application.config.logger.info('Application logging to STDOUT') diff --git a/spec/config/initializers/log_stash_formatter_spec.rb b/spec/config/initializers/log_stash_formatter_spec.rb deleted file mode 100644 index 4ba11d3d..00000000 --- a/spec/config/initializers/log_stash_formatter_spec.rb +++ /dev/null @@ -1,111 +0,0 @@ -require "rails_helper" - -RSpec.describe LogStashFormatter do - let(:formatter) { described_class.new } - - let(:info_level_log) do - { - application: "AYTQ", - duration: 253.90900003910065, - duration_ms: nil, - environment: "development", - host: "example-host", - hosting_environment: "local", - level: :info, - level_index: 2, - message: "Completed #new", - name: "Users::SessionsController", - payload: { - action: "new", - allocations: 47_495, - controller: "Users::SessionsController", - db_runtime: 41.77, - format: "HTML", - method: "GET", - path: "/users/sign_in", - status: 200, - status_message: "OK", - view_runtime: 77.42 - }, - pid: 6092, - thread: "puma srv tp 002", - time: "2023-03-30T11:14:00.000+01:00", - type: "rails" - } - end - - let(:fatal_level_log) do - { - application: "Semantic Logger", - duration: nil, - duration_ms: nil, - environment: "development", - exception: { - name: "ActionController::RoutingError", - message: "No route matches [GET] \"/users/unknown\"", - stack_trace: [ - "stack trace line 1", - "stack trace line 2", - "stack trace line 3", - "stack trace line 4", - "stack trace line 5" - ] - }, - host: "example-host", - hosting_environment: "local", - level: :fatal, - level_index: 5, - line: 93, - name: "Rails", - pid: 6092, - thread: "puma srv tp 004", - time: "2023-03-23T12:17:42.923+00:00", - type: "rails" - } - end - - before { allow(formatter).to receive(:hash).and_return(log) } - - describe "#format_exception" do - before { formatter.format_exception } - - context "when there is an info log" do - let(:log) { info_level_log } - - it "does not change the log" do - expect(formatter.hash).to eq(log) - end - end - - context "when there is an fatal log" do - let(:log) { fatal_level_log } - - it "adds the error message under the `message` key in the log" do - expect(formatter.hash[:message]).to eq( - 'Exception occured: No route matches [GET] "/users/unknown"' - ) - end - end - end - - describe "#format_stacktrace" do - before { formatter.format_stacktrace } - - let(:log) { fatal_level_log } - - it "removes the stacktrace from the exception" do - expect(formatter.hash[:exception]).to eq( - { - name: "ActionController::RoutingError", - message: "No route matches [GET] \"/users/unknown\"" - } - ) - end - - it "adds a stacktrace key to the log" do - expect(formatter.hash[:stacktrace]).to eq( - ["stack trace line 1", "stack trace line 2", "stack trace line 3"] - ) - end - end -end