[7891] - apply/pubilsh logging config

app/lib/custom_log_formatter.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class CustomLogFormatter < SemanticLogger::Formatters::Json
+  def call(log, logger)
+    super
+
+    format_job_data
+    format_exception
+    format_json_message_context
+    format_backtrace
+    remove_post_params
+
+    hash.to_json
+  end
+
+private
+
+  def format_job_data
+    hash[:job_id] = RequestStore.store[:job_id] if RequestStore.store[:job_id].present?
+    hash[:job_queue] = RequestStore.store[:job_queue] if RequestStore.store[:job_queue].present?
+  end
+
+  def format_exception
+    exception_message = hash.dig(:exception, :message)
+    return if exception_message.nil?
+
+    hash[:message] = "Exception occurred: #{exception_message}"
+  end
+
+  def format_json_message_context
+    if hash[:message].present?
+      context = JSON.parse(hash[:message])["context"]
+      hash[:sidekiq_job_context] = hash[:message]
+      hash[:message] = context
+    end
+  rescue JSON::ParserError
+    nil
+  end
+
+  def format_backtrace
+    return unless hash[:message]&.start_with?("/")
+
+    message_lines = hash[:message].split("\n")
+    return unless message_lines.all? { |line| line.start_with?("/") }
+
+    hash[:backtrace] = hash[:message]
+    hash[:message] = "Exception occured: #{message_lines.first}"
+  end
+
+  def remove_post_params
+    return unless method_is_post_or_put_or_patch? && hash.dig(:payload, :params).present?
+
+    hash[:payload][:params].clear
+  end
+
+  def method_is_post_or_put_or_patch?
+    hash.dig(:payload, :method).in?(%w[PUT POST PATCH])
+  end
+end

config/application.rb

@@ -60,5 +60,8 @@ class Application < Rails::Application
     config.active_record.encryption.key_derivation_salt = ENV.fetch("ACTIVE_RECORD_ENCRYPTION__KEY_DERIVATION_SALT", nil)
 
     config.active_record.raise_on_assign_to_attr_readonly = false
+
+    config.log_tags = [:request_id]
+    config.log_level = Settings.log_level
   end
 end

config/environments/development.rb

@@ -3,91 +3,82 @@
 require "active_support/core_ext/integer/time"
 
 Rails.application.configure do
+  # Bullet gem configuration for detecting N+1 queries and more
   config.after_initialize do
     Bullet.enable = true
     Bullet.bullet_logger = true
     Bullet.rails_logger = true
     Bullet.unused_eager_loading_enable = false
   end
 
-  # Settings specified here will take precedence over those in config/application.rb.
-
-  # In the development environment your application's code is reloaded any time
-  # it changes. This slows down response time but is perfect for development
-  # since you don't have to restart the web server when you make code changes.
+  # Code reloading and eager loading settings
   config.cache_classes = false
-
-  # Do not eager load code on boot.
   config.eager_load = false
 
-  # Show full error reports.
+  # Error reporting and debugging
   config.consider_all_requests_local = true
-
-  # Enable server timing
   config.server_timing = true
 
-  # Enable/disable caching. By default caching is disabled.
-  # Run rails dev:cache to toggle caching.
+  # Caching configuration
   if Rails.root.join("tmp/caching-dev.txt").exist?
     config.action_controller.perform_caching = true
     config.action_controller.enable_fragment_cache_logging = true
-
     config.cache_store = :redis_cache_store
     config.public_file_server.headers = {
       "Cache-Control" => "public, max-age=#{2.days.to_i}",
     }
   else
     config.action_controller.perform_caching = false
-
     config.cache_store = :null_store
   end
 
-  # Store uploaded files on the local file system (see config/storage.yml for options).
+  # Active Storage configuration
   config.active_storage.service = :local
 
-  # Don't care if the mailer can't send.
+  # Mailer configuration
   config.action_mailer.raise_delivery_errors = false
-
   config.action_mailer.perform_caching = false
 
-  # Print deprecation notices to the Rails logger.
+  # Deprecation and migration settings
   config.active_support.deprecation = :log
-
-  # Raise exceptions for disallowed deprecations.
   config.active_support.disallowed_deprecation = :raise
-
-  # Tell Active Support which deprecation messages to disallow.
   config.active_support.disallowed_deprecation_warnings = []
-
-  # Raise an error on page load if there are pending migrations.
   config.active_record.migration_error = :page_load
 
-  # Highlight code that triggered database queries in logs.
+  # Database query logs
   config.active_record.verbose_query_logs = true
 
-  # Suppress logger output for asset requests.
-  # config.assets.quiet = true
-
-  # Raises error for missing translations.
+  # Internationalization
   config.i18n.raise_on_missing_translations = true
 
-  # Annotate rendered view with file names.
-  # config.action_view.annotate_rendered_view_with_filenames = true
-
-  # Uncomment if you wish to allow Action Cable access from any origin.
-  # config.action_cable.disable_request_forgery_protection = true
-
-  # Use an evented file watcher to asynchronously detect changes in source code,
-  # routes, locales, etc. This feature depends on the listen gem.
+  # File watcher
   config.file_watcher = ActiveSupport::EventedFileUpdateChecker
 
-  # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
+  # Security and SSL
   config.force_ssl = Settings.features.use_ssl
 
-  # We don't use hosts authorization in deployed environments and there's no value to having it in dev.
+  # Hosts authorization
   config.hosts.clear
 
+  # Active Job
   config.active_job.queue_adapter = :sidekiq
+
+  # Public file server
   config.public_file_server.enabled = true
+
+  # Logging configuration
   config.colorize_logging = true
+  config.rails_semantic_logger.semantic   = false
+  config.rails_semantic_logger.started    = true
+  config.rails_semantic_logger.processing = true
+  config.rails_semantic_logger.rendered   = true
+
+  # Uncomment to suppress logger output for asset requests
+  # config.assets.quiet = true
+
+  # Uncomment to annotate rendered view with file names
+  # config.action_view.annotate_rendered_view_with_filenames = true
+
+  # Uncomment to allow Action Cable access from any origin
+  # config.action_cable.disable_request_forgery_protection = true
 end

config/environments/production.rb

@@ -2,6 +2,7 @@
 
 require "active_support/core_ext/integer/time"
 require Rails.root.join("config/initializers/redis")
+require_dependency Rails.root.join("app/lib/custom_log_formatter")
 
 Rails.application.configure do
   # Settings specified here will take precedence over those in config/application.rb.
@@ -77,9 +78,15 @@
   ##################
   # logging config #
   ##################
-  config.log_level = :info # less chatter in prod
-  config.active_record.logger = nil # Don't log SQL
-  config.active_support.report_deprecations = false # Don't log any deprecations.
-  config.semantic_logger.backtrace_level = nil # no backtrace in prod
-  config.colorize_logging = false
+  config.rails_semantic_logger.add_file_appender = false
+  config.rails_semantic_logger.format = CustomLogFormatter.new
+  config.rails_semantic_logger.filter = proc { |log| log.name != "DfE::Analytics::SendEvents" }
+  config.semantic_logger.add_appender(
+    io: $stdout,
+    level: config.log_level,
+    formatter: CustomLogFormatter.new,
+    filter: config.rails_semantic_logger.filter,
+  )
+
+  config.active_record.logger = nil # Don't log SQL in production
 end

config/initializers/sidekiq.rb

@@ -1,25 +1,10 @@
 # frozen_string_literal: true
 
-if ENV.key?("VCAP_SERVICES")
-  service_config = JSON.parse(ENV["VCAP_SERVICES"])
-  redis_config = service_config["redis"]
-  redis_worker_config = redis_config.select { |r| r["instance_name"].include?("worker") }.first
-  redis_credentials = redis_worker_config["credentials"]
-  queue_url = redis_credentials["uri"]
-else
-  queue_url = ENV.fetch("REDIS_QUEUE_URL", nil)
-end
+QUEUE_URL = ENV.fetch("REDIS_QUEUE_URL", nil)
 
 Sidekiq.configure_server do |config|
   config.redis = {
-    url: queue_url,
-  }
-  config.logger = Rails.logger
-end
-
-Sidekiq.configure_client do |config|
-  config.redis = {
-    url: queue_url,
+    url: QUEUE_URL,
   }
 end
 

config/settings.yml

@@ -10,6 +10,8 @@ feedback_link_url: "https://forms.office.com/e/Q6LVwtEKje"
 support_email: [email protected]
 data_email: [email protected]
 
+log_level: info
+
 dttp:
   client_id: "application-registration-client-id-from-env"
   scope: "https://dttp-dev.crm4.dynamics.com/.default"

config/settings/development.yml

@@ -1,3 +1,5 @@
+log_level: debug
+
 dttp:
   api_base_url: "https://dttp-dev.api.crm4.dynamics.com"
   portal_host: traineeteacherportal-local.education.gov.uk