Configure the Semantic Logger to output JSON (#2876)

Dockerfile

@@ -53,22 +53,7 @@ WORKDIR ${APP_HOME}
 
 EXPOSE 3000
 
-CMD /filebeat/filebeat -c /filebeat/filebeat.yml & bundle exec rails server
-
-# Download and install filebeat for sending logs to logstash
-ENV FILEBEAT_VERSION=7.6.2
-ENV FILEBEAT_DOWNLOAD_PATH=/tmp/filebeat.tar.gz
-ENV FILEBEAT_CHECKSUM=482304509aed80db78ef63a0fed88e4453ebe7b11f6b4ab3168036a78f6a413e2f6a5c039f405e13984653b1a094c23f7637ac7daf3da75a032692d1c34a9b65
-
-RUN curl https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz -o ${FILEBEAT_DOWNLOAD_PATH} && \
-  [ "$(sha512sum ${FILEBEAT_DOWNLOAD_PATH})" = "${FILEBEAT_CHECKSUM}  ${FILEBEAT_DOWNLOAD_PATH}" ] && \
-  tar xzvf ${FILEBEAT_DOWNLOAD_PATH} && \
-  rm ${FILEBEAT_DOWNLOAD_PATH} && \
-  mv filebeat-${FILEBEAT_VERSION}-linux-x86_64 /filebeat && \
-  rm -f /filebeat/filebeat.yml
-
-# Copy our local filebeat config to the installation
-COPY filebeat.yml /filebeat/filebeat.yml
+CMD bundle exec rails server
 
 # Copy dependencies (relying on dependencies using the same base image as this)
 COPY --from=dependencies ${DEPS_HOME}/Gemfile ${APP_HOME}/Gemfile
@@ -96,7 +81,7 @@ RUN DFE_SIGN_IN_API_CLIENT_ID= \
   SUPPRESS_DFE_ANALYTICS_INIT= \
   bundle exec rake assets:precompile
 
-RUN chown -hR appuser:appgroup ${APP_HOME} /filebeat
+RUN chown -hR appuser:appgroup ${APP_HOME}
 
 USER appuser
 

bin/start-worker

@@ -1,6 +1,4 @@
 #!/bin/bash
 
 echo "Starting workers..."
-/filebeat/filebeat -c /filebeat/filebeat.yml \
-  & bundle exec bin/delayed_job start -n "$WORKER_COUNT" \
-  & tail -f "log/$RAILS_ENV.log"
+bundle exec bin/delayed_job run -n "$WORKER_COUNT"

config/environments/development.rb

@@ -87,4 +87,9 @@
   # Add '127.0.0.1 ecp.test' to your /etc/hosts to use this over localhost.
   # This allows you to use https://ecp.test:3000/ in your browser.
   config.hosts << "ecp.test"
+
+  # https://technical-guidance.education.gov.uk/infrastructure/monitoring/logit/#ruby-on-rails
+  config.log_level = :debug                       # Or :info
+  config.log_format = :color                      # Console colorised non-json output
+  config.semantic_logger.backtrace_level = :debug # Show file and line number (expensive: not for production)
 end

config/environments/production.rb

@@ -47,8 +47,11 @@
   # information to avoid inadvertent exposure of personally identifiable information (PII).
   config.log_level = :info
 
-  # Prepend all log lines with the following tags.
-  config.log_tags = [:request_id]
+  # https://technical-guidance.education.gov.uk/infrastructure/monitoring/logit/#ruby-on-rails
+  config.log_format = :json                               # For parsing in Logit
+  config.rails_semantic_logger.add_file_appender = false  # Don't log to file
+  config.active_record.logger = nil                       # Don't log SQL
+  config.rails_semantic_logger.filter = proc { |log| log.name != "DfE::Analytics::SendEvents" }
 
   # Use a different cache store in production.
   # config.cache_store = :mem_cache_store
@@ -89,16 +92,6 @@
   # require "syslog/logger"
   # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new "app-name")
 
-  if ENV["RAILS_LOG_TO_STDOUT"].present?
-    logger = ActiveSupport::Logger.new($stdout)
-    logger.formatter = config.log_formatter
-    config.logger = ActiveSupport::TaggedLogging.new(logger)
-
-    $stdout.sync = true
-    config.rails_semantic_logger.add_file_appender = false
-    config.semantic_logger.add_appender(io: $stdout, formatter: config.rails_semantic_logger.format)
-  end
-
   # Do not dump schema after migrations.
   config.active_record.dump_schema_after_migration = false
 end

config/initializers/semantic_logger.rb

@@ -1,3 +1,10 @@
+# https://technical-guidance.education.gov.uk/infrastructure/monitoring/logit/#ruby-on-rails
 Rails.application.configure do
-  config.semantic_logger.application = "Claim Additional Payments for Teaching"
+  config.semantic_logger.application = "" # This is added by logstash from its tags
+  config.log_tags = [:request_id]         # Prepend all log lines with the following tags
+end
+
+unless Rails.env.test?
+  SemanticLogger.add_appender(io: $stdout, level: Rails.application.config.log_level, formatter: Rails.application.config.log_format)
+  Rails.application.config.logger.info("Application logging to STDOUT")
 end

docs/logging.md

@@ -1,33 +1,24 @@
 # Logging
 
-# Application-level logging in logit.io
+The application logs to STDOUT in JSON format using [Semantic Logger](https://github.com/reidmorrison/rails_semantic_logger).
 
-The application sends all Rails logs to [logit.io](https://logit.io/). This
-gives us a hosted ELK stack. ELK stands for Elasticsearch, Logstash, Kibana.
-It’s a open-source stack frequently used for log aggregation and visualisation.
+# AKS/container logs
 
-Kibana is the tool that you’ll use to explore the logs.
+Logs are available via the standard Azure/AKS tooling.
 
-To view the production logs in Kibana:
-
-1. Log in to logit.io.
-2. Click on the “DfE Claim” account.
-3. Find the “Production” stack and click “Launch Kibana”.
-
-## Missing logs in logit.io
+The `az` Azure command line tool and `kubectl` have some useful commands for interacting with
+logs.
 
-At the time of writing (2020-03-26), we have a problem where some log messages
-seem to be missing from logit.io. We are trying to fix this in
-[this Trello card](https://trello.com/c/JN44De4l/1330-understand-why-expected-logs-arent-in-logit).
+```sh
+kubectl -n srtl-test get pods
+kubectl -n srtl-test logs claim-additional-payments-for-teaching-test-web-123456
+```
 
-# Azure logs
+You will need PIM elevated privileges to view logs for production.
 
-We should aim for logit.io to be the single place developers need to look to
-find logs. However, it’s useful to know about some of the logs that are
-available from Azure.
+# Logit.io
 
-The `az` Azure command line tool has some useful commands for interacting with
-logs.
+Logs are shipped to [Logit.io](https://logit.io/) automatically by AKS. You will need an account to access this service.
 
 ## App Service
 
@@ -38,27 +29,6 @@ There is some
 about the logs available for an App Service. Here we go over some of the ways to
 extract these logs.
 
-### Docker logs
-
-To view Docker logs in production, you will need to elevate your privileges
-using a [PIM request](privileged-identity-management-requests.md).
-
-You can use the `az webapp log` commands to tail and download the logs.
-
-Here’s an example command that will live tail the container logs and Docker host
-logs, for `production`:
-
-```
-az webapp log tail --name s118p01-app-as --resource-group s118p01-app --subscription s118-teacherpaymentsservice-production
-```
-
-Or, you can download them with a browser:
-
-- production Docker logs from the instances currently in the App Service:
-  https://s118p01-app-as.scm.azurewebsites.net/api/logs/docker
-- production Docker logs from current and previous instances:
-  https://s118d01-app-as.scm.azurewebsites.net/api/vfs/LogFiles/
-
 ### Application Insights
 
 The Rails application uses the `application_insights` gem, which sends
@@ -72,22 +42,3 @@ To view these logs:
 3. Click “Logs” under “Monitoring” on the left.
 4. You can now execute queries. For example, to view all requests, type
    `requests` in the query field, and click Run.
-
-## Container Instances
-
-The container instances perform other tasks like running a background job
-worker. The logs are lost after a deploy. You don’t need PIM-elevated privileges
-to view the container instance logs.
-
-To view the logs for a container instance, we can use `az container logs`:
-
-```
-az container logs --name s118p01-app-worker-aci --resource-group s118p01-app --subscription s118-teacherpaymentsservice-production
-```
-
-Or, you can view them in a browser:
-
-1. Visit `https://portal.azure.com`.
-2. Search for `s118p01-app-worker-aci`.
-3. Click “Containers” on the left.
-4. Click “Logs”.

terraform/application/application.tf

@@ -40,6 +40,8 @@ module "web_application" {
   command      = var.startup_command
 
   replicas = var.web_replicas
+
+  enable_logit = var.enable_logit
 }
 
 module "worker_application" {
@@ -60,4 +62,6 @@ module "worker_application" {
   command      = var.worker_command
 
   replicas = var.worker_replicas
+
+  enable_logit = var.enable_logit
 }

terraform/application/config/review.tfvars.json

@@ -5,5 +5,6 @@
   "deploy_azure_backing_services": false,
   "enable_postgres_ssl": false,
   "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],
-  "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"]
+  "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"],
+  "enable_logit": true
 }

terraform/application/config/test.tfvars.json

@@ -9,5 +9,6 @@
   "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"],
   "enable_monitoring": true,
   "statuscake_contact_groups": [195955, 282453],
-  "external_url": "https://claim-additional-payments-for-teaching-test-web.test.teacherservices.cloud/healthcheck"
+  "external_url": "https://claim-additional-payments-for-teaching-test-web.test.teacherservices.cloud/healthcheck",
+  "enable_logit": true
 }

terraform/application/variables.tf

@@ -77,6 +77,12 @@ variable "postgres_flexible_server_sku" {
 variable "postgres_enable_high_availability" {
   default = false
 }
+variable "enable_logit" {
+  type        = bool
+  default     = false
+  description = "A boolean to indicate whether to enable sending container logs to logit.io"
+  nullable    = false
+}
 
 locals {
   postgres_ssl_mode       = var.enable_postgres_ssl ? "require" : "disable"