Add heartbeat monitor for statuscake

README.md

@@ -299,6 +299,13 @@ Accessing a live console on production requires a
 [PIM (Privileged Identity Management) request](docs/privileged-identity-management-requests.md).
 Follow the steps outlined in [Connect to an instance running in Azure](docs/connecting-to-azure.md).
 
+### Restarting worker via the command line
+
+1. [Connect to the desired Kubernetes cluster](docs/connecting-to-azure.md#1-authenticate-to-the-kubernetes-cluster)
+1. Identify the worker deployment via kubectl on the command prompt: `kubectl -n <namespace> get deployments`
+1. Restart the deployment: `kubectl -n <namespace> rollout restart deployment <deployment-name>`
+1. Verify successful restart: `kubectl -n <namespace> get pods`. The column "AGE" should be updated.
+
 ### Usage
 
 When accessing the Rails console on a live system, do so in sandbox mode to

app/jobs/heartbeat_job.rb

@@ -1,12 +1,18 @@
-# Writes "Heartbeat job performed" to the logs every minute. This message is
-# evidence that the worker is working properly. We can use a log alerting tool
-# to alert us if too much time passes without seeing one of these messages.
+require "net/http"
+
 class HeartbeatJob < CronJob
   self.cron_expression = "* * * * *"
 
   queue_as :heartbeat
 
   def perform
-    logger.info "Heartbeat job performed"
+    if ENV.key?("HEARTBEAT_CHECK_URL") # Not available in dev and review environments
+      uri = URI(ENV["HEARTBEAT_CHECK_URL"])
+      res = Net::HTTP.get_response(uri)
+
+      unless res.is_a?(Net::HTTPSuccess)
+        Rails.logger.error "Error connecting to StatusCake: #{res.code} #{res.msg}"
+      end
+    end
   end
 end

terraform/application/application.tf

@@ -17,9 +17,14 @@ module "application_configuration" {
       PGSSLMODE          = local.postgres_ssl_mode
       CANONICAL_HOSTNAME = local.canonical_hostname
   })
-  secret_variables = {
-    DATABASE_URL = module.postgres.url
-  }
+  secret_variables = merge(
+    {
+      DATABASE_URL        = module.postgres.url
+    },
+    var.enable_monitoring ? {
+      HEARTBEAT_CHECK_URL = module.statuscake[0].heartbeat_check_urls[local.heartbeat_check_name]
+    } : {}
+  )
 }
 
 module "web_application" {

terraform/application/statuscake.tf

@@ -7,4 +7,6 @@ module "statuscake" {
   ssl_urls    = compact([var.external_url])
 
   contact_groups = var.statuscake_contact_groups
+
+  heartbeat_names  = [local.heartbeat_check_name]
 }

terraform/application/variables.tf

@@ -89,4 +89,5 @@ locals {
   canonical_hostname      = var.canonical_hostname != null ? var.canonical_hostname : "${var.service_name}-${var.environment}-web.test.teacherservices.cloud"
   app_env_values_from_yml = yamldecode(file("${path.module}/config/${var.config}_app_env.yml"))
   app_env_values          = merge(local.app_env_values_from_yml)
+  heartbeat_check_name    = "${var.service_name}-${var.environment}-worker"
 }