diff --git a/README.md b/README.md index a0101a2b8b..ab0449a47b 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,13 @@ Accessing a live console on production requires a [PIM (Privileged Identity Management) request](docs/privileged-identity-management-requests.md). Follow the steps outlined in [Connect to an instance running in Azure](docs/connecting-to-azure.md). +### Restarting worker via the command line + +1. [Connect to the desired Kubernetes cluster](docs/connecting-to-azure.md#1-authenticate-to-the-kubernetes-cluster) +1. Identify the worker deployment via kubectl on the command prompt: `kubectl -n get deployments` +1. Restart the deployment: `kubectl -n rollout restart deployment ` +1. Verify successful restart: `kubectl -n get pods`. The column "AGE" should be updated. + ### Usage When accessing the Rails console on a live system, do so in sandbox mode to diff --git a/app/jobs/heartbeat_job.rb b/app/jobs/heartbeat_job.rb index 56626032cd..4af6dc1ed7 100644 --- a/app/jobs/heartbeat_job.rb +++ b/app/jobs/heartbeat_job.rb @@ -1,12 +1,18 @@ -# Writes "Heartbeat job performed" to the logs every minute. This message is -# evidence that the worker is working properly. We can use a log alerting tool -# to alert us if too much time passes without seeing one of these messages. +require "net/http" + class HeartbeatJob < CronJob self.cron_expression = "* * * * *" queue_as :heartbeat def perform - logger.info "Heartbeat job performed" + if ENV.key?("HEARTBEAT_CHECK_URL") # Not available in dev and review environments + uri = URI(ENV["HEARTBEAT_CHECK_URL"]) + res = Net::HTTP.get_response(uri) + + unless res.is_a?(Net::HTTPSuccess) + Rails.logger.error "Error connecting to StatusCake: #{res.code} #{res.msg}" + end + end end end diff --git a/terraform/application/application.tf b/terraform/application/application.tf index d4afd87ec6..34a7229e5a 100644 --- a/terraform/application/application.tf +++ b/terraform/application/application.tf @@ -17,9 +17,14 @@ module "application_configuration" { PGSSLMODE = local.postgres_ssl_mode CANONICAL_HOSTNAME = local.canonical_hostname }) - secret_variables = { - DATABASE_URL = module.postgres.url - } + secret_variables = merge( + { + DATABASE_URL = module.postgres.url + }, + var.enable_monitoring ? { + HEARTBEAT_CHECK_URL = module.statuscake[0].heartbeat_check_urls[local.heartbeat_check_name] + } : {} + ) } module "web_application" { diff --git a/terraform/application/statuscake.tf b/terraform/application/statuscake.tf index be1b3f03ef..5b56946d88 100644 --- a/terraform/application/statuscake.tf +++ b/terraform/application/statuscake.tf @@ -7,4 +7,6 @@ module "statuscake" { ssl_urls = compact([var.external_url]) contact_groups = var.statuscake_contact_groups + + heartbeat_names = [local.heartbeat_check_name] } diff --git a/terraform/application/variables.tf b/terraform/application/variables.tf index 4453323375..d95514bf47 100644 --- a/terraform/application/variables.tf +++ b/terraform/application/variables.tf @@ -89,4 +89,5 @@ locals { canonical_hostname = var.canonical_hostname != null ? var.canonical_hostname : "${var.service_name}-${var.environment}-web.test.teacherservices.cloud" app_env_values_from_yml = yamldecode(file("${path.module}/config/${var.config}_app_env.yml")) app_env_values = merge(local.app_env_values_from_yml) + heartbeat_check_name = "${var.service_name}-${var.environment}-worker" }