From eba76398b53b419f8f31d5963de179c0c90b1c93 Mon Sep 17 00:00:00 2001 From: Eddy Perez Date: Tue, 9 Jan 2024 14:35:47 -0400 Subject: [PATCH 1/3] fix: Add datadog-agent to ECS --- cloudwatch.tf | 6 ++++++ common.tf | 10 +++++++++ templates.tf | 14 +++++++++++++ templates/apiary-hms-readonly.json | 32 +++++++++++++++++++++++++++++ templates/apiary-hms-readwrite.json | 32 +++++++++++++++++++++++++++++ variables.tf | 17 +++++++++++++++ 6 files changed, 111 insertions(+) diff --git a/cloudwatch.tf b/cloudwatch.tf index ff7d224..c96882c 100644 --- a/cloudwatch.tf +++ b/cloudwatch.tf @@ -4,6 +4,12 @@ * Licensed under the Apache License, Version 2.0 (the "License"); */ +resource "aws_cloudwatch_log_group" "ecs" { + count = var.hms_instance_type == "ecs" ? 1 : 0 + name = local.instance_alias + tags = var.apiary_tags +} + data "template_file" "s3_widgets" { count = length(local.schemas_info) diff --git a/common.tf b/common.tf index 63e22d7..dc06dda 100644 --- a/common.tf +++ b/common.tf @@ -78,3 +78,13 @@ data "aws_route53_zone" "apiary_zone" { name = var.apiary_domain_name vpc_id = var.vpc_id } + +data "aws_secretsmanager_secret" "datadog_key" { + count = length(var.datadog_key_secret_name) == 0 ? 0 : 1 + name = var.datadog_key_secret_name +} + +data "aws_secretsmanager_secret_version" "datadog_key" { + count = length(var.datadog_key_secret_name) == 0 ? 0 : 1 + secret_id = data.aws_secretsmanager_secret.datadog_key[0].id +} diff --git a/templates.tf b/templates.tf index 985706e..2c0154e 100644 --- a/templates.tf +++ b/templates.tf @@ -61,6 +61,13 @@ locals{ mysql_permissions = "ALL" mysql_master_cred_arn = var.external_database_host == "" ? aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn : null mysql_user_cred_arn = data.aws_secretsmanager_secret.db_rw_user.arn + + # Datadog vars + datadog_secret_key = jsondecode(data.aws_secretsmanager_secret_version.datadog_key[0].secret_string).api_key + wd_instance_type = var.hms_instance_type + metrics_port = var.datadog_metrics_port + datadog_agent_version = var.datadog_agent_version + include_datadog_agent = var.include_datadog_agent }) hms_readonly_template = templatefile("${path.module}/templates/apiary-hms-readonly.json", { @@ -104,5 +111,12 @@ locals{ mysql_write_db = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}" mysql_master_cred_arn = var.external_database_host == "" ? aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn : null mysql_user_cred_arn = data.aws_secretsmanager_secret.db_ro_user.arn + + # Datadog vars + datadog_secret_key = jsondecode(data.aws_secretsmanager_secret_version.datadog_key[0].secret_string).api_key + wd_instance_type = var.hms_instance_type + metrics_port = var.datadog_metrics_port + datadog_agent_version = var.datadog_agent_version + include_datadog_agent = var.include_datadog_agent }) } diff --git a/templates/apiary-hms-readonly.json b/templates/apiary-hms-readonly.json index df313cf..2f635f0 100644 --- a/templates/apiary-hms-readonly.json +++ b/templates/apiary-hms-readonly.json @@ -179,4 +179,36 @@ %{ endfor } ] } +%{ if include_datadog_agent } + ,{ + "name": "datadog-agent", + "image": "public.ecr.aws/datadog/agent:${datadog_agent_version}", + "essential": true, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${loggroup}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "/" + } + }, + "environment": [ + { + "name": "DD_API_KEY", + "value": "${datadog_secret_key}" + }, + { + "name": "ECS_FARGATE", + "value": "true" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "curl -f http://localhost:18000/actuator/health || exit 1"], + "interval": 5, + "retries": 3, + "startPeriod": 60, + "timeout": 5 + } + } +%{ endif } ] diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json index 0afae54..b08b640 100644 --- a/templates/apiary-hms-readwrite.json +++ b/templates/apiary-hms-readwrite.json @@ -231,4 +231,36 @@ %{ endfor } ] } +%{ if include_datadog_agent } + ,{ + "name": "datadog-agent", + "image": "public.ecr.aws/datadog/agent:${datadog_agent_version}", + "essential": true, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${loggroup}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "/" + } + }, + "environment": [ + { + "name": "DD_API_KEY", + "value": "${datadog_secret_key}" + }, + { + "name": "ECS_FARGATE", + "value": "true" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "curl -f http://localhost:18000/actuator/health || exit 1"], + "interval": 5, + "retries": 3, + "startPeriod": 60, + "timeout": 5 + } + } +%{ endif } ] diff --git a/variables.tf b/variables.tf index f6de153..e71f8dd 100644 --- a/variables.tf +++ b/variables.tf @@ -704,6 +704,23 @@ variable "datadog_metrics_port" { default = "8080" } +variable "datadog_key_secret_name" { + description = "Name of the secret containing the DataDog API key. This needs to be created manually in AWS secrets manager. This is only applicable to ECS deployments." + type = string +} + +variable "datadog_agent_version" { + description = "Version of the Datadog Agent running in the ECS cluster. This is only applicable to ECS deployments." + type = string + default = "7.46.0-jmx" +} + +variable "include_datadog_agent" { + description = "Whether to include the datadog-agent container. This is only applicable to ECS deployments." + type = bool + default = false +} + variable "hms_rw_request_partition_limit" { description = "Read-write Hive metastore setting for size of the Hive metastore limit of request partitions." type = string From 7a3725e6d5a01d4d48d931f0d94d210ae9852dbc Mon Sep 17 00:00:00 2001 From: Eddy Perez Date: Fri, 12 Jan 2024 13:22:22 -0400 Subject: [PATCH 2/3] fix: Add datadog-agent to ECS --- variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/variables.tf b/variables.tf index e71f8dd..3a235bd 100644 --- a/variables.tf +++ b/variables.tf @@ -707,6 +707,7 @@ variable "datadog_metrics_port" { variable "datadog_key_secret_name" { description = "Name of the secret containing the DataDog API key. This needs to be created manually in AWS secrets manager. This is only applicable to ECS deployments." type = string + default = null } variable "datadog_agent_version" { From a7275c743302e68077f7f708e382cd8db289d7a7 Mon Sep 17 00:00:00 2001 From: Eddy Perez Date: Fri, 12 Jan 2024 13:34:21 -0400 Subject: [PATCH 3/3] fix: Add datadog-agent to ECS --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index a207acc..c7f5690 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,11 @@ module "apiary" { apiary_log_bucket = "s3-logs-bucket" db_instance_class = "db.t2.medium" db_backup_retention = "7" + + # Enable datadog for monitoring + datadog_key_secret_name = "datadog-secret-key-name" + include_datadog_agent = true + apiary_managed_schemas = [ { schema_name = "db1",