Feature/datadog (#247)

* Added datadog agent for BEX ECS * Added datadog agent for BEX ECS * Removed the health check * Added dockerLabel * Added changelog * Added the VARIABLES.md --------- Co-authored-by: Dhrubajyoti Sadhu <[email protected]>
ExpediaGroup · Jan 23, 2024 · 2a758e3 · 2a758e3
1 parent da49cc1
commit 2a758e3
Show file tree

Hide file tree

Showing 8 changed files with 239 additions and 114 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.0.1] - 2024-01-22
+### Added
+- Added `datadog-agent` for HMS-Readonly and HMS-Readwrite in ECS.
+
 ## [7.0.0] - 2023-11-16
 ### Changed
 - Changed `k8s` API to work with provider 2.x

diff --git a/VARIABLES.md b/VARIABLES.md
diff --git a/cloudwatch.tf b/cloudwatch.tf
@@ -4,6 +4,12 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  */
 
+resource "aws_cloudwatch_log_group" "ecs" {
+  count = var.hms_instance_type == "ecs" ? 1 : 0
+  name  = local.instance_alias
+  tags  = var.apiary_tags
+}
+
 data "template_file" "s3_widgets" {
   count = length(local.schemas_info)
 

diff --git a/common.tf b/common.tf
@@ -78,3 +78,23 @@ data "aws_route53_zone" "apiary_zone" {
   name   = var.apiary_domain_name
   vpc_id = var.vpc_id
 }
+
+data "aws_secretsmanager_secret" "datadog_key" {
+  count = length(var.datadog_key_secret_name) > 0 ? 1 : 0
+  name  = var.datadog_key_secret_name
+}
+
+data "aws_secretsmanager_secret_version" "datadog_key" {
+  count = length(var.datadog_key_secret_name) > 0 ? 1 : 0
+  secret_id = data.aws_secretsmanager_secret.datadog_key[0].id
+}
+
+data "external" "datadog_key" {
+  count = length(var.datadog_key_secret_name) > 0 ? 1 : 0
+  program = ["echo", "${data.aws_secretsmanager_secret_version.datadog_key[0].secret_string}"]
+}
+
+provider "datadog" {
+  api_key  = chomp(data.external.datadog_key[0].result["api_key"])
+  app_key  = chomp(data.external.datadog_key[0].result["app_key"])
+}
diff --git a/templates.tf b/templates.tf
@@ -61,6 +61,13 @@ locals{
     mysql_permissions      = "ALL"
     mysql_master_cred_arn  = var.external_database_host == "" ? aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn : null
     mysql_user_cred_arn    = data.aws_secretsmanager_secret.db_rw_user.arn
+
+    # Datadog variables
+    datadog_secret_key    = length(var.datadog_key_secret_name) > 0 ? chomp(data.external.datadog_key[0].result["api_key"]) : ""
+    wd_instance_type      = var.hms_instance_type
+    metrics_port          = var.datadog_metrics_port
+    datadog_agent_version = var.datadog_agent_version
+    datadog_agent_enabled = var.datadog_agent_enabled
   })
 
   hms_readonly_template = templatefile("${path.module}/templates/apiary-hms-readonly.json", {
@@ -104,5 +111,12 @@ locals{
     mysql_write_db         = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}"
     mysql_master_cred_arn  = var.external_database_host == "" ? aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn : null
     mysql_user_cred_arn    = data.aws_secretsmanager_secret.db_ro_user.arn
+
+    # Datadog variables
+    datadog_agent_enabled = var.datadog_agent_enabled
+    datadog_secret_key    = length(var.datadog_key_secret_name) > 0 ? chomp(data.external.datadog_key[0].result["api_key"]) : ""
+    wd_instance_type      = var.hms_instance_type
+    metrics_port          = var.datadog_metrics_port
+    datadog_agent_version = var.datadog_agent_version
   })
 }
diff --git a/templates/apiary-hms-readonly.json b/templates/apiary-hms-readonly.json
@@ -39,7 +39,7 @@
     ],
     "command": ["sh", "/allow-grant.sh"]
   },
-%{ endif } 
+%{ endif }
   {
     "name": "apiary-hms-readonly",
     "image": "${hms_docker_image}:${hms_docker_version}",
@@ -60,6 +60,11 @@
             "awslogs-stream-prefix": "/"
         }
     },
+    "dockerLabels": {
+        "com.datadoghq.ad.instances": "[{ \"openmetrics_endpoint\": \"http://%%host%%:8080/actuator/prometheus\", \"namespace\": \"hmsreadonlylegacy\", \"metrics\": [\"metrics_classloading_loaded_value\", \"metrics_threads_count_value\", \"metrics_memory_heap_max_value\", \"metrics_init_total_count_tables_value\", \"metrics_init_total_count_dbs_value\", \"metrics_memory_heap_used_value\", \"metrics_init_total_count_partitions_value\", \"jvm_threads_current\", \"jvm_threads_started_total\", \"jvm_memory_bytes_used\", \"jvm_memory_bytes_init\", \"jvm_gc_collection_seconds_count\", \"jvm_gc_collection_seconds\", \"process_cpu_seconds_total\", \"java_lang_operatingsystem_processcpuload\", \"java_lang_operatingsystem_processcputime\", \"metrics_threads_runnable_count_value\", \"metrics_threads_waiting_count_value\", \"java_lang_memory_heapmemoryusage_used\", \"metrics_memory_heap_init_value\", \"metrics_api_get_partition_by_name_count\", \"metrics_api_get_partitions_by_names_count\", \"metrics_api_get_partition_names_count\", \"metrics_api_get_partitions_by_expr_count\", \"metrics_api_get_partitions_count\", \"metrics_api_get_partition_count\", \"metrics_api_get_partitions_by_filter_count\", \"metrics_api_add_partitions_count\", \"metrics_api_add_partitions_req_count\", \"metrics_api_drop_partition_by_name_count\", \"metrics_api_add_partition_count\", \"metrics_api_alter_partitions_count\", \"metrics_api_create_table_count\", \"metrics_api_alter_table_with_cascade_count\", \"metrics_api_get_table_meta_count\", \"metrics_api_get_table_metas_count\", \"metrics_api_get_table_count\", \"metrics_api_alter_table_count\", \"metrics_api_get_tables_count\", \"metrics_api_get_all_tables_count\", \"metrics_api_drop_table_count\", \"metrics_api_get_multi_table_count\", \"metrics_api_get_database_count\", \"metrics_api_get_all_databases_count\", \"metrics_api_get_databases_count\", \"metrics_api_create_function_count\", \"metrics_api_getmetaconf_count\", \"metrics_api_alter_table_with_environment_context_count\", \"metrics_api_delete_column_statistics_by_table_count\", \"metrics_api_get_functions_count\", \"metrics_api_get_function_count\", \"metrics_api_shutdown_count\", \"metrics_api_flushcache_count\", \"metrics_api_get_indexes_count\", \"metrics_api_get_config_value_count\", \"metrics_api_set_ugi_count\", \"metrics_api_get_all_functions_count\", \"metrics_api_get_table_req_95thpercentile\", \"metrics_api_get_table_req_50thpercentile\", \"metrics_api_get_table_req_count\", \"metrics_api_get_table_req_max\", \"metrics_api_get_databases_count\", \"metrics_api_get_databases_95thpercentile\", \"metrics_api_get_databases_50thpercentile\", \"metrics_api_get_databases_max\", \"metrics_api_get_partitions_95thpercentile\", \"metrics_api_get_partitions_50thpercentile\", \"metrics_api_get_partitions_count\", \"metrics_api_get_partitions_max\", \"metrics_api_get_partitions_50thpercentile\", \"metrics_api_get_table_req_50thpercentile\", \"metrics_api_get_database_95thpercentile\", \"metrics_api_get_database_50thpercentile\", \"metrics_kafka_listener_failures_count\", \"metrics_kafka_listener_successes_count\", \"metrics_api_get_table_objects_by_name_req_max\" ], \"type_overrides\": { \"metrics_classloading_loaded_value\": \"gauge\", \"metrics_threads_count_value\": \"gauge\", \"metrics_memory_heap_max_value\": \"gauge\", \"metrics_init_total_count_tables_value\": \"gauge\", \"metrics_init_total_count_dbs_value\": \"gauge\", \"metrics_memory_heap_used_value\": \"gauge\", \"metrics_init_total_count_partitions_value\": \"gauge\", \"jvm_threads_current\": \"gauge\", \"jvm_threads_started_total\": \"gauge\", \"jvm_memory_bytes_used\": \"gauge\", \"jvm_memory_bytes_init\": \"gauge\", \"jvm_gc_collection_seconds_count\": \"gauge\", \"jvm_gc_collection_seconds\": \"gauge\", \"process_cpu_seconds_total\": \"gauge\", \"java_lang_operatingsystem_processcpuload\": \"gauge\", \"java_lang_operatingsystem_processcputime\": \"gauge\", \"metrics_threads_runnable_count_value\": \"gauge\", \"metrics_threads_waiting_count_value\": \"gauge\", \"java_lang_memory_heapmemoryusage_used\": \"gauge\", \"metrics_memory_heap_init_value\": \"gauge\", \"metrics_api_get_partition_by_name_count\": \"gauge\", \"metrics_api_get_partitions_by_names_count\": \"gauge\", \"metrics_api_get_partition_names_count\": \"gauge\", \"metrics_api_get_partitions_by_expr_count\": \"gauge\", \"metrics_api_get_partitions_count\": \"gauge\", \"metrics_api_get_partition_count\": \"gauge\", \"metrics_api_get_partitions_by_filter_count\": \"gauge\", \"metrics_api_add_partitions_count\": \"gauge\", \"metrics_api_add_partitions_req_count\": \"gauge\", \"metrics_api_drop_partition_by_name_count\": \"gauge\", \"metrics_api_add_partition_count\": \"gauge\", \"metrics_api_alter_partitions_count\": \"gauge\", \"metrics_api_create_table_count\": \"gauge\", \"metrics_api_alter_table_with_cascade_count\": \"gauge\", \"metrics_api_get_table_meta_count\": \"gauge\", \"metrics_api_get_table_metas_count\": \"gauge\", \"metrics_api_get_table_count\": \"gauge\", \"metrics_api_alter_table_count\": \"gauge\", \"metrics_api_get_tables_count\": \"gauge\", \"metrics_api_get_all_tables_count\": \"gauge\", \"metrics_api_drop_table_count\": \"gauge\", \"metrics_api_get_multi_table_count\": \"gauge\", \"metrics_api_get_database_count\": \"gauge\", \"metrics_api_get_all_databases_count\": \"gauge\", \"metrics_api_get_databases_count\": \"gauge\", \"metrics_api_create_function_count\": \"gauge\", \"metrics_api_getmetaconf_count\": \"gauge\", \"metrics_api_alter_table_with_environment_context_count\": \"gauge\", \"metrics_api_delete_column_statistics_by_table_count\": \"gauge\", \"metrics_api_get_functions_count\": \"gauge\", \"metrics_api_get_function_count\": \"gauge\", \"metrics_api_shutdown_count\": \"gauge\", \"metrics_api_flushcache_count\": \"gauge\", \"metrics_api_get_indexes_count\": \"gauge\", \"metrics_api_get_config_value_count\": \"gauge\", \"metrics_api_set_ugi_count\": \"gauge\", \"metrics_api_get_all_functions_count\": \"gauge\", \"metrics_api_get_table_req_95thpercentile\": \"gauge\", \"metrics_api_get_table_req_50thpercentile\": \"gauge\", \"metrics_api_get_table_req_count\": \"gauge\", \"metrics_api_get_table_req_max\": \"gauge\", \"metrics_api_get_databases_count\": \"gauge\", \"metrics_api_get_databases_95thpercentile\": \"gauge\", \"metrics_api_get_databases_50thpercentile\": \"gauge\", \"metrics_api_get_databases_max\": \"gauge\", \"metrics_api_get_partitions_95thpercentile\": \"gauge\", \"metrics_api_get_partitions_50thpercentile\": \"gauge\", \"metrics_api_get_partitions_count\": \"gauge\", \"metrics_api_get_partitions_max\": \"gauge\", \"metrics_api_get_partitions_50thpercentile\": \"gauge\", \"metrics_api_get_table_req_50thpercentile\": \"gauge\",\"metrics_api_get_database_95thpercentile\": \"gauge\",\"metrics_api_get_database_50thpercentile\": \"gauge\", \"metrics_kafka_listener_failures_count\": \"gauge\", \"metrics_kafka_listener_successes_count\": \"gauge\", \"metrics_api_get_table_objects_by_name_req_max\": \"gauge\"} }]",
+        "com.datadoghq.ad.check_names": "[\"openmetrics\"]",
+        "com.datadoghq.ad.init_configs": "[{}]"
+    },
     "portMappings": [
       {
         "containerPort": 9083,
@@ -73,7 +78,7 @@
         "condition": "SUCCESS"
       }
     ],
-%{ endif } 
+%{ endif }
     "environment":[
       {
         "name": "MYSQL_DB_HOST",
@@ -179,4 +184,29 @@
 %{ endfor }
    ]
   }
+%{ if datadog_agent_enabled }
+  ,{
+    "name": "datadog-agent",
+    "image": "public.ecr.aws/datadog/agent:${datadog_agent_version}",
+    "essential": true,
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "${loggroup}",
+        "awslogs-region": "${region}",
+        "awslogs-stream-prefix": "/"
+      }
+    },
+    "environment": [
+      {
+        "name": "DD_API_KEY",
+        "value": "${datadog_secret_key}"
+      },
+      {
+        "name": "ECS_FARGATE",
+        "value": "true"
+      }
+    ]
+  }
+%{ endif }
 ]
diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json
@@ -60,6 +60,11 @@
             "awslogs-stream-prefix": "/"
         }
     },
+    "dockerLabels": {
+        "com.datadoghq.ad.instances": "[{ \"openmetrics_endpoint\": \"http://%%host%%:8080/actuator/prometheus\", \"namespace\": \"hmsreadwritelegacy\", \"metrics\": [\"metrics_classloading_loaded_value\", \"metrics_threads_count_value\", \"metrics_memory_heap_max_value\", \"metrics_init_total_count_tables_value\", \"metrics_init_total_count_dbs_value\", \"metrics_memory_heap_used_value\", \"metrics_init_total_count_partitions_value\", \"jvm_threads_current\", \"jvm_threads_started_total\", \"jvm_memory_bytes_used\", \"jvm_memory_bytes_init\", \"jvm_gc_collection_seconds_count\", \"jvm_gc_collection_seconds\", \"process_cpu_seconds_total\", \"java_lang_operatingsystem_processcpuload\", \"java_lang_operatingsystem_processcputime\", \"metrics_threads_runnable_count_value\", \"metrics_threads_waiting_count_value\", \"java_lang_memory_heapmemoryusage_used\", \"metrics_memory_heap_init_value\", \"metrics_api_get_partition_by_name_count\", \"metrics_api_get_partitions_by_names_count\", \"metrics_api_get_partition_names_count\", \"metrics_api_get_partitions_by_expr_count\", \"metrics_api_get_partitions_count\", \"metrics_api_get_partition_count\", \"metrics_api_get_partitions_by_filter_count\", \"metrics_api_add_partitions_count\", \"metrics_api_add_partitions_req_count\", \"metrics_api_drop_partition_by_name_count\", \"metrics_api_add_partition_count\", \"metrics_api_alter_partitions_count\", \"metrics_api_create_table_count\", \"metrics_api_alter_table_with_cascade_count\", \"metrics_api_get_table_meta_count\", \"metrics_api_get_table_metas_count\", \"metrics_api_get_table_count\", \"metrics_api_alter_table_count\", \"metrics_api_get_tables_count\", \"metrics_api_get_all_tables_count\", \"metrics_api_drop_table_count\", \"metrics_api_get_multi_table_count\", \"metrics_api_get_database_count\", \"metrics_api_get_all_databases_count\", \"metrics_api_get_databases_count\", \"metrics_api_create_function_count\", \"metrics_api_getmetaconf_count\", \"metrics_api_alter_table_with_environment_context_count\", \"metrics_api_delete_column_statistics_by_table_count\", \"metrics_api_get_functions_count\", \"metrics_api_get_function_count\", \"metrics_api_shutdown_count\", \"metrics_api_flushcache_count\", \"metrics_api_get_indexes_count\", \"metrics_api_get_config_value_count\", \"metrics_api_set_ugi_count\", \"metrics_api_get_all_functions_count\", \"metrics_api_get_table_req_95thpercentile\", \"metrics_api_get_table_req_50thpercentile\", \"metrics_api_get_table_req_count\", \"metrics_api_get_table_req_max\", \"metrics_api_get_databases_count\", \"metrics_api_get_databases_95thpercentile\", \"metrics_api_get_databases_50thpercentile\", \"metrics_api_get_databases_max\", \"metrics_api_get_partitions_95thpercentile\", \"metrics_api_get_partitions_50thpercentile\", \"metrics_api_get_partitions_count\", \"metrics_api_get_partitions_max\", \"metrics_api_get_partitions_50thpercentile\", \"metrics_api_get_table_req_50thpercentile\", \"metrics_api_get_database_95thpercentile\", \"metrics_api_get_database_50thpercentile\", \"metrics_kafka_listener_failures_count\", \"metrics_kafka_listener_successes_count\", \"metrics_api_get_table_objects_by_name_req_max\" ], \"type_overrides\": { \"metrics_classloading_loaded_value\": \"gauge\", \"metrics_threads_count_value\": \"gauge\", \"metrics_memory_heap_max_value\": \"gauge\", \"metrics_init_total_count_tables_value\": \"gauge\", \"metrics_init_total_count_dbs_value\": \"gauge\", \"metrics_memory_heap_used_value\": \"gauge\", \"metrics_init_total_count_partitions_value\": \"gauge\", \"jvm_threads_current\": \"gauge\", \"jvm_threads_started_total\": \"gauge\", \"jvm_memory_bytes_used\": \"gauge\", \"jvm_memory_bytes_init\": \"gauge\", \"jvm_gc_collection_seconds_count\": \"gauge\", \"jvm_gc_collection_seconds\": \"gauge\", \"process_cpu_seconds_total\": \"gauge\", \"java_lang_operatingsystem_processcpuload\": \"gauge\", \"java_lang_operatingsystem_processcputime\": \"gauge\", \"metrics_threads_runnable_count_value\": \"gauge\", \"metrics_threads_waiting_count_value\": \"gauge\", \"java_lang_memory_heapmemoryusage_used\": \"gauge\", \"metrics_memory_heap_init_value\": \"gauge\", \"metrics_api_get_partition_by_name_count\": \"gauge\", \"metrics_api_get_partitions_by_names_count\": \"gauge\", \"metrics_api_get_partition_names_count\": \"gauge\", \"metrics_api_get_partitions_by_expr_count\": \"gauge\", \"metrics_api_get_partitions_count\": \"gauge\", \"metrics_api_get_partition_count\": \"gauge\", \"metrics_api_get_partitions_by_filter_count\": \"gauge\", \"metrics_api_add_partitions_count\": \"gauge\", \"metrics_api_add_partitions_req_count\": \"gauge\", \"metrics_api_drop_partition_by_name_count\": \"gauge\", \"metrics_api_add_partition_count\": \"gauge\", \"metrics_api_alter_partitions_count\": \"gauge\", \"metrics_api_create_table_count\": \"gauge\", \"metrics_api_alter_table_with_cascade_count\": \"gauge\", \"metrics_api_get_table_meta_count\": \"gauge\", \"metrics_api_get_table_metas_count\": \"gauge\", \"metrics_api_get_table_count\": \"gauge\", \"metrics_api_alter_table_count\": \"gauge\", \"metrics_api_get_tables_count\": \"gauge\", \"metrics_api_get_all_tables_count\": \"gauge\", \"metrics_api_drop_table_count\": \"gauge\", \"metrics_api_get_multi_table_count\": \"gauge\", \"metrics_api_get_database_count\": \"gauge\", \"metrics_api_get_all_databases_count\": \"gauge\", \"metrics_api_get_databases_count\": \"gauge\", \"metrics_api_create_function_count\": \"gauge\", \"metrics_api_getmetaconf_count\": \"gauge\", \"metrics_api_alter_table_with_environment_context_count\": \"gauge\", \"metrics_api_delete_column_statistics_by_table_count\": \"gauge\", \"metrics_api_get_functions_count\": \"gauge\", \"metrics_api_get_function_count\": \"gauge\", \"metrics_api_shutdown_count\": \"gauge\", \"metrics_api_flushcache_count\": \"gauge\", \"metrics_api_get_indexes_count\": \"gauge\", \"metrics_api_get_config_value_count\": \"gauge\", \"metrics_api_set_ugi_count\": \"gauge\", \"metrics_api_get_all_functions_count\": \"gauge\", \"metrics_api_get_table_req_95thpercentile\": \"gauge\", \"metrics_api_get_table_req_50thpercentile\": \"gauge\", \"metrics_api_get_table_req_count\": \"gauge\", \"metrics_api_get_table_req_max\": \"gauge\", \"metrics_api_get_databases_count\": \"gauge\", \"metrics_api_get_databases_95thpercentile\": \"gauge\", \"metrics_api_get_databases_50thpercentile\": \"gauge\", \"metrics_api_get_databases_max\": \"gauge\", \"metrics_api_get_partitions_95thpercentile\": \"gauge\", \"metrics_api_get_partitions_50thpercentile\": \"gauge\", \"metrics_api_get_partitions_count\": \"gauge\", \"metrics_api_get_partitions_max\": \"gauge\", \"metrics_api_get_partitions_50thpercentile\": \"gauge\", \"metrics_api_get_table_req_50thpercentile\": \"gauge\",\"metrics_api_get_database_95thpercentile\": \"gauge\",\"metrics_api_get_database_50thpercentile\": \"gauge\", \"metrics_kafka_listener_failures_count\": \"gauge\", \"metrics_kafka_listener_successes_count\": \"gauge\", \"metrics_api_get_table_objects_by_name_req_max\": \"gauge\"} }]",
+        "com.datadoghq.ad.check_names": "[\"openmetrics\"]",
+        "com.datadoghq.ad.init_configs": "[{}]"
+  },
     "portMappings": [
       {
         "containerPort": 9083,
@@ -231,4 +236,29 @@
 %{ endfor }
     ]
   }
+%{ if datadog_agent_enabled }
+  ,{
+   "name": "datadog-agent",
+   "image": "public.ecr.aws/datadog/agent:${datadog_agent_version}",
+   "essential": true,
+   "logConfiguration": {
+     "logDriver": "awslogs",
+     "options": {
+       "awslogs-group": "${loggroup}",
+       "awslogs-region": "${region}",
+       "awslogs-stream-prefix": "/"
+     }
+   },
+   "environment": [
+     {
+       "name": "DD_API_KEY",
+       "value": "${datadog_secret_key}"
+     },
+     {
+       "name": "ECS_FARGATE",
+       "value": "true"
+     }
+   ]
+ }
+%{ endif }
 ]
diff --git a/variables.tf b/variables.tf
@@ -715,3 +715,21 @@ variable "hms_ro_request_partition_limit" {
   type        = string
   default     = ""
 }
+
+variable "datadog_key_secret_name" {
+  description = "Name of the secret containing the DataDog API key. This needs to be created manually in AWS secrets manager. This is only applicable to ECS deployments."
+  type        = string
+  default     = null
+}
+
+variable "datadog_agent_version" {
+  description = "Version of the Datadog Agent running in the ECS cluster. This is only applicable to ECS deployments."
+  type        = string
+  default     = "7.50.3-jmx"
+}
+
+variable "datadog_agent_enabled" {
+  description = "Whether to include the datadog-agent container. This is only applicable to ECS deployments."
+  type        = bool
+  default     = false
+}