Merge pull request #65 from ExpediaGroup/feature/add_ext_db_host

feat: add support to use external db host

CHANGELOG.md

@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [5.2.0] - 2024-11-25
+### Added
+- Added new variable `beekeeper_db_external_hostname` to support external DB host.
+
 ## [5.1.1] - 2024-11-07
 ### Added
 - Kubernetes variables to adjust number replicas on deployments.

config-templates.tf

@@ -19,9 +19,9 @@ data "template_file" "beekeeper_scheduler_apiary_config" {
   template = file("${path.module}/files/beekeeper-scheduler-apiary-config.json")
 
   vars = {
-    db_endpoint     = aws_db_instance.beekeeper.endpoint
-    db_name         = aws_db_instance.beekeeper.name
-    db_username     = aws_db_instance.beekeeper.username
+    db_endpoint     = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}"
+    db_name         = var.beekeeper_db_name
+    db_username     = var.db_username
     queue           = aws_sqs_queue.beekeeper.id
     graphite_config = var.graphite_enabled == "false" ? "" : data.template_file.beekeeper_graphite_config.rendered
   }
@@ -31,9 +31,9 @@ data "template_file" "beekeeper_path_cleanup_config" {
   template = file("${path.module}/files/beekeeper-path-cleanup-config.json")
 
   vars = {
-    db_endpoint        = aws_db_instance.beekeeper.endpoint
-    db_name            = aws_db_instance.beekeeper.name
-    db_username        = aws_db_instance.beekeeper.username
+    db_endpoint        = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}"
+    db_name            = var.beekeeper_db_name
+    db_username        = var.db_username
     scheduler_delay_ms = var.scheduler_delay_ms
     dry_run_enabled    = var.path_cleanup_dry_run_enabled
     graphite_config    = var.graphite_enabled == "false" ? "" : data.template_file.beekeeper_graphite_config.rendered
@@ -44,9 +44,9 @@ data "template_file" "beekeeper_metadata_cleanup_config" {
   template = file("${path.module}/files/beekeeper-metadata-cleanup-config.json")
 
   vars = {
-    db_endpoint        = aws_db_instance.beekeeper.endpoint
-    db_name            = aws_db_instance.beekeeper.name
-    db_username        = aws_db_instance.beekeeper.username
+    db_endpoint        = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}"
+    db_name            = var.beekeeper_db_name
+    db_username        = var.db_username
     scheduler_delay_ms = var.scheduler_delay_ms
     dry_run_enabled    = var.metadata_cleanup_dry_run_enabled
     metastore_uri      = var.metastore_uri
@@ -58,9 +58,9 @@ data "template_file" "beekeeper_api_config" {
   template = file("${path.module}/files/beekeeper-api-config.json")
 
   vars = {
-    db_endpoint        = aws_db_instance.beekeeper.endpoint
-    db_name            = aws_db_instance.beekeeper.name
-    db_username        = aws_db_instance.beekeeper.username
+    db_endpoint        = var.beekeeper_db_external_hostname == "" ? aws_db_instance.beekeeper[0].endpoint : "${var.beekeeper_db_external_hostname}:${var.beekeeper_db_port}"
+    db_name            = var.beekeeper_db_name
+    db_username        = var.db_username
     scheduler_delay_ms = var.scheduler_delay_ms
     metastore_uri      = var.metastore_uri
     graphite_config    = var.graphite_enabled == "false" ? "" : data.template_file.beekeeper_graphite_config.rendered

rds.tf

@@ -5,6 +5,7 @@
  */
 
 resource "aws_db_subnet_group" "beekeeper_db_subnet_group" {
+  count       = var.beekeeper_db_external_hostname == "" ? 1 : 0
   name        = "${local.instance_alias}-db-subnet-group"
   subnet_ids  = var.rds_subnets
   description = "Beekeeper DB Subnet Group for ${local.instance_alias}"
@@ -13,6 +14,7 @@ resource "aws_db_subnet_group" "beekeeper_db_subnet_group" {
 }
 
 resource "aws_security_group" "beekeeper_db_sg" {
+  count       = var.beekeeper_db_external_hostname == "" ? 1 : 0
   name   = "${local.instance_alias}-db"
   vpc_id = var.vpc_id
   tags   = var.beekeeper_tags
@@ -46,9 +48,10 @@ resource "random_id" "snapshot_id" {
 }
 
 resource "aws_db_instance" "beekeeper" {
+  count                        = var.beekeeper_db_external_hostname == "" ? 1 : 0
   identifier                   = local.instance_alias
-  db_subnet_group_name         = aws_db_subnet_group.beekeeper_db_subnet_group.name
-  vpc_security_group_ids       = [aws_security_group.beekeeper_db_sg.id]
+  db_subnet_group_name         = aws_db_subnet_group.beekeeper_db_subnet_group[0].name
+  vpc_security_group_ids       = [aws_security_group.beekeeper_db_sg[0].id]
   allocated_storage            = var.rds_allocated_storage
   max_allocated_storage        = var.rds_max_allocated_storage
   storage_type                 = var.rds_storage_type
@@ -57,7 +60,7 @@ resource "aws_db_instance" "beekeeper" {
   instance_class               = var.rds_instance_class
   // Don't use instance alias as part of default DB name since all the Flyway scripts expect "beekeeper" as the db name.
   // No reason to parameterize the db name anyway, since we have a different RDS instance per Beekeeper instance.
-  db_name                      = "beekeeper"
+  db_name                      = var.beekeeper_db_name
   username                     = var.db_username
   password                     = chomp(data.aws_secretsmanager_secret_version.beekeeper_db.secret_string)
   parameter_group_name         = var.rds_parameter_group_name

variables.tf

@@ -43,6 +43,7 @@ variable "beekeeper_tags" {
 variable "rds_subnets" {
   description = "Subnets in which to provision Beekeeper RDS DB."
   type        = list(string)
+  default     = []
 }
 
 variable "rds_allocated_storage" {
@@ -622,10 +623,20 @@ variable "beekeeper_path_cleanup_metrics" {
   ]
 }
 
-variable "beekeeper_perf_scheduler_apiary_metrics" {
-  description = "Beekeeper metrics to be sent to Datadog."
-  type        = list(string)
-  default = [
-    "metadata_scheduled_seconds_count*"
-  ]
+variable "beekeeper_db_external_hostname" {
+  description = "Hostname of the background Database. When not null, RDS DB wont be created."
+  type        = string
+  default     = ""
+}
+
+variable "beekeeper_db_name" {
+  description = "Name of the beekeeper background Database"
+  type        = string
+  default     = "beekeeper"
+}
+
+variable "beekeeper_db_port" {
+  description = "Port number of the beekeeper background Database"
+  type        = number
+  default     = 3306
 }