Add support for additional Hive Metastore env variables (#221)

ExpediaGroup · Nov 15, 2022 · 0db76df · 0db76df
1 parent 0bb7fd4
commit 0db76df
Show file tree

Hide file tree

Showing 9 changed files with 48 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.15.0] - 2022-11-15
+### Added
+- Add variable to set custom environment variables for the Hive Metastore
+
 ## [6.14.2] - 2022-07-19
 ### Changed
 - Add support for wildcards in consumer iam roles.

diff --git a/VARIABLES.md b/VARIABLES.md
@@ -52,6 +52,7 @@
 | external\_data\_buckets                                   | Buckets that are not managed by Apiary but added to Hive Metastore IAM role access.                                                                                                                                  | `list(any)`              | `[]`                                                                                                                                                                                                                                                                                           |    no    |
 | external\_database\_host                                  | External Metastore database host to support legacy installations, MySQL database won't be created by Apiary when this option is specified.                                                                           | `string`                 | `""`                                                                                                                                                                                                                                                                                           |    no    |
 | hive\_metastore\_port                                     | Port on which both Hive Metastore readwrite and readonly will run.                                                                                                                                                   | `number`                 | `9083`                                                                                                                                                                                                                                                                                         |    no    |
+| hms\_additional\_environment\_variables                   | Additional environment variables for the Hive Metastore.                                                                                                                                                             | `map(any)`               | `{}`                                                                                                                                                                                                                                                                                           |    no    |                                  
 | hms\_autogather\_stats                                    | Read-write Hive metastore setting to enable/disable statistics auto-gather on table/partition creation.                                                                                                              | `bool`                   | `true`                                                                                                                                                                                                                                                                                         |    no    |
 | hms\_docker\_image                                        | Docker image ID for the Hive Metastore.                                                                                                                                                                              | `string`                 | n/a                                                                                                                                                                                                                                                                                            |   yes    |
 | hms\_docker\_version                                      | Version of the Docker image for the Hive Metastore.                                                                                                                                                                  | `string`                 | n/a                                                                                                                                                                                                                                                                                            |   yes    |

diff --git a/ecs.tf b/ecs.tf
@@ -25,7 +25,7 @@ resource "aws_ecs_task_definition" "apiary_hms_readwrite" {
   memory                   = var.hms_rw_heapsize
   cpu                      = var.hms_rw_cpu
   requires_compatibilities = ["EC2", "FARGATE"]
-  container_definitions    = data.template_file.hms_readwrite.rendered
+  container_definitions    = local.hms_readwrite_template
   tags                     = var.apiary_tags
 }
 
@@ -38,7 +38,7 @@ resource "aws_ecs_task_definition" "apiary_hms_readonly" {
   memory                   = var.hms_ro_heapsize
   cpu                      = var.hms_ro_cpu
   requires_compatibilities = ["EC2", "FARGATE"]
-  container_definitions    = data.template_file.hms_readonly.rendered
+  container_definitions    = local.hms_readonly_template
   tags                     = var.apiary_tags
 }
 

diff --git a/k8s-readonly.tf b/k8s-readonly.tf
@@ -179,6 +179,14 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
             name  = "HMS_AUTOGATHER_STATS"
             value = "false"
           }
+          dynamic "env" {
+            for_each = var.hms_additional_environment_variables
+
+            content {
+              name  = env.key
+              value = env.value
+            }
+          }
 
           liveness_probe {
             tcp_socket {

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -219,6 +219,14 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
             name  = "HMS_AUTOGATHER_STATS"
             value = var.hms_autogather_stats
           }
+          dynamic "env" {
+            for_each = var.hms_additional_environment_variables
+
+            content {
+              name  = env.key
+              value = env.value
+            }
+          }
 
           liveness_probe {
             tcp_socket {

diff --git a/templates.tf b/templates.tf
@@ -4,10 +4,8 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  */
 
-data "template_file" "hms_readwrite" {
-  template = file("${path.module}/templates/apiary-hms-readwrite.json")
-
-  vars = {
+locals{
+  hms_readwrite_template = templatefile("${path.module}/templates/apiary-hms-readwrite.json", {
     mysql_db_host              = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}"
     mysql_db_name              = "${var.apiary_database_name}"
     mysql_secret_arn           = "${data.aws_secretsmanager_secret.db_rw_user.arn}"
@@ -17,6 +15,7 @@ data "template_file" "hms_readwrite" {
     hms_maxthreads             = local.hms_ro_maxthreads
     hms_docker_image           = "${var.hms_docker_image}"
     hms_docker_version         = "${var.hms_docker_version}"
+    hms_additional_vars        = var.hms_additional_environment_variables
     region                     = "${var.aws_region}"
     loggroup                   = "${join("", aws_cloudwatch_log_group.apiary_ecs.*.name)}"
     hive_metastore_log_level   = "${var.hms_log_level}"
@@ -62,13 +61,9 @@ data "template_file" "hms_readwrite" {
     mysql_permissions      = "ALL"
     mysql_master_cred_arn  = var.external_database_host == "" ? aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn : null
     mysql_user_cred_arn    = data.aws_secretsmanager_secret.db_rw_user.arn
-  }
-}
-
-data "template_file" "hms_readonly" {
-  template = file("${path.module}/templates/apiary-hms-readonly.json")
+  })
 
-  vars = {
+  hms_readonly_template = templatefile("${path.module}/templates/apiary-hms-readonly.json", {
     mysql_db_host              = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.reader_endpoint) : var.external_database_host}"
     mysql_db_name              = "${var.apiary_database_name}"
     mysql_secret_arn           = "${data.aws_secretsmanager_secret.db_ro_user.arn}"
@@ -78,6 +73,7 @@ data "template_file" "hms_readonly" {
     hms_maxthreads             = local.hms_rw_maxthreads
     hms_docker_image           = "${var.hms_docker_image}"
     hms_docker_version         = "${var.hms_docker_version}"
+    hms_additional_vars        = var.hms_additional_environment_variables
     region                     = "${var.aws_region}"
     loggroup                   = "${join("", aws_cloudwatch_log_group.apiary_ecs.*.name)}"
     hive_metastore_log_level   = "${var.hms_log_level}"
@@ -107,5 +103,5 @@ data "template_file" "hms_readonly" {
     mysql_write_db         = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}"
     mysql_master_cred_arn  = var.external_database_host == "" ? aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn : null
     mysql_user_cred_arn    = data.aws_secretsmanager_secret.db_ro_user.arn
-  }
+  })
 }
diff --git a/templates/apiary-hms-readonly.json b/templates/apiary-hms-readonly.json
@@ -167,6 +167,12 @@
         "name": "HMS_AUTOGATHER_STATS",
         "value": "false"
       }
+%{ for env_key, env_value in hms_additional_vars }
+     ,{
+        "name": "${env_key}",
+        "value": "${env_value}"
+      }
+%{ endfor }
    ]
   }
 ]
diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json
@@ -219,6 +219,12 @@
        "name": "HMS_AUTOGATHER_STATS",
        "value": "${hms_autogather_stats}"
      }
+%{ for env_key, env_value in hms_additional_vars }
+    ,{
+      "name": "${env_key}",
+      "value": "${env_value}"
+     }
+%{ endfor }
     ]
   }
 ]
diff --git a/variables.tf b/variables.tf
@@ -653,3 +653,9 @@ variable "db_enhanced_monitoring_interval" {
   type        = number
   default     = 0
 }
+
+variable "hms_additional_environment_variables" {
+  description = "Additional environment variables for Hive metastore."
+  type        = map(any)
+  default     = {}
+}