Configure HMS MySQL thread-pool size and stats auto-gathering (#214)

* Configure HMS MySQL thread-pool size and stats auto-gathering * Configure HMS MySQL thread-pool size and stats auto-gathering * fix var name * explicitly set autogather=false for RO HMS Co-authored-by: Scott Barnhart <[email protected]>
ExpediaGroup · May 20, 2022 · 975f7f5 · 975f7f5
1 parent a0bdc36
commit 975f7f5
Show file tree

Hide file tree

Showing 8 changed files with 162 additions and 102 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.12.2] - 2022-05-20
+### Added
+- Add ability to configure size of HMS MySQL connection pool, and configure stats computation on table/partition creation.
+
 ## [6.12.1] - 2022-03-17
 ### Fixed
 - Fixed type error in `apiary_consumer_prefix_iamroles` variable.

diff --git a/VARIABLES.md b/VARIABLES.md
diff --git a/k8s-readonly.tf b/k8s-readonly.tf
@@ -171,6 +171,14 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
             name  = "HMS_MAX_THREADS"
             value = local.hms_ro_maxthreads
           }
+          env {
+            name  = "MYSQL_CONNECTION_POOL_SIZE"
+            value = var.hms_ro_db_connection_pool_size
+          }
+          env {
+            name  = "HMS_AUTOGATHER_STATS"
+            value = "false"
+          }
 
           liveness_probe {
             tcp_socket {

diff --git a/k8s-readwrite.tf b/k8s-readwrite.tf
@@ -211,6 +211,14 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
             name  = "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES"
             value = var.disallow_incompatible_col_type_changes
           }
+          env {
+            name  = "MYSQL_CONNECTION_POOL_SIZE"
+            value = var.hms_rw_db_connection_pool_size
+          }
+          env {
+            name  = "HMS_AUTOGATHER_STATS"
+            value = var.hms_autogather_stats
+          }
 
           liveness_probe {
             tcp_socket {

diff --git a/templates.tf b/templates.tf
@@ -44,6 +44,8 @@ data "template_file" "hms_readwrite" {
     kafka_topic_name                       = var.kafka_topic_name
     system_schema_name                     = var.system_schema_name
     disallow_incompatible_col_type_changes = var.disallow_incompatible_col_type_changes
+    hms_autogather_stats                   = var.hms_autogather_stats
+    hms_rw_db_connection_pool_size         = var.hms_rw_db_connection_pool_size
 
     #to instruct docker to turn off upgrading hive db schema when using external database
     external_database = "${var.external_database_host == "" ? "" : "1"}"
@@ -94,6 +96,8 @@ data "template_file" "hms_readonly" {
     ldap_base                 = "${var.ldap_base}"
     ldap_secret_arn           = "${var.ldap_url == "" ? "" : join("", data.aws_secretsmanager_secret.ldap_user.*.arn)}"
 
+    hms_ro_db_connection_pool_size = var.hms_ro_db_connection_pool_size
+
     #to instruct ECS to use repositoryCredentials for private docker registry
     docker_auth = "${var.docker_registry_auth_secret_name == "" ? "" : format("\"repositoryCredentials\" :{\n \"credentialsParameter\":\"%s\"\n},", join("\",\"", concat(data.aws_secretsmanager_secret.docker_registry.*.arn)))}"
 

diff --git a/templates/apiary-hms-readonly.json b/templates/apiary-hms-readonly.json
@@ -158,7 +158,15 @@
       {
         "name": "HMS_MAX_THREADS",
         "value": "${hms_maxthreads}"
+      },
+      {
+        "name": "MYSQL_CONNECTION_POOL_SIZE",
+        "value": "${hms_ro_db_connection_pool_size}"
+      },
+      {
+        "name": "HMS_AUTOGATHER_STATS",
+        "value": "false"
       }
-    ]
+   ]
   }
 ]
diff --git a/templates/apiary-hms-readwrite.json b/templates/apiary-hms-readwrite.json
@@ -208,8 +208,16 @@
        "value": "${system_schema_name}"
      },
      {
-      "name": "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES",
-      "value": "${disallow_incompatible_col_type_changes}"
+       "name": "DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES",
+       "value": "${disallow_incompatible_col_type_changes}"
+     },
+     {
+       "name": "MYSQL_CONNECTION_POOL_SIZE",
+       "value": "${hms_rw_db_connection_pool_size}"
+     },
+     {
+       "name": "HMS_AUTOGATHER_STATS",
+       "value": "${hms_autogather_stats}"
      }
     ]
   }

diff --git a/variables.tf b/variables.tf
@@ -593,5 +593,22 @@ variable "rds_family" {
   type        = string
   default     = "aurora5.6"
 }
-
-
+
+variable "hms_autogather_stats" {
+  description = "Read-write Hive metastore setting to enable/disable statistics auto-gather on table/partition creation."
+  type        = bool
+  default     = true
+}
+
+variable "hms_ro_db_connection_pool_size" {
+  description = "Read-only Hive metastore setting for size of the MySQL connection pool. Default is 10."
+  type        = number
+  default     = 10
+}
+
+variable "hms_rw_db_connection_pool_size" {
+  description = "Read-write Hive metastore setting for size of the MySQL connection pool. Default is 10."
+  type        = number
+  default     = 10
+}
+