Support Waggledance "latency" parameter. (#88)

* Support for setting the latency param on each metastore. * add documentation Co-authored-by: Scott Barnhart <[email protected]>
ExpediaGroup · Oct 18, 2021 · 13430c0 · 13430c0
1 parent 24a4d3d
commit 13430c0
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,12 +3,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [3.3.6] - 2021-010-18
+### Added
+- Support Waggledance `latency` parameter
+
 ## [3.3.5] - 2021-09-21
-### Change
+### Changed
 - Fixes to deploy on k8s clusters requiring serviceaccount for all deployments.
 
 ## [3.3.4] - 2021-08-13
-### Change
+### Changed
 - Enable health check in k8s deployment.
 
 ## [3.3.3] - 2021-04-06

diff --git a/README.md b/README.md
@@ -10,6 +10,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
 | aws_region | AWS region to use for resources. | string | - | yes |
 | bastion_ssh_key_secret_name | Secret name in AWS Secrets Manager which stores the private key used to log in to bastions. The secret's key should be `private_key` and the value should be stored as a base64 encoded string. Max character limit for a secret's value is 4096. | string | `` | no |
 | cpu | The number of CPU units to reserve for the Waggle Dance container. Valid values can be 256, 512, 1024, 2048 and 4096. Reference: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `1024` | no |
+| default_latency | Latency used for the primary metastore, and other other metastores that don't override it in their own configurations. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `0` | no |
 | docker_image | Full path Waggle Dance Docker image. | string | - | yes |
 | docker_registry_auth_secret_name | Docker Registry authentication SecretManager secret name. | string | `` | no |
 | docker_version | Waggle Dance Docker image version. | string | - | yes |
@@ -70,6 +71,8 @@ module "apiary-waggledance" {
   primary_metastore_host      = "primary-metastore.yourdomain.com"
   primary_metastore_whitelist = ["test_.*", "team_.*"]
 
+  default_latency = 2000
+
   remote_metastores = [
     {
       endpoint              = "com.amazonaws.vpce.us-west-2.vpce-svc-1"
@@ -78,6 +81,7 @@ module "apiary-waggledance" {
       mapped-databases      = "default,test"
       database-name-mapping = "test:test_alias,default:default_alias"
       writable-whitelist    = "test"
+      latency               = 5000
     },
     {
       endpoint         = "com.amazonaws.vpce.us-east-1.vpce-svc-2"
@@ -126,6 +130,7 @@ An example entry looks like:
 local_metastores = [
     {
       host                  = "hms-readonly.metastore.svc.cluster.local"
+      latency               = 2000
       port                  = "9083"
       prefix                = "local1"
       mapped-databases      = "default,test"
@@ -139,6 +144,7 @@ local_metastores = [
 Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
 | host | Host name of the Hive metastore server on the local network. | string | - | yes |
+| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no |
 | port | IP port that the Thrift server of the Hive metastore listens on. | string | `"9083"` | no |
 | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes |
 | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no |
@@ -169,6 +175,7 @@ remote_metastores = [
 Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
 | endpoint | AWS VPC endpoint name that is connected to the remote Hive metastore. | string | - | yes |
+| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no |
 | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no |
 | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes |
 | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no |
@@ -202,6 +209,7 @@ remote_region_metastores = [
 Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
 | endpoint | AWS VPC endpoint service name that is connected to the remote Hive metastore. | string | - | yes |
+| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no |
 | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no |
 | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes |
 | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no |
@@ -237,6 +245,7 @@ ssh_metastores = [
 
 Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no |
 | metastore-host | Host name of the Hive metastore that can be resolved/reached from the bastion host. | string | - | yes |
 | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no |
 | bastion-host | Host name of the bastion host. | string | - | yes |

diff --git a/templates.tf b/templates.tf
@@ -58,6 +58,7 @@ data "template_file" "local_metastores_yaml" {
     writable_whitelist     = lookup(var.local_metastores[count.index], "writable-whitelist", "")
     enable_path_conversion = lookup(var.local_metastores[count.index], "enable_path_conversion", false)
     metastore_enabled      = lookup(var.local_metastores[count.index], "enabled", true)
+    latency                = lookup(var.local_metastores[count.index], "latency", var.default_latency)
   }
 }
 
@@ -74,6 +75,7 @@ data "template_file" "remote_metastores_yaml" {
     writable_whitelist     = lookup(var.remote_metastores[count.index], "writable-whitelist", "")
     enable_path_conversion = lookup(var.remote_metastores[count.index], "enable_path_conversion", false)
     metastore_enabled      = lookup(var.remote_metastores[count.index], "enabled", true)
+    latency                = lookup(var.remote_metastores[count.index], "latency", var.default_latency)
   }
 }
 
@@ -90,6 +92,7 @@ data "template_file" "remote_region_metastores_yaml" {
     writable_whitelist     = lookup(var.remote_region_metastores[count.index], "writable-whitelist", "")
     enable_path_conversion = lookup(var.remote_region_metastores[count.index], "enable_path_conversion", true)
     metastore_enabled      = lookup(var.remote_region_metastores[count.index], "enabled", true)
+    latency                = lookup(var.remote_region_metastores[count.index], "latency", var.default_latency)
   }
 }
 
@@ -109,6 +112,7 @@ data "template_file" "ssh_metastores_yaml" {
     database_name_mapping = lookup(var.ssh_metastores[count.index], "database-name-mapping", "")
     writable_whitelist    = lookup(var.ssh_metastores[count.index], "writable-whitelist", "")
     metastore_enabled     = lookup(var.ssh_metastores[count.index], "enabled", true)
+    latency               = lookup(var.ssh_metastores[count.index], "latency", var.default_latency)
   }
 }
 
@@ -118,6 +122,7 @@ data "template_file" "federation_yaml" {
   vars = {
     primary_metastore_host      = var.primary_metastore_host
     primary_metastore_port      = var.primary_metastore_port
+    primary_metastore_latency   = var.default_latency
     primary_metastore_whitelist = join("", data.template_file.primary_metastore_whitelist.*.rendered)
     local_metastores            = join("", data.template_file.local_metastores_yaml.*.rendered)
     remote_metastores           = join("", data.template_file.remote_metastores_yaml.*.rendered)

diff --git a/templates/waggle-dance-federation-local.yml.tmpl b/templates/waggle-dance-federation-local.yml.tmpl
@@ -3,6 +3,7 @@
     access-control-type: ${ writable_whitelist == "" ? "READ_ONLY" : "READ_AND_WRITE_ON_DATABASE_WHITELIST" }
     database-prefix: ${prefix}_
     remote-meta-store-uris: thrift://${metastore_host}:${metastore_port}
+    latency: ${latency}
 %{if enable_path_conversion ~}
     hive-metastore-filter-hook: com.expediagroup.apiary.extensions.hooks.filters.ApiaryMetastoreFilter
 %{~endif}

diff --git a/templates/waggle-dance-federation-remote.yml.tmpl b/templates/waggle-dance-federation-remote.yml.tmpl
@@ -3,6 +3,7 @@
     access-control-type: ${ writable_whitelist == "" ? "READ_ONLY" : "READ_AND_WRITE_ON_DATABASE_WHITELIST" }
     database-prefix: ${prefix}_
     remote-meta-store-uris: thrift://${metastore_host}:${metastore_port}
+    latency: ${latency}
 %{if enable_path_conversion ~}
     hive-metastore-filter-hook: com.expediagroup.apiary.extensions.hooks.filters.ApiaryMetastoreFilter
 %{~endif}

diff --git a/templates/waggle-dance-federation-ssh.yml.tmpl b/templates/waggle-dance-federation-ssh.yml.tmpl
@@ -3,6 +3,7 @@
     access-control-type: ${ writable_whitelist == "" ? "READ_ONLY" : "READ_AND_WRITE_ON_DATABASE_WHITELIST" }
     database-prefix: ${prefix}_
     remote-meta-store-uris: thrift://${metastore_host}:${metastore_port}
+    latency: ${latency}
     metastore-tunnel:
       route: ${user}@${bastion_host}
       private-keys: /root/.ssh/bastion_ssh

diff --git a/templates/waggle-dance-federation.yml.tmpl b/templates/waggle-dance-federation.yml.tmpl
@@ -3,6 +3,7 @@ primary-meta-store:
   database-prefix: ''
   name: primary
   remote-meta-store-uris: thrift://${primary_metastore_host}:${primary_metastore_port}
+  latency: ${primary_metastore_latency}
   writable-database-white-list:
 ${primary_metastore_whitelist}
 federated-meta-stores:

diff --git a/variables.tf b/variables.tf
@@ -238,3 +238,9 @@ variable "alluxio_endpoints" {
   type    = list(map(string))
   default = []
 }
+
+variable "default_latency" {
+  type        = number
+  default     = 0
+  description = "HMS latency (in ms.) that Waggledance will tolerate.  See \"latency\" parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md."
+}