From 13430c0093f53e8b1d1c44452db7d323d73b6f45 Mon Sep 17 00:00:00 2001 From: Scott Barnhart Date: Mon, 18 Oct 2021 14:39:02 +0200 Subject: [PATCH] Support Waggledance "latency" parameter. (#88) * Support for setting the latency param on each metastore. * add documentation Co-authored-by: Scott Barnhart --- CHANGELOG.md | 8 ++++++-- README.md | 9 +++++++++ templates.tf | 5 +++++ templates/waggle-dance-federation-local.yml.tmpl | 1 + templates/waggle-dance-federation-remote.yml.tmpl | 1 + templates/waggle-dance-federation-ssh.yml.tmpl | 1 + templates/waggle-dance-federation.yml.tmpl | 1 + variables.tf | 6 ++++++ 8 files changed, 30 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23eec87..21c4ce3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [3.3.6] - 2021-010-18 +### Added +- Support Waggledance `latency` parameter + ## [3.3.5] - 2021-09-21 -### Change +### Changed - Fixes to deploy on k8s clusters requiring serviceaccount for all deployments. ## [3.3.4] - 2021-08-13 -### Change +### Changed - Enable health check in k8s deployment. ## [3.3.3] - 2021-04-06 diff --git a/README.md b/README.md index 058d74f..2e14383 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi | aws_region | AWS region to use for resources. | string | - | yes | | bastion_ssh_key_secret_name | Secret name in AWS Secrets Manager which stores the private key used to log in to bastions. The secret's key should be `private_key` and the value should be stored as a base64 encoded string. Max character limit for a secret's value is 4096. | string | `` | no | | cpu | The number of CPU units to reserve for the Waggle Dance container. Valid values can be 256, 512, 1024, 2048 and 4096. Reference: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `1024` | no | +| default_latency | Latency used for the primary metastore, and other other metastores that don't override it in their own configurations. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `0` | no | | docker_image | Full path Waggle Dance Docker image. | string | - | yes | | docker_registry_auth_secret_name | Docker Registry authentication SecretManager secret name. | string | `` | no | | docker_version | Waggle Dance Docker image version. | string | - | yes | @@ -70,6 +71,8 @@ module "apiary-waggledance" { primary_metastore_host = "primary-metastore.yourdomain.com" primary_metastore_whitelist = ["test_.*", "team_.*"] + default_latency = 2000 + remote_metastores = [ { endpoint = "com.amazonaws.vpce.us-west-2.vpce-svc-1" @@ -78,6 +81,7 @@ module "apiary-waggledance" { mapped-databases = "default,test" database-name-mapping = "test:test_alias,default:default_alias" writable-whitelist = "test" + latency = 5000 }, { endpoint = "com.amazonaws.vpce.us-east-1.vpce-svc-2" @@ -126,6 +130,7 @@ An example entry looks like: local_metastores = [ { host = "hms-readonly.metastore.svc.cluster.local" + latency = 2000 port = "9083" prefix = "local1" mapped-databases = "default,test" @@ -139,6 +144,7 @@ local_metastores = [ Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | host | Host name of the Hive metastore server on the local network. | string | - | yes | +| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no | | port | IP port that the Thrift server of the Hive metastore listens on. | string | `"9083"` | no | | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes | | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no | @@ -169,6 +175,7 @@ remote_metastores = [ Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | endpoint | AWS VPC endpoint name that is connected to the remote Hive metastore. | string | - | yes | +| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no | | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no | | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes | | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no | @@ -202,6 +209,7 @@ remote_region_metastores = [ Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | endpoint | AWS VPC endpoint service name that is connected to the remote Hive metastore. | string | - | yes | +| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no | | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no | | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes | | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no | @@ -237,6 +245,7 @@ ssh_metastores = [ Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| latency | Latency used for this metastore. See `latency` parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md. | number | `var.default_latency` | no | | metastore-host | Host name of the Hive metastore that can be resolved/reached from the bastion host. | string | - | yes | | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no | | bastion-host | Host name of the bastion host. | string | - | yes | diff --git a/templates.tf b/templates.tf index 3b665be..6ba24da 100644 --- a/templates.tf +++ b/templates.tf @@ -58,6 +58,7 @@ data "template_file" "local_metastores_yaml" { writable_whitelist = lookup(var.local_metastores[count.index], "writable-whitelist", "") enable_path_conversion = lookup(var.local_metastores[count.index], "enable_path_conversion", false) metastore_enabled = lookup(var.local_metastores[count.index], "enabled", true) + latency = lookup(var.local_metastores[count.index], "latency", var.default_latency) } } @@ -74,6 +75,7 @@ data "template_file" "remote_metastores_yaml" { writable_whitelist = lookup(var.remote_metastores[count.index], "writable-whitelist", "") enable_path_conversion = lookup(var.remote_metastores[count.index], "enable_path_conversion", false) metastore_enabled = lookup(var.remote_metastores[count.index], "enabled", true) + latency = lookup(var.remote_metastores[count.index], "latency", var.default_latency) } } @@ -90,6 +92,7 @@ data "template_file" "remote_region_metastores_yaml" { writable_whitelist = lookup(var.remote_region_metastores[count.index], "writable-whitelist", "") enable_path_conversion = lookup(var.remote_region_metastores[count.index], "enable_path_conversion", true) metastore_enabled = lookup(var.remote_region_metastores[count.index], "enabled", true) + latency = lookup(var.remote_region_metastores[count.index], "latency", var.default_latency) } } @@ -109,6 +112,7 @@ data "template_file" "ssh_metastores_yaml" { database_name_mapping = lookup(var.ssh_metastores[count.index], "database-name-mapping", "") writable_whitelist = lookup(var.ssh_metastores[count.index], "writable-whitelist", "") metastore_enabled = lookup(var.ssh_metastores[count.index], "enabled", true) + latency = lookup(var.ssh_metastores[count.index], "latency", var.default_latency) } } @@ -118,6 +122,7 @@ data "template_file" "federation_yaml" { vars = { primary_metastore_host = var.primary_metastore_host primary_metastore_port = var.primary_metastore_port + primary_metastore_latency = var.default_latency primary_metastore_whitelist = join("", data.template_file.primary_metastore_whitelist.*.rendered) local_metastores = join("", data.template_file.local_metastores_yaml.*.rendered) remote_metastores = join("", data.template_file.remote_metastores_yaml.*.rendered) diff --git a/templates/waggle-dance-federation-local.yml.tmpl b/templates/waggle-dance-federation-local.yml.tmpl index fb11ed5..02a40a8 100644 --- a/templates/waggle-dance-federation-local.yml.tmpl +++ b/templates/waggle-dance-federation-local.yml.tmpl @@ -3,6 +3,7 @@ access-control-type: ${ writable_whitelist == "" ? "READ_ONLY" : "READ_AND_WRITE_ON_DATABASE_WHITELIST" } database-prefix: ${prefix}_ remote-meta-store-uris: thrift://${metastore_host}:${metastore_port} + latency: ${latency} %{if enable_path_conversion ~} hive-metastore-filter-hook: com.expediagroup.apiary.extensions.hooks.filters.ApiaryMetastoreFilter %{~endif} diff --git a/templates/waggle-dance-federation-remote.yml.tmpl b/templates/waggle-dance-federation-remote.yml.tmpl index fb11ed5..02a40a8 100644 --- a/templates/waggle-dance-federation-remote.yml.tmpl +++ b/templates/waggle-dance-federation-remote.yml.tmpl @@ -3,6 +3,7 @@ access-control-type: ${ writable_whitelist == "" ? "READ_ONLY" : "READ_AND_WRITE_ON_DATABASE_WHITELIST" } database-prefix: ${prefix}_ remote-meta-store-uris: thrift://${metastore_host}:${metastore_port} + latency: ${latency} %{if enable_path_conversion ~} hive-metastore-filter-hook: com.expediagroup.apiary.extensions.hooks.filters.ApiaryMetastoreFilter %{~endif} diff --git a/templates/waggle-dance-federation-ssh.yml.tmpl b/templates/waggle-dance-federation-ssh.yml.tmpl index 890f07b..1afe76f 100644 --- a/templates/waggle-dance-federation-ssh.yml.tmpl +++ b/templates/waggle-dance-federation-ssh.yml.tmpl @@ -3,6 +3,7 @@ access-control-type: ${ writable_whitelist == "" ? "READ_ONLY" : "READ_AND_WRITE_ON_DATABASE_WHITELIST" } database-prefix: ${prefix}_ remote-meta-store-uris: thrift://${metastore_host}:${metastore_port} + latency: ${latency} metastore-tunnel: route: ${user}@${bastion_host} private-keys: /root/.ssh/bastion_ssh diff --git a/templates/waggle-dance-federation.yml.tmpl b/templates/waggle-dance-federation.yml.tmpl index 4bea7b5..1838277 100644 --- a/templates/waggle-dance-federation.yml.tmpl +++ b/templates/waggle-dance-federation.yml.tmpl @@ -3,6 +3,7 @@ primary-meta-store: database-prefix: '' name: primary remote-meta-store-uris: thrift://${primary_metastore_host}:${primary_metastore_port} + latency: ${primary_metastore_latency} writable-database-white-list: ${primary_metastore_whitelist} federated-meta-stores: diff --git a/variables.tf b/variables.tf index d3980f6..771ec4e 100644 --- a/variables.tf +++ b/variables.tf @@ -238,3 +238,9 @@ variable "alluxio_endpoints" { type = list(map(string)) default = [] } + +variable "default_latency" { + type = number + default = 0 + description = "HMS latency (in ms.) that Waggledance will tolerate. See \"latency\" parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md." +} \ No newline at end of file