Support Waggledance "mapped-tables" option. (#96)

* Add support for mapped-tables feature * bug fix Co-authored-by: Scott Barnhart <[email protected]>
ExpediaGroup · Nov 18, 2021 · 6b28240 · 6b28240
1 parent f896162
commit 6b28240
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [3.3.13] - 2021-11-18
+### Added
+- Added suppport for Waggledance `mapped-tables` [configuration](https://github.com/ExpediaGroup/waggle-dance#mapped-tables).  
+
 ## [3.3.12] - 2021-11-16
 ### Fixed
 - Replace aws.remote provider proxy with configuration_aliases as provider proxy is not compatible with module count.

diff --git a/README.md b/README.md
@@ -138,6 +138,7 @@ local_metastores = [
       prefix                = "local1"
       mapped-databases      = "default,test"
       database-name-mapping = "test:test_alias,default:default_alias"
+      mapped-tables         = "test:test_table1,test_table1;default:default_table1.*,default_table2"
       writable-whitelist    = "test"
     }
 ]
@@ -151,10 +152,11 @@ Name | Description | Type | Default | Required |
 | port | IP port that the Thrift server of the Hive metastore listens on. | string | `"9083"` | no |
 | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes |
 | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no |
-| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/HotelsDotCom/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
+| mapped-tables | Semicolon-separated/comma-separated list of databases and DB tables from this metastore to expose to federation. If not specified, *all* tables for each database are exposed. See [Waggle Dance Mapped Tables](https://github.com/ExpediaGroup/waggle-dance#mapped-tables) for more information.| string | `""` | no |
+| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/ExpediaGroup/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
 | writable-whitelist | Comma-separated list of databases from this metastore that can be in read-write mode. If not specified, all databases are read-only. Use `.*` to allow all databases to be written to. | string | `""` | no |
 
-See [Waggle Dance README](https://github.com/HotelsDotCom/waggle-dance/README.md) for more information on all these parameters.
+See [Waggle Dance README](https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md) for more information on all these parameters.
 
 ### remote_metastores
 
@@ -169,6 +171,7 @@ remote_metastores = [
       prefix                = "remote1"
       mapped-databases      = "default,test"
       database-name-mapping = "test:test_alias,default:default_alias"
+      mapped-tables         = "test:test_table1,test_table1;default:default_table1.*,default_table2"
       writable-whitelist    = ".*"
     }
 ]
@@ -182,10 +185,11 @@ Name | Description | Type | Default | Required |
 | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no |
 | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes |
 | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no |
-| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/HotelsDotCom/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
+| mapped-tables | Semicolon-separated/comma-separated list of databases and DB tables from this metastore to expose to federation. If not specified, *all* tables for each database are exposed. See [Waggle Dance Mapped Tables](https://github.com/ExpediaGroup/waggle-dance#mapped-tables) for more information.| string | `""` | no |
+| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/ExpediaGroup/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
 | writable-whitelist | Comma-separated list of databases from this metastore that can be in read-write mode. If not specified, all databases are read-only. Use `.*` to allow all databases to be written to. | string | `""` | no |
 
-See [Waggle Dance README](https://github.com/HotelsDotCom/waggle-dance/README.md) for more information on all these parameters.
+See [Waggle Dance README](https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md) for more information on all these parameters.
 
 ### remote_region_metastores
 
@@ -199,6 +203,7 @@ remote_region_metastores = [
       port                  = "9083"
       prefix                = "remote1"
       mapped-databases      = "default,test"
+      mapped-tables         = "test:test_table1,test_table1;default:default_table1.*,default_table2"
       database-name-mapping = "test:test_alias,default:default_alias"
       writable-whitelist    = ".*"
       vpc_id                = "vpc-123456"
@@ -216,13 +221,14 @@ Name | Description | Type | Default | Required |
 | port | IP port that the Thrift server of the remote Hive metastore listens on. | string | `"9083"` | no |
 | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes |
 | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no |
-| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/HotelsDotCom/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
+| mapped-tables | Semicolon-separated/comma-separated list of databases and DB tables from this metastore to expose to federation. If not specified, *all* tables for each database are exposed. See [Waggle Dance Mapped Tables](https://github.com/ExpediaGroup/waggle-dance#mapped-tables) for more information.| string | `""` | no |
+| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/ExpediaGroup/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
 | writable-whitelist | Comma-separated list of databases from this metastore that can be in read-write mode. If not specified, all databases are read-only. Use `.*` to allow all databases to be written to. | string | `""` | no |
 | vpc_id | Remote region AWS VPC id. | string | - | yes |
 | subnets | AWS VPC subnets in remote region. | string | - | yes |
 | security_group_id | AWS EC2 security group in remote region. | string | - | yes |
 
-See [Waggle Dance README](https://github.com/HotelsDotCom/waggle-dance/README.md) for more information on all these parameters.
+See [Waggle Dance README](https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md) for more information on all these parameters.
 
 An example entry looks like:
 ### ssh_metastores
@@ -241,6 +247,7 @@ ssh_metastores = [
       prefix                = "ssh_metastore1"
       mapped-databases      = "default,test"
       database-name-mapping = "test:test_alias,default:default_alias"
+      mapped-tables         = "test:test_table1,test_table1;default:default_table1.*,default_table2"
     }
 ]
 ```
@@ -256,10 +263,11 @@ Name | Description | Type | Default | Required |
 | timeout | The SSH session timeout in milliseconds, 0 means no timeout. Default is 60000 milliseconds, i.e. 1 minute. | string | `"60000"` | no |
 | prefix | Prefix added to the database names from this metastore. Must be unique among all local, remote, and SSH federated metastores in this Waggle Dance instance. | string | - | yes |
 | mapped-databases | Comma-separated list of databases from this metastore to expose to federation. If not specified, *all* databases are exposed.| string | `""` | no |
-| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/HotelsDotCom/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
+| mapped-tables | Semicolon-separated/comma-separated list of databases and DB tables from this metastore to expose to federation. If not specified, *all* tables for each database are exposed. See [Waggle Dance Mapped Tables](https://github.com/ExpediaGroup/waggle-dance#mapped-tables) for more information.| string | `""` | no |
+| database-name-mapping | Comma-separated list of `<database>:<alias>` key/value pairs to add aliases for the given databases. Default is no aliases. This is used primarily in migration scenarios where a database has been renamed/relocated. See [Waggle Dance Database Name Mapping](https://github.com/ExpediaGroup/waggle-dance#database-name-mapping) for more information.  | string | `""` | no |
 | writable-whitelist | Comma-separated list of databases from this metastore that can be in read-write mode. If not specified, all databases are read-only. Use `.*` to allow all databases to be written to. | string | `""` | no |
 
-See [Waggle Dance README](https://github.com/HotelsDotCom/waggle-dance/README.md) for more information on all these parameters.
+See [Waggle Dance README](https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md) for more information on all these parameters.
 
 # Contact
 

diff --git a/templates.tf b/templates.tf
@@ -62,6 +62,7 @@ data "template_file" "local_metastores_yaml" {
     metastore_host         = var.local_metastores[count.index].host
     metastore_port         = lookup(var.local_metastores[count.index], "port", "9083")
     mapped_databases       = lookup(var.local_metastores[count.index], "mapped-databases", "")
+    mapped_tables          = lookup(var.local_metastores[count.index], "mapped-tables", "")
     database_name_mapping  = lookup(var.local_metastores[count.index], "database-name-mapping", "")
     writable_whitelist     = lookup(var.local_metastores[count.index], "writable-whitelist", "")
     enable_path_conversion = lookup(var.local_metastores[count.index], "enable_path_conversion", false)
@@ -79,6 +80,7 @@ data "template_file" "remote_metastores_yaml" {
     metastore_host         = aws_vpc_endpoint.remote_metastores[count.index].dns_entry[0].dns_name
     metastore_port         = lookup(var.remote_metastores[count.index], "port", "9083")
     mapped_databases       = lookup(var.remote_metastores[count.index], "mapped-databases", "")
+    mapped_tables          = lookup(var.remote_metastores[count.index], "mapped-tables", "")
     database_name_mapping  = lookup(var.remote_metastores[count.index], "database-name-mapping", "")
     writable_whitelist     = lookup(var.remote_metastores[count.index], "writable-whitelist", "")
     enable_path_conversion = lookup(var.remote_metastores[count.index], "enable_path_conversion", false)
@@ -96,6 +98,7 @@ data "template_file" "remote_region_metastores_yaml" {
     metastore_host         = aws_vpc_endpoint.remote_region_metastores[var.remote_region_metastores[count.index]["endpoint"]].dns_entry[0].dns_name
     metastore_port         = lookup(var.remote_region_metastores[count.index], "port", "9083")
     mapped_databases       = lookup(var.remote_region_metastores[count.index], "mapped-databases", "")
+    mapped_tables          = lookup(var.remote_region_metastores[count.index], "mapped-tables", "")
     database_name_mapping  = lookup(var.remote_region_metastores[count.index], "database-name-mapping", "")
     writable_whitelist     = lookup(var.remote_region_metastores[count.index], "writable-whitelist", "")
     enable_path_conversion = lookup(var.remote_region_metastores[count.index], "enable_path_conversion", true)
@@ -117,6 +120,7 @@ data "template_file" "ssh_metastores_yaml" {
     user                  = lookup(var.ssh_metastores[count.index], "user")
     timeout               = lookup(var.ssh_metastores[count.index], "timeout", "60000")
     mapped_databases      = lookup(var.ssh_metastores[count.index], "mapped-databases", "")
+    mapped_tables         = lookup(var.ssh_metastores[count.index], "mapped-tables", "")
     database_name_mapping = lookup(var.ssh_metastores[count.index], "database-name-mapping", "")
     writable_whitelist    = lookup(var.ssh_metastores[count.index], "writable-whitelist", "")
     metastore_enabled     = lookup(var.ssh_metastores[count.index], "enabled", true)

diff --git a/templates/waggle-dance-federation-local.yml.tmpl b/templates/waggle-dance-federation-local.yml.tmpl
@@ -13,4 +13,14 @@ ${ database_name_mapping == "" ? "" : "    database-name-mapping:" }
 ${ database_name_mapping == "" ? "" : join("\n", formatlist("      %s", split(",", replace(replace(database_name_mapping, " ", ""), ":", ": ")))) }
 ${ writable_whitelist == "" ? "" : "    writable-database-white-list:" }
 ${ writable_whitelist == "" ? "" : join("\n",formatlist("    - %s",split(",",writable_whitelist))) }
+%{~if mapped_tables != ""}
+    mapped-tables:
+%{~for table_mapping in split(";", mapped_tables)}
+    - database: ${element(split(":", table_mapping),0)}
+      mapped-tables:
+%{~for table in split(",",element(split(":", table_mapping),1))}
+      - ${table}
+%{~endfor}
+%{~endfor}
+%{~endif}
 %{~endif}
diff --git a/templates/waggle-dance-federation-remote.yml.tmpl b/templates/waggle-dance-federation-remote.yml.tmpl
@@ -13,4 +13,14 @@ ${ database_name_mapping == "" ? "" : "    database-name-mapping:" }
 ${ database_name_mapping == "" ? "" : join("\n", formatlist("      %s", split(",", replace(replace(database_name_mapping, " ", ""), ":", ": ")))) }
 ${ writable_whitelist == "" ? "" : "    writable-database-white-list:" }
 ${ writable_whitelist == "" ? "" : join("\n",formatlist("    - %s",split(",",writable_whitelist))) }
+%{~if mapped_tables != ""}
+    mapped-tables:
+%{~for table_mapping in split(";", mapped_tables)}
+    - database: ${element(split(":", table_mapping),0)}
+      mapped-tables:
+%{~for table in split(",",element(split(":", table_mapping),1))}
+      - ${table}
+%{~endfor}
+%{~endfor}
+%{~endif}
 %{~endif}
diff --git a/templates/waggle-dance-federation-ssh.yml.tmpl b/templates/waggle-dance-federation-ssh.yml.tmpl
@@ -16,4 +16,14 @@ ${ database_name_mapping == "" ? "" : "    database-name-mapping:" }
 ${ database_name_mapping == "" ? "" : join("\n", formatlist("      %s", split(",", replace(replace(database_name_mapping, " ", ""), ":", ": ")))) }
 ${ writable_whitelist == "" ? "" : "    writable-database-white-list:" }
 ${ writable_whitelist == "" ? "" : join("\n",formatlist("    - %s",split(",",writable_whitelist))) }
+%{~if mapped_tables != ""}
+    mapped-tables:
+%{~for table_mapping in split(";", mapped_tables)}
+    - database: ${element(split(":", table_mapping),0)}
+      mapped-tables:
+%{~for table in split(",",element(split(":", table_mapping),1))}
+      - ${table}
+%{~endfor}
+%{~endfor}
+%{~endif}
 %{~endif}