diff --git a/CHANGELOG.md b/CHANGELOG.md index bc4a1ed..fb8ee48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [1.0.0] - TBD +## [1.1.0] - TBD +### Added +- New variable for passing Circus Train common configurations to Shunting Yard. eg. Graphite. +## [1.0.0] - 2019-02-06 ### Added - First version of Shunting Yard Integration with Apiary. diff --git a/README.md b/README.md index 8af556b..ccf577f 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,21 @@ For more information please refer to the main [Apiary](https://github.com/Expedi ## Variables | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| aws_region | AWS region to use for resources. | string | - | yes | -| cpu | The number of CPU units to reserve for the Shunting Yard container. Valid values can be 256, 512, 1024, 2048 and 4096. Reference: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `1024` | no | -| memory | The amount of memory (in MiB) used to allocate for the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `4096` | no | -| docker_image | Full path to Shunting Yard Docker image. | string | - | yes | -| docker_version | Shunting Yard Docker image version. | string | - | yes | -| allowed_s3_buckets | List of S3 Buckets to which Shunting Yard will have read-write access. eg. `["bucket-1", "bucket-2"]` | list | `` | no | -| instance_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `` | no | -| subnets | ECS container subnets. | list | - | yes | +| allowed\_s3\_buckets | List of S3 Buckets to which Shunting Yard will have read-write access. eg. `["bucket-1", "bucket-2"]`. | list | `n/a` | yes | +| aws\_region | AWS region to use for resources. | string | n/a | yes | +| cpu | The number of CPU units to reserve for the Shunting Yard container. Valid values can be 256, 512, 1024, 2048 and 4096. Reference: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"1024"` | no | +| ct\_common\_config\_yaml | Common Circus Train configuration to be passed to internal Circus Train instance. It can be used, for example to configure Graphite for Circus Train. Refer to [Circus Train README](https://github.com/HotelsDotCom/circus-train/blob/master/README.md) for an exhaustive list of options supported by Circus Train. | string | n/a | yes | +| docker\_image | Full path of Shunting Yard Docker image. | string | n/a | yes | +| docker\_version | Shunting Yard Docker image version. | string | n/a | yes | +| instance\_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `""` | no | +| memory | The amount of memory (in MiB) allocated to the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"4096"` | no | +| metastore\_events\_sns\_topic | SNS Topic for Hive Metastore events. | string | n/a | yes | +| selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2` | string | n/a | yes | +| source\_metastore\_uri | Source Metastore URI for Shunting Yard. | string | n/a | yes | +| subnets | ECS container subnets. | list | n/a | yes | | tags | A map of tags to apply to resources. | map | `` | no | -| vpc_id | VPC ID. | string | - | yes | -| source_metastore_uri | Source Metastore URI for Shunting Yard. | string | - | yes | -| target_metastore_uri | Target Metastore URI for Shunting Yard. | string | - | yes | -| metastore_events_sns_topic | SNS Topic for Hive Metastore events. | string | - | yes | -| selected_tables | Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2` | string | - | yes | +| target\_metastore\_uri | Target Metastore URI for Shunting Yard. | string | n/a | yes | +| vpc\_id | VPC ID. | string | n/a | yes | ## Usage diff --git a/main.tf b/main.tf index a779cf3..b9cddec 100644 --- a/main.tf +++ b/main.tf @@ -128,7 +128,7 @@ data "template_file" "shuntingyard_config_yaml" { vars { source_metastore_uri = "${var.source_metastore_uri}" target_metastore_uri = "${var.target_metastore_uri}" - shuntingyard_sqs_queue = "${aws_sqs_queue.shuntingyard_sqs_queue.id}" + shuntingyard_sqs_queue = "${aws_sqs_queue.shuntingyard_sqs_queue.id}" selected_tables = "${var.selected_tables}" } } @@ -143,6 +143,7 @@ data "template_file" "shuntingyard" { region = "${var.aws_region}" loggroup = "${aws_cloudwatch_log_group.shuntingyard_ecs.name}" shuntingyard_config_yaml = "${base64encode(data.template_file.shuntingyard_config_yaml.rendered)}" + ct_common_config_yaml = "${base64encode(var.ct_common_config_yaml)}" } } diff --git a/templates/shuntingyard.json b/templates/shuntingyard.json index 644d852..98e0fa8 100644 --- a/templates/shuntingyard.json +++ b/templates/shuntingyard.json @@ -19,6 +19,10 @@ { "name": "SHUNTINGYARD_CONFIG_YAML", "value": "${shuntingyard_config_yaml}" + }, + { + "name": "CT_COMMON_CONFIG_YAML", + "value": "${ct_common_config_yaml}" } ] } diff --git a/variables.tf b/variables.tf index 6ccc4d8..53032cf 100644 --- a/variables.tf +++ b/variables.tf @@ -68,8 +68,13 @@ variable "docker_version" { type = "string" } +variable "ct_common_config_yaml" { + description = "Common Circus Train configuration to be passed to internal Circus Train instance. It can be used, for example to configure Graphite for Circus Train. Refer to [Circus Train README](https://github.com/HotelsDotCom/circus-train/blob/master/README.md) for an exhaustive list of options supported by Circus Train." + type = "string" +} + variable "allowed_s3_buckets" { - description = "List of S3 Buckets to which SY will have read-write access." + description = "List of S3 Buckets to which Shunting Yard will have read-write access." type = "list" } @@ -89,6 +94,6 @@ variable "metastore_events_sns_topic" { } variable "selected_tables" { - description = "Tables selected for Shunting Yard Replication." + description = "Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2`" type = "string" }