From 56f1a17954ec71d8744e3a6417a0f8b2e7560d30 Mon Sep 17 00:00:00 2001 From: Georgi Ivanov Date: Wed, 21 Aug 2019 14:56:40 +0100 Subject: [PATCH 1/8] Adding features * Adding support for Docker Registry Auth * Adding variable docker_registry_auth_secret_name * Add access to secretsmanager to ECS Service * Convert selected_tables filter to list * Adding cloudwatch dashboard * Adding cloudwatch alerts --- README.md | 41 +++--- cloudwatch.tf | 169 ++++++++++++++++++++++ common.tf | 5 + ecs.tf | 40 ++++++ iam-policy-s3-buckets.tf | 39 +++++ iam-policy-secretsmanager.tf | 22 +++ iam-policy-sqs.tf | 25 ++++ iam.tf | 50 +++++++ main.tf | 192 ------------------------- sg.tf | 29 ++++ sns.tf | 15 ++ sqs.tf | 34 +++++ templates.tf | 34 +++++ templates/shuntingyard-config.yml.tmpl | 9 +- templates/shuntingyard.json | 1 + variables.tf | 34 +++-- 16 files changed, 513 insertions(+), 226 deletions(-) create mode 100644 cloudwatch.tf create mode 100644 ecs.tf create mode 100644 iam-policy-s3-buckets.tf create mode 100644 iam-policy-secretsmanager.tf create mode 100644 iam-policy-sqs.tf create mode 100644 iam.tf delete mode 100644 main.tf create mode 100644 sg.tf create mode 100644 sns.tf create mode 100644 sqs.tf create mode 100644 templates.tf diff --git a/README.md b/README.md index 301e01c..725d706 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,16 @@ For more information please refer to the main [Apiary](https://github.com/Expedi | ct\_common\_config\_yaml | Common Circus Train configuration to be passed to internal Circus Train instance. It can be used, for example to configure Graphite for Circus Train. Refer to [Circus Train README](https://github.com/HotelsDotCom/circus-train/blob/master/README.md) for an exhaustive list of options supported by Circus Train. | string | n/a | yes | | docker\_image | Full path of Shunting Yard Docker image. | string | n/a | yes | | docker\_version | Shunting Yard Docker image version. | string | n/a | yes | +| docker\_registry\_auth\_secret\_name | Docker Registry authentication SecretManager secret name. | string | `` | no | | instance\_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `""` | no | | memory | The amount of memory (in MiB) allocated to the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"4096"` | no | | metastore\_events\_sns\_topic | SNS Topic for Hive Metastore events. | string | n/a | yes | -| selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `database_1.table_1, database_2.table_2` | string | n/a | yes | +| shuntingyard_sqs_queue_wait_timeout | Shunting Yard SQS queue wait timeout | string | 15 | no | +| shuntingyard_sqs_queue_stale_messages_timeout | Shunting Yard SQS queue stale messages alert timeout | string | 300 | no | +| selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `[ "database_1.table_1", "database_2.table_2" ]` | list | [] | no | | source\_metastore\_uri | Source Metastore URI for Shunting Yard. | string | n/a | yes | | subnets | ECS container subnets. | list | n/a | yes | -| tags | A map of tags to apply to resources. | map | `` | no | +| shuntingyard_tags | A map of tags to apply to resources. | map | `` | no | | target\_metastore\_uri | Target Metastore URI for Shunting Yard. | string | n/a | yes | | vpc\_id | VPC ID. | string | n/a | yes | @@ -29,26 +32,20 @@ For more information please refer to the main [Apiary](https://github.com/Expedi Example module invocation: ``` module "apiary-shuntingyard" { - source = "git::https://github.com/ExpediaInc/apiary-replication.git?ref=master" - instance_name = "shuntingyard-test" - aws_region = "us-west-2" - vpc_id = "vpc-1" - subnets = ["subnet-1", "subnet-2"] - allowed_s3_buckets = ["bucket-1", "bucket-2"] - - tags = { - Name = "Apiary-Shuntingyard" - Team = "Operations" - } - - source_metastore_uri = "thrift://ip-address:9083" - target_metastore_uri = "thrift://ip-address:9083" - metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic" - selected_tables = "database_1.table_1, database_2.table_2" - ct_common_config_yaml = "${data.template_file.ct_common_config_yaml.rendered}" - - docker_image = "your.docker.repo/apiary-shuntingyard" - docker_version = "latest" + source = "git::https://github.com/ExpediaInc/apiary-replication.git" + aws_region = "us-west-2" + instance_name = "shuntingyard-test" + shuntingyard_tags = "${var.tags}" + vpc_id = "vpc-1" + subnets = ["subnet-1", "subnet-2"] + source_metastore_uri = "thrift://ip-address:9083" + target_metastore_uri = "thrift://ip-address:9083" + metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic" + selected_tables = [ "database_1.table_1, database_2.table_2" ] + allowed_s3_buckets = [ "bucket-1", "bucket-2" ] + ct_common_config_yaml = "${data.template_file.ct_common_config_yaml.rendered}" + docker_image = "your.docker.repo/apiary-shuntingyard" + docker_version = "latest" } ``` diff --git a/cloudwatch.tf b/cloudwatch.tf new file mode 100644 index 0000000..e45789d --- /dev/null +++ b/cloudwatch.tf @@ -0,0 +1,169 @@ +/** + * Copyright (C) 2019 Expedia Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + */ + +data "template_file" "ecs_widgets" { + template = < Date: Tue, 27 Aug 2019 17:38:28 +0100 Subject: [PATCH 2/8] updating Changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8699e32..23d97f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [1.2.0] - 2019-08-27 +### Added +- Support for Docker Auth +- Ability to store credentials for docker auth in AWS secrets manager +- Cloudwatch dashboard +- Cloudwatch alerts +- SQS queue permissions + +### Changed +- `selected\_tables` variable is now a list + ## [1.1.0] - 2019-02-08 ### Added - New variable for passing Circus Train common configurations to Shunting Yard. eg. Graphite. From 935b1245c567e257c55c087b2ef80a3cb247d98d Mon Sep 17 00:00:00 2001 From: Georgi Ivanov Date: Tue, 27 Aug 2019 17:53:42 +0100 Subject: [PATCH 3/8] updating README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 725d706..d3ad73c 100644 --- a/README.md +++ b/README.md @@ -34,18 +34,18 @@ Example module invocation: module "apiary-shuntingyard" { source = "git::https://github.com/ExpediaInc/apiary-replication.git" aws_region = "us-west-2" - instance_name = "shuntingyard-test" - shuntingyard_tags = "${var.tags}" vpc_id = "vpc-1" subnets = ["subnet-1", "subnet-2"] + instance_name = "shuntingyard-test" + docker_image = "your.docker.repo/apiary-shuntingyard" + docker_version = "latest" + ct_common_config_yaml = "${data.template_file.ct_common_config_yaml.rendered}" source_metastore_uri = "thrift://ip-address:9083" target_metastore_uri = "thrift://ip-address:9083" metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic" - selected_tables = [ "database_1.table_1, database_2.table_2" ] + selected_tables = [ "database_1.table_1", "database_2.table_2" ] allowed_s3_buckets = [ "bucket-1", "bucket-2" ] - ct_common_config_yaml = "${data.template_file.ct_common_config_yaml.rendered}" - docker_image = "your.docker.repo/apiary-shuntingyard" - docker_version = "latest" + shuntingyard_tags = "${var.tags}" } ``` From cc469ede272d87d8874ef37a99f39d679566d771 Mon Sep 17 00:00:00 2001 From: Georgi Ivanov Date: Tue, 27 Aug 2019 18:00:24 +0100 Subject: [PATCH 4/8] update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d3ad73c..525d4b9 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,10 @@ module "apiary-shuntingyard" { metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic" selected_tables = [ "database_1.table_1", "database_2.table_2" ] allowed_s3_buckets = [ "bucket-1", "bucket-2" ] - shuntingyard_tags = "${var.tags}" + shuntingyard_tags = { + Name = "Apiary Replication" + Team = "Operations" + } } ``` From ae968ec8247baddb450a76bf1c4124b4b6416e61 Mon Sep 17 00:00:00 2001 From: Georgi Ivanov Date: Wed, 28 Aug 2019 17:25:21 +0100 Subject: [PATCH 5/8] updating with PR comments --- CHANGELOG.md | 4 ++-- README.md | 8 ++++---- cloudwatch.tf | 2 +- common.tf | 2 +- ecs.tf | 2 +- iam-policy-s3-buckets.tf | 2 +- iam-policy-secretsmanager.tf | 2 +- iam-policy-sqs.tf | 2 +- iam.tf | 2 +- sg.tf | 4 ++-- sns.tf | 8 +------- sqs.tf | 8 +++++++- templates.tf | 2 +- templates/shuntingyard-config.yml.tmpl | 2 +- variables.tf | 4 ++-- version.tf | 2 +- 16 files changed, 28 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23d97f3..30b5daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [1.2.0] - 2019-08-27 +## [1.2.0] - TBD ### Added - Support for Docker Auth - Ability to store credentials for docker auth in AWS secrets manager @@ -12,7 +12,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - SQS queue permissions ### Changed -- `selected\_tables` variable is now a list +- `selected_tables` variable is now a list ## [1.1.0] - 2019-02-08 ### Added diff --git a/README.md b/README.md index 525d4b9..b5c8923 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Terraform module for setting up infrastructure for [Shunting Yard](https://github.com/HotelsDotCom/shunting-yard). -For more information please refer to the main [Apiary](https://github.com/ExpediaInc/apiary) project page. +For more information please refer to the main [Apiary](https://github.com/ExpediaGroup/apiary) project page. ## Variables | Name | Description | Type | Default | Required | @@ -18,8 +18,8 @@ For more information please refer to the main [Apiary](https://github.com/Expedi | instance\_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `""` | no | | memory | The amount of memory (in MiB) allocated to the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"4096"` | no | | metastore\_events\_sns\_topic | SNS Topic for Hive Metastore events. | string | n/a | yes | -| shuntingyard_sqs_queue_wait_timeout | Shunting Yard SQS queue wait timeout | string | 15 | no | -| shuntingyard_sqs_queue_stale_messages_timeout | Shunting Yard SQS queue stale messages alert timeout | string | 300 | no | +| shuntingyard_sqs_queue_wait_timeout | Shunting Yard SQS queue wait timeout (in seconds) | string | 15 | no | +| shuntingyard_sqs_queue_stale_messages_timeout | Shunting Yard SQS queue stale messages alert timeout (in seconds) | string | 300 | no | | selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `[ "database_1.table_1", "database_2.table_2" ]` | list | [] | no | | source\_metastore\_uri | Source Metastore URI for Shunting Yard. | string | n/a | yes | | subnets | ECS container subnets. | list | n/a | yes | @@ -32,7 +32,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi Example module invocation: ``` module "apiary-shuntingyard" { - source = "git::https://github.com/ExpediaInc/apiary-replication.git" + source = "git::https://github.com/ExpediaGroup/apiary-replication.git" aws_region = "us-west-2" vpc_id = "vpc-1" subnets = ["subnet-1", "subnet-2"] diff --git a/cloudwatch.tf b/cloudwatch.tf index e45789d..ccd397a 100644 --- a/cloudwatch.tf +++ b/cloudwatch.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/common.tf b/common.tf index e558ee0..3b9ecca 100644 --- a/common.tf +++ b/common.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/ecs.tf b/ecs.tf index d012020..7a7cc42 100644 --- a/ecs.tf +++ b/ecs.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/iam-policy-s3-buckets.tf b/iam-policy-s3-buckets.tf index 33c2dd9..33b043b 100644 --- a/iam-policy-s3-buckets.tf +++ b/iam-policy-s3-buckets.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/iam-policy-secretsmanager.tf b/iam-policy-secretsmanager.tf index 8c8eb9d..8524a03 100644 --- a/iam-policy-secretsmanager.tf +++ b/iam-policy-secretsmanager.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/iam-policy-sqs.tf b/iam-policy-sqs.tf index e06d853..7c2c04a 100644 --- a/iam-policy-sqs.tf +++ b/iam-policy-sqs.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/iam.tf b/iam.tf index a1ebe19..6e55b6c 100644 --- a/iam.tf +++ b/iam.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/sg.tf b/sg.tf index 0c0935c..c0b2e15 100644 --- a/sg.tf +++ b/sg.tf @@ -1,11 +1,11 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ resource "aws_security_group" "shuntingyard_sg" { - name = "${local.instance_alias}-sg2" + name = "${local.instance_alias}-sg" vpc_id = "${var.vpc_id}" tags = "${var.shuntingyard_tags}" diff --git a/sns.tf b/sns.tf index b49a0f0..723dc5d 100644 --- a/sns.tf +++ b/sns.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ @@ -7,9 +7,3 @@ resource "aws_sns_topic" "shuntingyard_ops_sns" { name = "${local.instance_alias}-operational-events" } - -resource "aws_sns_topic_subscription" "sqs_hive_metastore_sns_subscription" { - topic_arn = "${var.metastore_events_sns_topic}" - protocol = "sqs" - endpoint = "${aws_sqs_queue.shuntingyard_sqs_queue.arn}" -} diff --git a/sqs.tf b/sqs.tf index 1ecbb8d..de21c5b 100644 --- a/sqs.tf +++ b/sqs.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ @@ -32,3 +32,9 @@ resource "aws_sqs_queue_policy" "shuntingyard_sqs_queue_policy" { } POLICY } + +resource "aws_sns_topic_subscription" "sqs_hive_metastore_sns_subscription" { + topic_arn = "${var.metastore_events_sns_topic}" + protocol = "sqs" + endpoint = "${aws_sqs_queue.shuntingyard_sqs_queue.arn}" +} diff --git a/templates.tf b/templates.tf index 74b1377..c794299 100644 --- a/templates.tf +++ b/templates.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ diff --git a/templates/shuntingyard-config.yml.tmpl b/templates/shuntingyard-config.yml.tmpl index a2451f9..e71305d 100644 --- a/templates/shuntingyard-config.yml.tmpl +++ b/templates/shuntingyard-config.yml.tmpl @@ -1,5 +1,5 @@ --- -# Copyright (C) 2019 Expedia Inc. +# Copyright (C) 2019 Expedia, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); source-catalog: diff --git a/variables.tf b/variables.tf index 69c0e87..5e38859 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ @@ -88,7 +88,7 @@ variable "metastore_events_sns_topic" { } variable "shuntingyard_sqs_queue_wait_timeout" { - description = "Wait timeout for connecting to the Shunting Yard SQS queue" + description = "Wait timeout for connecting to the Shunting Yard SQS queue (in seconds)" type = "string" default = 15 } diff --git a/version.tf b/version.tf index d570be0..e871fa7 100644 --- a/version.tf +++ b/version.tf @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019 Expedia Inc. + * Copyright (C) 2019 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); */ From 0747d5aa40b95b07fde6446cabdcd50cb21d0c89 Mon Sep 17 00:00:00 2001 From: Adrian Woodhead Date: Thu, 29 Aug 2019 11:55:35 +0100 Subject: [PATCH 6/8] added full stops --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30b5daa..0a26543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [1.2.0] - TBD ### Added -- Support for Docker Auth -- Ability to store credentials for docker auth in AWS secrets manager -- Cloudwatch dashboard -- Cloudwatch alerts -- SQS queue permissions +- Support for Docker Auth. +- Ability to store credentials for docker auth in AWS secrets manager. +- Cloudwatch dashboard. +- Cloudwatch alerts. +- SQS queue permissions. ### Changed - `selected_tables` variable is now a list From f698eb29f0324fc45aa3f9894f8cad737abfe47b Mon Sep 17 00:00:00 2001 From: Abhimanyu Gupta Date: Thu, 29 Aug 2019 13:48:16 +0100 Subject: [PATCH 7/8] Adding escape \ to new parameters in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b5c8923..94865c4 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,12 @@ For more information please refer to the main [Apiary](https://github.com/Expedi | instance\_name | Shunting Yard instance name to identify resources in multi-instance deployments. | string | `""` | no | | memory | The amount of memory (in MiB) allocated to the Shunting Yard container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `"4096"` | no | | metastore\_events\_sns\_topic | SNS Topic for Hive Metastore events. | string | n/a | yes | -| shuntingyard_sqs_queue_wait_timeout | Shunting Yard SQS queue wait timeout (in seconds) | string | 15 | no | -| shuntingyard_sqs_queue_stale_messages_timeout | Shunting Yard SQS queue stale messages alert timeout (in seconds) | string | 300 | no | +| shuntingyard\_sqs\_queue\_wait\_timeout | Shunting Yard SQS queue wait timeout (in seconds) | string | 15 | no | +| shuntingyard\_sqs\_queue\_stale\_messages\_timeout | Shunting Yard SQS queue stale messages alert timeout (in seconds) | string | 300 | no | | selected\_tables | Tables selected for Shunting Yard Replication. Supported Format: `[ "database_1.table_1", "database_2.table_2" ]` | list | [] | no | | source\_metastore\_uri | Source Metastore URI for Shunting Yard. | string | n/a | yes | | subnets | ECS container subnets. | list | n/a | yes | -| shuntingyard_tags | A map of tags to apply to resources. | map | `` | no | +| shuntingyard\_tags | A map of tags to apply to resources. | map | `` | no | | target\_metastore\_uri | Target Metastore URI for Shunting Yard. | string | n/a | yes | | vpc\_id | VPC ID. | string | n/a | yes | From 868cd670db08c7373f226f1ff7fc87f022dd96a5 Mon Sep 17 00:00:00 2001 From: Adrian Woodhead Date: Thu, 29 Aug 2019 15:07:33 +0100 Subject: [PATCH 8/8] small fixes --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a26543..e236c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [1.2.0] - TBD ### Added - Support for Docker Auth. -- Ability to store credentials for docker auth in AWS secrets manager. +- Ability to store credentials for Docker auth in AWS secrets manager. - Cloudwatch dashboard. - Cloudwatch alerts. - SQS queue permissions. ### Changed -- `selected_tables` variable is now a list +- `selected_tables` variable is now a list. ## [1.1.0] - 2019-02-08 ### Added