Merge pull request #4 from givanovexpe/fixes

Minor fixes

CHANGELOG.md

@@ -3,6 +3,19 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [2.0.0] - TBD
+### Added
+- SQS permissions policy.
+- tags to the lambda.
+
+### Changed
+- converting the filter_policy to a template.
+- updating filter vars from string to list.
+- updating the variable `pg_jars_s3_key` to `pg_lambda_s3_key`.
+
+### Removed
+- removed the `pg_lambda_version` module variable (but left an example of using it client-side in the README.md)
+
 ## [1.0.0] - 2019-06-27
 ### Added
 - Terraform scripts for Privilege Grantor Apiary Extension.

privileges-grantor/README.md

@@ -11,12 +11,11 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
 | instance\_name | Privileges Grantor instance name to identify resources in multi-instance deployments. | string | `""` | no |
 | lambda\_timeout | The time after which the lambda execution stops. | string | `"200"` | no |
 | memory | The amount of memory (in MiB) to be used by Lambda | string | `"512"` | no |
-| metastore\_events\_filter | List of metastore event types to be added to SNS filter. Supported format: `<<EOD "CREATE_TABLE","ALTER_TABLE" EOD` | string | `"\"CREATE_TABLE\",\"ALTER_TABLE\""` | no |
-| database\_filter | List of database names to be added to SNS filter. Supported format: `<<EOD \"DB_NAME_1\",\"DB_NAME_2\" EOD` | string | n/a | yes |
+| metastore\_events\_filter | List of metastore event types to be added to SNS filter. Supported format: `[ "CREATE_TABLE","ALTER_TABLE" ]` | list | `[ "CREATE_TABLE", "ALTER_TABLE" ]` | no |
+| database\_filter | List of database names to be added to SNS filter. Supported format: `[ "DB_NAME_1", "DB_NAME_2" ]` | list | n/a | yes |
 | metastore\_events\_sns\_topic | SNS Topic for Hive Metastore events. | string | n/a | yes |
-| pg\_jars\_s3\_key | S3 key where zip file is located. | string | n/a | yes |
-| pg\_lambda\_bucket | Bucket where the Lambda zip can be found, for example 'bucket_name'. Used together with `pg_jars_s3_key`. | string | n/a | yes |
-| pg\_lambda\_version | Version of the Privileges Grantor Lambda. | string | n/a | yes |
+| pg\_lambda\_s3\_key | S3 key where privilege grantor lambda jar/zip file is located. | string | n/a | yes |
+| pg\_lambda\_bucket | Bucket where the privilege grantor lambda jar/zip can be found, for example 'bucket\_name'. Used together with `pg_lambda_s3_key` to construct the full S3 path. | string | n/a | yes |
 | pg\_metastore\_uri | Thrift URI of the metastore to which Lambda will connect to. | string | n/a | yes |
 | security\_groups | Security groups in which Lambda will have access to. | list | n/a | yes |
 | subnets | Subnets in which Lambda will have access to. | list | n/a | yes |
@@ -29,11 +28,10 @@ Example module invocation:
 module "apiary-privileges-grantor" {
   source = "[email protected]:ExpediaGroup/apiary-extensions-terraform.git/privileges-grantor"
   pg_lambda_bucket           = "pg-s3-bucket"
-  pg_jars_s3_key             = "pg-s3-key"
-  pg_lambda_version          = "4.1.0"
+  pg_lambda_s3_key           = "pg-s3-key"
   pg_metastore_uri           = "thrift://ip-address:9083"
   metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic"
-  database_filter            = "\"db_1\",\"db_2\""
+  database_filter            = [ "db_1", "db_2" ]
   subnets                    = ["subnet-1", "subnet-2"]
   security_groups            = ["security-group-1", "security-group-2"]
   tags = {
@@ -44,6 +42,54 @@ module "apiary-privileges-grantor" {
 
 ```
 
+The apiary-privileges-grantor lambda can be found in the public [maven repository](https://mvnrepository.com/artifact/com.expediagroup.apiary/apiary-privileges-grantor-lambda).
+The jars can be downloaded from the link provided above and uploaded to S3 via terraform as follows:
+
+```
+variable "pg_lambda_version" {
+  description = "Version of the Privilege Grantor Lambda."
+  type        = "string"
+  default     = "4.2.0"
+}
+
+data "aws_s3_bucket" "apiary-extensions" {
+  bucket = "pg-s3-bucket"
+}
+
+resource "null_resource" "apiary-privileges-grantor-jar" {
+  depends_on = ["data.aws_s3_bucket.apiary_extensions"]
+
+  provisioner "local-exec" {
+    command = <<CMD
+        curl -sLo apiary-privileges-grantor-core-${var.pg_lambda_version}.jar https://repo1.maven.org/maven2/com/expediagroup/apiary/apiary-privileges-grantor-core/${var.pg_lambda_version}/apiary-privileges-grantor-core-${var.pg_lambda_version}.jar
+CMD
+  }
+}
+
+resource "aws_s3_bucket_object" "apiary-privileges-grantor-jar" {
+  depends_on = ["null_resource.apiary-privileges-grantor-jar"]
+
+  bucket = "${data.aws_s3_bucket.apiary_extensions.id}"
+  key    = "apiary-privileges-grantor-lambda-${var.pg_lambda_version}.jar"
+  source = "apiary-privileges-grantor-core-${var.pg_lambda_version}.jar"
+}
+
+module "apiary-privileges-grantor" {
+  source = "[email protected]:ExpediaGroup/apiary-extensions-terraform.git/privileges-grantor"
+  pg_lambda_bucket           = "${data.aws_s3_bucket.apiary-extensions.id}"
+  pg_lambda_s3_key           = "${aws_s3_bucket_object.apiary-privileges-grantor-jar.id}"
+  pg_metastore_uri           = "thrift://ip-address:9083"
+  metastore_events_sns_topic = "arn:aws:sns:us-west-2:1234567:metastore-events-sns-topic"
+  database_filter            = [ "db_1", "db_2" ]
+  subnets                    = ["subnet-1", "subnet-2"]
+  security_groups            = ["security-group-1", "security-group-2"]
+  tags = {
+    Name = "Apiary-Privileges-Grantor"
+    Team = "Operations"
+  }
+}
+``` 
+
 # Contact
 
 ## Mailing List

privileges-grantor/main.tf

@@ -53,6 +53,31 @@ resource "aws_sqs_queue" "privilege_grantor_sqs_queue" {
   visibility_timeout_seconds = "${var.lambda_timeout}"
 }
 
+resource "aws_sqs_queue_policy" "privilege_grantor_sqs_queue_policy" {
+  queue_url = "${aws_sqs_queue.privilege_grantor_sqs_queue.id}"
+
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Id": "AllowSNSSendMessage",
+  "Statement": [
+    {
+      "Sid": "Allow Apiary Metadata Events",
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:SendMessage",
+      "Resource": "${aws_sqs_queue.privilege_grantor_sqs_queue.arn}",
+      "Condition": {
+        "ArnEquals": {
+          "aws:SourceArn": "${var.metastore_events_sns_topic}"
+        }
+      }
+    }
+  ]
+}
+POLICY
+}
+
 resource "aws_iam_role_policy" "sqs_for_privilege_grantor" {
   name = "${local.instance_alias}-sqs-policy"
   role = "${aws_iam_role.iam_for_privilege_grantor.id}"
@@ -73,22 +98,29 @@ resource "aws_iam_role_policy" "sqs_for_privilege_grantor" {
 EOF
 }
 
+data "template_file" "filter_policy" {
+  template = <<JSON
+$${val}
+JSON
+    vars {
+        val = "${jsonencode(map(
+                    "eventType", "${compact(split(",",upper(join(",",var.metastore_events_filter))))}",
+                    "dbName", "${compact(split(",",lower(join(",",var.database_filter))))}"
+                ))}"
+      }
+}
+
 resource "aws_sns_topic_subscription" "sqs_hive_metastore_sns_subscription" {
   topic_arn = "${var.metastore_events_sns_topic}"
   protocol  = "sqs"
   endpoint  = "${aws_sqs_queue.privilege_grantor_sqs_queue.arn}"
 
-  filter_policy = <<EOF
-{
-   "eventType": [${upper(var.metastore_events_filter)}],
-   "dbName": [${lower(var.database_filter)}]
-}
-EOF
+  filter_policy = "${data.template_file.filter_policy.rendered}"
 }
 
 resource "aws_lambda_function" "privilege_grantor_fn" {
   s3_bucket     = "${var.pg_lambda_bucket}"
-  s3_key        = "${var.pg_jars_s3_key}/apiary-privileges-grantor-lambda-${var.pg_lambda_version}.zip"
+  s3_key        = "${var.pg_lambda_s3_key}"
   function_name = "${local.instance_alias}-fn"
   role          = "${aws_iam_role.iam_for_privilege_grantor.arn}"
   handler       = "com.expediagroup.apiary.extensions.events.metastore.consumer.privilegesgrantor.lambda.PrivilegesGrantorLambda::handleRequest"
@@ -107,6 +139,8 @@ resource "aws_lambda_function" "privilege_grantor_fn" {
     subnet_ids         = ["${var.subnets}"]
     security_group_ids = ["${var.security_groups}"]
   }
+
+  tags = "${var.tags}"
 }
 
 resource "aws_lambda_event_source_mapping" "sqs_lambda_mapping" {

privileges-grantor/variables.tf

@@ -21,20 +21,15 @@ variable "security_groups" {
 }
 
 variable "pg_lambda_bucket" {
-  description = "Bucket where the Lambda zip can be found, for example 'bucket_name'. Used together with `pg_jars_s3_key`."
+  description = "Bucket where the Lambda zip can be found, for example 'bucket_name'. Used together with `pg_lambda_s3_key`."
   type        = "string"
 }
 
-variable "pg_jars_s3_key" {
+variable "pg_lambda_s3_key" {
   description = "S3 key where zip file is located."
   type        = "string"
 }
 
-variable "pg_lambda_version" {
-  description = "Version of the Privilege Grantor Lambda."
-  type        = "string"
-}
-
 variable "pg_metastore_uri" {
   description = "Thrift URI of the metastore to which Lambda will connect to."
   type        = "string"
@@ -47,13 +42,13 @@ variable "metastore_events_sns_topic" {
 
 variable "metastore_events_filter" {
   description = "List of metastore event types to be added to SNS filter. Supported format: `<<EOD \"CREATE_TABLE\",\"ALTER_TABLE\" EOD`"
-  type        = "string"
-  default     = "\"CREATE_TABLE\",\"ALTER_TABLE\""
+  type        = "list"
+  default     = [ "CREATE_TABLE","ALTER_TABLE" ]
 }
 
 variable "database_filter" {
   description = "List of database names to be added to SNS filter. Supported format: `<<EOD \"DB_NAME_1\",\"DB_NAME_2\" EOD`"
-  type        = "string"
+  type        = "list"
 }
 
 # Tags