From 9b515ad91bf2ac1e7228f2feba64cf3193799f30 Mon Sep 17 00:00:00 2001 From: Garland Kan Date: Thu, 10 Mar 2022 17:55:30 -0800 Subject: [PATCH] Msk update (#270) --- terraform-modules/aws/msk/main.tf | 43 ++++++++++++++++---------- terraform-modules/aws/msk/variables.tf | 29 +++++++++++++---- 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/terraform-modules/aws/msk/main.tf b/terraform-modules/aws/msk/main.tf index 195f23eb5..79a205d98 100644 --- a/terraform-modules/aws/msk/main.tf +++ b/terraform-modules/aws/msk/main.tf @@ -20,21 +20,28 @@ resource "aws_s3_bucket" "this" { tags = var.tags } -resource "aws_s3_bucket_acl" "this" { - bucket = aws_s3_bucket.this.id - acl = "private" -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "this" { - bucket = aws_s3_bucket.this.bucket - - rule { - apply_server_side_encryption_by_default { - kms_master_key_id = aws_kms_key.this.arn - sse_algorithm = "aws:kms" - } - } -} +# resource "aws_s3_bucket_acl" "this" { +# bucket = aws_s3_bucket.this.id +# acl = "private" +# } + +# When turning on server side encryption the ACM creation failes with: +# │ Error: error creating ACM PCA Certificate Authority: ValidationException: Permission error with your S3 bucket '476264532441-us-west-2-msk-logs'. Check that your bucket policy, encryption settings, S3 Block Public Access settings, and global account permissions are configured correctly. For more information, check the service documentation. +# │ status code: 400, request id: 3ba26851-f96a-48b6-a9a2-ca7a68be8e5f +# │ +# │ with aws_acmpca_certificate_authority.this, +# │ on main.tf line 91, in resource "aws_acmpca_certificate_authority" "this": +# │ 91: resource "aws_acmpca_certificate_authority" "this" { +# resource "aws_s3_bucket_server_side_encryption_configuration" "this" { +# bucket = aws_s3_bucket.this.bucket + +# rule { +# apply_server_side_encryption_by_default { +# kms_master_key_id = aws_kms_key.this.arn +# sse_algorithm = "aws:kms" +# } +# } +# } data "aws_iam_policy_document" "acmpca_bucket_access" { statement { @@ -119,7 +126,7 @@ resource "aws_acmpca_certificate_authority" "this" { ####################################### module "msk" { source = "cloudposse/msk-apache-kafka-cluster/aws" - version = "v0.8.3" + version = "v0.8.4" namespace = var.namespace name = var.name vpc_id = var.vpc_id @@ -134,12 +141,14 @@ module "msk" { tags = var.tags certificate_authority_arns = [aws_acmpca_certificate_authority.this.arn] client_tls_auth_enabled = var.client_tls_auth_enabled + client_sasl_iam_enabled = var.client_sasl_iam_enabled encryption_in_cluster = var.encryption_in_cluster - encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key_arn + encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key_arn != null ? var.encryption_at_rest_kms_key_arn : aws_kms_key.this.arn cloudwatch_logs_enabled = var.cloudwatch_logs_enabled cloudwatch_logs_log_group = var.cloudwatch_logs_enabled == true ? var.cloudwatch_logs_log_group : "" enhanced_monitoring = var.enhanced_monitoring node_exporter_enabled = var.node_exporter_enabled + jmx_exporter_enabled = var.jmx_exporter_enabled s3_logs_bucket = var.s3_logs_enabled == true ? aws_s3_bucket.this.id : "" s3_logs_enabled = var.s3_logs_enabled s3_logs_prefix = var.s3_logs_enabled == true ? var.s3_logs_prefix : "" diff --git a/terraform-modules/aws/msk/variables.tf b/terraform-modules/aws/msk/variables.tf index d2a4d1cda..cc2075141 100644 --- a/terraform-modules/aws/msk/variables.tf +++ b/terraform-modules/aws/msk/variables.tf @@ -34,8 +34,10 @@ variable "subnet_ids" { description = "Subnet IDs for Client Broker" } +# Supported versions: https://docs.aws.amazon.com/msk/latest/developerguide/supported-kafka-versions.html variable "kafka_version" { type = string + default = "2.8.1" description = "The desired Kafka software version" } @@ -44,8 +46,10 @@ variable "number_of_broker_nodes" { description = "The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets." } +# https://docs.aws.amazon.com/msk/latest/developerguide/msk-create-cluster.html#broker-instance-types variable "broker_instance_type" { type = string + default = "kafka.t3.small" description = "The instance type to use for the Kafka brokers" } @@ -66,7 +70,8 @@ variable "encryption_in_cluster" { variable "encryption_at_rest_kms_key_arn" { type = string - description = "You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest" + default = null + description = "You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If null the key created in this module will be used." } variable "cloudwatch_logs_enabled" { @@ -99,11 +104,6 @@ variable "s3_logs_prefix" { description = "Prefix to append to the S3 folder name logs are delivered to" } -variable "node_exporter_enabled" { - type = bool - description = "Set true to enable the Node Exporter" -} - variable "security_groups" { type = list(string) description = "The security_group_id_list output from the security_groups module" @@ -114,6 +114,12 @@ variable "client_tls_auth_enabled" { description = "Set true to enable the Client TLS Authentication" } +variable "client_sasl_iam_enabled" { + type = bool + default = false + description = "Enables client authentication via IAM policies (cannot be set to true at the same time as client_sasl_*_enabled)." +} + variable "common_name" { type = string description = "The common name for the CA" @@ -138,3 +144,14 @@ variable "signing_algorithm" { default = "SHA512WITHRSA" } +variable "node_exporter_enabled" { + type = bool + default = false + description = "Set true to enable the Prometheus Node Exporter" +} + +variable "jmx_exporter_enabled" { + type = bool + default = false + description = "Set true to enable the Prometheus JMX Exporter" +}