Msk update (#270)

terraform-modules/aws/msk/main.tf

@@ -20,21 +20,28 @@ resource "aws_s3_bucket" "this" {
   tags   = var.tags
 }
 
-resource "aws_s3_bucket_acl" "this" {
-  bucket = aws_s3_bucket.this.id
-  acl    = "private"
-}
-
-resource "aws_s3_bucket_server_side_encryption_configuration" "this" {
-  bucket = aws_s3_bucket.this.bucket
-
-  rule {
-    apply_server_side_encryption_by_default {
-      kms_master_key_id = aws_kms_key.this.arn
-      sse_algorithm     = "aws:kms"
-    }
-  }
-}
+# resource "aws_s3_bucket_acl" "this" {
+#   bucket = aws_s3_bucket.this.id
+#   acl    = "private"
+# }
+
+# When turning on server side encryption the ACM creation failes with:
+# │ Error: error creating ACM PCA Certificate Authority: ValidationException: Permission error with your S3 bucket '476264532441-us-west-2-msk-logs'. Check that your bucket policy, encryption settings, S3 Block Public Access settings, and global account permissions are configured correctly. For more information, check the service documentation.
+# │       status code: 400, request id: 3ba26851-f96a-48b6-a9a2-ca7a68be8e5f
+# │ 
+# │   with aws_acmpca_certificate_authority.this,
+# │   on main.tf line 91, in resource "aws_acmpca_certificate_authority" "this":
+# │   91: resource "aws_acmpca_certificate_authority" "this" {
+# resource "aws_s3_bucket_server_side_encryption_configuration" "this" {
+#   bucket = aws_s3_bucket.this.bucket
+
+#   rule {
+#     apply_server_side_encryption_by_default {
+#       kms_master_key_id = aws_kms_key.this.arn
+#       sse_algorithm     = "aws:kms"
+#     }
+#   }
+# }
 
 data "aws_iam_policy_document" "acmpca_bucket_access" {
   statement {
@@ -119,7 +126,7 @@ resource "aws_acmpca_certificate_authority" "this" {
 #######################################
 module "msk" {
   source                         = "cloudposse/msk-apache-kafka-cluster/aws"
-  version                        = "v0.8.3"
+  version                        = "v0.8.4"
   namespace                      = var.namespace
   name                           = var.name
   vpc_id                         = var.vpc_id
@@ -134,12 +141,14 @@ module "msk" {
   tags                           = var.tags
   certificate_authority_arns     = [aws_acmpca_certificate_authority.this.arn]
   client_tls_auth_enabled        = var.client_tls_auth_enabled
+  client_sasl_iam_enabled        = var.client_sasl_iam_enabled
   encryption_in_cluster          = var.encryption_in_cluster
-  encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key_arn
+  encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key_arn != null ? var.encryption_at_rest_kms_key_arn : aws_kms_key.this.arn
   cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
   cloudwatch_logs_log_group      = var.cloudwatch_logs_enabled == true ? var.cloudwatch_logs_log_group : ""
   enhanced_monitoring            = var.enhanced_monitoring
   node_exporter_enabled          = var.node_exporter_enabled
+  jmx_exporter_enabled           = var.jmx_exporter_enabled
   s3_logs_bucket                 = var.s3_logs_enabled == true ? aws_s3_bucket.this.id : ""
   s3_logs_enabled                = var.s3_logs_enabled
   s3_logs_prefix                 = var.s3_logs_enabled == true ? var.s3_logs_prefix : ""

terraform-modules/aws/msk/variables.tf

@@ -34,8 +34,10 @@ variable "subnet_ids" {
   description = "Subnet IDs for Client Broker"
 }
 
+# Supported versions: https://docs.aws.amazon.com/msk/latest/developerguide/supported-kafka-versions.html
 variable "kafka_version" {
   type        = string
+  default     = "2.8.1"
   description = "The desired Kafka software version"
 }
 
@@ -44,8 +46,10 @@ variable "number_of_broker_nodes" {
   description = "The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets."
 }
 
+# https://docs.aws.amazon.com/msk/latest/developerguide/msk-create-cluster.html#broker-instance-types
 variable "broker_instance_type" {
   type        = string
+  default     = "kafka.t3.small"
   description = "The instance type to use for the Kafka brokers"
 }
 
@@ -66,7 +70,8 @@ variable "encryption_in_cluster" {
 
 variable "encryption_at_rest_kms_key_arn" {
   type        = string
-  description = "You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest"
+  default     = null
+  description = "You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest.  If null the key created in this module will be used."
 }
 
 variable "cloudwatch_logs_enabled" {
@@ -99,11 +104,6 @@ variable "s3_logs_prefix" {
   description = "Prefix to append to the S3 folder name logs are delivered to"
 }
 
-variable "node_exporter_enabled" {
-  type        = bool
-  description = "Set true to enable the Node Exporter"
-}
-
 variable "security_groups" {
   type        = list(string)
   description = "The security_group_id_list output from the security_groups module"
@@ -114,6 +114,12 @@ variable "client_tls_auth_enabled" {
   description = "Set true to enable the Client TLS Authentication"
 }
 
+variable "client_sasl_iam_enabled" {
+  type        = bool
+  default     = false
+  description = "Enables client authentication via IAM policies (cannot be set to true at the same time as client_sasl_*_enabled)."
+}
+
 variable "common_name" {
  type        = string
  description = "The common name for the CA"
@@ -138,3 +144,14 @@ variable "signing_algorithm" {
  default     = "SHA512WITHRSA"
 }
 
+variable "node_exporter_enabled" {
+  type        = bool
+  default     = false
+  description = "Set true to enable the Prometheus Node Exporter"
+}
+
+variable "jmx_exporter_enabled" {
+  type        = bool
+  default     = false
+  description = "Set true to enable the Prometheus JMX Exporter"
+}