Added config for the dev shared-kafka msk cluster (#157)

* Added config for the dev shared-kafka msk cluster

* Update pubsub examples topic name

dev-aws/kafka-shared-msk/README.md

@@ -0,0 +1 @@
+../../prod-aws/kafka-shared/README.md

dev-aws/kafka-shared-msk/TERRAFORM_LOCALLY.md

@@ -0,0 +1 @@
+../../prod-aws/kafka-shared/TERRAFORM_LOCALLY.md

dev-aws/kafka-shared-msk/__env.tf

@@ -0,0 +1,17 @@
+terraform {
+  backend "s3" {}
+
+  required_providers {
+    kafka = {
+      source = "Mongey/kafka"
+    }
+  }
+}
+
+provider "kafka" {
+  bootstrap_servers = [
+    "b-1.devenablementpubsubmsk.xmhf7r.c8.kafka.eu-west-1.amazonaws.com:9094",
+    "b-2.devenablementpubsubmsk.xmhf7r.c8.kafka.eu-west-1.amazonaws.com:9094",
+    "b-3.devenablementpubsubmsk.xmhf7r.c8.kafka.eu-west-1.amazonaws.com:9094",
+  ]
+}

dev-aws/kafka-shared-msk/pubsub/Makefile

@@ -0,0 +1 @@
+include ../../../lib/kafka-shared/Makefile

dev-aws/kafka-shared-msk/pubsub/__env.tf

@@ -0,0 +1 @@
+../__env.tf

dev-aws/kafka-shared-msk/pubsub/brokers.tf

@@ -0,0 +1,17 @@
+resource "kafka_acl" "brokers_topic" {
+  resource_name       = "*"
+  resource_type       = "Topic"
+  acl_principal       = "User:CN=*.devenablementpubsubmsk.xmhf7r.c8.kafka.eu-west-1.amazonaws.com"
+  acl_host            = "*"
+  acl_operation       = "Describe"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "brokers_group" {
+  resource_name       = "*"
+  resource_type       = "Group"
+  acl_principal       = "User:CN=*.devenablementpubsubmsk.xmhf7r.c8.kafka.eu-west-1.amazonaws.com"
+  acl_host            = "*"
+  acl_operation       = "Describe"
+  acl_permission_type = "Allow"
+}

dev-aws/kafka-shared-msk/pubsub/kafka-exporter.tf

@@ -0,0 +1,23 @@
+# kafka-exporter ACLs
+# Using resourceType = "Any" triggers kafka to complain:
+# `java.lang.IllegalArgumentException: resourceType must not be ANY`
+# Looks like we'd have to create ACLs per resource type
+# We should need the following according to:
+# https://github.com/danielqsj/kafka_exporter/issues/40#issuecomment-832122188
+resource "kafka_acl" "kafka_exporter_topic" {
+  resource_name       = "*"
+  resource_type       = "Topic"
+  acl_principal       = "User:CN=pubsub/msk-exporter"
+  acl_host            = "*"
+  acl_operation       = "Describe"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "kafka_exporter_group" {
+  resource_name       = "*"
+  resource_type       = "Group"
+  acl_principal       = "User:CN=pubsub/msk-exporter"
+  acl_host            = "*"
+  acl_operation       = "Describe"
+  acl_permission_type = "Allow"
+}

dev-aws/kafka-shared-msk/pubsub/kafka-ui.tf

@@ -0,0 +1,29 @@
+resource "kafka_acl" "kafka_ui_topic" {
+  resource_name       = "*"
+  resource_type       = "Topic"
+  acl_principal       = "User:CN=pubsub/kafka-ui"
+  acl_host            = "*"
+  acl_operation       = "All"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "kafka_ui_group" {
+  resource_name       = "*"
+  resource_type       = "Group"
+  acl_principal       = "User:CN=pubsub/kafka-ui"
+  acl_host            = "*"
+  acl_operation       = "All"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "kafka_ui_cluster" {
+  resource_name                = "kafka-cluster"
+  resource_type                = "Cluster"
+  acl_principal                = "User:CN=pubsub/kafka-ui"
+  acl_host                     = "*"
+  acl_operation                = "All"
+  acl_permission_type          = "Allow"
+  resource_pattern_type_filter = "Literal"
+
+  depends_on = [kafka_acl.tf_applier_cluster]
+}

dev-aws/kafka-shared-msk/pubsub/mirror-maker.tf

@@ -0,0 +1,27 @@
+resource "kafka_acl" "mirror_maker_topic_access" {
+  resource_name       = "*"
+  resource_type       = "Topic"
+  acl_principal       = "User:CN=pubsub/mirror-maker"
+  acl_host            = "*"
+  acl_operation       = "All"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "mirror_maker_group_access" {
+  resource_name       = "*"
+  resource_type       = "Group"
+  acl_principal       = "User:CN=pubsub/mirror-maker"
+  acl_host            = "*"
+  acl_operation       = "All"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "mirror_maker_cluster_access" {
+  resource_name                = "kafka-cluster"
+  resource_type                = "Cluster"
+  acl_principal                = "User:CN=pubsub/mirror-maker"
+  acl_host                     = "*"
+  acl_operation                = "All"
+  acl_permission_type          = "Allow"
+  resource_pattern_type_filter = "Literal"
+}

dev-aws/kafka-shared-msk/pubsub/pubsub-examples.tf

@@ -0,0 +1,33 @@
+resource "kafka_topic" "pubsub_examples" {
+  name               = "pubsub.pubsub-examples"
+  replication_factor = 3
+  partitions         = 10
+  config = {
+    # retain 100MB on each partition
+    "retention.bytes" = "104857600"
+    # keep data for 2 days
+    "retention.ms" = "172800000"
+    # allow max 1 MB for a message
+    "max.message.bytes" = "1048576"
+    "compression.type"  = "zstd"
+    "cleanup.policy"    = "delete"
+  }
+}
+
+module "example_producer" {
+  source           = "../../../modules/tls-app"
+  produce_topics   = [kafka_topic.pubsub_examples.name]
+  cert_common_name = "pubsub/example-producer"
+}
+
+module "example_process_individually_consumer" {
+  source           = "../../../modules/tls-app"
+  consume_topics   = { (kafka_topic.pubsub_examples.name) : "pubsub.example-consume-process-individually" }
+  cert_common_name = "pubsub/example-consume-process-individually"
+}
+
+module "example_process_batch_consumer" {
+  source           = "../../../modules/tls-app"
+  consume_topics   = { (kafka_topic.pubsub_examples.name) : "pubsub.example-consume-process-batch" }
+  cert_common_name = "pubsub/example-consume-process-batch"
+}

dev-aws/kafka-shared-msk/pubsub/tf-applier.tf

@@ -0,0 +1,27 @@
+resource "kafka_acl" "tf_applier_topic" {
+  resource_name       = "*"
+  resource_type       = "Topic"
+  acl_principal       = "User:CN=pubsub/tf-applier"
+  acl_host            = "*"
+  acl_operation       = "All"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "tf_applier_group" {
+  resource_name       = "*"
+  resource_type       = "Group"
+  acl_principal       = "User:CN=pubsub/tf-applier"
+  acl_host            = "*"
+  acl_operation       = "All"
+  acl_permission_type = "Allow"
+}
+
+resource "kafka_acl" "tf_applier_cluster" {
+  resource_name                = "kafka-cluster"
+  resource_type                = "Cluster"
+  acl_principal                = "User:CN=pubsub/tf-applier"
+  acl_host                     = "*"
+  acl_operation                = "All"
+  acl_permission_type          = "Allow"
+  resource_pattern_type_filter = "Literal"
+}

exp-1-merit/pubsub-msk/pubsub-examples.tf

@@ -1,5 +1,5 @@
 resource "kafka_topic" "pubsub_examples" {
-  name               = "pubsub.pubsub-examples"
+  name               = "pubsub.examples"
   replication_factor = 3
   partitions         = 10
   config = {
@@ -18,7 +18,7 @@ resource "kafka_topic" "pubsub_examples" {
 }
 
 resource "kafka_acl" "example_producer_topic_access" {
-  resource_name       = "pubsub.pubsub-examples"
+  resource_name       = "pubsub.examples"
   resource_type       = "Topic"
   acl_principal       = "User:CN=pubsub/example-producer"
   acl_host            = "*"
@@ -38,7 +38,7 @@ resource "kafka_quota" "example_producer_quota" {
 }
 
 resource "kafka_acl" "example_consume_process_individually_topic_access" {
-  resource_name       = "pubsub.pubsub-examples"
+  resource_name       = "pubsub.examples"
   resource_type       = "Topic"
   acl_principal       = "User:CN=pubsub/example-consume-process-individually"
   acl_host            = "*"