From d45498d9a6606e8863a22082f8d7f4e9da0a01ad Mon Sep 17 00:00:00 2001 From: Victoria Eyres Date: Tue, 10 Sep 2024 11:05:40 +0100 Subject: [PATCH 1/5] chore(FIN-1204): create topic for transactions-auditor-diff and module --- .../transactions-auditor-diff-events.tf | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf diff --git a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf new file mode 100644 index 00000000..d07d2824 --- /dev/null +++ b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf @@ -0,0 +1,28 @@ +resource "kafka_topic" "transactions_auditor_diff_events" { + name = "billing.transactions-auditor-diff.events" + replication_factor = 3 + partitions = 10 + config = { + "compression.type" = "zstd" + # 805 GB + "retention.bytes" = "805306368000" + # 105 MB + "max.message.bytes" = "104857600" + # Use tiered storage + "remote.storage.enable" = "true" + # keep data in hot storage for 2 days + "local.retention.ms" = "172800000" + "retention.ms" = "-1" + "cleanup.policy" = "delete" + } +} + +module "transactions-auditor-api" { + source = "../../../modules/tls-app" + // consume_topics = [ index? ] + produce_topics = [ + kafka_topic.transactions-auditor-diff.events.name, + ] + // consume_groups = ["billing.finance-bigquery-connector"] + cert_common_name = "billing/finance-bigquery-connector" +} From 8015b1a69df3defb4ff5ea229c6e7d6c5f4870a6 Mon Sep 17 00:00:00 2001 From: Victoria Eyres Date: Tue, 10 Sep 2024 11:11:19 +0100 Subject: [PATCH 2/5] chore(FIN-1204): update cert common name in module --- .../billing/transactions-auditor-diff-events.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf index d07d2824..2a59789b 100644 --- a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf +++ b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf @@ -23,6 +23,6 @@ module "transactions-auditor-api" { produce_topics = [ kafka_topic.transactions-auditor-diff.events.name, ] - // consume_groups = ["billing.finance-bigquery-connector"] - cert_common_name = "billing/finance-bigquery-connector" + // consume_groups = ["?"] + cert_common_name = "billing/transactions-auditor-api" } From ad4fa66204855b6e92907f2e026f33300a728990 Mon Sep 17 00:00:00 2001 From: Victoria Eyres Date: Tue, 10 Sep 2024 14:26:45 +0100 Subject: [PATCH 3/5] chore(FIN-1204): draft consumer groups --- .../billing/transactions-auditor-diff-events.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf index 2a59789b..589120b1 100644 --- a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf +++ b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf @@ -17,12 +17,13 @@ resource "kafka_topic" "transactions_auditor_diff_events" { } } + module "transactions-auditor-api" { source = "../../../modules/tls-app" - // consume_topics = [ index? ] + // consume_topics = [transactions-auditor-events-indexer.name,] produce_topics = [ kafka_topic.transactions-auditor-diff.events.name, ] - // consume_groups = ["?"] + // consume_groups = ["billing.transactions-auditor-events-indexer"] cert_common_name = "billing/transactions-auditor-api" } From 55a31c6a3e4354e326eef0a8e6c709caeede6618 Mon Sep 17 00:00:00 2001 From: Victoria Eyres Date: Tue, 10 Sep 2024 14:44:08 +0100 Subject: [PATCH 4/5] chore(FIN-1204): error correcting --- .../billing/transactions-auditor-diff-events.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf index 589120b1..f1f577cd 100644 --- a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf +++ b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf @@ -1,4 +1,4 @@ -resource "kafka_topic" "transactions_auditor_diff_events" { +resource "kafka_topic" "transactions_auditor_diff.events" { name = "billing.transactions-auditor-diff.events" replication_factor = 3 partitions = 10 @@ -18,12 +18,12 @@ resource "kafka_topic" "transactions_auditor_diff_events" { } -module "transactions-auditor-api" { +module "transactions_auditor_api" { source = "../../../modules/tls-app" - // consume_topics = [transactions-auditor-events-indexer.name,] + # consume_topics = [transactions-auditor-events-indexer.name,] produce_topics = [ - kafka_topic.transactions-auditor-diff.events.name, + kafka_topic.transactions_auditor_diff.events.name ] - // consume_groups = ["billing.transactions-auditor-events-indexer"] + # consume_groups = ["billing.transactions-auditor-events-indexer"] cert_common_name = "billing/transactions-auditor-api" } From 7161280864043d333b3b683b2b266dbbbc2a9f22 Mon Sep 17 00:00:00 2001 From: Victoria Eyres Date: Tue, 10 Sep 2024 14:50:16 +0100 Subject: [PATCH 5/5] linting issue --- .../billing/transactions-auditor-diff-events.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf index f1f577cd..1a713dec 100644 --- a/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf +++ b/dev-aws/kafka-shared-msk/billing/transactions-auditor-diff-events.tf @@ -1,4 +1,4 @@ -resource "kafka_topic" "transactions_auditor_diff.events" { +resource "kafka_topic" "transactions_auditor_diff_events" { name = "billing.transactions-auditor-diff.events" replication_factor = 3 partitions = 10 @@ -22,7 +22,7 @@ module "transactions_auditor_api" { source = "../../../modules/tls-app" # consume_topics = [transactions-auditor-events-indexer.name,] produce_topics = [ - kafka_topic.transactions_auditor_diff.events.name + kafka_topic.transactions_auditor_diff_events.name ] # consume_groups = ["billing.transactions-auditor-events-indexer"] cert_common_name = "billing/transactions-auditor-api"