add bex fulfilment large invoice topics to dev (#700)

context - utilitywarehouse/kubernetes-manifests#99563
utilitywarehouse · Nov 18, 2024 · e237f78 · e237f78
1 parent 904717c
commit e237f78
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 1 deletion.
diff --git a/dev-aws/kafka-shared-msk/customer-billing/modules.tf b/dev-aws/kafka-shared-msk/customer-billing/modules.tf
@@ -47,19 +47,23 @@ module "invoice_fulfillment" {
     kafka_topic.invoice_fulfillment.name,
     kafka_topic.internal_bex_fulfilment_retry_1.name,
     kafka_topic.internal_bex_fulfilment_retry_2.name,
+    kafka_topic.internal_bex_fulfilment_large_invoice.name,
     kafka_topic.invoice_fulfillment_deadletter.name,
     kafka_topic.internal_invoice_fulfilment_deadletter.name,
     kafka_topic.internal_bex_bill_regeneration_retry_1.name,
     kafka_topic.internal_bex_bill_regeneration_retry_2.name,
+    kafka_topic.internal_bex_bill_regeneration_large_invoice.name,
     kafka_topic.internal_bex_bill_regeneration_deadletter.name
   ]
   consume_topics = [
     kafka_topic.transition_bex_fulfilment_request.name,
     kafka_topic.internal_bex_fulfilment_retry_1.name,
     kafka_topic.internal_bex_fulfilment_retry_2.name,
+    kafka_topic.internal_bex_fulfilment_large_invoice.name,
     kafka_topic.internal_bex_bill_regeneration.name,
     kafka_topic.internal_bex_bill_regeneration_retry_1.name,
-    kafka_topic.internal_bex_bill_regeneration_retry_2.name
+    kafka_topic.internal_bex_bill_regeneration_retry_2.name,
+    kafka_topic.internal_bex_bill_regeneration_large_invoice.name
   ]
   consume_groups = [
     "bex.invoice-fulfillment",

diff --git a/dev-aws/kafka-shared-msk/customer-billing/topics.tf b/dev-aws/kafka-shared-msk/customer-billing/topics.tf
@@ -227,6 +227,25 @@ resource "kafka_topic" "internal_bex_fulfilment_retry_2" {
   }
 }
 
+# this topic is used in the bill generation pipeline
+# used for processing large invoices
+resource "kafka_topic" "internal_bex_fulfilment_large_invoice" {
+  name               = "bex.internal.fulfilment_large_invoice"
+  replication_factor = 3
+  partitions         = 1
+  config = {
+    "remote.storage.enable" = "true"
+    # keep data in primary storage for 1 day
+    "local.retention.ms" = "86400000"
+    "compression.type"   = "zstd"
+    "retention.bytes"    = "8053063680" # keep on each partition 7.5GiB
+    # allow for a batch of records maximum 1MiB
+    "max.message.bytes" = "1048576"
+    "cleanup.policy"    = "delete"
+    # keep data for 3 days
+    "retention.ms" = "259200000"
+  }
+}
 
 # this topic is used in the bill regeneration pipeline
 resource "kafka_topic" "internal_bex_bill_regeneration" {
@@ -287,6 +306,26 @@ resource "kafka_topic" "internal_bex_bill_regeneration_retry_2" {
   }
 }
 
+# this topic is used in the bill regeneration pipeline
+# used for processing large invoices
+resource "kafka_topic" "internal_bex_bill_regeneration_large_invoice" {
+  name               = "bex.internal.bill_regeneration_large_invoice"
+  replication_factor = 3
+  partitions         = 1
+  config = {
+    "remote.storage.enable" = "true"
+    # keep data in primary storage for 1 day
+    "local.retention.ms" = "86400000"
+    "compression.type"   = "zstd"
+    "retention.bytes"    = "8053063680" # keep on each partition 7.5GiB
+    # allow for a batch of records maximum 1MiB
+    "max.message.bytes" = "1048576"
+    "cleanup.policy"    = "delete"
+    # keep data for 3 days
+    "retention.ms" = "259200000"
+  }
+}
+
 # this topic is used in the bill regeneration pipeline
 # used for retrying the bill regeneration on failure
 resource "kafka_topic" "internal_bex_bill_regeneration_deadletter" {