Skip to content

Commit

Permalink
chore: DENA-424 - add config for kafka connect (#741)
Browse files Browse the repository at this point in the history
Using kafka connect to consume from particular topics
* `customer-proposition.service-status.events.v3` - for account to
service(s) mapping
* `account-identity.legacy.account.events` - for denormalising the
account model
  • Loading branch information
fillyD authored Dec 20, 2024
1 parent 66e66ab commit 78a283d
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 0 deletions.
6 changes: 6 additions & 0 deletions dev-aws/kafka-shared-msk/account-identity/kafka-connect.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
module "kafka_connect_read_legacy_account_events" {
source = "../../../modules/tls-app"
consume_topics = ["account-identity.legacy.account.events"]
consume_groups = ["dev-enablement.kafka-connect-group"]
cert_common_name = "dev-enablement/kafka-connect"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
module "kafka_connect_read_service_status_events" {
source = "../../../modules/tls-app"
consume_topics = ["customer-proposition.service-status.events.v3"]
consume_groups = ["dev-enablement.kafka-connect-group"]
cert_common_name = "dev-enablement/kafka-connect"
}
47 changes: 47 additions & 0 deletions dev-aws/kafka-shared-msk/dev-enablement/kafka-connect.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# https://docs.confluent.io/platform/7.8/connect/references/allconfigs.html#distributed-worker-configuration
resource "kafka_topic" "connect_configs" {
name = "dev-enablement.connect-configs"
partitions = 1
replication_factor = 3

config = {
"cleanup.policy" = "compact"
"compression.type" = "zstd"

}
}

resource "kafka_topic" "connect_offsets" {
name = "dev-enablement.connect-offsets"
partitions = 25
replication_factor = 3

config = {
"cleanup.policy" = "compact"
"compression.type" = "zstd"

}
}

resource "kafka_topic" "connect_status" {
name = "dev-enablement.connect-status"
partitions = 5
replication_factor = 3

config = {
"cleanup.policy" = "compact"
"compression.type" = "zstd"

}
}


# https://docs.confluent.io/platform/7.8/connect/security.html#worker-acl-requirements

# Allow Kafka Connect full access to internal topics
module "kafka_connect_full_internal_topics" {
source = "../../../modules/tls-app"
consume_topics = ["dev-enablement.connect-configs", "dev-enablement.connect-offsets", "dev-enablement.connect-status"]
consume_groups = ["dev-enablement.kafka-connect-group"]
cert_common_name = "dev-enablement/kafka-connect"
}

0 comments on commit 78a283d

Please sign in to comment.