From 94b5f07e8f51316ffcebe0d2ba54c6b7e0f51af7 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 12 Dec 2024 14:11:51 +0000 Subject: [PATCH] Add topic and app for new otel collector The new tail sampler will: * Read from the current `otel.otlp_spans` topic * Produce to the new `otel.otlp_sampled_spans` Later `tempo` will also be configure to read from the new topic and not the old one, but we'll leave it alone for now. Ticket: DENA-1126 --- dev-aws/otel/otel.tf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dev-aws/otel/otel.tf b/dev-aws/otel/otel.tf index 9ed81f25..c068a3ce 100644 --- a/dev-aws/otel/otel.tf +++ b/dev-aws/otel/otel.tf @@ -30,3 +30,30 @@ module "tempo_distributor" { consume_groups = ["processor-tempo"] cert_common_name = "otel/tempo-distributor" } + +resource "kafka_topic" "otlp_sampled_spans" { + name = "otel.otlp_sampled_spans" + replication_factor = 3 + partitions = 200 + config = { + # retain 5GB on each partition + "retention.bytes" = "5368709120" + # keep data for 12 hours + "retention.ms" = "43200000" + # allow max 128 MB for a message + "max.message.bytes" = "134217728" + # roll log at 3h max + "segment.ms" = "10800000" + # max log size of 250 MB + "segment.bytes" = "262144000" + "compression.type" = "zstd" + "cleanup.policy" = "delete" + } +} + +module "otel_tail_sampling_collector" { + source = "../../modules/tls-app" + consume_topics = [kafka_topic.otlp_spans.name] + produce_topics = [kafka_topic.otlp_sampled_spans.name] + cert_common_name = "otel/tail-sampling-collector" +}