Bring dev changes for Otel to prod

I.e. bring in equivalent changes as:

* 94b5f07
* e6749ae
* 0b607e7

Ticket: DENA-1126

prod-aws/otel/otel.tf

@@ -32,8 +32,39 @@ module "otel_collector_web" {
 }
 
 module "tempo_distributor" {
-  source           = "../../modules/tls-app"
-  consume_topics   = [kafka_topic.otlp_spans.name]
+  source = "../../modules/tls-app"
+  consume_topics = [
+    kafka_topic.otlp_spans.name,
+    kafka_topic.otlp_sampled_spans.name,
+  ]
   consume_groups   = ["processor-tempo"]
   cert_common_name = "otel/tempo-distributor"
 }
+
+resource "kafka_topic" "otlp_sampled_spans" {
+  name               = "otel.otlp_sampled_spans"
+  replication_factor = 3
+  partitions         = 200
+  config = {
+    # retain 5GB on each partition
+    "retention.bytes" = "5368709120"
+    # keep data for 12 hours
+    "retention.ms" = "43200000"
+    # allow max 128 MB for a message
+    "max.message.bytes" = "134217728"
+    # roll log at 3h max
+    "segment.ms" = "10800000"
+    # max log size of 250 MB
+    "segment.bytes"    = "262144000"
+    "compression.type" = "zstd"
+    "cleanup.policy"   = "delete"
+  }
+}
+
+module "otel_tail_sampling_collector" {
+  source           = "../../modules/tls-app"
+  consume_topics   = [kafka_topic.otlp_spans.name]
+  produce_topics   = [kafka_topic.otlp_sampled_spans.name]
+  consume_groups   = ["otel.tail-sampling-collector"]
+  cert_common_name = "otel/tail-sampling-collector"
+}