From f120a987f2ee4a89dd63eda18f8b6d6ab9a389c1 Mon Sep 17 00:00:00 2001 From: Matt Richardson Date: Thu, 28 Nov 2024 13:26:34 +1100 Subject: [PATCH 1/2] Increase throughput of the BatchSpanProcessor --- .../endpoints/honeycomb/HoneycombOTELEndpointHandler.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java index 9dc77b1..046e880 100644 --- a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java +++ b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java @@ -94,7 +94,12 @@ private SpanProcessor buildGrpcSpanProcessor(Map headers, String .setMeterProvider(meterProvider) .build(); - return BatchSpanProcessor.builder(spanExporter).build(); + return BatchSpanProcessor.builder(spanExporter) + .setMaxQueueSize(32768) // Default is 2048. Increasing it to limit dropped spans. + .setScheduleDelay(Duration.ofSeconds(5)) // Default is 5s. This is another lever we can tweak. + .setMaxExportBatchSize(8192) // Default is 512. Increasing it to limit dropped spans. + .setMeterProvider(meterProvider) + .build(); } @Override From 514e0773ef83f1d18b43b3be66942f770a5aa4a9 Mon Sep 17 00:00:00 2001 From: Matt Richardson Date: Thu, 28 Nov 2024 13:35:36 +1100 Subject: [PATCH 2/2] Increase BatchSpanProcessor queue and batch size To reduce the occurrence of dropped spans --- .../teamcity/opentelemetry/common/PluginConstants.java | 5 +++++ .../endpoints/custom/CustomOTELEndpointHandler.java | 6 +++++- .../endpoints/honeycomb/HoneycombOTELEndpointHandler.java | 6 +++--- .../endpoints/zipkin/ZipKinOTELEndpointHandler.java | 8 ++++++-- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/com/octopus/teamcity/opentelemetry/common/PluginConstants.java b/common/src/main/java/com/octopus/teamcity/opentelemetry/common/PluginConstants.java index 4aae058..d9fc324 100644 --- a/common/src/main/java/com/octopus/teamcity/opentelemetry/common/PluginConstants.java +++ b/common/src/main/java/com/octopus/teamcity/opentelemetry/common/PluginConstants.java @@ -1,5 +1,7 @@ package com.octopus.teamcity.opentelemetry.common; +import java.time.Duration; + public class PluginConstants { private PluginConstants() {} public static final String TRACER_INSTRUMENTATION_NAME = "octopus.teamcity.opentelemetry"; @@ -15,6 +17,9 @@ private PluginConstants() {} public static final String PROPERTY_KEY_HONEYCOMB_APIKEY = "octopus.teamcity.opentelemetry.plugin.honeycomb.apikey"; public static final String PROPERTY_KEY_HONEYCOMB_METRICS_ENABLED = "octopus.teamcity.opentelemetry.plugin.honeycomb.metrics.enabled"; + public static final int BATCH_SPAN_PROCESSOR_MAX_QUEUE_SIZE = 32768; // Default is 2048. Increasing it to limit dropped spans. + public static final Duration BATCH_SPAN_PROCESSOR_MAX_SCHEDULE_DELAY = Duration.ofSeconds(5); // Default is 5s. This is another lever we can tweak. + public static final int BATCH_SPAN_PROCESSOR_MAX_EXPORT_BATCH_SIZE = 8192; // Default is 512. Increasing it to limit dropped spans. public static final String ATTRIBUTE_SERVICE_NAME = "service_name"; public static final String ATTRIBUTE_NAME = "name"; diff --git a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/custom/CustomOTELEndpointHandler.java b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/custom/CustomOTELEndpointHandler.java index ebdeed8..f68711f 100644 --- a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/custom/CustomOTELEndpointHandler.java +++ b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/custom/CustomOTELEndpointHandler.java @@ -81,6 +81,10 @@ private SpanProcessor buildGrpcSpanProcessor(Map headers, String LOG.debug("OTEL_PLUGIN: Opentelemetry export headers: " + LogMasker.mask(headers.toString())); LOG.debug("OTEL_PLUGIN: Opentelemetry export endpoint: " + exporterEndpoint); - return BatchSpanProcessor.builder(spanExporter).build(); + return BatchSpanProcessor.builder(spanExporter) + .setMaxQueueSize(BATCH_SPAN_PROCESSOR_MAX_QUEUE_SIZE) + .setScheduleDelay(BATCH_SPAN_PROCESSOR_MAX_SCHEDULE_DELAY) + .setMaxExportBatchSize(BATCH_SPAN_PROCESSOR_MAX_EXPORT_BATCH_SIZE) + .build(); } } diff --git a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java index 046e880..2651980 100644 --- a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java +++ b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java @@ -95,9 +95,9 @@ private SpanProcessor buildGrpcSpanProcessor(Map headers, String .build(); return BatchSpanProcessor.builder(spanExporter) - .setMaxQueueSize(32768) // Default is 2048. Increasing it to limit dropped spans. - .setScheduleDelay(Duration.ofSeconds(5)) // Default is 5s. This is another lever we can tweak. - .setMaxExportBatchSize(8192) // Default is 512. Increasing it to limit dropped spans. + .setMaxQueueSize(BATCH_SPAN_PROCESSOR_MAX_QUEUE_SIZE) + .setScheduleDelay(BATCH_SPAN_PROCESSOR_MAX_SCHEDULE_DELAY) + .setMaxExportBatchSize(BATCH_SPAN_PROCESSOR_MAX_EXPORT_BATCH_SIZE) .setMeterProvider(meterProvider) .build(); } diff --git a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/zipkin/ZipKinOTELEndpointHandler.java b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/zipkin/ZipKinOTELEndpointHandler.java index 316cd36..f9702bf 100644 --- a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/zipkin/ZipKinOTELEndpointHandler.java +++ b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/zipkin/ZipKinOTELEndpointHandler.java @@ -13,7 +13,7 @@ import javax.servlet.http.HttpServletRequest; import java.util.Map; -import static com.octopus.teamcity.opentelemetry.common.PluginConstants.PROPERTY_KEY_ENDPOINT; +import static com.octopus.teamcity.opentelemetry.common.PluginConstants.*; public class ZipKinOTELEndpointHandler implements IOTELEndpointHandler { private final PluginDescriptor pluginDescriptor; @@ -44,7 +44,11 @@ private SpanProcessor buildZipkinSpanProcessor(String exporterEndpoint) { .setEndpoint(endpoint) .build(); - return BatchSpanProcessor.builder(zipkinExporter).build(); + return BatchSpanProcessor.builder(zipkinExporter) + .setMaxQueueSize(BATCH_SPAN_PROCESSOR_MAX_QUEUE_SIZE) + .setScheduleDelay(BATCH_SPAN_PROCESSOR_MAX_SCHEDULE_DELAY) + .setMaxExportBatchSize(BATCH_SPAN_PROCESSOR_MAX_EXPORT_BATCH_SIZE) + .build(); } @Override