Add support for OTEL metrics export

So we can diagnose dropped spans
OctopusDeploy · Nov 26, 2024 · 1cf7d55 · 1cf7d55
1 parent 411061d
commit 1cf7d55
Show file tree

Hide file tree

Showing 12 changed files with 109 additions and 6 deletions.
diff --git a/common/src/main/java/com/octopus/teamcity/opentelemetry/common/PluginConstants.java b/common/src/main/java/com/octopus/teamcity/opentelemetry/common/PluginConstants.java
@@ -13,6 +13,8 @@ private PluginConstants() {}
     public static final String PROPERTY_KEY_HONEYCOMB_TEAM = "octopus.teamcity.opentelemetry.plugin.honeycomb.team";
     public static final String PROPERTY_KEY_HONEYCOMB_DATASET = "octopus.teamcity.opentelemetry.plugin.honeycomb.dataset";
     public static final String PROPERTY_KEY_HONEYCOMB_APIKEY = "octopus.teamcity.opentelemetry.plugin.honeycomb.apikey";
+    public static final String PROPERTY_KEY_HONEYCOMB_METRICS_ENABLED = "octopus.teamcity.opentelemetry.plugin.honeycomb.metrics.enabled";
+
 
     public static final String ATTRIBUTE_SERVICE_NAME = "service_name";
     public static final String ATTRIBUTE_NAME = "name";

diff --git a/server/build.gradle b/server/build.gradle
@@ -18,6 +18,8 @@ dependencies {
     implementation('io.opentelemetry:opentelemetry-api:1.44.1')
     implementation('io.opentelemetry:opentelemetry-sdk:1.44.1')
     implementation('io.opentelemetry:opentelemetry-exporter-otlp:1.44.1')
+    implementation('io.opentelemetry:opentelemetry-exporter-logging:1.44.1')
+    implementation 'io.opentelemetry:opentelemetry-sdk-metrics:1.28.0'
     implementation('io.opentelemetry:opentelemetry-semconv:1.31.0-alpha-SNAPSHOT')
     implementation('io.opentelemetry:opentelemetry-exporter-zipkin')
     implementation 'io.grpc:grpc-netty-shaded:1.68.1'

diff --git a/...c/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/IOTELEndpointHandler.java b/...c/main/java/com/octopus/teamcity/opentelemetry/server/endpoints/IOTELEndpointHandler.java
@@ -1,17 +1,21 @@
 package com.octopus.teamcity.opentelemetry.server.endpoints;
 
 import com.octopus.teamcity.opentelemetry.server.SetProjectConfigurationSettingsRequest;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
 import io.opentelemetry.sdk.trace.SpanProcessor;
 import jetbrains.buildServer.serverSide.SBuild;
 import org.springframework.web.servlet.ModelAndView;
 
+import javax.annotation.Nullable;
 import javax.servlet.http.HttpServletRequest;
 import java.util.Map;
 
 public interface IOTELEndpointHandler {
     ModelAndView getBuildOverviewModelAndView(SBuild build, Map<String, String> params, String traceId);
 
     SpanProcessor buildSpanProcessor(String endpoint, Map<String, String> params);
+    @Nullable
+    MetricExporter buildMetricsExporter(String endpoint, Map<String, String> params);
 
     SetProjectConfigurationSettingsRequest getSetProjectConfigurationSettingsRequest(HttpServletRequest request);
 

diff --git a/...com/octopus/teamcity/opentelemetry/server/endpoints/custom/CustomOTELEndpointHandler.java b/...com/octopus/teamcity/opentelemetry/server/endpoints/custom/CustomOTELEndpointHandler.java
@@ -4,13 +4,15 @@
 import com.octopus.teamcity.opentelemetry.server.endpoints.IOTELEndpointHandler;
 import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
 import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporterBuilder;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
 import io.opentelemetry.sdk.trace.SpanProcessor;
 import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
 import jetbrains.buildServer.serverSide.SBuild;
 import jetbrains.buildServer.serverSide.crypt.EncryptUtil;
 import jetbrains.buildServer.web.openapi.PluginDescriptor;
 import org.apache.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
 import org.springframework.web.servlet.ModelAndView;
 
 import javax.servlet.http.HttpServletRequest;
@@ -47,6 +49,12 @@ public SpanProcessor buildSpanProcessor(String endpoint, Map<String, String> par
         return buildGrpcSpanProcessor(headers, endpoint);
     }
 
+    @Nullable
+    @Override
+    public MetricExporter buildMetricsExporter(String endpoint, Map<String, String> params) {
+        return null;
+    }
+
     @Override
     public SetProjectConfigurationSettingsRequest getSetProjectConfigurationSettingsRequest(HttpServletRequest request) {
         return new SetCustomProjectConfigurationSettingsRequest(request);

diff --git a/...topus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java b/...topus/teamcity/opentelemetry/server/endpoints/honeycomb/HoneycombOTELEndpointHandler.java
@@ -2,8 +2,10 @@
 
 import com.octopus.teamcity.opentelemetry.server.*;
 import com.octopus.teamcity.opentelemetry.server.endpoints.IOTELEndpointHandler;
+import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter;
 import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
 import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporterBuilder;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
 import io.opentelemetry.sdk.trace.SpanProcessor;
 import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
@@ -15,6 +17,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.springframework.web.servlet.ModelAndView;
 
+import javax.annotation.Nullable;
 import javax.servlet.http.HttpServletRequest;
 import java.util.ArrayList;
 import java.util.Date;
@@ -62,6 +65,19 @@ public SpanProcessor buildSpanProcessor(String endpoint, Map<String, String> par
         return buildGrpcSpanProcessor(headers, endpoint);
     }
 
+    @Override
+    @Nullable
+    public MetricExporter buildMetricsExporter(String endpoint, Map<String, String> params) {
+        if (params.getOrDefault(PROPERTY_KEY_HONEYCOMB_METRICS_ENABLED, "false").equals("true")) {
+            return OtlpGrpcMetricExporter.builder()
+                    .setEndpoint(endpoint)
+                    .addHeader("x-honeycomb-team", EncryptUtil.unscramble(params.get(PROPERTY_KEY_HONEYCOMB_APIKEY)))
+                    .addHeader("x-honeycomb-dataset", params.get(PROPERTY_KEY_HONEYCOMB_DATASET))
+                    .build();
+        }
+        return null;
+    }
+
     private SpanProcessor buildGrpcSpanProcessor(Map<String, String> headers, String exporterEndpoint) {
 
         OtlpGrpcSpanExporterBuilder spanExporterBuilder = OtlpGrpcSpanExporter.builder();
@@ -85,6 +101,7 @@ public void mapParamsToModel(Map<String, String> params, Map<String, Object> mod
         model.put("otelEndpoint", params.get(PROPERTY_KEY_ENDPOINT));
         model.put("otelHoneycombTeam", params.get(PROPERTY_KEY_HONEYCOMB_TEAM));
         model.put("otelHoneycombDataset", params.get(PROPERTY_KEY_HONEYCOMB_DATASET));
+        model.put("otelHoneycombMetricsEnabled", params.get(PROPERTY_KEY_HONEYCOMB_METRICS_ENABLED));
         if (params.get(PROPERTY_KEY_HONEYCOMB_APIKEY) == null) {
             model.put("otelHoneycombApiKey", null);
         }

diff --git a/...telemetry/server/endpoints/honeycomb/SetHoneycombProjectConfigurationSettingsRequest.java b/...telemetry/server/endpoints/honeycomb/SetHoneycombProjectConfigurationSettingsRequest.java
@@ -17,11 +17,13 @@ public class SetHoneycombProjectConfigurationSettingsRequest extends SetProjectC
     public final String honeycombTeam;
     public final String honeycombDataset;
     public final String honeycombApiKey;
+    public final String honeycombMetricsEnabled;
 
     public SetHoneycombProjectConfigurationSettingsRequest(HttpServletRequest request) {
         super(request);
         this.honeycombTeam = request.getParameter("honeycombTeam");
         this.honeycombDataset = request.getParameter("honeycombDataset");
+        this.honeycombMetricsEnabled = request.getParameter("honeycombMetricsEnabled");
         this.honeycombApiKey = RSACipher.decryptWebRequestData(request.getParameter("encryptedHoneycombApiKey"));
     }
 
@@ -39,6 +41,7 @@ protected void serviceSpecificValidate(ActionErrors errors) {
     protected void mapServiceSpecificParams(HashMap<String, String> params, ArrayList<HeaderDto> headers) {
         params.put(PROPERTY_KEY_HONEYCOMB_DATASET, honeycombDataset);
         params.put(PROPERTY_KEY_HONEYCOMB_TEAM, honeycombTeam);
+        params.put(PROPERTY_KEY_HONEYCOMB_METRICS_ENABLED, honeycombMetricsEnabled);
         params.put(PROPERTY_KEY_HONEYCOMB_APIKEY, EncryptUtil.scramble(honeycombApiKey));
     }
 }
diff --git a/...com/octopus/teamcity/opentelemetry/server/endpoints/zipkin/ZipKinOTELEndpointHandler.java b/...com/octopus/teamcity/opentelemetry/server/endpoints/zipkin/ZipKinOTELEndpointHandler.java
@@ -3,11 +3,13 @@
 import com.octopus.teamcity.opentelemetry.server.SetProjectConfigurationSettingsRequest;
 import com.octopus.teamcity.opentelemetry.server.endpoints.IOTELEndpointHandler;
 import io.opentelemetry.exporter.zipkin.ZipkinSpanExporter;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
 import io.opentelemetry.sdk.trace.SpanProcessor;
 import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
 import jetbrains.buildServer.serverSide.SBuild;
 import jetbrains.buildServer.web.openapi.PluginDescriptor;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 import org.springframework.web.servlet.ModelAndView;
 
 import javax.servlet.http.HttpServletRequest;
@@ -38,6 +40,12 @@ public SpanProcessor buildSpanProcessor(String endpoint, Map<String, String> par
         return buildZipkinSpanProcessor(endpoint);
     }
 
+    @Nullable
+    @Override
+    public MetricExporter buildMetricsExporter(String endpoint, Map<String, String> params) {
+        return null;
+    }
+
     private SpanProcessor buildZipkinSpanProcessor(String exporterEndpoint) {
         String endpoint = String.format("%s/api/v2/spans", exporterEndpoint);
         ZipkinSpanExporter zipkinExporter = ZipkinSpanExporter.builder()

diff --git a/...va/com/octopus/teamcity/opentelemetry/server/helpers/HelperPerBuildOTELHelperFactory.java b/...va/com/octopus/teamcity/opentelemetry/server/helpers/HelperPerBuildOTELHelperFactory.java
@@ -43,13 +43,13 @@ public OTELHelper getOTELHelper(BuildPromotion build) {
                 var params = feature.getParameters();
                 if (params.get(PROPERTY_KEY_ENABLED).equals("true")) {
                     var endpoint = params.get(PROPERTY_KEY_ENDPOINT);
-                    SpanProcessor spanProcessor;
 
                     var otelHandler = otelEndpointFactory.getOTELEndpointHandler(params.get(PROPERTY_KEY_SERVICE));
-                    spanProcessor = otelHandler.buildSpanProcessor(endpoint, params);
+                    var spanProcessor = otelHandler.buildSpanProcessor(endpoint, params);
+                    var metricsExporter = otelHandler.buildMetricsExporter(endpoint, params);
 
                     long startTime = System.nanoTime();
-                    var otelHelper = new OTELHelperImpl(spanProcessor, String.valueOf(buildId));
+                    var otelHelper = new OTELHelperImpl(spanProcessor, metricsExporter, String.valueOf(buildId));
                     long endTime = System.nanoTime();
 
                     long duration = (endTime - startTime);

diff --git a/server/src/main/java/com/octopus/teamcity/opentelemetry/server/helpers/OTELHelperImpl.java b/server/src/main/java/com/octopus/teamcity/opentelemetry/server/helpers/OTELHelperImpl.java
@@ -1,21 +1,29 @@
 package com.octopus.teamcity.opentelemetry.server.helpers;
 
 import com.octopus.teamcity.opentelemetry.common.PluginConstants;
+import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.Meter;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
 import io.opentelemetry.context.Context;
 import io.opentelemetry.context.propagation.ContextPropagators;
+import io.opentelemetry.exporter.logging.LoggingMetricExporter;
 import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
+import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
 import io.opentelemetry.sdk.resources.Resource;
 import io.opentelemetry.sdk.trace.SdkTracerProvider;
 import io.opentelemetry.sdk.trace.SpanProcessor;
 import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;
 import org.apache.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
 
 import javax.annotation.Nullable;
+import java.time.Duration;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
@@ -27,10 +35,16 @@ public class OTELHelperImpl implements OTELHelper {
     private final SdkTracerProvider sdkTracerProvider;
     private final String helperName;
 
-    public OTELHelperImpl(SpanProcessor spanProcessor, String helperName) {
+    public OTELHelperImpl(
+            @NotNull SpanProcessor spanProcessor,
+            @Nullable MetricExporter metricExporter,
+            @NotNull String helperName) {
         this.helperName = helperName;
         Resource serviceNameResource = Resource
                 .create(Attributes.of(ResourceAttributes.SERVICE_NAME, PluginConstants.SERVICE_NAME));
+
+        configureMetricsExport(metricExporter, serviceNameResource);
+
         this.sdkTracerProvider = SdkTracerProvider.builder()
                 .setResource(Resource.getDefault().merge(serviceNameResource))
                 .addSpanProcessor(spanProcessor)
@@ -43,6 +57,30 @@ public OTELHelperImpl(SpanProcessor spanProcessor, String helperName) {
         this.spanMap = new ConcurrentHashMap<>();
     }
 
+    private static void configureMetricsExport(@Nullable MetricExporter metricExporter, Resource serviceNameResource) {
+        var loggingMetricExporter = LoggingMetricExporter.create();
+        var meterProviderBuilder = SdkMeterProvider.builder()
+                .setResource(Resource.getDefault().merge(serviceNameResource))
+                .registerMetricReader(PeriodicMetricReader.builder(loggingMetricExporter)
+                        .setInterval(Duration.ofSeconds(10))
+                        .build());
+        if (metricExporter != null) {
+            meterProviderBuilder
+                    .registerMetricReader(PeriodicMetricReader.builder(metricExporter)
+                            .setInterval(Duration.ofSeconds(10))
+                            .build());
+        }
+        var sdkMeterProvider = meterProviderBuilder.build();
+        var globalOpenTelemetry = OpenTelemetrySdk.builder()
+                .setMeterProvider(sdkMeterProvider)
+                .build();
+
+        GlobalOpenTelemetry.set(globalOpenTelemetry);
+
+        // Shutdown hooks to close resources properly
+        Runtime.getRuntime().addShutdownHook(new Thread(sdkMeterProvider::close));
+    }
+
     @Override
     public boolean isReady() {
         return this.openTelemetry != null && this.tracer != null && this.spanMap != null;
@@ -56,6 +94,7 @@ public Span getOrCreateParentSpan(String buildId) {
     @Override
     public Span createSpan(String spanName, Span parentSpan, String parentSpanName) {
         LOG.info("Creating child span " + spanName + " under parent " + parentSpanName);
+        IncrementCounter("spans-created", "Number of spans created");
         return this.spanMap.computeIfAbsent(spanName, key -> this.tracer
                 .spanBuilder(spanName)
                 .setParent(Context.current().with(parentSpan))
@@ -64,6 +103,7 @@ public Span createSpan(String spanName, Span parentSpan, String parentSpanName)
 
     @Override
     public Span createTransientSpan(String spanName, Span parentSpan, long startTime) {
+        IncrementCounter("spans-created", "Number of spans created");
         return this.tracer.spanBuilder(spanName)
                 .setParent(Context.current().with(parentSpan))
                 .setStartTimestamp(startTime, TimeUnit.MILLISECONDS)
@@ -94,4 +134,15 @@ public void release(String helperName) {
         this.sdkTracerProvider.close();
         this.spanMap.clear();
     }
+
+    public void IncrementCounter(String counterName, String counterDescription)
+    {
+        //likely temporary while I confirm that metrics is working as expected
+        Meter meter = openTelemetry.getMeter("teamcity-opentelemetry-plugin");
+        meter.counterBuilder(counterName)
+                .setDescription(counterDescription)
+                .setUnit("1")
+                .build()
+                .add(1);
+    }
 }
diff --git a/server/src/main/resources/buildServerResources/projectConfigurationSettingsHoneycomb.jspf b/server/src/main/resources/buildServerResources/projectConfigurationSettingsHoneycomb.jspf
@@ -19,3 +19,9 @@
         <span class="error" id="error_honeycombDataset"></span>
     </td>
 </tr>
+<tr <c:if test='${otelService != "honeycomb.io"}'>style="display: none"</c:if>>
+    <th><label for="honeycombMetricsEnabled">Send Metrics?</label></th>
+    <td>
+        <forms:checkbox name="honeycombMetricsEnabled" checked="${otelHoneycombMetricsEnabled}" >&nbsp;</forms:checkbox>
+    </td>
+</tr>
diff --git a/server/src/test/java/com/octopus/teamcity/opentelemetry/server/OTELHelperTest.java b/server/src/test/java/com/octopus/teamcity/opentelemetry/server/OTELHelperTest.java
@@ -6,6 +6,7 @@
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
 import io.opentelemetry.sdk.trace.SpanProcessor;
 import jetbrains.buildServer.serverSide.BuildPromotion;
 import jetbrains.buildServer.serverSide.SRunningBuild;
@@ -32,7 +33,7 @@ class OTELHelperTest {
     @BeforeEach
     void setUp() {
         GlobalOpenTelemetry.resetForTest();
-        this.otelHelper = new OTELHelperImpl(mock(SpanProcessor.class, RETURNS_DEEP_STUBS), "helperNamr");
+        this.otelHelper = new OTELHelperImpl(mock(SpanProcessor.class, RETURNS_DEEP_STUBS),  mock(MetricExporter.class, RETURNS_DEEP_STUBS), "helperNamr");
     }
 
 

diff --git a/...er/src/test/java/com/octopus/teamcity/opentelemetry/server/TeamCityBuildListenerTest.java b/...er/src/test/java/com/octopus/teamcity/opentelemetry/server/TeamCityBuildListenerTest.java
@@ -5,6 +5,7 @@
 import com.octopus.teamcity.opentelemetry.server.helpers.OTELHelperImpl;
 import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
 import io.opentelemetry.sdk.trace.SpanProcessor;
 import jetbrains.buildServer.serverSide.BuildPromotion;
 import jetbrains.buildServer.serverSide.BuildServerListener;
@@ -33,7 +34,7 @@ class TeamCityBuildListenerTest {
     @BeforeEach
     void setUp(@Mock EventDispatcher<BuildServerListener> buildServerListenerEventDispatcher) {
         GlobalOpenTelemetry.resetForTest();
-        this.otelHelper = new OTELHelperImpl(mock(SpanProcessor.class, RETURNS_DEEP_STUBS), "helper");
+        this.otelHelper = new OTELHelperImpl(mock(SpanProcessor.class, RETURNS_DEEP_STUBS),  mock(MetricExporter.class, RETURNS_DEEP_STUBS), "helper");
         this.factory = mock(OTELHelperFactory.class, RETURNS_DEEP_STUBS);
 
         var buildStorageManager = mock(BuildStorageManager.class, RETURNS_DEEP_STUBS);