feat: remove high cardinality metrics (#406)

* removing high cardinality metrics * removing content_length as well * adding smoke test
hypertrace · Sep 9, 2024 · 5813589 · 5813589
1 parent cbf3c2f
commit 5813589
Show file tree

Hide file tree

Showing 4 changed files with 115 additions and 0 deletions.
diff --git a/...xtensions/src/main/java/org/hypertrace/agent/otel/extensions/MetricViewConfiguration.java b/...xtensions/src/main/java/org/hypertrace/agent/otel/extensions/MetricViewConfiguration.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hypertrace.agent.otel.extensions;
+
+import io.opentelemetry.sdk.metrics.View;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+public class MetricViewConfiguration {
+
+  public static View createView() {
+    // Attributes to exclude
+    Set<String> excludedAttributes =
+        new HashSet<>(
+            Arrays.asList(
+                "net.sock.peer.addr",
+                "net.sock.peer.port",
+                "http.user_agent",
+                "enduser.id",
+                "http.client_ip",
+                "http.request_content_length",
+                "http.response_content_length",
+                "user_agent.original"));
+
+    // Build the view
+    return View.builder()
+        .setAttributeFilter(
+            attributes -> {
+              for (String attribute : excludedAttributes) {
+                if (attributes.contains(attribute)) {
+                  return false;
+                }
+              }
+              return true;
+            })
+        .build();
+  }
+}
diff --git a/...c/main/java/org/hypertrace/agent/otel/extensions/config/HypertraceAgentConfiguration.java b/...c/main/java/org/hypertrace/agent/otel/extensions/config/HypertraceAgentConfiguration.java
@@ -20,6 +20,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer;
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
+import io.opentelemetry.sdk.metrics.InstrumentSelector;
+import io.opentelemetry.sdk.metrics.View;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -28,6 +30,7 @@
 import org.hypertrace.agent.config.v1.Config.MetricReporterType;
 import org.hypertrace.agent.config.v1.Config.PropagationFormat;
 import org.hypertrace.agent.config.v1.Config.TraceReporterType;
+import org.hypertrace.agent.otel.extensions.MetricViewConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -113,5 +116,12 @@ public static String toOtelPropagators(List<PropagationFormat> propagationFormat
   @Override
   public void customize(AutoConfigurationCustomizer autoConfigurationCustomizer) {
     autoConfigurationCustomizer.addPropertiesSupplier(this::getProperties);
+    autoConfigurationCustomizer.addMeterProviderCustomizer(
+        (builder, config) -> {
+          View httpServerDurationView = MetricViewConfiguration.createView();
+          builder.registerView(
+              InstrumentSelector.builder().setName("*").build(), httpServerDurationView);
+          return builder;
+        });
   }
 }
diff --git a/smoke-tests/src/test/java/org/hypertrace/agent/smoketest/AbstractSmokeTest.java b/smoke-tests/src/test/java/org/hypertrace/agent/smoketest/AbstractSmokeTest.java
@@ -20,11 +20,14 @@
 import com.google.protobuf.util.JsonFormat;
 import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
 import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.metrics.v1.Metric;
 import io.opentelemetry.proto.trace.v1.ScopeSpans;
 import io.opentelemetry.proto.trace.v1.Span;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Collection;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.jar.Attributes;
@@ -244,6 +247,46 @@ protected boolean hasMetricNamed(
         .anyMatch(metric -> metric.getName().equals(metricName));
   }
 
+  // Checks if a metric with the given name contains the specified attribute
+  protected boolean hasMetricWithAttribute(
+      String metricName,
+      String attributeName,
+      Collection<ExportMetricsServiceRequest> metricRequests) {
+
+    return metricRequests.stream()
+        .flatMap(metricRequest -> metricRequest.getResourceMetricsList().stream())
+        .flatMap(resourceMetrics -> resourceMetrics.getScopeMetricsList().stream())
+        .flatMap(scopeMetrics -> scopeMetrics.getMetricsList().stream())
+        .filter(metric -> metric.getName().equals(metricName))
+        .anyMatch(metric -> metricHasAttribute(metric, attributeName));
+  }
+
+  private boolean metricHasAttribute(Metric metric, String attributeName) {
+    switch (metric.getDataCase()) {
+      case GAUGE:
+        return metric.getGauge().getDataPointsList().stream()
+            .anyMatch(dataPoint -> hasAttribute(dataPoint.getAttributesList(), attributeName));
+      case SUM:
+        return metric.getSum().getDataPointsList().stream()
+            .anyMatch(dataPoint -> hasAttribute(dataPoint.getAttributesList(), attributeName));
+      case HISTOGRAM:
+        return metric.getHistogram().getDataPointsList().stream()
+            .anyMatch(dataPoint -> hasAttribute(dataPoint.getAttributesList(), attributeName));
+      case EXPONENTIAL_HISTOGRAM:
+        return metric.getExponentialHistogram().getDataPointsList().stream()
+            .anyMatch(dataPoint -> hasAttribute(dataPoint.getAttributesList(), attributeName));
+      case SUMMARY:
+        return metric.getSummary().getDataPointsList().stream()
+            .anyMatch(dataPoint -> hasAttribute(dataPoint.getAttributesList(), attributeName));
+      default:
+        return false;
+    }
+  }
+
+  private boolean hasAttribute(List<KeyValue> attributes, String attributeName) {
+    return attributes.stream().anyMatch(attribute -> attribute.getKey().equals(attributeName));
+  }
+
   public static String getPropertyOrEnv(String propName) {
     String property = System.getProperty(propName);
     if (property != null && !property.isEmpty()) {

diff --git a/smoke-tests/src/test/java/org/hypertrace/agent/smoketest/SpringBootSmokeTest.java b/smoke-tests/src/test/java/org/hypertrace/agent/smoketest/SpringBootSmokeTest.java
@@ -174,6 +174,15 @@ public void postJson() throws IOException, InterruptedException {
     Assertions.assertTrue(hasMetricNamed("processedSpans", metrics));
     Assertions.assertTrue(hasMetricNamed("queueSize", metrics));
     Assertions.assertTrue(hasMetricNamed("http.server.duration", metrics));
+
+    Assertions.assertTrue(hasMetricWithAttribute("http.server.duration", "http.method", metrics));
+    Assertions.assertTrue(hasMetricWithAttribute("http.server.duration", "http.route", metrics));
+    Assertions.assertFalse(
+        hasMetricWithAttribute("http.server.duration", "net.sock.peer.addr", metrics));
+    Assertions.assertFalse(
+        hasMetricWithAttribute("http.server.duration", "http.request_content_length", metrics));
+    Assertions.assertFalse(
+        hasMetricWithAttribute("http.server.duration", "http.response_content_length", metrics));
   }
 
   @Test