add CircuitBreaker timesTripped metric (#203)

* add CircuitBreaker timesTripped metric * remove String.format * toLowerCase add locale * tidy * move metrics management to its own map * use stdlib Map.copyOf * tidy
cowpaths · May 31, 2024 · 9302dcf · 9302dcf
1 parent 9892b9b
commit 9302dcf
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 0 deletions.
diff --git a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.servlet;
 
+import static org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry.getTimesTrippedMetrics;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayOutputStream;
@@ -92,6 +94,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
       caller.call(qTime, metrics, request);
     }
     getCompressingDirectoryPoolMetrics(metrics);
+    getCircuitBreakerMetrics(metrics);
     getSharedCacheMetrics(metrics, getSolrDispatchFilter(request).getCores(), cacheMetricTypes);
     metrics.add(
         new PrometheusMetric(
@@ -193,6 +196,19 @@ private void getCompressingDirectoryPoolMetrics(List<PrometheusMetric> metrics)
     }
   }
 
+  private void getCircuitBreakerMetrics(List<PrometheusMetric> metrics) {
+    getTimesTrippedMetrics()
+        .forEach(
+            (k, v) -> {
+              metrics.add(
+                  new PrometheusMetric(
+                      "times_tripped" + k,
+                      PrometheusMetricType.COUNTER,
+                      "number of times circuit has been tripped",
+                      v));
+            });
+  }
+
   @SuppressWarnings({"rawtypes", "unchecked"})
   static void getSharedCacheMetrics(
       List<PrometheusMetric> results,

diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
@@ -27,6 +27,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;
 import org.slf4j.Logger;
@@ -43,6 +44,7 @@ public class CircuitBreakerRegistry implements Closeable {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   private final Map<SolrRequestType, List<CircuitBreaker>> circuitBreakerMap = new HashMap<>();
+  private static final Map<String, Long> circuitBreakerTrippedMetrics = new ConcurrentHashMap<>();
 
   public CircuitBreakerRegistry() {}
 
@@ -82,6 +84,7 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
     for (CircuitBreaker circuitBreaker :
         circuitBreakerMap.getOrDefault(requestType, Collections.emptyList())) {
       if (circuitBreaker.isTripped()) {
+        incrementTripped(requestType, circuitBreaker);
         if (circuitBreaker.isDebugMode()) {
           if (log.isInfoEnabled()) {
             log.info(
@@ -102,6 +105,18 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
     return triggeredCircuitBreakers;
   }
 
+  private void incrementTripped(SolrRequestType requestType, CircuitBreaker circuitBreaker) {
+    String metricKey =
+        circuitBreaker.getClass().getSimpleName()
+            + "_"
+            + requestType.name().toLowerCase(Locale.ROOT);
+    circuitBreakerTrippedMetrics.merge(metricKey, 1L, Long::sum);
+  }
+
+  public static Map<String, Long> getTimesTrippedMetrics() {
+    return Map.copyOf(circuitBreakerTrippedMetrics);
+  }
+
   /**
    * Construct the final error message to be printed when circuit breakers trip.
    *