From 9302dcf45452395a9d638dc66666225c2c043a73 Mon Sep 17 00:00:00 2001
From: Nick Ginther <104029658+nginthfs@users.noreply.github.com>
Date: Fri, 31 May 2024 10:08:20 -0500
Subject: [PATCH] add CircuitBreaker timesTripped metric (#203)

* add CircuitBreaker timesTripped metric

* remove String.format

* toLowerCase add locale

* tidy

* move metrics management to its own map

* use stdlib Map.copyOf

* tidy
---
 .../solr/servlet/PrometheusMetricsServlet.java   | 16 ++++++++++++++++
 .../circuitbreaker/CircuitBreakerRegistry.java   | 15 +++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java
index 3239788e078..ee7d6d7f13e 100644
--- a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java
+++ b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.servlet;
 
+import static org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry.getTimesTrippedMetrics;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayOutputStream;
@@ -92,6 +94,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
       caller.call(qTime, metrics, request);
     }
     getCompressingDirectoryPoolMetrics(metrics);
+    getCircuitBreakerMetrics(metrics);
     getSharedCacheMetrics(metrics, getSolrDispatchFilter(request).getCores(), cacheMetricTypes);
     metrics.add(
         new PrometheusMetric(
@@ -193,6 +196,19 @@ private void getCompressingDirectoryPoolMetrics(List<PrometheusMetric> metrics)
     }
   }
 
+  private void getCircuitBreakerMetrics(List<PrometheusMetric> metrics) {
+    getTimesTrippedMetrics()
+        .forEach(
+            (k, v) -> {
+              metrics.add(
+                  new PrometheusMetric(
+                      "times_tripped" + k,
+                      PrometheusMetricType.COUNTER,
+                      "number of times circuit has been tripped",
+                      v));
+            });
+  }
+
   @SuppressWarnings({"rawtypes", "unchecked"})
   static void getSharedCacheMetrics(
       List<PrometheusMetric> results,
diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
index a63432f7aed..44afd6e5332 100644
--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
@@ -27,6 +27,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;
 import org.slf4j.Logger;
@@ -43,6 +44,7 @@ public class CircuitBreakerRegistry implements Closeable {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   private final Map<SolrRequestType, List<CircuitBreaker>> circuitBreakerMap = new HashMap<>();
+  private static final Map<String, Long> circuitBreakerTrippedMetrics = new ConcurrentHashMap<>();
 
   public CircuitBreakerRegistry() {}
 
@@ -82,6 +84,7 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
     for (CircuitBreaker circuitBreaker :
         circuitBreakerMap.getOrDefault(requestType, Collections.emptyList())) {
       if (circuitBreaker.isTripped()) {
+        incrementTripped(requestType, circuitBreaker);
         if (circuitBreaker.isDebugMode()) {
           if (log.isInfoEnabled()) {
             log.info(
@@ -102,6 +105,18 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
     return triggeredCircuitBreakers;
   }
 
+  private void incrementTripped(SolrRequestType requestType, CircuitBreaker circuitBreaker) {
+    String metricKey =
+        circuitBreaker.getClass().getSimpleName()
+            + "_"
+            + requestType.name().toLowerCase(Locale.ROOT);
+    circuitBreakerTrippedMetrics.merge(metricKey, 1L, Long::sum);
+  }
+
+  public static Map<String, Long> getTimesTrippedMetrics() {
+    return Map.copyOf(circuitBreakerTrippedMetrics);
+  }
+
   /**
    * Construct the final error message to be printed when circuit breakers trip.
    *