From 06fa954361be013aa4ecbb73959626122aaf5e4e Mon Sep 17 00:00:00 2001
From: Nick Ginther <104029658+nginthfs@users.noreply.github.com>
Date: Fri, 31 May 2024 10:08:20 -0500
Subject: [PATCH] add CircuitBreaker timesTripped metric (#203)

* add CircuitBreaker timesTripped metric

* remove String.format

* toLowerCase add locale

* tidy

* move metrics management to its own map

* use stdlib Map.copyOf

* tidy

(cherry picked from commit 9302dcf45452395a9d638dc66666225c2c043a73)
---
 .../solr/servlet/PrometheusMetricsServlet.java   | 16 ++++++++++++++++
 .../circuitbreaker/CircuitBreakerRegistry.java   | 15 +++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java
index 3239788e078..ee7d6d7f13e 100644
--- a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java
+++ b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.servlet;
 
+import static org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry.getTimesTrippedMetrics;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayOutputStream;
@@ -92,6 +94,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
       caller.call(qTime, metrics, request);
     }
     getCompressingDirectoryPoolMetrics(metrics);
+    getCircuitBreakerMetrics(metrics);
     getSharedCacheMetrics(metrics, getSolrDispatchFilter(request).getCores(), cacheMetricTypes);
     metrics.add(
         new PrometheusMetric(
@@ -193,6 +196,19 @@ private void getCompressingDirectoryPoolMetrics(List<PrometheusMetric> metrics)
     }
   }
 
+  private void getCircuitBreakerMetrics(List<PrometheusMetric> metrics) {
+    getTimesTrippedMetrics()
+        .forEach(
+            (k, v) -> {
+              metrics.add(
+                  new PrometheusMetric(
+                      "times_tripped" + k,
+                      PrometheusMetricType.COUNTER,
+                      "number of times circuit has been tripped",
+                      v));
+            });
+  }
+
   @SuppressWarnings({"rawtypes", "unchecked"})
   static void getSharedCacheMetrics(
       List<PrometheusMetric> results,
diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
index d602706a2cd..286d0e673a4 100644
--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java
@@ -27,6 +27,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -62,6 +63,7 @@ public class CircuitBreakerRegistry implements Closeable {
   public static final String SYSPROP_QUERY_MEM = SYSPROP_PREFIX + "query.mem";
   public static final String SYSPROP_QUERY_LOADAVG = SYSPROP_PREFIX + "query.loadavg";
   public static final String SYSPROP_WARN_ONLY_SUFFIX = ".warnonly";
+  private static final Map<String, Long> circuitBreakerTrippedMetrics = new ConcurrentHashMap<>();
 
   public CircuitBreakerRegistry(CoreContainer coreContainer) {
     initGlobal(coreContainer);
@@ -192,6 +194,7 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
     List<CircuitBreaker> triggeredCircuitBreakers = null;
     for (CircuitBreaker circuitBreaker : breakersOfType) {
       if (circuitBreaker.isTripped()) {
+        incrementTripped(requestType, circuitBreaker);
         if (triggeredCircuitBreakers == null) {
           triggeredCircuitBreakers = new ArrayList<>();
         }
@@ -203,6 +206,18 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
     return triggeredCircuitBreakers;
   }
 
+  private void incrementTripped(SolrRequestType requestType, CircuitBreaker circuitBreaker) {
+    String metricKey =
+        circuitBreaker.getClass().getSimpleName()
+            + "_"
+            + requestType.name().toLowerCase(Locale.ROOT);
+    circuitBreakerTrippedMetrics.merge(metricKey, 1L, Long::sum);
+  }
+
+  public static Map<String, Long> getTimesTrippedMetrics() {
+    return Map.copyOf(circuitBreakerTrippedMetrics);
+  }
+
   /**
    * Construct the final error message to be printed when circuit breakers trip.
    *