Skip to content

Commit

Permalink
add CircuitBreaker timesTripped metric (#203)
Browse files Browse the repository at this point in the history
* add CircuitBreaker timesTripped metric

* remove String.format

* toLowerCase add locale

* tidy

* move metrics management to its own map

* use stdlib Map.copyOf

* tidy

(cherry picked from commit 9302dcf)
  • Loading branch information
nginthfs authored and magibney committed Oct 10, 2024
1 parent c143929 commit 06fa954
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package org.apache.solr.servlet;

import static org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry.getTimesTrippedMetrics;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.ByteArrayOutputStream;
Expand Down Expand Up @@ -92,6 +94,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
caller.call(qTime, metrics, request);
}
getCompressingDirectoryPoolMetrics(metrics);
getCircuitBreakerMetrics(metrics);
getSharedCacheMetrics(metrics, getSolrDispatchFilter(request).getCores(), cacheMetricTypes);
metrics.add(
new PrometheusMetric(
Expand Down Expand Up @@ -193,6 +196,19 @@ private void getCompressingDirectoryPoolMetrics(List<PrometheusMetric> metrics)
}
}

private void getCircuitBreakerMetrics(List<PrometheusMetric> metrics) {
getTimesTrippedMetrics()
.forEach(
(k, v) -> {
metrics.add(
new PrometheusMetric(
"times_tripped" + k,
PrometheusMetricType.COUNTER,
"number of times circuit has been tripped",
v));
});
}

@SuppressWarnings({"rawtypes", "unchecked"})
static void getSharedCacheMetrics(
List<PrometheusMetric> results,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -62,6 +63,7 @@ public class CircuitBreakerRegistry implements Closeable {
public static final String SYSPROP_QUERY_MEM = SYSPROP_PREFIX + "query.mem";
public static final String SYSPROP_QUERY_LOADAVG = SYSPROP_PREFIX + "query.loadavg";
public static final String SYSPROP_WARN_ONLY_SUFFIX = ".warnonly";
private static final Map<String, Long> circuitBreakerTrippedMetrics = new ConcurrentHashMap<>();

public CircuitBreakerRegistry(CoreContainer coreContainer) {
initGlobal(coreContainer);
Expand Down Expand Up @@ -192,6 +194,7 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
List<CircuitBreaker> triggeredCircuitBreakers = null;
for (CircuitBreaker circuitBreaker : breakersOfType) {
if (circuitBreaker.isTripped()) {
incrementTripped(requestType, circuitBreaker);
if (triggeredCircuitBreakers == null) {
triggeredCircuitBreakers = new ArrayList<>();
}
Expand All @@ -203,6 +206,18 @@ public List<CircuitBreaker> checkTripped(SolrRequestType requestType) {
return triggeredCircuitBreakers;
}

private void incrementTripped(SolrRequestType requestType, CircuitBreaker circuitBreaker) {
String metricKey =
circuitBreaker.getClass().getSimpleName()
+ "_"
+ requestType.name().toLowerCase(Locale.ROOT);
circuitBreakerTrippedMetrics.merge(metricKey, 1L, Long::sum);
}

public static Map<String, Long> getTimesTrippedMetrics() {
return Map.copyOf(circuitBreakerTrippedMetrics);
}

/**
* Construct the final error message to be printed when circuit breakers trip.
*
Expand Down

0 comments on commit 06fa954

Please sign in to comment.