From 254cba48f004192ba6f262fad71c5ce985945abc Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Thu, 7 Sep 2023 17:49:42 -0400 Subject: [PATCH 1/2] get node-level cache metrics into Prometheus --- .../servlet/PrometheusMetricsServlet.java | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java index 1240822e568..dac9c33110f 100644 --- a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java +++ b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java @@ -31,6 +31,7 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -72,9 +73,10 @@ public final class PrometheusMetricsServlet extends BaseSolrServlet { new OsMetricsApiCaller(), new ThreadMetricsApiCaller(), new StatusCodeMetricsApiCaller(), - new CoresMetricsApiCaller())); + new CoresMetricsApiCaller(), + new NodeCacheMetricsApiCaller())); - private final Map cacheMetricTypes = + private static final Map cacheMetricTypes = Map.of( "bytesUsed", PrometheusMetricType.GAUGE, "lookups", PrometheusMetricType.COUNTER, @@ -143,6 +145,41 @@ static void getSharedCacheMetrics( } } + static class NodeCacheMetricsApiCaller extends MetricsApiCaller { + private static final String PREFIX = "CACHE.nodeLevelCache/"; + NodeCacheMetricsApiCaller() { + super( + "node", + PREFIX, + ""); + } + + @Override + protected void handle(List results, JsonNode metrics) throws IOException { + JsonNode parent = metrics.path("solr.node"); + Iterator> caches = parent.fields(); + while (caches.hasNext()) { + Map.Entry cacheEntry = caches.next(); + String cacheName = cacheEntry.getKey().substring(PREFIX.length()).toLowerCase(Locale.ROOT); + Iterator> fields = cacheEntry.getValue().fields(); + while (fields.hasNext()) { + Map.Entry next = fields.next(); + String fieldName = next.getKey(); + PrometheusMetricType type = cacheMetricTypes.get(fieldName); + if (type != null) { + results.add( + new PrometheusMetric( + String.format(Locale.ROOT, "cache_%s_%s", cacheName, fieldName), + type, + String.format( + Locale.ROOT, "%s %s for cache %s", fieldName, type.getDisplayName(), cacheName), + next.getValue().numberValue())); + } + } + } + } + } + static class GarbageCollectorMetricsApiCaller extends MetricsApiCaller { GarbageCollectorMetricsApiCaller() { From 823fb238acdaad38633bfb1a8a5fed76c6099a25 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Fri, 8 Sep 2023 09:44:05 -0400 Subject: [PATCH 2/2] add test; compatibility mapping of field names --- .../servlet/PrometheusMetricsServlet.java | 36 +++++++++++---- .../servlet/PrometheusMetricsServletTest.java | 44 +++++++++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java index dac9c33110f..98f1dbf95c2 100644 --- a/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java +++ b/solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java @@ -25,6 +25,7 @@ import java.lang.invoke.MethodHandles; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -76,7 +77,7 @@ public final class PrometheusMetricsServlet extends BaseSolrServlet { new CoresMetricsApiCaller(), new NodeCacheMetricsApiCaller())); - private static final Map cacheMetricTypes = + private final Map cacheMetricTypes = Map.of( "bytesUsed", PrometheusMetricType.GAUGE, "lookups", PrometheusMetricType.COUNTER, @@ -84,6 +85,19 @@ public final class PrometheusMetricsServlet extends BaseSolrServlet { "puts", PrometheusMetricType.COUNTER, "evictions", PrometheusMetricType.COUNTER); + /** + * node-level caches use slightly different terminology for some fields, so we map to old + * terminology for consistency. + */ + private static final Map> + nodeLevelCacheMetricTypes = + Map.of( + "ramBytesUsed", new SimpleImmutableEntry<>("bytesUsed", PrometheusMetricType.GAUGE), + "lookups", new SimpleImmutableEntry<>("lookups", PrometheusMetricType.COUNTER), + "hits", new SimpleImmutableEntry<>("hits", PrometheusMetricType.COUNTER), + "inserts", new SimpleImmutableEntry<>("puts", PrometheusMetricType.COUNTER), + "evictions", new SimpleImmutableEntry<>("evictions", PrometheusMetricType.COUNTER)); + @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, UnavailableException { @@ -147,11 +161,9 @@ static void getSharedCacheMetrics( static class NodeCacheMetricsApiCaller extends MetricsApiCaller { private static final String PREFIX = "CACHE.nodeLevelCache/"; + NodeCacheMetricsApiCaller() { - super( - "node", - PREFIX, - ""); + super("node", PREFIX, ""); } @Override @@ -164,15 +176,21 @@ protected void handle(List results, JsonNode metrics) throws I Iterator> fields = cacheEntry.getValue().fields(); while (fields.hasNext()) { Map.Entry next = fields.next(); - String fieldName = next.getKey(); - PrometheusMetricType type = cacheMetricTypes.get(fieldName); - if (type != null) { + Map.Entry typeEntry = + nodeLevelCacheMetricTypes.get(next.getKey()); + if (typeEntry != null) { + String fieldName = typeEntry.getKey(); + PrometheusMetricType type = typeEntry.getValue(); results.add( new PrometheusMetric( String.format(Locale.ROOT, "cache_%s_%s", cacheName, fieldName), type, String.format( - Locale.ROOT, "%s %s for cache %s", fieldName, type.getDisplayName(), cacheName), + Locale.ROOT, + "%s %s for cache %s", + fieldName, + type.getDisplayName(), + cacheName), next.getValue().numberValue())); } } diff --git a/solr/core/src/test/org/apache/solr/servlet/PrometheusMetricsServletTest.java b/solr/core/src/test/org/apache/solr/servlet/PrometheusMetricsServletTest.java index 57a029bb976..0bf4ffe520a 100644 --- a/solr/core/src/test/org/apache/solr/servlet/PrometheusMetricsServletTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/PrometheusMetricsServletTest.java @@ -478,4 +478,48 @@ public void testCoresMetricsApiCallerMissingIndex() throws Exception { + "deletes_by_query 66\n"; assertMetricsApiCaller(new PrometheusMetricsServlet.CoresMetricsApiCaller(), json, 25, output); } + + @Test + public void testNodeCacheMetricsApiCaller() throws Exception { + String json = + "{\n" + + " \"responseHeader\":{\n" + + " \"status\":0,\n" + + " \"QTime\":25},\n" + + " \"metrics\":{\n" + + " \"solr.node\":{\n" + + " \"CACHE.nodeLevelCache/myCache\":{\n" + + " \"lookups\":2,\n" + + " \"hits\":1,\n" + + " \"hitratio\":0.5,\n" + + " \"inserts\":1,\n" + + " \"evictions\":0,\n" + + " \"size\":1,\n" + + " \"warmupTime\":0,\n" + + " \"ramBytesUsed\":623,\n" + + " \"maxRamMB\":-1,\n" + + " \"cumulative_lookups\":2,\n" + + " \"cumulative_hits\":1,\n" + + " \"cumulative_hitratio\":0.5,\n" + + " \"cumulative_inserts\":1,\n" + + " \"cumulative_evictions\":0}}}}"; + String output = + "# HELP cache_mycache_lookups lookups counter for cache mycache\n" + + "# TYPE cache_mycache_lookups counter\n" + + "cache_mycache_lookups 2\n" + + "# HELP cache_mycache_hits hits counter for cache mycache\n" + + "# TYPE cache_mycache_hits counter\n" + + "cache_mycache_hits 1\n" + + "# HELP cache_mycache_puts puts counter for cache mycache\n" + + "# TYPE cache_mycache_puts counter\n" + + "cache_mycache_puts 1\n" + + "# HELP cache_mycache_evictions evictions counter for cache mycache\n" + + "# TYPE cache_mycache_evictions counter\n" + + "cache_mycache_evictions 0\n" + + "# HELP cache_mycache_bytes_used bytesUsed gauge for cache mycache\n" + + "# TYPE cache_mycache_bytes_used gauge\n" + + "cache_mycache_bytes_used 623\n"; + assertMetricsApiCaller( + new PrometheusMetricsServlet.NodeCacheMetricsApiCaller(), json, 25, output); + } }