diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index ed28fffb05c75..35f3b58813794 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -518,6 +518,7 @@ public void testClusterStatsMetricFiltering() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.FS.metricName(), Metric.JVM.metricName(), Metric.PLUGINS.metricName(), Metric.OS.metricName())) + .applyMetricFiltering(true) .get(); assertNotNull(response.getNodesStats()); assertNotNull(response.getNodesStats().getJvm()); @@ -531,6 +532,8 @@ public void testClusterStatsMetricFiltering() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.INDICES.metricName())) + .indexMetrics(IndexMetrics.allIndicesMetrics()) + .applyMetricFiltering(true) .get(); assertNotNull(response.getIndicesStats()); assertNotNull(response.getIndicesStats().getMappings()); @@ -565,6 +568,7 @@ public void testClusterStatsMetricFiltering() { IndexMetrics.ANALYSIS.metricName() ) ) + .applyMetricFiltering(true) .get(); assertNotNull(response.getIndicesStats()); assertNotNull(response.getIndicesStats().getShards()); @@ -583,6 +587,7 @@ public void testClusterStatsMetricFiltering() { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of(Metric.OS.metricName(), Metric.PROCESS.metricName(), Metric.INDICES.metricName())) .indexMetrics(Set.of(IndexMetrics.SHARDS.metricName(), IndexMetrics.MAPPINGS.metricName())) + .applyMetricFiltering(true) .get(); assertNotNull(response.getNodesStats()); assertNotNull(response.getNodesStats().getOs()); @@ -602,6 +607,7 @@ public void testClusterStatsMetricFiltering() { .prepareClusterStats() .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Set.of("random_metric")) + .applyMetricFiltering(true) .get() ); @@ -613,6 +619,7 @@ public void testClusterStatsMetricFiltering() { .useAggregatedNodeLevelResponses(randomBoolean()) .requestMetrics(Metric.allMetrics()) .indexMetrics(Set.of("random_metric")) + .applyMetricFiltering(true) .get() ); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 224948027d8f2..9ebda488d3ac7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -90,44 +90,52 @@ public class ClusterStatsNodes implements ToXContentFragment { private final PackagingTypes packagingTypes; private final IngestStats ingestStats; + public static final Set NODE_STATS_METRICS = Set.of( + Metric.OS.metricName(), + Metric.PROCESS.metricName(), + Metric.JVM.metricName(), + Metric.FS.metricName(), + Metric.PLUGINS.metricName(), + Metric.INGEST.metricName(), + Metric.NETWORK_TYPES.metricName(), + Metric.DISCOVERY_TYPES.metricName(), + Metric.PACKAGING_TYPES.metricName() + ); + ClusterStatsNodes(List nodeResponses) { this(Metric.allMetrics(), nodeResponses); } ClusterStatsNodes(Set requestedMetrics, List nodeResponses) { this.versions = new HashSet<>(); - boolean isFSInfoRequested = ClusterStatsRequest.Metric.FS.containedIn(requestedMetrics); - boolean isPluginsInfoRequested = ClusterStatsRequest.Metric.PLUGINS.containedIn(requestedMetrics); - this.fs = isFSInfoRequested ? new FsInfo.Path() : null; - this.plugins = isPluginsInfoRequested ? new HashSet<>() : null; + this.fs = ClusterStatsRequest.Metric.FS.containedIn(requestedMetrics) ? new FsInfo.Path() : null; + this.plugins = ClusterStatsRequest.Metric.PLUGINS.containedIn(requestedMetrics) ? new HashSet<>() : null; Set seenAddresses = new HashSet<>(nodeResponses.size()); List nodeInfos = new ArrayList<>(nodeResponses.size()); - List nodesStats = new ArrayList<>(nodeResponses.size()); + List nodeStats = new ArrayList<>(nodeResponses.size()); for (ClusterStatsNodeResponse nodeResponse : nodeResponses) { - NodeInfo nodeInfo = nodeResponse.nodeInfo(); - NodeStats nodeStats = nodeResponse.nodeStats(); - nodeInfos.add(nodeInfo); - nodesStats.add(nodeStats); - this.versions.add(nodeInfo.getVersion()); - if (isPluginsInfoRequested) { - this.plugins.addAll(nodeInfo.getInfo(PluginsAndModules.class).getPluginInfos()); + nodeInfos.add(nodeResponse.nodeInfo()); + nodeStats.add(nodeResponse.nodeStats()); + this.versions.add(nodeResponse.nodeInfo().getVersion()); + if (this.plugins != null) { + this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos()); } // now do the stats that should be deduped by hardware (implemented by ip deduping) - TransportAddress publishAddress = nodeInfo.getInfo(TransportInfo.class).address().publishAddress(); + TransportAddress publishAddress = nodeResponse.nodeInfo().getInfo(TransportInfo.class).address().publishAddress(); final InetAddress inetAddress = publishAddress.address().getAddress(); if (!seenAddresses.add(inetAddress)) { continue; } - if (isFSInfoRequested && nodeStats.getFs() != null) { - this.fs.add(nodeStats.getFs().getTotal()); + if (this.fs != null && nodeResponse.nodeStats().getFs() != null) { + this.fs.add(nodeResponse.nodeStats().getFs().getTotal()); } } this.counts = new Counts(nodeInfos); - this.os = ClusterStatsRequest.Metric.OS.containedIn(requestedMetrics) ? new OsStats(nodeInfos, nodesStats) : null; - this.process = ClusterStatsRequest.Metric.PROCESS.containedIn(requestedMetrics) ? new ProcessStats(nodesStats) : null; - this.jvm = ClusterStatsRequest.Metric.JVM.containedIn(requestedMetrics) ? new JvmStats(nodeInfos, nodesStats) : null; + this.os = ClusterStatsRequest.Metric.OS.containedIn(requestedMetrics) ? new OsStats(nodeInfos, nodeStats) : null; + this.process = ClusterStatsRequest.Metric.PROCESS.containedIn(requestedMetrics) ? new ProcessStats(nodeStats) : null; + this.jvm = ClusterStatsRequest.Metric.JVM.containedIn(requestedMetrics) ? new JvmStats(nodeInfos, nodeStats) : null; this.networkTypes = ClusterStatsRequest.Metric.NETWORK_TYPES.containedIn(requestedMetrics) ? new NetworkTypes(nodeInfos) : null; this.discoveryTypes = ClusterStatsRequest.Metric.DISCOVERY_TYPES.containedIn(requestedMetrics) ? new DiscoveryTypes(nodeInfos) @@ -135,7 +143,7 @@ public class ClusterStatsNodes implements ToXContentFragment { this.packagingTypes = ClusterStatsRequest.Metric.PACKAGING_TYPES.containedIn(requestedMetrics) ? new PackagingTypes(nodeInfos) : null; - this.ingestStats = ClusterStatsRequest.Metric.INGEST.containedIn(requestedMetrics) ? new IngestStats(nodesStats) : null; + this.ingestStats = ClusterStatsRequest.Metric.INGEST.containedIn(requestedMetrics) ? new IngestStats(nodeStats) : null; } public Counts getCounts() { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 5abd69265f3bb..1526c817ddbdb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -52,8 +52,9 @@ @PublicApi(since = "1.0.0") public class ClusterStatsRequest extends BaseNodesRequest { - private final Set requestedMetrics = new HashSet<>(Metric.allMetrics()); - private final Set indicesMetrics = new HashSet<>(IndexMetrics.allIndicesMetrics()); + private final Set requestedMetrics = new HashSet<>(); + private final Set indexMetricsRequested = new HashSet<>(); + private Boolean applyMetricFiltering = false; public ClusterStatsRequest(StreamInput in) throws IOException { super(in); @@ -61,10 +62,9 @@ public ClusterStatsRequest(StreamInput in) throws IOException { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } if (in.getVersion().onOrAfter(Version.V_3_0_0)) { - requestedMetrics.clear(); - indicesMetrics.clear(); + applyMetricFiltering = in.readOptionalBoolean(); requestedMetrics.addAll(in.readStringList()); - indicesMetrics.addAll(in.readStringList()); + indexMetricsRequested.addAll(in.readStringList()); } } @@ -86,6 +86,14 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon this.useAggregatedNodeLevelResponses = useAggregatedNodeLevelResponses; } + public boolean applyMetricFiltering() { + return applyMetricFiltering; + } + + public void applyMetricFiltering(boolean honourMetricFiltering) { + this.applyMetricFiltering = honourMetricFiltering; + } + /** * Add Metric */ @@ -111,12 +119,12 @@ public ClusterStatsRequest addIndexMetric(String indexMetric) { if (IndexMetrics.allIndicesMetrics().contains(indexMetric) == false) { throw new IllegalStateException("Used an illegal index metric: " + indexMetric); } - indicesMetrics.add(indexMetric); + indexMetricsRequested.add(indexMetric); return this; } public Set indicesMetrics() { - return new HashSet<>(indicesMetrics); + return new HashSet<>(indexMetricsRequested); } public void clearRequestedMetrics() { @@ -124,7 +132,7 @@ public void clearRequestedMetrics() { } public void clearIndicesMetrics() { - indicesMetrics.clear(); + indexMetricsRequested.clear(); } @Override @@ -134,8 +142,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalBoolean(applyMetricFiltering); out.writeStringArray(requestedMetrics.toArray(new String[0])); - out.writeStringArray(indicesMetrics.toArray(new String[0])); + out.writeStringArray(indexMetricsRequested.toArray(new String[0])); } } @@ -180,6 +189,7 @@ public boolean containedIn(Set metricNames) { * An enumeration of the "core" sections of indices metrics that may be requested * from the cluster stats endpoint. */ + @PublicApi(since = "3.0.0") public enum IndexMetrics { SHARDS("shards"), DOCS("docs"), diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index 619cca950b786..63305825d3ed3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -58,14 +58,17 @@ public final ClusterStatsRequestBuilder useAggregatedNodeLevelResponses(boolean return this; } + public final ClusterStatsRequestBuilder applyMetricFiltering(boolean applyMetricFiltering) { + request.applyMetricFiltering(applyMetricFiltering); + return this; + } + public final ClusterStatsRequestBuilder requestMetrics(Set requestMetrics) { - request.clearRequestedMetrics(); requestMetrics.forEach(request::addMetric); return this; } public final ClusterStatsRequestBuilder indexMetrics(Set indexMetrics) { - request.clearIndicesMetrics(); indexMetrics.forEach(request::addIndexMetric); return this; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java index fd2a72992b4d5..f994b0ddb78b1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -116,7 +116,7 @@ public ClusterStatsResponse( super(clusterName, nodes, failures); this.clusterUUID = clusterUUID; this.timestamp = timestamp; - nodesStats = !requestedMetrics.isEmpty() && !requestedMetrics.equals(Set.of(Metric.INDICES.metricName())) + nodesStats = requestedMetrics.stream().anyMatch(ClusterStatsNodes.NODE_STATS_METRICS::contains) ? new ClusterStatsNodes(requestedMetrics, nodes) : null; MappingStats mappingStats = ClusterStatsRequest.IndexMetrics.MAPPINGS.containedIn(indicesMetrics) ? MappingStats.of(state) : null; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 403b30268b684..5bae001c15bd1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -65,6 +65,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -78,13 +79,20 @@ public class TransportClusterStatsAction extends TransportNodesAction< TransportClusterStatsAction.ClusterStatsNodeRequest, ClusterStatsNodeResponse> { - private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags( + private static final Map INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP = Map.of( + ClusterStatsRequest.IndexMetrics.DOCS.metricName(), CommonStatsFlags.Flag.Docs, + ClusterStatsRequest.IndexMetrics.STORE.metricName(), CommonStatsFlags.Flag.Store, + ClusterStatsRequest.IndexMetrics.FIELDDATA.metricName(), CommonStatsFlags.Flag.FieldData, + ClusterStatsRequest.IndexMetrics.QUERY_CACHE.metricName(), CommonStatsFlags.Flag.QueryCache, + ClusterStatsRequest.IndexMetrics.COMPLETION.metricName(), CommonStatsFlags.Flag.Completion, + ClusterStatsRequest.IndexMetrics.SEGMENTS.metricName(), CommonStatsFlags.Flag.Segments + ); private final NodeService nodeService; @@ -126,16 +134,27 @@ protected ClusterStatsResponse newResponse( + " the cluster state that are too slow for a transport thread" ); ClusterState state = clusterService.state(); - return new ClusterStatsResponse( - System.currentTimeMillis(), - state.metadata().clusterUUID(), - clusterService.getClusterName(), - responses, - failures, - state, - request.requestedMetrics(), - request.indicesMetrics() - ); + if (request.applyMetricFiltering()) { + return new ClusterStatsResponse( + System.currentTimeMillis(), + state.metadata().clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + state, + request.requestedMetrics(), + request.indicesMetrics() + ); + } else { + return new ClusterStatsResponse( + System.currentTimeMillis(), + state.metadata().clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + state + ); + } } @Override @@ -151,20 +170,21 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce @Override protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) { NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false, false); + boolean applyMetricFiltering = nodeRequest.request.applyMetricFiltering(); Set requestedMetrics = nodeRequest.request.requestedMetrics(); NodeStats nodeStats = nodeService.stats( CommonStatsFlags.NONE, - Metric.OS.containedIn(requestedMetrics), - Metric.PROCESS.containedIn(requestedMetrics), - Metric.JVM.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.OS.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.PROCESS.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.JVM.containedIn(requestedMetrics), false, - Metric.FS.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.FS.containedIn(requestedMetrics), false, false, false, false, false, - Metric.INGEST.containedIn(requestedMetrics), + !applyMetricFiltering || Metric.INGEST.containedIn(requestedMetrics), false, false, false, @@ -182,7 +202,13 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false ); List shardsStats = new ArrayList<>(); - if (Metric.INDICES.containedIn(requestedMetrics)) { + if (!applyMetricFiltering || Metric.INDICES.containedIn(requestedMetrics)) { + CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); + for (String metric : nodeRequest.request.indicesMetrics()) { + if (INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.containsKey(metric)) { + commonStatsFlags.set(INDEX_METRIC_TO_SHARDS_STATS_FLAG_MAP.get(metric), true); + } + } for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { @@ -204,7 +230,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq new ShardStats( indexShard.routingEntry(), indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, commonStatsFlags), commitStats, seqNoStats, retentionLeaseStats diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index 398b75de38e06..e3858c27815ab 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -100,36 +100,38 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - String[] nodeIds = request.paramAsStringArray("nodeId", null); - Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", null)); Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", null)); - logger.info("Request URI path params, metrics : {}, index_metrics : {}, nodes : {}", metrics, indexMetrics, nodeIds); + String[] nodeIds = request.paramAsStringArray("nodeId", null); ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); clusterStatsRequest.timeout(request.param("timeout")); clusterStatsRequest.useAggregatedNodeLevelResponses(true); - if (metrics.size() > 1 && metrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ); - } else if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual index metrics [%s]", - request.path(), - request.param("sub_metric") - ) - ); - } else { - clusterStatsRequest.clearRequestedMetrics(); - clusterStatsRequest.clearIndicesMetrics(); + if (!metrics.isEmpty()) { + if (metrics.size() > 1 && metrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ); + } + + if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual index metrics [%s]", + request.path(), + request.param("sub_metric") + ) + ); + } + + clusterStatsRequest.applyMetricFiltering(true); final Set metricsRequested = new HashSet<>(); if (metrics.contains("_all")) { @@ -160,16 +162,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } - logger.info("Computed metrics: {}", metricsRequested); if (ClusterStatsRequest.Metric.INDICES.containedIn(metricsRequested)) { - logger.info("Stats request contains indices metric"); final Set indexMetricsRequested = new HashSet<>(); if (indexMetrics.isEmpty() || indexMetrics.contains("_all")) { indexMetricsRequested.addAll(ClusterStatsRequest.IndexMetrics.allIndicesMetrics()); } else { indexMetricsRequested.addAll(indexMetrics); } - logger.info("Computed index metrics: {}", indexMetricsRequested); final Set invalidIndexMetrics = new TreeSet<>(); for (String indexMetric : indexMetricsRequested) { Consumer clusterStatsRequestConsumer = INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.get(indexMetric); @@ -186,7 +185,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } } - } return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java index c167cf0bbe46b..1c64c01eb0e49 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilderTests.java @@ -44,6 +44,16 @@ public void testUseAggregatedNodeLevelResponses() { assertFalse(clusterStatsRequestBuilder.request().useAggregatedNodeLevelResponses()); } + public void testApplyMetricFiltering() { + ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( + this.testClient, + ClusterStatsAction.INSTANCE + ); + assertFalse(clusterStatsRequestBuilder.request().applyMetricFiltering()); + clusterStatsRequestBuilder.applyMetricFiltering(true); + assertTrue(clusterStatsRequestBuilder.request().applyMetricFiltering()); + } + public void testRequestedMetrics() { ClusterStatsRequestBuilder clusterStatsRequestBuilder = new ClusterStatsRequestBuilder( this.testClient,