From e4329b8e51da1613a446756f31096d2e4ecda883 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Fri, 12 Jul 2024 17:12:48 +0530 Subject: [PATCH 01/15] Optimize Cluster Stats Indices to precomute node level stats Signed-off-by: Pranshu Shukla --- .../admin/cluster/stats/ClusterStatsIT.java | 74 ++++-- .../cluster/stats/ClusterStatsIndices.java | 86 ++++-- .../stats/ClusterStatsNodeResponse.java | 43 ++- .../cluster/stats/ClusterStatsRequest.java | 17 ++ .../stats/ClusterStatsRequestBuilder.java | 5 + .../cluster/stats/NodeIndexShardStats.java | 98 +++++++ .../stats/TransportClusterStatsAction.java | 10 +- .../admin/cluster/RestClusterStatsAction.java | 1 + .../cluster/stats/ClusterStatsNodesTests.java | 244 ++++++++++++++++++ 9 files changed, 536 insertions(+), 42 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/stats/NodeIndexShardStats.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index 085a32593063a..c0b1539b3b8d8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -88,7 +88,11 @@ public void testNodeCounts() { Map expectedCounts = getExpectedCounts(1, 1, 1, 1, 1, 0, 0); int numNodes = randomIntBetween(1, 5); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertCounts(response.getNodesStats().getCounts(), total, expectedCounts); for (int i = 0; i < numNodes; i++) { @@ -153,7 +157,11 @@ public void testNodeCountsWithDeprecatedMasterRole() throws ExecutionException, Map expectedCounts = getExpectedCounts(0, 1, 1, 0, 0, 0, 0); Client client = client(); - ClusterStatsResponse response = client.admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse response = client.admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertCounts(response.getNodesStats().getCounts(), total, expectedCounts); Set expectedRoles = Set.of(DiscoveryNodeRole.MASTER_ROLE.roleName()); @@ -179,7 +187,11 @@ private void assertShardStats(ClusterStatsIndices.ShardStats stats, int indices, public void testIndicesShardStats() throws ExecutionException, InterruptedException { internalCluster().startNode(); ensureGreen(); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get(); @@ -195,14 +207,14 @@ public void testIndicesShardStats() throws ExecutionException, InterruptedExcept ensureGreen(); index("test1", "type", "1", "f", "f"); refresh(); // make the doc visible - response = client().admin().cluster().prepareClusterStats().get(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(1L)); assertShardStats(response.getIndicesStats().getShards(), 1, 4, 2, 1.0); prepareCreate("test2").setSettings(Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 0)).get(); ensureGreen(); - response = client().admin().cluster().prepareClusterStats().get(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(2)); assertShardStats(response.getIndicesStats().getShards(), 2, 7, 5, 2.0 / 5); @@ -225,7 +237,11 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte internalCluster().startNodes(randomIntBetween(1, 3)); index("test1", "type", "1", "f", "f"); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); String msg = response.toString(); assertThat(msg, response.getTimestamp(), Matchers.greaterThan(946681200000L)); // 1 Jan 2000 assertThat(msg, response.indicesStats.getStore().getSizeInBytes(), Matchers.greaterThan(0L)); @@ -265,13 +281,21 @@ public void testAllocatedProcessors() throws Exception { internalCluster().startNode(Settings.builder().put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 7).build()); waitForNodes(1); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertThat(response.getNodesStats().getOs().getAllocatedProcessors(), equalTo(7)); } public void testClusterStatusWhenStateNotRecovered() throws Exception { internalCluster().startClusterManagerOnlyNode(Settings.builder().put("gateway.recover_after_nodes", 2).build()); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED)); if (randomBoolean()) { @@ -281,14 +305,18 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception { } // wait for the cluster status to settle ensureGreen(); - response = client().admin().cluster().prepareClusterStats().get(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); } public void testFieldTypes() { internalCluster().startNode(); ensureGreen(); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); @@ -301,7 +329,7 @@ public void testFieldTypes() { + "\"eggplant\":{\"type\":\"integer\"}}}}}" ) .get(); - response = client().admin().cluster().prepareClusterStats().get(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3)); Set stats = response.getIndicesStats().getMappings().getFieldTypeStats(); for (IndexFeatureStats stat : stats) { @@ -329,7 +357,11 @@ public void testNodeRolesWithMasterLegacySettings() throws ExecutionException, I Map expectedCounts = getExpectedCounts(0, 1, 1, 0, 1, 0, 0); Client client = client(); - ClusterStatsResponse clusterStatsResponse = client.admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse clusterStatsResponse = client.admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedCounts); Set expectedRoles = Set.of( @@ -359,7 +391,11 @@ public void testNodeRolesWithClusterManagerRole() throws ExecutionException, Int Map expectedCounts = getExpectedCounts(0, 1, 1, 0, 1, 0, 0); Client client = client(); - ClusterStatsResponse clusterStatsResponse = client.admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse clusterStatsResponse = client.admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedCounts); Set expectedRoles = Set.of( @@ -383,7 +419,11 @@ public void testNodeRolesWithSeedDataNodeLegacySettings() throws ExecutionExcept Map expectedRoleCounts = getExpectedCounts(1, 1, 1, 0, 1, 0, 0); Client client = client(); - ClusterStatsResponse clusterStatsResponse = client.admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse clusterStatsResponse = client.admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedRoleCounts); Set expectedRoles = Set.of( @@ -410,7 +450,11 @@ public void testNodeRolesWithDataNodeLegacySettings() throws ExecutionException, Map expectedRoleCounts = getExpectedCounts(1, 1, 1, 0, 1, 0, 0); Client client = client(); - ClusterStatsResponse clusterStatsResponse = client.admin().cluster().prepareClusterStats().get(); + ClusterStatsResponse clusterStatsResponse = client.admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(randomBoolean()) + .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedRoleCounts); Set> expectedNodesRoles = Set.of( diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 26e554f44fca1..06678e335df01 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -34,6 +34,9 @@ import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.cache.query.QueryCacheStats; @@ -78,26 +81,44 @@ public ClusterStatsIndices(List nodeResponses, Mapping this.segments = new SegmentsStats(); for (ClusterStatsNodeResponse r : nodeResponses) { - for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { - ShardStats indexShardStats = countsPerIndex.get(shardStats.getShardRouting().getIndexName()); - if (indexShardStats == null) { - indexShardStats = new ShardStats(); - countsPerIndex.put(shardStats.getShardRouting().getIndexName(), indexShardStats); - } - - indexShardStats.total++; - - CommonStats shardCommonStats = shardStats.getStats(); - - if (shardStats.getShardRouting().primary()) { - indexShardStats.primaries++; - docs.add(shardCommonStats.docs); + // Optimized response from the node + if (r.getNodeIndexShardStats() != null) { + r.getNodeIndexShardStats().indexStatsMap.forEach( + (index, indexCountStats) -> countsPerIndex.merge(index, indexCountStats, (v1, v2) -> { + v1.addStatsFrom(v2); + return v1; + }) + ); + + docs.add(r.getNodeIndexShardStats().docs); + store.add(r.getNodeIndexShardStats().store); + fieldData.add(r.getNodeIndexShardStats().fieldData); + queryCache.add(r.getNodeIndexShardStats().queryCache); + completion.add(r.getNodeIndexShardStats().completion); + segments.add(r.getNodeIndexShardStats().segments); + } else { + // Default response from the node + for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { + ShardStats indexShardStats = countsPerIndex.get(shardStats.getShardRouting().getIndexName()); + if (indexShardStats == null) { + indexShardStats = new ShardStats(); + countsPerIndex.put(shardStats.getShardRouting().getIndexName(), indexShardStats); + } + + indexShardStats.total++; + + CommonStats shardCommonStats = shardStats.getStats(); + + if (shardStats.getShardRouting().primary()) { + indexShardStats.primaries++; + docs.add(shardCommonStats.docs); + } + store.add(shardCommonStats.store); + fieldData.add(shardCommonStats.fieldData); + queryCache.add(shardCommonStats.queryCache); + completion.add(shardCommonStats.completion); + segments.add(shardCommonStats.segments); } - store.add(shardCommonStats.store); - fieldData.add(shardCommonStats.fieldData); - queryCache.add(shardCommonStats.queryCache); - completion.add(shardCommonStats.completion); - segments.add(shardCommonStats.segments); } } @@ -185,11 +206,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws * @opensearch.api */ @PublicApi(since = "1.0.0") - public static class ShardStats implements ToXContentFragment { + public static class ShardStats implements ToXContentFragment, Writeable { - int indices; - int total; - int primaries; + int indices = 0; + int total = 0; + int primaries = 0; // min/max int minIndexShards = -1; @@ -202,6 +223,12 @@ public static class ShardStats implements ToXContentFragment { public ShardStats() {} + public ShardStats(StreamInput in) throws IOException { + indices = in.readVInt(); + total = in.readVInt(); + primaries = in.readVInt(); + } + /** * number of indices in the cluster */ @@ -329,6 +356,19 @@ public void addIndexShardCount(ShardStats indexShardCount) { } } + public void addStatsFrom(ShardStats incomingStats) { + this.total += incomingStats.getTotal(); + this.indices += incomingStats.getIndices(); + this.primaries += incomingStats.getPrimaries(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(indices); + out.writeVInt(total); + out.writeVInt(primaries); + } + /** * Inner Fields used for creating XContent and parsing * diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index 1b25bf84356d6..6df24f44432e1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.cluster.stats; +import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.indices.stats.ShardStats; @@ -55,6 +56,7 @@ public class ClusterStatsNodeResponse extends BaseNodeResponse { private final NodeStats nodeStats; private final ShardStats[] shardsStats; private ClusterHealthStatus clusterStatus; + private NodeIndexShardStats nodeIndexShardStats; public ClusterStatsNodeResponse(StreamInput in) throws IOException { super(in); @@ -64,7 +66,12 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException { } this.nodeInfo = new NodeInfo(in); this.nodeStats = new NodeStats(in); - shardsStats = in.readArray(ShardStats::new, ShardStats[]::new); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.shardsStats = in.readOptionalArray(ShardStats::new, ShardStats[]::new); + this.nodeIndexShardStats = in.readOptionalWriteable(NodeIndexShardStats::new); + } else { + this.shardsStats = in.readArray(ShardStats::new, ShardStats[]::new); + } } public ClusterStatsNodeResponse( @@ -81,6 +88,24 @@ public ClusterStatsNodeResponse( this.clusterStatus = clusterStatus; } + public ClusterStatsNodeResponse( + DiscoveryNode node, + @Nullable ClusterHealthStatus clusterStatus, + NodeInfo nodeInfo, + NodeStats nodeStats, + ShardStats[] shardsStats, + boolean optimized + ) { + super(node); + this.nodeInfo = nodeInfo; + this.nodeStats = nodeStats; + if (optimized) { + this.nodeIndexShardStats = new NodeIndexShardStats(node, shardsStats); + } + this.shardsStats = shardsStats; + this.clusterStatus = clusterStatus; + } + public NodeInfo nodeInfo() { return this.nodeInfo; } @@ -101,6 +126,10 @@ public ShardStats[] shardsStats() { return this.shardsStats; } + public NodeIndexShardStats getNodeIndexShardStats() { + return nodeIndexShardStats; + } + public static ClusterStatsNodeResponse readNodeResponse(StreamInput in) throws IOException { return new ClusterStatsNodeResponse(in); } @@ -116,6 +145,16 @@ public void writeTo(StreamOutput out) throws IOException { } nodeInfo.writeTo(out); nodeStats.writeTo(out); - out.writeArray(shardsStats); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (nodeIndexShardStats != null) { + out.writeOptionalArray(null); + out.writeOptionalWriteable(nodeIndexShardStats); + } else { + out.writeOptionalArray(shardsStats); + out.writeOptionalWriteable(null); + } + } else { + out.writeArray(shardsStats); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 6a99451c596ed..f85c964c1eea8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.cluster.stats; +import org.opensearch.Version; import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; @@ -49,8 +50,13 @@ public class ClusterStatsRequest extends BaseNodesRequest { public ClusterStatsRequest(StreamInput in) throws IOException { super(in); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + useOptimizedClusterStatsResponse = in.readOptionalBoolean(); + } } + private Boolean useOptimizedClusterStatsResponse = false; + /** * Get stats from nodes based on the nodes ids specified. If none are passed, stats * based on all nodes will be returned. @@ -59,9 +65,20 @@ public ClusterStatsRequest(String... nodesIds) { super(nodesIds); } + public boolean useOptimizedClusterStatsResponse() { + return useOptimizedClusterStatsResponse; + } + + public void useOptimizedClusterStatsResponse(boolean useOptimizedClusterStatsResponse) { + this.useOptimizedClusterStatsResponse = useOptimizedClusterStatsResponse; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalBoolean(useOptimizedClusterStatsResponse); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index 0dcb03dc26d0e..3a07382cbf93e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -50,4 +50,9 @@ public class ClusterStatsRequestBuilder extends NodesOperationRequestBuilder< public ClusterStatsRequestBuilder(OpenSearchClient client, ClusterStatsAction action) { super(client, action, new ClusterStatsRequest()); } + + public final ClusterStatsRequestBuilder useOptimizedClusterStatsResponse(boolean useOptimizedClusterStatsResponse) { + request.useOptimizedClusterStatsResponse(useOptimizedClusterStatsResponse); + return (ClusterStatsRequestBuilder) this; + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/NodeIndexShardStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/NodeIndexShardStats.java new file mode 100644 index 0000000000000..e782e21730125 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/NodeIndexShardStats.java @@ -0,0 +1,98 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.stats; + +import org.opensearch.action.admin.indices.stats.CommonStats; +import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.index.cache.query.QueryCacheStats; +import org.opensearch.index.engine.SegmentsStats; +import org.opensearch.index.fielddata.FieldDataStats; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.store.StoreStats; +import org.opensearch.search.suggest.completion.CompletionStats; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Node level statistics used for ClusterStatsIndices for _cluster/stats call. + */ +public class NodeIndexShardStats extends BaseNodeResponse { + + DocsStats docs; + StoreStats store; + FieldDataStats fieldData; + QueryCacheStats queryCache; + CompletionStats completion; + SegmentsStats segments; + Map indexStatsMap; + + protected NodeIndexShardStats(StreamInput in) throws IOException { + super(in); + docs = in.readOptionalWriteable(DocsStats::new); + store = in.readOptionalWriteable(StoreStats::new); + fieldData = in.readOptionalWriteable(FieldDataStats::new); + queryCache = in.readOptionalWriteable(QueryCacheStats::new); + completion = in.readOptionalWriteable(CompletionStats::new); + segments = in.readOptionalWriteable(SegmentsStats::new); + indexStatsMap = in.readMap(StreamInput::readString, ClusterStatsIndices.ShardStats::new); + } + + protected NodeIndexShardStats(DiscoveryNode node, ShardStats[] indexShardsStats) { + super(node); + + this.docs = new DocsStats(); + this.store = new StoreStats(); + this.fieldData = new FieldDataStats(); + this.queryCache = new QueryCacheStats(); + this.completion = new CompletionStats(); + this.segments = new SegmentsStats(); + this.indexStatsMap = new HashMap<>(); + + // Index Level Stats + for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : indexShardsStats) { + ClusterStatsIndices.ShardStats indexShardStats = this.indexStatsMap.get(shardStats.getShardRouting().getIndexName()); + if (indexShardStats == null) { + indexShardStats = new ClusterStatsIndices.ShardStats(); + this.indexStatsMap.put(shardStats.getShardRouting().getIndexName(), indexShardStats); + } + + indexShardStats.total++; + + CommonStats shardCommonStats = shardStats.getStats(); + + if (shardStats.getShardRouting().primary()) { + indexShardStats.primaries++; + this.docs.add(shardCommonStats.docs); + } + this.store.add(shardCommonStats.store); + this.fieldData.add(shardCommonStats.fieldData); + this.queryCache.add(shardCommonStats.queryCache); + this.completion.add(shardCommonStats.completion); + this.segments.add(shardCommonStats.segments); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalWriteable(docs); + out.writeOptionalWriteable(store); + out.writeOptionalWriteable(fieldData); + out.writeOptionalWriteable(queryCache); + out.writeOptionalWriteable(completion); + out.writeOptionalWriteable(segments); + out.writeMap(indexStatsMap, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream)); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index e4f483f796f44..22e4fbcfb3999 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -212,8 +212,14 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq clusterStatus = new ClusterStateHealth(clusterService.state()).getStatus(); } - return new ClusterStatsNodeResponse(nodeInfo.getNode(), clusterStatus, nodeInfo, nodeStats, shardsStats.toArray(new ShardStats[0])); - + return new ClusterStatsNodeResponse( + nodeInfo.getNode(), + clusterStatus, + nodeInfo, + nodeStats, + shardsStats.toArray(new ShardStats[0]), + nodeRequest.request.useOptimizedClusterStatsResponse() + ); } /** diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index 0766e838210fa..804fa594b1905 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -66,6 +66,7 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null)); clusterStatsRequest.timeout(request.param("timeout")); + clusterStatsRequest.useOptimizedClusterStatsResponse(true); return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index 40a30342b86b9..70509e053d11e 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -32,16 +32,35 @@ package org.opensearch.action.admin.cluster.stats; +import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodeStatsTests; +import org.opensearch.action.admin.indices.stats.CommonStats; +import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.index.cache.query.QueryCacheStats; +import org.opensearch.index.engine.SegmentsStats; +import org.opensearch.index.fielddata.FieldDataStats; +import org.opensearch.index.flush.FlushStats; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.shard.IndexingStats; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.StoreStats; +import org.opensearch.search.suggest.completion.CompletionStats; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -158,6 +177,231 @@ public void testIngestStats() throws Exception { ); } + public void testMultiVersionScenario() { + // Assuming the default behavior will be the type of response expected from a node of version prior to version containing optimized + // output + int numberOfNodes = randomIntBetween(1, 4); + Index testIndex = new Index("test-index", "_na_"); + + List defaultClusterStatsNodeResponses = new ArrayList<>(); + List optimizedClusterStatsNodeResponses = new ArrayList<>(); + + boolean optimiseClusterStats = randomBoolean(); + + for (int i = 0; i < numberOfNodes; i++) { + DiscoveryNode node = new DiscoveryNode("node-" + i, buildNewFakeTransportAddress(), Version.CURRENT); + CommonStats commonStats = createRandomCommonStats(); + ShardStats[] shardStats = createshardStats(node, testIndex, commonStats); + ClusterStatsNodeResponse customClusterStatsResponse = createClusterStatsNodeResponse(node, shardStats, testIndex, true, false); + ClusterStatsNodeResponse customOptimizedClusterStatsResponse = createClusterStatsNodeResponse( + node, + shardStats, + testIndex, + false, + optimiseClusterStats + ); + defaultClusterStatsNodeResponses.add(customClusterStatsResponse); + optimizedClusterStatsNodeResponses.add(customOptimizedClusterStatsResponse); + } + + ClusterStatsIndices defaultClusterStatsIndices = new ClusterStatsIndices(defaultClusterStatsNodeResponses, null, null); + ClusterStatsIndices optimzedClusterStatsIndices = new ClusterStatsIndices(optimizedClusterStatsNodeResponses, null, null); + + assertEquals(defaultClusterStatsIndices.getIndexCount(), optimzedClusterStatsIndices.getIndexCount()); + + assertEquals(defaultClusterStatsIndices.getShards().getIndices(), optimzedClusterStatsIndices.getShards().getIndices()); + assertEquals(defaultClusterStatsIndices.getShards().getTotal(), optimzedClusterStatsIndices.getShards().getTotal()); + assertEquals(defaultClusterStatsIndices.getShards().getPrimaries(), optimzedClusterStatsIndices.getShards().getPrimaries()); + assertEquals( + defaultClusterStatsIndices.getShards().getMinIndexShards(), + optimzedClusterStatsIndices.getShards().getMaxIndexShards() + ); + assertEquals( + defaultClusterStatsIndices.getShards().getMinIndexPrimaryShards(), + optimzedClusterStatsIndices.getShards().getMinIndexPrimaryShards() + ); + + // As AssertEquals with double is deprecated and can only be used to compare floating-point numbers + assertTrue(defaultClusterStatsIndices.getShards().getReplication() == optimzedClusterStatsIndices.getShards().getReplication()); + assertTrue( + defaultClusterStatsIndices.getShards().getAvgIndexShards() == optimzedClusterStatsIndices.getShards().getAvgIndexShards() + ); + assertTrue( + defaultClusterStatsIndices.getShards().getMaxIndexPrimaryShards() == optimzedClusterStatsIndices.getShards() + .getMaxIndexPrimaryShards() + ); + assertTrue( + defaultClusterStatsIndices.getShards().getAvgIndexPrimaryShards() == optimzedClusterStatsIndices.getShards() + .getAvgIndexPrimaryShards() + ); + assertTrue( + defaultClusterStatsIndices.getShards().getMinIndexReplication() == optimzedClusterStatsIndices.getShards() + .getMinIndexReplication() + ); + assertTrue( + defaultClusterStatsIndices.getShards().getAvgIndexReplication() == optimzedClusterStatsIndices.getShards() + .getAvgIndexReplication() + ); + assertTrue( + defaultClusterStatsIndices.getShards().getMaxIndexReplication() == optimzedClusterStatsIndices.getShards() + .getMaxIndexReplication() + ); + + // Docs stats + assertEquals( + defaultClusterStatsIndices.getDocs().getAverageSizeInBytes(), + optimzedClusterStatsIndices.getDocs().getAverageSizeInBytes() + ); + assertEquals(defaultClusterStatsIndices.getDocs().getDeleted(), optimzedClusterStatsIndices.getDocs().getDeleted()); + assertEquals(defaultClusterStatsIndices.getDocs().getCount(), optimzedClusterStatsIndices.getDocs().getCount()); + assertEquals( + defaultClusterStatsIndices.getDocs().getTotalSizeInBytes(), + optimzedClusterStatsIndices.getDocs().getTotalSizeInBytes() + ); + + // Store Stats + assertEquals(defaultClusterStatsIndices.getStore().getSizeInBytes(), optimzedClusterStatsIndices.getStore().getSizeInBytes()); + assertEquals(defaultClusterStatsIndices.getStore().getSize(), optimzedClusterStatsIndices.getStore().getSize()); + assertEquals(defaultClusterStatsIndices.getStore().getReservedSize(), optimzedClusterStatsIndices.getStore().getReservedSize()); + + // Query Cache + assertEquals( + defaultClusterStatsIndices.getQueryCache().getCacheCount(), + optimzedClusterStatsIndices.getQueryCache().getCacheCount() + ); + assertEquals(defaultClusterStatsIndices.getQueryCache().getCacheSize(), optimzedClusterStatsIndices.getQueryCache().getCacheSize()); + assertEquals(defaultClusterStatsIndices.getQueryCache().getEvictions(), optimzedClusterStatsIndices.getQueryCache().getEvictions()); + assertEquals(defaultClusterStatsIndices.getQueryCache().getHitCount(), optimzedClusterStatsIndices.getQueryCache().getHitCount()); + assertEquals( + defaultClusterStatsIndices.getQueryCache().getTotalCount(), + optimzedClusterStatsIndices.getQueryCache().getTotalCount() + ); + assertEquals(defaultClusterStatsIndices.getQueryCache().getMissCount(), optimzedClusterStatsIndices.getQueryCache().getMissCount()); + assertEquals( + defaultClusterStatsIndices.getQueryCache().getMemorySize(), + optimzedClusterStatsIndices.getQueryCache().getMemorySize() + ); + assertEquals( + defaultClusterStatsIndices.getQueryCache().getMemorySizeInBytes(), + optimzedClusterStatsIndices.getQueryCache().getMemorySizeInBytes() + ); + + // Completion Stats + assertEquals( + defaultClusterStatsIndices.getCompletion().getSizeInBytes(), + optimzedClusterStatsIndices.getCompletion().getSizeInBytes() + ); + assertEquals(defaultClusterStatsIndices.getCompletion().getSize(), optimzedClusterStatsIndices.getCompletion().getSize()); + + // Segment Stats + assertEquals( + defaultClusterStatsIndices.getSegments().getBitsetMemory(), + optimzedClusterStatsIndices.getSegments().getBitsetMemory() + ); + assertEquals(defaultClusterStatsIndices.getSegments().getCount(), optimzedClusterStatsIndices.getSegments().getCount()); + assertEquals( + defaultClusterStatsIndices.getSegments().getBitsetMemoryInBytes(), + optimzedClusterStatsIndices.getSegments().getBitsetMemoryInBytes() + ); + assertEquals(defaultClusterStatsIndices.getSegments().getFileSizes(), optimzedClusterStatsIndices.getSegments().getFileSizes()); + assertEquals( + defaultClusterStatsIndices.getSegments().getIndexWriterMemoryInBytes(), + optimzedClusterStatsIndices.getSegments().getIndexWriterMemoryInBytes() + ); + assertEquals( + defaultClusterStatsIndices.getSegments().getVersionMapMemory(), + optimzedClusterStatsIndices.getSegments().getVersionMapMemory() + ); + assertEquals( + defaultClusterStatsIndices.getSegments().getVersionMapMemoryInBytes(), + optimzedClusterStatsIndices.getSegments().getVersionMapMemoryInBytes() + ); + + } + + private ClusterStatsNodeResponse createClusterStatsNodeResponse( + DiscoveryNode node, + ShardStats[] shardStats, + Index index, + boolean defaultBehavior, + boolean optimized + ) { + if (defaultBehavior) { + return new ClusterStatsNodeResponse(node, null, null, null, shardStats); + } else { + return new ClusterStatsNodeResponse(node, null, null, null, shardStats, optimized); + } + + } + + private CommonStats createRandomCommonStats() { + CommonStats commonStats = new CommonStats(CommonStatsFlags.NONE); + commonStats.docs = new DocsStats(randomLongBetween(0, 10000), randomLongBetween(0, 100), randomLongBetween(0, 1000)); + commonStats.store = new StoreStats(randomLongBetween(0, 100), randomLongBetween(0, 1000)); + commonStats.indexing = new IndexingStats(); + commonStats.completion = new CompletionStats(); + commonStats.flush = new FlushStats(randomLongBetween(0, 100), randomLongBetween(0, 100), randomLongBetween(0, 100)); + commonStats.fieldData = new FieldDataStats(randomLongBetween(0, 100), randomLongBetween(0, 100), null); + commonStats.queryCache = new QueryCacheStats( + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100) + ); + commonStats.segments = new SegmentsStats(); + + return commonStats; + } + + private ShardStats[] createshardStats(DiscoveryNode localNode, Index index, CommonStats commonStats) { + List shardStatsList = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + index.getName(), + i, + localNode.getId(), + randomBoolean(), + shardRoutingState + ); + + Path path = createTempDir().resolve("indices") + .resolve(shardRouting.shardId().getIndex().getUUID()) + .resolve(String.valueOf(shardRouting.shardId().id())); + + ShardStats shardStats = new ShardStats( + shardRouting, + new ShardPath(false, path, path, shardRouting.shardId()), + commonStats, + null, + null, + null + ); + shardStatsList.add(shardStats); + } + + return shardStatsList.toArray(new ShardStats[0]); + } + + private class MockShardStats extends ClusterStatsIndices.ShardStats { + public boolean equals(ClusterStatsIndices.ShardStats shardStats) { + return this.getIndices() == shardStats.getIndices() + && this.getTotal() == shardStats.getTotal() + && this.getPrimaries() == shardStats.getPrimaries() + && this.getReplication() == shardStats.getReplication() + && this.getMaxIndexShards() == shardStats.getMaxIndexShards() + && this.getMinIndexShards() == shardStats.getMinIndexShards() + && this.getAvgIndexShards() == shardStats.getAvgIndexShards() + && this.getMaxIndexPrimaryShards() == shardStats.getMaxIndexPrimaryShards() + && this.getMinIndexPrimaryShards() == shardStats.getMinIndexPrimaryShards() + && this.getAvgIndexPrimaryShards() == shardStats.getAvgIndexPrimaryShards() + && this.getMinIndexReplication() == shardStats.getMinIndexReplication() + && this.getAvgIndexReplication() == shardStats.getAvgIndexReplication() + && this.getMaxIndexReplication() == shardStats.getMaxIndexReplication(); + } + } + private static NodeInfo createNodeInfo(String nodeId, String transportType, String httpType) { Settings.Builder settings = Settings.builder(); if (transportType != null) { From 42654567e7221feac6ae00eb104c780b251e6d17 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Fri, 12 Jul 2024 19:20:25 +0530 Subject: [PATCH 02/15] Add change logs and update code coverage tests Signed-off-by: Pranshu Shukla --- CHANGELOG.md | 1 + .../admin/cluster/stats/ClusterStatsIT.java | 59 +++++++++++++++++-- .../cluster/stats/ClusterStatsNodesTests.java | 1 - 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62bb73d80f2c1..492280d5a0ece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Optimize Cluster Stats Indices to precomute node level stats ([#14426](https://github.com/opensearch-project/OpenSearch/pull/14426)) - Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724)) - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/)) - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865)) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index c0b1539b3b8d8..6558b0831804f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -184,19 +184,19 @@ private void assertShardStats(ClusterStatsIndices.ShardStats stats, int indices, assertThat(stats.getReplication(), Matchers.equalTo(replicationFactor)); } - public void testIndicesShardStats() throws ExecutionException, InterruptedException { + public void testIndicesShardStatsDefault() { internalCluster().startNode(); ensureGreen(); ClusterStatsResponse response = client().admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useOptimizedClusterStatsResponse(false) .get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get(); - response = client().admin().cluster().prepareClusterStats().get(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.YELLOW)); assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(0L)); assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(1)); @@ -207,14 +207,63 @@ public void testIndicesShardStats() throws ExecutionException, InterruptedExcept ensureGreen(); index("test1", "type", "1", "f", "f"); refresh(); // make the doc visible - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(1L)); assertShardStats(response.getIndicesStats().getShards(), 1, 4, 2, 1.0); prepareCreate("test2").setSettings(Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 0)).get(); ensureGreen(); - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(2)); + assertShardStats(response.getIndicesStats().getShards(), 2, 7, 5, 2.0 / 5); + + assertThat(response.getIndicesStats().getShards().getAvgIndexPrimaryShards(), Matchers.equalTo(2.5)); + assertThat(response.getIndicesStats().getShards().getMinIndexPrimaryShards(), Matchers.equalTo(2)); + assertThat(response.getIndicesStats().getShards().getMaxIndexPrimaryShards(), Matchers.equalTo(3)); + + assertThat(response.getIndicesStats().getShards().getAvgIndexShards(), Matchers.equalTo(3.5)); + assertThat(response.getIndicesStats().getShards().getMinIndexShards(), Matchers.equalTo(3)); + assertThat(response.getIndicesStats().getShards().getMaxIndexShards(), Matchers.equalTo(4)); + + assertThat(response.getIndicesStats().getShards().getAvgIndexReplication(), Matchers.equalTo(0.5)); + assertThat(response.getIndicesStats().getShards().getMinIndexReplication(), Matchers.equalTo(0.0)); + assertThat(response.getIndicesStats().getShards().getMaxIndexReplication(), Matchers.equalTo(1.0)); + + } + + public void testIndicesShardStatsOptimised() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useOptimizedClusterStatsResponse(true) + .get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + + prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get(); + + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.YELLOW)); + assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(0L)); + assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(1)); + assertShardStats(response.getIndicesStats().getShards(), 1, 2, 2, 0.0); + + // add another node, replicas should get assigned + internalCluster().startNode(); + ensureGreen(); + index("test1", "type", "1", "f", "f"); + refresh(); // make the doc visible + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(1L)); + assertShardStats(response.getIndicesStats().getShards(), 1, 4, 2, 1.0); + + prepareCreate("test2").setSettings(Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 0)).get(); + ensureGreen(); + response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(2)); assertShardStats(response.getIndicesStats().getShards(), 2, 7, 5, 2.0 / 5); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index 70509e053d11e..ced72ced92a65 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -316,7 +316,6 @@ public void testMultiVersionScenario() { defaultClusterStatsIndices.getSegments().getVersionMapMemoryInBytes(), optimzedClusterStatsIndices.getSegments().getVersionMapMemoryInBytes() ); - } private ClusterStatsNodeResponse createClusterStatsNodeResponse( From 0ddbe47507df23737a90d83802ae8a271e757241 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Fri, 12 Jul 2024 19:59:26 +0530 Subject: [PATCH 03/15] Fix spotless checks Signed-off-by: Pranshu Shukla --- .../action/admin/cluster/stats/ClusterStatsIT.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index 6558b0831804f..420dc02bd4c24 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -187,11 +187,7 @@ private void assertShardStats(ClusterStatsIndices.ShardStats stats, int indices, public void testIndicesShardStatsDefault() { internalCluster().startNode(); ensureGreen(); - ClusterStatsResponse response = client().admin() - .cluster() - .prepareClusterStats() - .useOptimizedClusterStatsResponse(false) - .get(); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get(); @@ -236,11 +232,7 @@ public void testIndicesShardStatsDefault() { public void testIndicesShardStatsOptimised() { internalCluster().startNode(); ensureGreen(); - ClusterStatsResponse response = client().admin() - .cluster() - .prepareClusterStats() - .useOptimizedClusterStatsResponse(true) - .get(); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get(); From 989fbc431b7c7ea83a652fb45546b1df4c33fffc Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Mon, 15 Jul 2024 12:00:38 +0530 Subject: [PATCH 04/15] Retry Build Signed-off-by: Pranshu Shukla From aac6776628125e06164d72773c507fcac7ba7783 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Wed, 17 Jul 2024 21:37:01 +0530 Subject: [PATCH 05/15] Fix test naming Signed-off-by: Pranshu Shukla --- .../opensearch/action/admin/cluster/stats/ClusterStatsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index 420dc02bd4c24..b76f8dc596ad9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -184,7 +184,7 @@ private void assertShardStats(ClusterStatsIndices.ShardStats stats, int indices, assertThat(stats.getReplication(), Matchers.equalTo(replicationFactor)); } - public void testIndicesShardStatsDefault() { + public void testIndicesShardStatsWithoutNodeIndexShardStats() { internalCluster().startNode(); ensureGreen(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); @@ -229,7 +229,7 @@ public void testIndicesShardStatsDefault() { } - public void testIndicesShardStatsOptimised() { + public void testIndicesShardStatsWithNodeIndexShardStats() { internalCluster().startNode(); ensureGreen(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); From 79a7c192820c7aef1abbfd0cf1c5a0ba894ee6e9 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Thu, 18 Jul 2024 11:50:52 +0530 Subject: [PATCH 06/15] Adding UTs for Cluster Stats Signed-off-by: Pranshu Shukla --- .../cluster/stats/ClusterStatsNodesTests.java | 91 ++++++++++++++++++- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index ced72ced92a65..d6127add4a1d4 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.cluster.stats; +import org.opensearch.Build; import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.stats.NodeStats; @@ -43,8 +44,10 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.cache.query.QueryCacheStats; @@ -186,8 +189,6 @@ public void testMultiVersionScenario() { List defaultClusterStatsNodeResponses = new ArrayList<>(); List optimizedClusterStatsNodeResponses = new ArrayList<>(); - boolean optimiseClusterStats = randomBoolean(); - for (int i = 0; i < numberOfNodes; i++) { DiscoveryNode node = new DiscoveryNode("node-" + i, buildNewFakeTransportAddress(), Version.CURRENT); CommonStats commonStats = createRandomCommonStats(); @@ -198,7 +199,7 @@ public void testMultiVersionScenario() { shardStats, testIndex, false, - optimiseClusterStats + true ); defaultClusterStatsNodeResponses.add(customClusterStatsResponse); optimizedClusterStatsNodeResponses.add(customOptimizedClusterStatsResponse); @@ -207,6 +208,10 @@ public void testMultiVersionScenario() { ClusterStatsIndices defaultClusterStatsIndices = new ClusterStatsIndices(defaultClusterStatsNodeResponses, null, null); ClusterStatsIndices optimzedClusterStatsIndices = new ClusterStatsIndices(optimizedClusterStatsNodeResponses, null, null); + assertClusterStatsIndices(defaultClusterStatsIndices, optimzedClusterStatsIndices); + } + + public void assertClusterStatsIndices(ClusterStatsIndices defaultClusterStatsIndices, ClusterStatsIndices optimzedClusterStatsIndices) { assertEquals(defaultClusterStatsIndices.getIndexCount(), optimzedClusterStatsIndices.getIndexCount()); assertEquals(defaultClusterStatsIndices.getShards().getIndices(), optimzedClusterStatsIndices.getShards().getIndices()); @@ -318,6 +323,33 @@ public void testMultiVersionScenario() { ); } + public void testNodeIndexShardStatsSuccessfulSerializationDeserialization() throws IOException { + Index testIndex = new Index("test-index", "_na_"); + + DiscoveryNode node = new DiscoveryNode("node", buildNewFakeTransportAddress(), Version.CURRENT); + CommonStats commonStats = createRandomCommonStats(); + ShardStats[] shardStats = createshardStats(node, testIndex, commonStats); + ClusterStatsNodeResponse customOptimizedClusterStatsResponse = createClusterStatsNodeResponse( + node, + shardStats, + testIndex, + false, + true + ); + + BytesStreamOutput out = new BytesStreamOutput(); + customOptimizedClusterStatsResponse.writeTo(out); + StreamInput in = out.bytes().streamInput(); + + ClusterStatsNodeResponse newClusterStatsNodeRequest = new ClusterStatsNodeResponse(in); + + ClusterStatsIndices beforeSerialization = new ClusterStatsIndices(List.of(customOptimizedClusterStatsResponse), null, null); + ClusterStatsIndices afterSerialization = new ClusterStatsIndices(List.of(newClusterStatsNodeRequest), null, null); + + assertClusterStatsIndices(beforeSerialization, afterSerialization); + + } + private ClusterStatsNodeResponse createClusterStatsNodeResponse( DiscoveryNode node, ShardStats[] shardStats, @@ -325,10 +357,59 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse( boolean defaultBehavior, boolean optimized ) { + NodeInfo nodeInfo = new NodeInfo( + Version.CURRENT, + Build.CURRENT, + node, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + + NodeStats nodeStats = new NodeStats( + node, + randomNonNegativeLong(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); if (defaultBehavior) { - return new ClusterStatsNodeResponse(node, null, null, null, shardStats); + return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats); } else { - return new ClusterStatsNodeResponse(node, null, null, null, shardStats, optimized); + return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats, optimized); } } From 10a78f7476f41a53dba3a05bb473b61192872ea0 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Thu, 18 Jul 2024 19:00:54 +0530 Subject: [PATCH 07/15] Refactor naming Signed-off-by: Pranshu Shukla --- .../admin/cluster/stats/ClusterStatsIT.java | 44 ++--- ...ats.java => AggregatedNodeLevelStats.java} | 6 +- .../cluster/stats/ClusterStatsIndices.java | 18 +- .../stats/ClusterStatsNodeResponse.java | 18 +- .../cluster/stats/ClusterStatsRequest.java | 14 +- .../stats/ClusterStatsRequestBuilder.java | 6 +- .../stats/TransportClusterStatsAction.java | 2 +- .../admin/cluster/RestClusterStatsAction.java | 2 +- .../cluster/stats/ClusterStatsNodesTests.java | 167 ++++++------------ 9 files changed, 111 insertions(+), 166 deletions(-) rename server/src/main/java/org/opensearch/action/admin/cluster/stats/{NodeIndexShardStats.java => AggregatedNodeLevelStats.java} (94%) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index b76f8dc596ad9..f23cdbb50b37a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -91,7 +91,7 @@ public void testNodeCounts() { ClusterStatsResponse response = client().admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertCounts(response.getNodesStats().getCounts(), total, expectedCounts); @@ -160,7 +160,7 @@ public void testNodeCountsWithDeprecatedMasterRole() throws ExecutionException, ClusterStatsResponse response = client.admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertCounts(response.getNodesStats().getCounts(), total, expectedCounts); @@ -184,15 +184,15 @@ private void assertShardStats(ClusterStatsIndices.ShardStats stats, int indices, assertThat(stats.getReplication(), Matchers.equalTo(replicationFactor)); } - public void testIndicesShardStatsWithoutNodeIndexShardStats() { + public void testIndicesShardStatsWithoutNodeLevelAggregations() { internalCluster().startNode(); ensureGreen(); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(false).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get(); - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(false).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.YELLOW)); assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(0L)); assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(1)); @@ -203,14 +203,14 @@ public void testIndicesShardStatsWithoutNodeIndexShardStats() { ensureGreen(); index("test1", "type", "1", "f", "f"); refresh(); // make the doc visible - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(false).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(1L)); assertShardStats(response.getIndicesStats().getShards(), 1, 4, 2, 1.0); prepareCreate("test2").setSettings(Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 0)).get(); ensureGreen(); - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(false).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(false).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(2)); assertShardStats(response.getIndicesStats().getShards(), 2, 7, 5, 2.0 / 5); @@ -229,15 +229,15 @@ public void testIndicesShardStatsWithoutNodeIndexShardStats() { } - public void testIndicesShardStatsWithNodeIndexShardStats() { + public void testIndicesShardStatsWithNodeLevelAggregations() { internalCluster().startNode(); ensureGreen(); - ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); + ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(true).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get(); - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(true).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.YELLOW)); assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(0L)); assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(1)); @@ -248,14 +248,14 @@ public void testIndicesShardStatsWithNodeIndexShardStats() { ensureGreen(); index("test1", "type", "1", "f", "f"); refresh(); // make the doc visible - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(true).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(1L)); assertShardStats(response.getIndicesStats().getShards(), 1, 4, 2, 1.0); prepareCreate("test2").setSettings(Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 0)).get(); ensureGreen(); - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(true).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(true).get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertThat(response.indicesStats.getIndexCount(), Matchers.equalTo(2)); assertShardStats(response.getIndicesStats().getShards(), 2, 7, 5, 2.0 / 5); @@ -281,7 +281,7 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte ClusterStatsResponse response = client().admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); String msg = response.toString(); assertThat(msg, response.getTimestamp(), Matchers.greaterThan(946681200000L)); // 1 Jan 2000 @@ -325,7 +325,7 @@ public void testAllocatedProcessors() throws Exception { ClusterStatsResponse response = client().admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertThat(response.getNodesStats().getOs().getAllocatedProcessors(), equalTo(7)); } @@ -335,7 +335,7 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception { ClusterStatsResponse response = client().admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED)); @@ -346,7 +346,7 @@ public void testClusterStatusWhenStateNotRecovered() throws Exception { } // wait for the cluster status to settle ensureGreen(); - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(randomBoolean()).get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN)); } @@ -356,7 +356,7 @@ public void testFieldTypes() { ClusterStatsResponse response = client().admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); @@ -370,7 +370,7 @@ public void testFieldTypes() { + "\"eggplant\":{\"type\":\"integer\"}}}}}" ) .get(); - response = client().admin().cluster().prepareClusterStats().useOptimizedClusterStatsResponse(randomBoolean()).get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(randomBoolean()).get(); assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3)); Set stats = response.getIndicesStats().getMappings().getFieldTypeStats(); for (IndexFeatureStats stat : stats) { @@ -401,7 +401,7 @@ public void testNodeRolesWithMasterLegacySettings() throws ExecutionException, I ClusterStatsResponse clusterStatsResponse = client.admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedCounts); @@ -435,7 +435,7 @@ public void testNodeRolesWithClusterManagerRole() throws ExecutionException, Int ClusterStatsResponse clusterStatsResponse = client.admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedCounts); @@ -463,7 +463,7 @@ public void testNodeRolesWithSeedDataNodeLegacySettings() throws ExecutionExcept ClusterStatsResponse clusterStatsResponse = client.admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedRoleCounts); @@ -494,7 +494,7 @@ public void testNodeRolesWithDataNodeLegacySettings() throws ExecutionException, ClusterStatsResponse clusterStatsResponse = client.admin() .cluster() .prepareClusterStats() - .useOptimizedClusterStatsResponse(randomBoolean()) + .useAggregatedNodeLevelResponses(randomBoolean()) .get(); assertCounts(clusterStatsResponse.getNodesStats().getCounts(), total, expectedRoleCounts); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/NodeIndexShardStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/AggregatedNodeLevelStats.java similarity index 94% rename from server/src/main/java/org/opensearch/action/admin/cluster/stats/NodeIndexShardStats.java rename to server/src/main/java/org/opensearch/action/admin/cluster/stats/AggregatedNodeLevelStats.java index e782e21730125..ab549aeef9e37 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/NodeIndexShardStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/AggregatedNodeLevelStats.java @@ -28,7 +28,7 @@ /** * Node level statistics used for ClusterStatsIndices for _cluster/stats call. */ -public class NodeIndexShardStats extends BaseNodeResponse { +public class AggregatedNodeLevelStats extends BaseNodeResponse { DocsStats docs; StoreStats store; @@ -38,7 +38,7 @@ public class NodeIndexShardStats extends BaseNodeResponse { SegmentsStats segments; Map indexStatsMap; - protected NodeIndexShardStats(StreamInput in) throws IOException { + protected AggregatedNodeLevelStats(StreamInput in) throws IOException { super(in); docs = in.readOptionalWriteable(DocsStats::new); store = in.readOptionalWriteable(StoreStats::new); @@ -49,7 +49,7 @@ protected NodeIndexShardStats(StreamInput in) throws IOException { indexStatsMap = in.readMap(StreamInput::readString, ClusterStatsIndices.ShardStats::new); } - protected NodeIndexShardStats(DiscoveryNode node, ShardStats[] indexShardsStats) { + protected AggregatedNodeLevelStats(DiscoveryNode node, ShardStats[] indexShardsStats) { super(node); this.docs = new DocsStats(); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 06678e335df01..94d170494f0f0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -81,21 +81,21 @@ public ClusterStatsIndices(List nodeResponses, Mapping this.segments = new SegmentsStats(); for (ClusterStatsNodeResponse r : nodeResponses) { - // Optimized response from the node - if (r.getNodeIndexShardStats() != null) { - r.getNodeIndexShardStats().indexStatsMap.forEach( + // Aggregated response from the node + if (r.getAggregatedNodeLevelStats() != null) { + r.getAggregatedNodeLevelStats().indexStatsMap.forEach( (index, indexCountStats) -> countsPerIndex.merge(index, indexCountStats, (v1, v2) -> { v1.addStatsFrom(v2); return v1; }) ); - docs.add(r.getNodeIndexShardStats().docs); - store.add(r.getNodeIndexShardStats().store); - fieldData.add(r.getNodeIndexShardStats().fieldData); - queryCache.add(r.getNodeIndexShardStats().queryCache); - completion.add(r.getNodeIndexShardStats().completion); - segments.add(r.getNodeIndexShardStats().segments); + docs.add(r.getAggregatedNodeLevelStats().docs); + store.add(r.getAggregatedNodeLevelStats().store); + fieldData.add(r.getAggregatedNodeLevelStats().fieldData); + queryCache.add(r.getAggregatedNodeLevelStats().queryCache); + completion.add(r.getAggregatedNodeLevelStats().completion); + segments.add(r.getAggregatedNodeLevelStats().segments); } else { // Default response from the node for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index 6df24f44432e1..a10df22a0acc9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -56,7 +56,7 @@ public class ClusterStatsNodeResponse extends BaseNodeResponse { private final NodeStats nodeStats; private final ShardStats[] shardsStats; private ClusterHealthStatus clusterStatus; - private NodeIndexShardStats nodeIndexShardStats; + private AggregatedNodeLevelStats aggregatedNodeLevelStats; public ClusterStatsNodeResponse(StreamInput in) throws IOException { super(in); @@ -68,7 +68,7 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException { this.nodeStats = new NodeStats(in); if (in.getVersion().onOrAfter(Version.V_3_0_0)) { this.shardsStats = in.readOptionalArray(ShardStats::new, ShardStats[]::new); - this.nodeIndexShardStats = in.readOptionalWriteable(NodeIndexShardStats::new); + this.aggregatedNodeLevelStats = in.readOptionalWriteable(AggregatedNodeLevelStats::new); } else { this.shardsStats = in.readArray(ShardStats::new, ShardStats[]::new); } @@ -94,13 +94,13 @@ public ClusterStatsNodeResponse( NodeInfo nodeInfo, NodeStats nodeStats, ShardStats[] shardsStats, - boolean optimized + boolean useAggregatedNodeLevelResponses ) { super(node); this.nodeInfo = nodeInfo; this.nodeStats = nodeStats; - if (optimized) { - this.nodeIndexShardStats = new NodeIndexShardStats(node, shardsStats); + if (useAggregatedNodeLevelResponses) { + this.aggregatedNodeLevelStats = new AggregatedNodeLevelStats(node, shardsStats); } this.shardsStats = shardsStats; this.clusterStatus = clusterStatus; @@ -126,8 +126,8 @@ public ShardStats[] shardsStats() { return this.shardsStats; } - public NodeIndexShardStats getNodeIndexShardStats() { - return nodeIndexShardStats; + public AggregatedNodeLevelStats getAggregatedNodeLevelStats() { + return aggregatedNodeLevelStats; } public static ClusterStatsNodeResponse readNodeResponse(StreamInput in) throws IOException { @@ -146,9 +146,9 @@ public void writeTo(StreamOutput out) throws IOException { nodeInfo.writeTo(out); nodeStats.writeTo(out); if (out.getVersion().onOrAfter(Version.V_3_0_0)) { - if (nodeIndexShardStats != null) { + if (aggregatedNodeLevelStats != null) { out.writeOptionalArray(null); - out.writeOptionalWriteable(nodeIndexShardStats); + out.writeOptionalWriteable(aggregatedNodeLevelStats); } else { out.writeOptionalArray(shardsStats); out.writeOptionalWriteable(null); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index f85c964c1eea8..fdeb82a3466f2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -51,11 +51,11 @@ public class ClusterStatsRequest extends BaseNodesRequest { public ClusterStatsRequest(StreamInput in) throws IOException { super(in); if (in.getVersion().onOrAfter(Version.V_3_0_0)) { - useOptimizedClusterStatsResponse = in.readOptionalBoolean(); + useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } } - private Boolean useOptimizedClusterStatsResponse = false; + private Boolean useAggregatedNodeLevelResponses = false; /** * Get stats from nodes based on the nodes ids specified. If none are passed, stats @@ -65,19 +65,19 @@ public ClusterStatsRequest(String... nodesIds) { super(nodesIds); } - public boolean useOptimizedClusterStatsResponse() { - return useOptimizedClusterStatsResponse; + public boolean useAggregatedNodeLevelResponses() { + return useAggregatedNodeLevelResponses; } - public void useOptimizedClusterStatsResponse(boolean useOptimizedClusterStatsResponse) { - this.useOptimizedClusterStatsResponse = useOptimizedClusterStatsResponse; + public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelResponses) { + this.useAggregatedNodeLevelResponses = useAggregatedNodeLevelResponses; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); if (out.getVersion().onOrAfter(Version.V_3_0_0)) { - out.writeOptionalBoolean(useOptimizedClusterStatsResponse); + out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index 3a07382cbf93e..4d0932bd3927d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -51,8 +51,8 @@ public ClusterStatsRequestBuilder(OpenSearchClient client, ClusterStatsAction ac super(client, action, new ClusterStatsRequest()); } - public final ClusterStatsRequestBuilder useOptimizedClusterStatsResponse(boolean useOptimizedClusterStatsResponse) { - request.useOptimizedClusterStatsResponse(useOptimizedClusterStatsResponse); - return (ClusterStatsRequestBuilder) this; + public final ClusterStatsRequestBuilder useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelResponses) { + request.useAggregatedNodeLevelResponses(useAggregatedNodeLevelResponses); + return this; } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 22e4fbcfb3999..9a79c8c3603bc 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -218,7 +218,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq nodeInfo, nodeStats, shardsStats.toArray(new ShardStats[0]), - nodeRequest.request.useOptimizedClusterStatsResponse() + nodeRequest.request.useAggregatedNodeLevelResponses() ); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index 804fa594b1905..ee33bd18db05d 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -66,7 +66,7 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null)); clusterStatsRequest.timeout(request.param("timeout")); - clusterStatsRequest.useOptimizedClusterStatsResponse(true); + clusterStatsRequest.useAggregatedNodeLevelResponses(true); return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index d6127add4a1d4..1c4a77905d73f 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -180,21 +180,21 @@ public void testIngestStats() throws Exception { ); } - public void testMultiVersionScenario() { - // Assuming the default behavior will be the type of response expected from a node of version prior to version containing optimized - // output + public void testMultiVersionScenarioWithAggregatedNodeLevelStats() { + // Assuming the default behavior will be the type of response expected from a node of version prior to version containing + // aggregated node level information int numberOfNodes = randomIntBetween(1, 4); Index testIndex = new Index("test-index", "_na_"); List defaultClusterStatsNodeResponses = new ArrayList<>(); - List optimizedClusterStatsNodeResponses = new ArrayList<>(); + List aggregatedNodeLevelClusterStatsNodeResponses = new ArrayList<>(); for (int i = 0; i < numberOfNodes; i++) { DiscoveryNode node = new DiscoveryNode("node-" + i, buildNewFakeTransportAddress(), Version.CURRENT); CommonStats commonStats = createRandomCommonStats(); ShardStats[] shardStats = createshardStats(node, testIndex, commonStats); ClusterStatsNodeResponse customClusterStatsResponse = createClusterStatsNodeResponse(node, shardStats, testIndex, true, false); - ClusterStatsNodeResponse customOptimizedClusterStatsResponse = createClusterStatsNodeResponse( + ClusterStatsNodeResponse customNodeLevelAggregatedClusterStatsResponse = createClusterStatsNodeResponse( node, shardStats, testIndex, @@ -202,125 +202,70 @@ public void testMultiVersionScenario() { true ); defaultClusterStatsNodeResponses.add(customClusterStatsResponse); - optimizedClusterStatsNodeResponses.add(customOptimizedClusterStatsResponse); + aggregatedNodeLevelClusterStatsNodeResponses.add(customNodeLevelAggregatedClusterStatsResponse); } ClusterStatsIndices defaultClusterStatsIndices = new ClusterStatsIndices(defaultClusterStatsNodeResponses, null, null); - ClusterStatsIndices optimzedClusterStatsIndices = new ClusterStatsIndices(optimizedClusterStatsNodeResponses, null, null); + ClusterStatsIndices aggregatedNodeLevelClusterStatsIndices = new ClusterStatsIndices( + aggregatedNodeLevelClusterStatsNodeResponses, + null, + null + ); - assertClusterStatsIndices(defaultClusterStatsIndices, optimzedClusterStatsIndices); + assertClusterStatsIndicesEqual(defaultClusterStatsIndices, aggregatedNodeLevelClusterStatsIndices); } - public void assertClusterStatsIndices(ClusterStatsIndices defaultClusterStatsIndices, ClusterStatsIndices optimzedClusterStatsIndices) { - assertEquals(defaultClusterStatsIndices.getIndexCount(), optimzedClusterStatsIndices.getIndexCount()); + public void assertClusterStatsIndicesEqual(ClusterStatsIndices first, ClusterStatsIndices second) { + assertEquals(first.getIndexCount(), second.getIndexCount()); - assertEquals(defaultClusterStatsIndices.getShards().getIndices(), optimzedClusterStatsIndices.getShards().getIndices()); - assertEquals(defaultClusterStatsIndices.getShards().getTotal(), optimzedClusterStatsIndices.getShards().getTotal()); - assertEquals(defaultClusterStatsIndices.getShards().getPrimaries(), optimzedClusterStatsIndices.getShards().getPrimaries()); - assertEquals( - defaultClusterStatsIndices.getShards().getMinIndexShards(), - optimzedClusterStatsIndices.getShards().getMaxIndexShards() - ); - assertEquals( - defaultClusterStatsIndices.getShards().getMinIndexPrimaryShards(), - optimzedClusterStatsIndices.getShards().getMinIndexPrimaryShards() - ); + assertEquals(first.getShards().getIndices(), second.getShards().getIndices()); + assertEquals(first.getShards().getTotal(), second.getShards().getTotal()); + assertEquals(first.getShards().getPrimaries(), second.getShards().getPrimaries()); + assertEquals(first.getShards().getMinIndexShards(), second.getShards().getMaxIndexShards()); + assertEquals(first.getShards().getMinIndexPrimaryShards(), second.getShards().getMinIndexPrimaryShards()); // As AssertEquals with double is deprecated and can only be used to compare floating-point numbers - assertTrue(defaultClusterStatsIndices.getShards().getReplication() == optimzedClusterStatsIndices.getShards().getReplication()); - assertTrue( - defaultClusterStatsIndices.getShards().getAvgIndexShards() == optimzedClusterStatsIndices.getShards().getAvgIndexShards() - ); - assertTrue( - defaultClusterStatsIndices.getShards().getMaxIndexPrimaryShards() == optimzedClusterStatsIndices.getShards() - .getMaxIndexPrimaryShards() - ); - assertTrue( - defaultClusterStatsIndices.getShards().getAvgIndexPrimaryShards() == optimzedClusterStatsIndices.getShards() - .getAvgIndexPrimaryShards() - ); - assertTrue( - defaultClusterStatsIndices.getShards().getMinIndexReplication() == optimzedClusterStatsIndices.getShards() - .getMinIndexReplication() - ); - assertTrue( - defaultClusterStatsIndices.getShards().getAvgIndexReplication() == optimzedClusterStatsIndices.getShards() - .getAvgIndexReplication() - ); - assertTrue( - defaultClusterStatsIndices.getShards().getMaxIndexReplication() == optimzedClusterStatsIndices.getShards() - .getMaxIndexReplication() - ); + assertTrue(first.getShards().getReplication() == second.getShards().getReplication()); + assertTrue(first.getShards().getAvgIndexShards() == second.getShards().getAvgIndexShards()); + assertTrue(first.getShards().getMaxIndexPrimaryShards() == second.getShards().getMaxIndexPrimaryShards()); + assertTrue(first.getShards().getAvgIndexPrimaryShards() == second.getShards().getAvgIndexPrimaryShards()); + assertTrue(first.getShards().getMinIndexReplication() == second.getShards().getMinIndexReplication()); + assertTrue(first.getShards().getAvgIndexReplication() == second.getShards().getAvgIndexReplication()); + assertTrue(first.getShards().getMaxIndexReplication() == second.getShards().getMaxIndexReplication()); // Docs stats - assertEquals( - defaultClusterStatsIndices.getDocs().getAverageSizeInBytes(), - optimzedClusterStatsIndices.getDocs().getAverageSizeInBytes() - ); - assertEquals(defaultClusterStatsIndices.getDocs().getDeleted(), optimzedClusterStatsIndices.getDocs().getDeleted()); - assertEquals(defaultClusterStatsIndices.getDocs().getCount(), optimzedClusterStatsIndices.getDocs().getCount()); - assertEquals( - defaultClusterStatsIndices.getDocs().getTotalSizeInBytes(), - optimzedClusterStatsIndices.getDocs().getTotalSizeInBytes() - ); + assertEquals(first.getDocs().getAverageSizeInBytes(), second.getDocs().getAverageSizeInBytes()); + assertEquals(first.getDocs().getDeleted(), second.getDocs().getDeleted()); + assertEquals(first.getDocs().getCount(), second.getDocs().getCount()); + assertEquals(first.getDocs().getTotalSizeInBytes(), second.getDocs().getTotalSizeInBytes()); // Store Stats - assertEquals(defaultClusterStatsIndices.getStore().getSizeInBytes(), optimzedClusterStatsIndices.getStore().getSizeInBytes()); - assertEquals(defaultClusterStatsIndices.getStore().getSize(), optimzedClusterStatsIndices.getStore().getSize()); - assertEquals(defaultClusterStatsIndices.getStore().getReservedSize(), optimzedClusterStatsIndices.getStore().getReservedSize()); + assertEquals(first.getStore().getSizeInBytes(), second.getStore().getSizeInBytes()); + assertEquals(first.getStore().getSize(), second.getStore().getSize()); + assertEquals(first.getStore().getReservedSize(), second.getStore().getReservedSize()); // Query Cache - assertEquals( - defaultClusterStatsIndices.getQueryCache().getCacheCount(), - optimzedClusterStatsIndices.getQueryCache().getCacheCount() - ); - assertEquals(defaultClusterStatsIndices.getQueryCache().getCacheSize(), optimzedClusterStatsIndices.getQueryCache().getCacheSize()); - assertEquals(defaultClusterStatsIndices.getQueryCache().getEvictions(), optimzedClusterStatsIndices.getQueryCache().getEvictions()); - assertEquals(defaultClusterStatsIndices.getQueryCache().getHitCount(), optimzedClusterStatsIndices.getQueryCache().getHitCount()); - assertEquals( - defaultClusterStatsIndices.getQueryCache().getTotalCount(), - optimzedClusterStatsIndices.getQueryCache().getTotalCount() - ); - assertEquals(defaultClusterStatsIndices.getQueryCache().getMissCount(), optimzedClusterStatsIndices.getQueryCache().getMissCount()); - assertEquals( - defaultClusterStatsIndices.getQueryCache().getMemorySize(), - optimzedClusterStatsIndices.getQueryCache().getMemorySize() - ); - assertEquals( - defaultClusterStatsIndices.getQueryCache().getMemorySizeInBytes(), - optimzedClusterStatsIndices.getQueryCache().getMemorySizeInBytes() - ); + assertEquals(first.getQueryCache().getCacheCount(), second.getQueryCache().getCacheCount()); + assertEquals(first.getQueryCache().getCacheSize(), second.getQueryCache().getCacheSize()); + assertEquals(first.getQueryCache().getEvictions(), second.getQueryCache().getEvictions()); + assertEquals(first.getQueryCache().getHitCount(), second.getQueryCache().getHitCount()); + assertEquals(first.getQueryCache().getTotalCount(), second.getQueryCache().getTotalCount()); + assertEquals(first.getQueryCache().getMissCount(), second.getQueryCache().getMissCount()); + assertEquals(first.getQueryCache().getMemorySize(), second.getQueryCache().getMemorySize()); + assertEquals(first.getQueryCache().getMemorySizeInBytes(), second.getQueryCache().getMemorySizeInBytes()); // Completion Stats - assertEquals( - defaultClusterStatsIndices.getCompletion().getSizeInBytes(), - optimzedClusterStatsIndices.getCompletion().getSizeInBytes() - ); - assertEquals(defaultClusterStatsIndices.getCompletion().getSize(), optimzedClusterStatsIndices.getCompletion().getSize()); + assertEquals(first.getCompletion().getSizeInBytes(), second.getCompletion().getSizeInBytes()); + assertEquals(first.getCompletion().getSize(), second.getCompletion().getSize()); // Segment Stats - assertEquals( - defaultClusterStatsIndices.getSegments().getBitsetMemory(), - optimzedClusterStatsIndices.getSegments().getBitsetMemory() - ); - assertEquals(defaultClusterStatsIndices.getSegments().getCount(), optimzedClusterStatsIndices.getSegments().getCount()); - assertEquals( - defaultClusterStatsIndices.getSegments().getBitsetMemoryInBytes(), - optimzedClusterStatsIndices.getSegments().getBitsetMemoryInBytes() - ); - assertEquals(defaultClusterStatsIndices.getSegments().getFileSizes(), optimzedClusterStatsIndices.getSegments().getFileSizes()); - assertEquals( - defaultClusterStatsIndices.getSegments().getIndexWriterMemoryInBytes(), - optimzedClusterStatsIndices.getSegments().getIndexWriterMemoryInBytes() - ); - assertEquals( - defaultClusterStatsIndices.getSegments().getVersionMapMemory(), - optimzedClusterStatsIndices.getSegments().getVersionMapMemory() - ); - assertEquals( - defaultClusterStatsIndices.getSegments().getVersionMapMemoryInBytes(), - optimzedClusterStatsIndices.getSegments().getVersionMapMemoryInBytes() - ); + assertEquals(first.getSegments().getBitsetMemory(), second.getSegments().getBitsetMemory()); + assertEquals(first.getSegments().getCount(), second.getSegments().getCount()); + assertEquals(first.getSegments().getBitsetMemoryInBytes(), second.getSegments().getBitsetMemoryInBytes()); + assertEquals(first.getSegments().getFileSizes(), second.getSegments().getFileSizes()); + assertEquals(first.getSegments().getIndexWriterMemoryInBytes(), second.getSegments().getIndexWriterMemoryInBytes()); + assertEquals(first.getSegments().getVersionMapMemory(), second.getSegments().getVersionMapMemory()); + assertEquals(first.getSegments().getVersionMapMemoryInBytes(), second.getSegments().getVersionMapMemoryInBytes()); } public void testNodeIndexShardStatsSuccessfulSerializationDeserialization() throws IOException { @@ -329,7 +274,7 @@ public void testNodeIndexShardStatsSuccessfulSerializationDeserialization() thro DiscoveryNode node = new DiscoveryNode("node", buildNewFakeTransportAddress(), Version.CURRENT); CommonStats commonStats = createRandomCommonStats(); ShardStats[] shardStats = createshardStats(node, testIndex, commonStats); - ClusterStatsNodeResponse customOptimizedClusterStatsResponse = createClusterStatsNodeResponse( + ClusterStatsNodeResponse aggregatedNodeLevelClusterStatsNodeResponse = createClusterStatsNodeResponse( node, shardStats, testIndex, @@ -338,15 +283,15 @@ public void testNodeIndexShardStatsSuccessfulSerializationDeserialization() thro ); BytesStreamOutput out = new BytesStreamOutput(); - customOptimizedClusterStatsResponse.writeTo(out); + aggregatedNodeLevelClusterStatsNodeResponse.writeTo(out); StreamInput in = out.bytes().streamInput(); ClusterStatsNodeResponse newClusterStatsNodeRequest = new ClusterStatsNodeResponse(in); - ClusterStatsIndices beforeSerialization = new ClusterStatsIndices(List.of(customOptimizedClusterStatsResponse), null, null); + ClusterStatsIndices beforeSerialization = new ClusterStatsIndices(List.of(aggregatedNodeLevelClusterStatsNodeResponse), null, null); ClusterStatsIndices afterSerialization = new ClusterStatsIndices(List.of(newClusterStatsNodeRequest), null, null); - assertClusterStatsIndices(beforeSerialization, afterSerialization); + assertClusterStatsIndicesEqual(beforeSerialization, afterSerialization); } @@ -355,7 +300,7 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse( ShardStats[] shardStats, Index index, boolean defaultBehavior, - boolean optimized + boolean aggregateNodeLevelStats ) { NodeInfo nodeInfo = new NodeInfo( Version.CURRENT, @@ -409,7 +354,7 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse( if (defaultBehavior) { return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats); } else { - return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats, optimized); + return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats, aggregateNodeLevelStats); } } From 31fd6d86266ca15bae3d70ae538623897eb88f44 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Mon, 22 Jul 2024 17:36:43 +0530 Subject: [PATCH 08/15] Update ChangeLogs Signed-off-by: Pranshu Shukla --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b16576221b02b..7fc95e3e4e83e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- Optimize Cluster Stats Indices to precomute node level stats ([#14426](https://github.com/opensearch-project/OpenSearch/pull/14426)) - Add fingerprint ingest processor ([#13724](https://github.com/opensearch-project/OpenSearch/pull/13724)) - [Remote Store] Rate limiter for remote store low priority uploads ([#14374](https://github.com/opensearch-project/OpenSearch/pull/14374/)) - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865)) @@ -23,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Refactor remote-routing-table service inline with remote state interfaces([#14668](https://github.com/opensearch-project/OpenSearch/pull/14668)) - Add prefix mode verification setting for repository verification (([#14790](https://github.com/opensearch-project/OpenSearch/pull/14790))) - Optimize TransportNodesAction to not send DiscoveryNodes for NodeStats, NodesInfo and ClusterStats call ([14749](https://github.com/opensearch-project/OpenSearch/pull/14749)) +- Optimize Cluster Stats Indices to precomute node level stats ([#14426](https://github.com/opensearch-project/OpenSearch/pull/14426)) ### Dependencies - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442)) From 614a284131874f40f0077df25670990f4c99ec9f Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Mon, 22 Jul 2024 18:52:32 +0530 Subject: [PATCH 09/15] Retry Build Signed-off-by: Pranshu Shukla From 82f0beb93b327e8459b87d801278e51d3bf742fe Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Tue, 23 Jul 2024 11:15:39 +0530 Subject: [PATCH 10/15] Re-ordering classes Signed-off-by: Pranshu Shukla --- .../stats/AggregatedNodeLevelStats.java | 98 ------------------- .../cluster/stats/ClusterStatsIndices.java | 65 ++++++------ .../stats/ClusterStatsNodeResponse.java | 66 +++++++++++++ .../admin/indices/stats/CommonStats.java | 25 +++++ 4 files changed, 120 insertions(+), 134 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/stats/AggregatedNodeLevelStats.java diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/AggregatedNodeLevelStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/AggregatedNodeLevelStats.java deleted file mode 100644 index ab549aeef9e37..0000000000000 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/AggregatedNodeLevelStats.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.action.admin.cluster.stats; - -import org.opensearch.action.admin.indices.stats.CommonStats; -import org.opensearch.action.admin.indices.stats.ShardStats; -import org.opensearch.action.support.nodes.BaseNodeResponse; -import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.index.cache.query.QueryCacheStats; -import org.opensearch.index.engine.SegmentsStats; -import org.opensearch.index.fielddata.FieldDataStats; -import org.opensearch.index.shard.DocsStats; -import org.opensearch.index.store.StoreStats; -import org.opensearch.search.suggest.completion.CompletionStats; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Node level statistics used for ClusterStatsIndices for _cluster/stats call. - */ -public class AggregatedNodeLevelStats extends BaseNodeResponse { - - DocsStats docs; - StoreStats store; - FieldDataStats fieldData; - QueryCacheStats queryCache; - CompletionStats completion; - SegmentsStats segments; - Map indexStatsMap; - - protected AggregatedNodeLevelStats(StreamInput in) throws IOException { - super(in); - docs = in.readOptionalWriteable(DocsStats::new); - store = in.readOptionalWriteable(StoreStats::new); - fieldData = in.readOptionalWriteable(FieldDataStats::new); - queryCache = in.readOptionalWriteable(QueryCacheStats::new); - completion = in.readOptionalWriteable(CompletionStats::new); - segments = in.readOptionalWriteable(SegmentsStats::new); - indexStatsMap = in.readMap(StreamInput::readString, ClusterStatsIndices.ShardStats::new); - } - - protected AggregatedNodeLevelStats(DiscoveryNode node, ShardStats[] indexShardsStats) { - super(node); - - this.docs = new DocsStats(); - this.store = new StoreStats(); - this.fieldData = new FieldDataStats(); - this.queryCache = new QueryCacheStats(); - this.completion = new CompletionStats(); - this.segments = new SegmentsStats(); - this.indexStatsMap = new HashMap<>(); - - // Index Level Stats - for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : indexShardsStats) { - ClusterStatsIndices.ShardStats indexShardStats = this.indexStatsMap.get(shardStats.getShardRouting().getIndexName()); - if (indexShardStats == null) { - indexShardStats = new ClusterStatsIndices.ShardStats(); - this.indexStatsMap.put(shardStats.getShardRouting().getIndexName(), indexShardStats); - } - - indexShardStats.total++; - - CommonStats shardCommonStats = shardStats.getStats(); - - if (shardStats.getShardRouting().primary()) { - indexShardStats.primaries++; - this.docs.add(shardCommonStats.docs); - } - this.store.add(shardCommonStats.store); - this.fieldData.add(shardCommonStats.fieldData); - this.queryCache.add(shardCommonStats.queryCache); - this.completion.add(shardCommonStats.completion); - this.segments.add(shardCommonStats.segments); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeOptionalWriteable(docs); - out.writeOptionalWriteable(store); - out.writeOptionalWriteable(fieldData); - out.writeOptionalWriteable(queryCache); - out.writeOptionalWriteable(completion); - out.writeOptionalWriteable(segments); - out.writeMap(indexStatsMap, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream)); - } -} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 94d170494f0f0..43d72689ddd6c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -34,9 +34,6 @@ import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.cache.query.QueryCacheStats; @@ -83,19 +80,23 @@ public ClusterStatsIndices(List nodeResponses, Mapping for (ClusterStatsNodeResponse r : nodeResponses) { // Aggregated response from the node if (r.getAggregatedNodeLevelStats() != null) { - r.getAggregatedNodeLevelStats().indexStatsMap.forEach( - (index, indexCountStats) -> countsPerIndex.merge(index, indexCountStats, (v1, v2) -> { - v1.addStatsFrom(v2); - return v1; - }) - ); - - docs.add(r.getAggregatedNodeLevelStats().docs); - store.add(r.getAggregatedNodeLevelStats().store); - fieldData.add(r.getAggregatedNodeLevelStats().fieldData); - queryCache.add(r.getAggregatedNodeLevelStats().queryCache); - completion.add(r.getAggregatedNodeLevelStats().completion); - segments.add(r.getAggregatedNodeLevelStats().segments); + + for (Map.Entry entry : r.getAggregatedNodeLevelStats().indexStatsMap.entrySet()) { + ShardStats indexShardStats = countsPerIndex.get(entry.getKey()); + if (indexShardStats == null) { + indexShardStats = new ShardStats(entry.getValue()); + countsPerIndex.put(entry.getKey(), indexShardStats); + } else { + indexShardStats.addStatsFrom(entry.getValue()); + } + } + + docs.add(r.getAggregatedNodeLevelStats().commonStats.docs); + store.add(r.getAggregatedNodeLevelStats().commonStats.store); + fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData); + queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache); + completion.add(r.getAggregatedNodeLevelStats().commonStats.completion); + segments.add(r.getAggregatedNodeLevelStats().commonStats.segments); } else { // Default response from the node for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { @@ -206,11 +207,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws * @opensearch.api */ @PublicApi(since = "1.0.0") - public static class ShardStats implements ToXContentFragment, Writeable { - - int indices = 0; - int total = 0; - int primaries = 0; + public static class ShardStats implements ToXContentFragment { + int indices; + int total; + int primaries; // min/max int minIndexShards = -1; @@ -223,10 +223,10 @@ public static class ShardStats implements ToXContentFragment, Writeable { public ShardStats() {} - public ShardStats(StreamInput in) throws IOException { - indices = in.readVInt(); - total = in.readVInt(); - primaries = in.readVInt(); + public ShardStats(CommonStats.AggregatedIndexStats aggregatedIndexStats) { + this.indices = aggregatedIndexStats.indices; + this.total = aggregatedIndexStats.total; + this.primaries = aggregatedIndexStats.primaries; } /** @@ -356,17 +356,10 @@ public void addIndexShardCount(ShardStats indexShardCount) { } } - public void addStatsFrom(ShardStats incomingStats) { - this.total += incomingStats.getTotal(); - this.indices += incomingStats.getIndices(); - this.primaries += incomingStats.getPrimaries(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(indices); - out.writeVInt(total); - out.writeVInt(primaries); + public void addStatsFrom(CommonStats.AggregatedIndexStats incomingStats) { + this.total += incomingStats.total; + this.indices += incomingStats.indices; + this.primaries += incomingStats.primaries; } /** diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index a10df22a0acc9..b5dd6c306142e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -35,6 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.health.ClusterHealthStatus; @@ -42,8 +43,16 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.index.cache.query.QueryCacheStats; +import org.opensearch.index.engine.SegmentsStats; +import org.opensearch.index.fielddata.FieldDataStats; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.store.StoreStats; +import org.opensearch.search.suggest.completion.CompletionStats; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; /** * Transport action for obtaining cluster stats from node level @@ -157,4 +166,61 @@ public void writeTo(StreamOutput out) throws IOException { out.writeArray(shardsStats); } } + + /** + * Node level statistics used for ClusterStatsIndices for _cluster/stats call. + */ + public class AggregatedNodeLevelStats extends BaseNodeResponse { + + CommonStats commonStats; + Map indexStatsMap; + + protected AggregatedNodeLevelStats(StreamInput in) throws IOException { + super(in); + commonStats = in.readOptionalWriteable(CommonStats::new); + indexStatsMap = in.readMap(StreamInput::readString, CommonStats.AggregatedIndexStats::new); + } + + protected AggregatedNodeLevelStats(DiscoveryNode node, ShardStats[] indexShardsStats) { + super(node); + this.commonStats = new CommonStats(); + this.commonStats.docs = new DocsStats(); + this.commonStats.store = new StoreStats(); + this.commonStats.fieldData = new FieldDataStats(); + this.commonStats.queryCache = new QueryCacheStats(); + this.commonStats.completion = new CompletionStats(); + this.commonStats.segments = new SegmentsStats(); + this.indexStatsMap = new HashMap<>(); + + // Index Level Stats + for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : indexShardsStats) { + CommonStats.AggregatedIndexStats indexShardStats = this.indexStatsMap.get(shardStats.getShardRouting().getIndexName()); + if (indexShardStats == null) { + indexShardStats = new CommonStats.AggregatedIndexStats(); + this.indexStatsMap.put(shardStats.getShardRouting().getIndexName(), indexShardStats); + } + + indexShardStats.total++; + + CommonStats shardCommonStats = shardStats.getStats(); + + if (shardStats.getShardRouting().primary()) { + indexShardStats.primaries++; + this.commonStats.docs.add(shardCommonStats.docs); + } + this.commonStats.store.add(shardCommonStats.store); + this.commonStats.fieldData.add(shardCommonStats.fieldData); + this.commonStats.queryCache.add(shardCommonStats.queryCache); + this.commonStats.completion.add(shardCommonStats.completion); + this.commonStats.segments.add(shardCommonStats.segments); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalWriteable(commonStats); + out.writeMap(indexStatsMap, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream)); + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java index 8bfeb13b253c3..e7e4cac5f52c8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java @@ -122,6 +122,9 @@ public class CommonStats implements Writeable, ToXContentFragment { @Nullable public RecoveryStats recoveryStats; + @Nullable + public AggregatedIndexStats aggregatedIndexStats; + public CommonStats() { this(CommonStatsFlags.NONE); } @@ -544,4 +547,26 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } return builder; } + + @PublicApi(since = "2.0.0") + public static class AggregatedIndexStats implements Writeable { + public int indices = 0; + public int total = 0; + public int primaries = 0; + + public AggregatedIndexStats(StreamInput in) throws IOException { + indices = in.readVInt(); + total = in.readVInt(); + primaries = in.readVInt(); + } + + public AggregatedIndexStats() {} + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(indices); + out.writeVInt(total); + out.writeVInt(primaries); + } + } } From d3aa38b0ef7821b64ab3ae14748b8c91b9c5dbff Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Tue, 23 Jul 2024 11:40:03 +0530 Subject: [PATCH 11/15] Add javadocs Signed-off-by: Pranshu Shukla --- .../opensearch/action/admin/indices/stats/CommonStats.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java index e7e4cac5f52c8..ba031c0749507 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java @@ -122,9 +122,6 @@ public class CommonStats implements Writeable, ToXContentFragment { @Nullable public RecoveryStats recoveryStats; - @Nullable - public AggregatedIndexStats aggregatedIndexStats; - public CommonStats() { this(CommonStatsFlags.NONE); } @@ -548,6 +545,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + /** + * Node level statistics used for ClusterStatsIndices for _cluster/stats call. + */ @PublicApi(since = "2.0.0") public static class AggregatedIndexStats implements Writeable { public int indices = 0; From 500495d5929d307cb4d74e4d572b0a980a359bd3 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Tue, 23 Jul 2024 12:09:54 +0530 Subject: [PATCH 12/15] Retry Build Signed-off-by: Pranshu Shukla From d543d54968726c6f646da66c19e3f8fcbfb73012 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Tue, 23 Jul 2024 17:47:55 +0530 Subject: [PATCH 13/15] Bump up version for AggregatedIndexStats to 2.16.0 Signed-off-by: Pranshu Shukla --- .../org/opensearch/action/admin/indices/stats/CommonStats.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java index ba031c0749507..ac26a3851f9c3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java @@ -548,7 +548,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws /** * Node level statistics used for ClusterStatsIndices for _cluster/stats call. */ - @PublicApi(since = "2.0.0") + @PublicApi(since = "2.16.0") public static class AggregatedIndexStats implements Writeable { public int indices = 0; public int total = 0; From e0f617856a3c9d8ace9fecd0070494dab248ca94 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Tue, 23 Jul 2024 18:22:32 +0530 Subject: [PATCH 14/15] Refactor AggregatedIndexStats to ClusterStatsNodeResponse Signed-off-by: Pranshu Shukla --- .../cluster/stats/ClusterStatsIndices.java | 10 +++--- .../stats/ClusterStatsNodeResponse.java | 32 ++++++++++++++++--- .../admin/indices/stats/CommonStats.java | 25 --------------- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 43d72689ddd6c..03a73f45ffe81 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -81,7 +81,8 @@ public ClusterStatsIndices(List nodeResponses, Mapping // Aggregated response from the node if (r.getAggregatedNodeLevelStats() != null) { - for (Map.Entry entry : r.getAggregatedNodeLevelStats().indexStatsMap.entrySet()) { + for (Map.Entry entry : r.getAggregatedNodeLevelStats().indexStatsMap + .entrySet()) { ShardStats indexShardStats = countsPerIndex.get(entry.getKey()); if (indexShardStats == null) { indexShardStats = new ShardStats(entry.getValue()); @@ -208,6 +209,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws */ @PublicApi(since = "1.0.0") public static class ShardStats implements ToXContentFragment { + int indices; int total; int primaries; @@ -223,8 +225,7 @@ public static class ShardStats implements ToXContentFragment { public ShardStats() {} - public ShardStats(CommonStats.AggregatedIndexStats aggregatedIndexStats) { - this.indices = aggregatedIndexStats.indices; + public ShardStats(ClusterStatsNodeResponse.AggregatedIndexStats aggregatedIndexStats) { this.total = aggregatedIndexStats.total; this.primaries = aggregatedIndexStats.primaries; } @@ -356,9 +357,8 @@ public void addIndexShardCount(ShardStats indexShardCount) { } } - public void addStatsFrom(CommonStats.AggregatedIndexStats incomingStats) { + public void addStatsFrom(ClusterStatsNodeResponse.AggregatedIndexStats incomingStats) { this.total += incomingStats.total; - this.indices += incomingStats.indices; this.primaries += incomingStats.primaries; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index b5dd6c306142e..133cf68f5f8c9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -41,8 +41,10 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.cache.query.QueryCacheStats; import org.opensearch.index.engine.SegmentsStats; import org.opensearch.index.fielddata.FieldDataStats; @@ -173,12 +175,12 @@ public void writeTo(StreamOutput out) throws IOException { public class AggregatedNodeLevelStats extends BaseNodeResponse { CommonStats commonStats; - Map indexStatsMap; + Map indexStatsMap; protected AggregatedNodeLevelStats(StreamInput in) throws IOException { super(in); commonStats = in.readOptionalWriteable(CommonStats::new); - indexStatsMap = in.readMap(StreamInput::readString, CommonStats.AggregatedIndexStats::new); + indexStatsMap = in.readMap(StreamInput::readString, AggregatedIndexStats::new); } protected AggregatedNodeLevelStats(DiscoveryNode node, ShardStats[] indexShardsStats) { @@ -194,9 +196,9 @@ protected AggregatedNodeLevelStats(DiscoveryNode node, ShardStats[] indexShardsS // Index Level Stats for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : indexShardsStats) { - CommonStats.AggregatedIndexStats indexShardStats = this.indexStatsMap.get(shardStats.getShardRouting().getIndexName()); + AggregatedIndexStats indexShardStats = this.indexStatsMap.get(shardStats.getShardRouting().getIndexName()); if (indexShardStats == null) { - indexShardStats = new CommonStats.AggregatedIndexStats(); + indexShardStats = new AggregatedIndexStats(); this.indexStatsMap.put(shardStats.getShardRouting().getIndexName(), indexShardStats); } @@ -223,4 +225,26 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(indexStatsMap, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream)); } } + + /** + * Node level statistics used for ClusterStatsIndices for _cluster/stats call. + */ + @PublicApi(since = "2.16.0") + public static class AggregatedIndexStats implements Writeable { + public int total = 0; + public int primaries = 0; + + public AggregatedIndexStats(StreamInput in) throws IOException { + total = in.readVInt(); + primaries = in.readVInt(); + } + + public AggregatedIndexStats() {} + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(total); + out.writeVInt(primaries); + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java index ac26a3851f9c3..8bfeb13b253c3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java @@ -544,29 +544,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } return builder; } - - /** - * Node level statistics used for ClusterStatsIndices for _cluster/stats call. - */ - @PublicApi(since = "2.16.0") - public static class AggregatedIndexStats implements Writeable { - public int indices = 0; - public int total = 0; - public int primaries = 0; - - public AggregatedIndexStats(StreamInput in) throws IOException { - indices = in.readVInt(); - total = in.readVInt(); - primaries = in.readVInt(); - } - - public AggregatedIndexStats() {} - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(indices); - out.writeVInt(total); - out.writeVInt(primaries); - } - } } From 1b350e5a5e3237c43c11e9b62c0cbb4d57e63c50 Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Tue, 23 Jul 2024 19:20:54 +0530 Subject: [PATCH 15/15] Retry Build Signed-off-by: Pranshu Shukla