From 0acfe3724f84692c1dc488a7d4b5d17338d35af1 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Mon, 11 Nov 2024 18:02:21 -0800 Subject: [PATCH 01/15] Added implementation for the stats calculation for search and regular replica in shards Signed-off-by: Vinay Krishna Pudyodu --- .../SegmentReplicationStatsIT.java | 100 ++++++++++++++++ .../SegmentReplicationShardStatsResponse.java | 18 +++ ...ransportSegmentReplicationStatsAction.java | 109 +++++++++++++++++- .../SegmentReplicationPerGroupStats.java | 6 +- .../index/SegmentReplicationShardStats.java | 4 +- .../replication/SegmentReplicationState.java | 11 ++ .../replication/SegmentReplicationTarget.java | 1 + 7 files changed, 241 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java index 89aef6f0be1a6..11f00fbbb97ab 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.ReplicationStats; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationShardStats; @@ -433,4 +434,103 @@ public void testSegmentReplicationNodeAndIndexStats() throws Exception { } + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, true).build(); + } + + public void testSegmentReplicationStatsResponseWithSearchReplica() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(); + + int numShards = 2; + assertAcked( + prepareCreate( + INDEX_NAME, + 0, + Settings.builder() + .put("number_of_shards", numShards) + .put("number_of_replicas", 1) + .put("number_of_search_only_replicas", 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + ) + ); + ensureGreen(); + final long numDocs = scaledRandomIntBetween(50, 100); + for (int i = 0; i < numDocs; i++) { + index(INDEX_NAME, "doc", Integer.toString(i)); + } + refresh(INDEX_NAME); + ensureSearchable(INDEX_NAME); + + assertBusy(() -> { + SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() + .indices() + .prepareSegmentReplicationStats(INDEX_NAME) + .setDetailed(true) + .execute() + .actionGet(); + SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); + final SegmentReplicationState currentReplicationState = perGroupStats.getReplicaStats() + .stream() + .findFirst() + .get() + .getCurrentReplicationState(); + assertEquals(segmentReplicationStatsResponse.getReplicationStats().size(), 1); + assertEquals(segmentReplicationStatsResponse.getTotalShards(), numShards * 3); + assertEquals(segmentReplicationStatsResponse.getSuccessfulShards(), numShards * 3); + assertNotNull(currentReplicationState); + assertEquals(currentReplicationState.getStage(), SegmentReplicationState.Stage.DONE); + assertTrue(currentReplicationState.getIndex().recoveredFileCount() > 0); + }, 1, TimeUnit.MINUTES); + } + + public void testSegmentReplicationStatsResponseWithOnlySearchReplica() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(); + + int numShards = 1; + assertAcked( + prepareCreate( + INDEX_NAME, + 0, + Settings.builder() + .put("number_of_shards", numShards) + .put("number_of_replicas", 0) + .put("number_of_search_only_replicas", 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + ) + ); + ensureGreen(); + final long numDocs = scaledRandomIntBetween(50, 100); + for (int i = 0; i < numDocs; i++) { + index(INDEX_NAME, "doc", Integer.toString(i)); + } + refresh(INDEX_NAME); + ensureSearchable(INDEX_NAME); + + assertBusy(() -> { + SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() + .indices() + .prepareSegmentReplicationStats(INDEX_NAME) + .setDetailed(true) + .execute() + .actionGet(); + SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); + final SegmentReplicationState currentReplicationState = perGroupStats.getReplicaStats() + .stream() + .findFirst() + .get() + .getCurrentReplicationState(); + assertEquals(segmentReplicationStatsResponse.getReplicationStats().size(), 1); + assertEquals(segmentReplicationStatsResponse.getTotalShards(), 2); + assertEquals(segmentReplicationStatsResponse.getSuccessfulShards(), 2); + assertNotNull(currentReplicationState); + assertEquals(currentReplicationState.getStage(), SegmentReplicationState.Stage.DONE); + assertTrue(currentReplicationState.getIndex().recoveredFileCount() > 0); + }, 1, TimeUnit.MINUTES); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java index ce17176a220ae..d5a05a60e8849 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java @@ -13,6 +13,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.SegmentReplicationPerGroupStats; +import org.opensearch.index.SegmentReplicationShardStats; import org.opensearch.indices.replication.SegmentReplicationState; import java.io.IOException; @@ -31,19 +32,31 @@ public class SegmentReplicationShardStatsResponse implements Writeable { @Nullable private final SegmentReplicationState replicaStats; + @Nullable + private final SegmentReplicationShardStats segmentReplicationShardStats; + public SegmentReplicationShardStatsResponse(StreamInput in) throws IOException { this.primaryStats = in.readOptionalWriteable(SegmentReplicationPerGroupStats::new); this.replicaStats = in.readOptionalWriteable(SegmentReplicationState::new); + this.segmentReplicationShardStats = in.readOptionalWriteable(SegmentReplicationShardStats::new); } public SegmentReplicationShardStatsResponse(SegmentReplicationPerGroupStats primaryStats) { this.primaryStats = primaryStats; this.replicaStats = null; + this.segmentReplicationShardStats = null; } public SegmentReplicationShardStatsResponse(SegmentReplicationState replicaStats) { this.replicaStats = replicaStats; this.primaryStats = null; + this.segmentReplicationShardStats = null; + } + + public SegmentReplicationShardStatsResponse(SegmentReplicationShardStats segmentReplicationShardStats) { + this.primaryStats = null; + this.replicaStats = null; + this.segmentReplicationShardStats = segmentReplicationShardStats; } public SegmentReplicationPerGroupStats getPrimaryStats() { @@ -54,10 +67,15 @@ public SegmentReplicationState getReplicaStats() { return replicaStats; } + public SegmentReplicationShardStats getSegmentReplicationShardStats() { + return segmentReplicationShardStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(primaryStats); out.writeOptionalWriteable(replicaStats); + out.writeOptionalWriteable(segmentReplicationShardStats); } @Override diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index fc97d67c6c3af..b15d83de4b3b7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -26,9 +26,12 @@ import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.store.Store; +import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -36,8 +39,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -101,6 +106,9 @@ protected SegmentReplicationStatsResponse newResponse( final Map replicaStats = new HashMap<>(); // map of index name to list of replication group stats. final Map> primaryStats = new HashMap<>(); + // search replica responses + final Set searchReplicaSegRepShardStats = new HashSet<>(); + for (SegmentReplicationShardStatsResponse response : responses) { if (response != null) { if (response.getReplicaStats() != null) { @@ -109,6 +117,11 @@ protected SegmentReplicationStatsResponse newResponse( replicaStats.putIfAbsent(shardRouting.allocationId().getId(), response.getReplicaStats()); } } + + if (response.getSegmentReplicationShardStats() != null) { + searchReplicaSegRepShardStats.add(response.getSegmentReplicationShardStats()); + } + if (response.getPrimaryStats() != null) { final ShardId shardId = response.getPrimaryStats().getShardId(); if (shardsToFetch.isEmpty() || shardsToFetch.contains(shardId.getId())) { @@ -134,6 +147,15 @@ protected SegmentReplicationStatsResponse newResponse( } } } + // combine the search replica stats with the stats of other replicas + for (Map.Entry> entry : primaryStats.entrySet()) { + for (SegmentReplicationPerGroupStats group : entry.getValue()) { + Set updatedSet = new HashSet<>(group.getReplicaStats()); + updatedSet.addAll(searchReplicaSegRepShardStats); + group.setReplicaStats(updatedSet); + } + } + return new SegmentReplicationStatsResponse(totalShards, successfulShards, failedShards, primaryStats, shardFailures); } @@ -154,13 +176,17 @@ protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplication if (shardRouting.primary()) { return new SegmentReplicationShardStatsResponse(pressureService.getStatsForShard(indexShard)); + } else if (shardRouting.isSearchOnly()) { + SegmentReplicationShardStats segmentReplicationShardStats = calcualteSegmentReplicationShardStats( + shardRouting, + indexShard, + shardId, + request.activeOnly() + ); + return new SegmentReplicationShardStatsResponse(segmentReplicationShardStats); + } else { + return new SegmentReplicationShardStatsResponse(getSegmentReplicationState(shardId, request.activeOnly())); } - - // return information about only on-going segment replication events. - if (request.activeOnly()) { - return new SegmentReplicationShardStatsResponse(targetService.getOngoingEventSegmentReplicationState(shardId)); - } - return new SegmentReplicationShardStatsResponse(targetService.getSegmentReplicationState(shardId)); } @Override @@ -181,4 +207,75 @@ protected ClusterBlockException checkRequestBlock( ) { return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); } + + private SegmentReplicationShardStats calcualteSegmentReplicationShardStats( + ShardRouting shardRouting, + IndexShard indexShard, + ShardId shardId, + boolean isActiveOnly + ) { + ReplicationCheckpoint indexReplicationCheckpoint = indexShard.getLatestReplicationCheckpoint(); + SegmentReplicationState segmentReplicationState = getSegmentReplicationState(shardId, isActiveOnly); + if (segmentReplicationState != null) { + ReplicationCheckpoint latestReplicationCheckpointReceived = segmentReplicationState.getLatestReplicationCheckpoint(); + + SegmentReplicationShardStats segmentReplicationShardStats = new SegmentReplicationShardStats( + shardRouting.allocationId().getId(), + calculateCheckpointsBehind(indexReplicationCheckpoint, latestReplicationCheckpointReceived), + calculateBytesBehind(indexReplicationCheckpoint, latestReplicationCheckpointReceived), + 0, + calculateCurrentReplicationLag(shardId), + getLastCompletedReplicationLag(shardId) + ); + + segmentReplicationShardStats.setCurrentReplicationState(segmentReplicationState); + return segmentReplicationShardStats; + } else { + return new SegmentReplicationShardStats(shardRouting.allocationId().getId(), 0, 0, 0, 0, 0); + } + } + + private SegmentReplicationState getSegmentReplicationState(ShardId shardId, boolean isActiveOnly) { + if (isActiveOnly) { + return targetService.getOngoingEventSegmentReplicationState(shardId); + } else { + return targetService.getSegmentReplicationState(shardId); + } + } + + private long calculateCheckpointsBehind( + ReplicationCheckpoint indexReplicationCheckpoint, + ReplicationCheckpoint latestReplicationCheckpointReceived + ) { + if (latestReplicationCheckpointReceived != null) { + return latestReplicationCheckpointReceived.getSegmentInfosVersion() - indexReplicationCheckpoint.getSegmentInfosVersion(); + } + return 0; + } + + private long calculateBytesBehind( + ReplicationCheckpoint indexReplicationCheckpoint, + ReplicationCheckpoint latestReplicationCheckpointReceived + ) { + if (latestReplicationCheckpointReceived != null) { + Store.RecoveryDiff diff = Store.segmentReplicationDiff( + latestReplicationCheckpointReceived.getMetadataMap(), + indexReplicationCheckpoint.getMetadataMap() + ); + return diff.missing.stream().mapToLong(StoreFileMetadata::length).sum(); + } + return 0; + } + + private long calculateCurrentReplicationLag(ShardId shardId) { + SegmentReplicationState ongoingEventSegmentReplicationState = targetService.getOngoingEventSegmentReplicationState(shardId); + return ongoingEventSegmentReplicationState != null ? ongoingEventSegmentReplicationState.getTimer().time() : 0; + } + + private long getLastCompletedReplicationLag(ShardId shardId) { + SegmentReplicationState lastCompletedSegmentReplicationState = targetService.getlatestCompletedEventSegmentReplicationState( + shardId + ); + return lastCompletedSegmentReplicationState != null ? lastCompletedSegmentReplicationState.getTimer().time() : 0; + } } diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java index 884686ee48fa1..98d8bfc7d66d8 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java @@ -28,7 +28,7 @@ public class SegmentReplicationPerGroupStats implements Writeable, ToXContentFragment { private final ShardId shardId; - private final Set replicaStats; + private Set replicaStats; private final long rejectedRequestCount; public SegmentReplicationPerGroupStats(ShardId shardId, Set replicaStats, long rejectedRequestCount) { @@ -55,6 +55,10 @@ public ShardId getShardId() { return shardId; } + public void setReplicaStats(Set replicaStats) { + this.replicaStats = replicaStats; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("rejected_requests", rejectedRequestCount); diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java index e381ade253422..40fac40f3ce54 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java @@ -64,6 +64,7 @@ public SegmentReplicationShardStats(StreamInput in) throws IOException { this.currentReplicationTimeMillis = in.readVLong(); this.lastCompletedReplicationTimeMillis = in.readVLong(); this.currentReplicationLagMillis = in.readVLong(); + this.currentReplicationState = in.readOptionalWriteable(SegmentReplicationState::new); } public String getAllocationId() { @@ -118,7 +119,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("current_replication_lag", new TimeValue(currentReplicationLagMillis)); builder.field("last_completed_replication_time", new TimeValue(lastCompletedReplicationTimeMillis)); if (currentReplicationState != null) { - builder.startObject(); + builder.startObject("current_replication_state"); currentReplicationState.toXContent(builder, params); builder.endObject(); } @@ -134,6 +135,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(currentReplicationTimeMillis); out.writeVLong(lastCompletedReplicationTimeMillis); out.writeVLong(currentReplicationLagMillis); + out.writeOptionalWriteable(currentReplicationState); } @Override diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 5fa123948c5ac..9e712b981d30a 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -18,6 +18,7 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationTimer; @@ -88,6 +89,8 @@ public static Stage fromId(byte id) { private String sourceDescription; private DiscoveryNode targetNode; + private ReplicationCheckpoint latestReplicationCheckpoint; + public ShardRouting getShardRouting() { return shardRouting; } @@ -148,6 +151,10 @@ public TimeValue getFinalizeReplicationStageTime() { return new TimeValue(time); } + public ReplicationCheckpoint getLatestReplicationCheckpoint() { + return this.latestReplicationCheckpoint; + } + public SegmentReplicationState( ShardRouting shardRouting, ReplicationLuceneIndex index, @@ -252,6 +259,10 @@ public void setStage(Stage stage) { } } + public void setLatestReplicationCheckpoint(ReplicationCheckpoint latestReplicationCheckpoint) { + this.latestReplicationCheckpoint = latestReplicationCheckpoint; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 7131b49a41834..bf86e316db8ec 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -177,6 +177,7 @@ public void startReplication(ActionListener listener) { source.getCheckpointMetadata(getId(), checkpoint, checkpointInfoListener); checkpointInfoListener.whenComplete(checkpointInfo -> { + state.setLatestReplicationCheckpoint(checkpointInfo.getCheckpoint()); final List filesToFetch = getFiles(checkpointInfo); state.setStage(SegmentReplicationState.Stage.GET_FILES); cancellableThreads.checkForCancel(); From 39576c383ee22504c1f4f6085ecc1393539722dc Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Mon, 18 Nov 2024 12:45:58 -0800 Subject: [PATCH 02/15] Updated changelog Signed-off-by: Vinay Krishna Pudyodu --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e9ecea13af60..a0ed8758a253d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489)) - Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271)) - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). +- Added implementation for the stats calculation for search and regular replica in shards ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678)) ### Dependencies - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) From 803c15b7ceead322147e15ae615d230bfb0f2378 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Fri, 22 Nov 2024 12:38:11 -0800 Subject: [PATCH 03/15] Added unit tests for TransportSegmentReplicationStatsAction Signed-off-by: Vinay Krishna Pudyodu --- ...ransportSegmentReplicationStatsAction.java | 108 ++-- .../replication/SegmentReplicationState.java | 10 +- .../replication/SegmentReplicationTarget.java | 2 +- ...ortSegmentReplicationStatsActionTests.java | 495 ++++++++++++++++++ 4 files changed, 556 insertions(+), 59 deletions(-) create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index b15d83de4b3b7..38a5b1c05c8cf 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -21,7 +21,6 @@ import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.index.IndexService; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; @@ -31,13 +30,13 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; -import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -166,9 +165,10 @@ protected SegmentReplicationStatsRequest readRequestFrom(StreamInput in) throws @Override protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplicationStatsRequest request, ShardRouting shardRouting) { - IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); - IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); ShardId shardId = shardRouting.shardId(); + IndexShard indexShard = indicesService + .indexServiceSafe(shardId.getIndex()) + .getShard(shardId.id()); if (indexShard.indexSettings().isSegRepEnabledOrRemoteNode() == false) { return null; @@ -177,12 +177,7 @@ protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplication if (shardRouting.primary()) { return new SegmentReplicationShardStatsResponse(pressureService.getStatsForShard(indexShard)); } else if (shardRouting.isSearchOnly()) { - SegmentReplicationShardStats segmentReplicationShardStats = calcualteSegmentReplicationShardStats( - shardRouting, - indexShard, - shardId, - request.activeOnly() - ); + SegmentReplicationShardStats segmentReplicationShardStats = calcualteSegmentReplicationShardStats(shardRouting); return new SegmentReplicationShardStatsResponse(segmentReplicationShardStats); } else { return new SegmentReplicationShardStatsResponse(getSegmentReplicationState(shardId, request.activeOnly())); @@ -208,31 +203,22 @@ protected ClusterBlockException checkRequestBlock( return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); } - private SegmentReplicationShardStats calcualteSegmentReplicationShardStats( - ShardRouting shardRouting, - IndexShard indexShard, - ShardId shardId, - boolean isActiveOnly - ) { - ReplicationCheckpoint indexReplicationCheckpoint = indexShard.getLatestReplicationCheckpoint(); - SegmentReplicationState segmentReplicationState = getSegmentReplicationState(shardId, isActiveOnly); - if (segmentReplicationState != null) { - ReplicationCheckpoint latestReplicationCheckpointReceived = segmentReplicationState.getLatestReplicationCheckpoint(); - - SegmentReplicationShardStats segmentReplicationShardStats = new SegmentReplicationShardStats( - shardRouting.allocationId().getId(), - calculateCheckpointsBehind(indexReplicationCheckpoint, latestReplicationCheckpointReceived), - calculateBytesBehind(indexReplicationCheckpoint, latestReplicationCheckpointReceived), - 0, - calculateCurrentReplicationLag(shardId), - getLastCompletedReplicationLag(shardId) - ); + private SegmentReplicationShardStats calcualteSegmentReplicationShardStats(ShardRouting shardRouting) { + ShardId shardId = shardRouting.shardId(); + SegmentReplicationState completedSegmentReplicationState = targetService.getlatestCompletedEventSegmentReplicationState(shardId); + SegmentReplicationState ongoingSegmentReplicationState = targetService.getOngoingEventSegmentReplicationState(shardId); - segmentReplicationShardStats.setCurrentReplicationState(segmentReplicationState); - return segmentReplicationShardStats; - } else { - return new SegmentReplicationShardStats(shardRouting.allocationId().getId(), 0, 0, 0, 0, 0); - } + SegmentReplicationShardStats segmentReplicationShardStats = new SegmentReplicationShardStats( + shardRouting.allocationId().getId(), + calculateCheckpointsBehind(completedSegmentReplicationState, ongoingSegmentReplicationState), + calculateBytesBehind(completedSegmentReplicationState, ongoingSegmentReplicationState), + 0, + getCurrentReplicationLag(ongoingSegmentReplicationState), + getLastCompletedReplicationLag(completedSegmentReplicationState) + ); + + segmentReplicationShardStats.setCurrentReplicationState(targetService.getSegmentReplicationState(shardId)); + return segmentReplicationShardStats; } private SegmentReplicationState getSegmentReplicationState(ShardId shardId, boolean isActiveOnly) { @@ -244,38 +230,54 @@ private SegmentReplicationState getSegmentReplicationState(ShardId shardId, bool } private long calculateCheckpointsBehind( - ReplicationCheckpoint indexReplicationCheckpoint, - ReplicationCheckpoint latestReplicationCheckpointReceived + SegmentReplicationState completedSegmentReplicationState, + SegmentReplicationState ongoingSegmentReplicationState ) { - if (latestReplicationCheckpointReceived != null) { - return latestReplicationCheckpointReceived.getSegmentInfosVersion() - indexReplicationCheckpoint.getSegmentInfosVersion(); + if (ongoingSegmentReplicationState == null || ongoingSegmentReplicationState.getReplicationCheckpoint() == null) { + return 0; } - return 0; + + if(completedSegmentReplicationState == null || + completedSegmentReplicationState.getReplicationCheckpoint() == null) { + return ongoingSegmentReplicationState + .getReplicationCheckpoint() + .getSegmentInfosVersion(); + } + + return ongoingSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion() - + completedSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion(); } private long calculateBytesBehind( - ReplicationCheckpoint indexReplicationCheckpoint, - ReplicationCheckpoint latestReplicationCheckpointReceived + SegmentReplicationState completedSegmentReplicationState, + SegmentReplicationState ongoingSegmentReplicationState ) { - if (latestReplicationCheckpointReceived != null) { + if (ongoingSegmentReplicationState == null || + ongoingSegmentReplicationState.getReplicationCheckpoint() == null) { + return 0; + } + + if (completedSegmentReplicationState == null || + completedSegmentReplicationState.getReplicationCheckpoint() == null) { Store.RecoveryDiff diff = Store.segmentReplicationDiff( - latestReplicationCheckpointReceived.getMetadataMap(), - indexReplicationCheckpoint.getMetadataMap() + ongoingSegmentReplicationState.getReplicationCheckpoint().getMetadataMap(), + Collections.emptyMap() ); return diff.missing.stream().mapToLong(StoreFileMetadata::length).sum(); } - return 0; + + Store.RecoveryDiff diff = Store.segmentReplicationDiff( + ongoingSegmentReplicationState.getReplicationCheckpoint().getMetadataMap(), + completedSegmentReplicationState.getReplicationCheckpoint().getMetadataMap() + ); + return diff.missing.stream().mapToLong(StoreFileMetadata::length).sum(); } - private long calculateCurrentReplicationLag(ShardId shardId) { - SegmentReplicationState ongoingEventSegmentReplicationState = targetService.getOngoingEventSegmentReplicationState(shardId); - return ongoingEventSegmentReplicationState != null ? ongoingEventSegmentReplicationState.getTimer().time() : 0; + private long getCurrentReplicationLag(SegmentReplicationState ongoingSegmentReplicationState) { + return ongoingSegmentReplicationState != null ? ongoingSegmentReplicationState.getTimer().time() : 0; } - private long getLastCompletedReplicationLag(ShardId shardId) { - SegmentReplicationState lastCompletedSegmentReplicationState = targetService.getlatestCompletedEventSegmentReplicationState( - shardId - ); - return lastCompletedSegmentReplicationState != null ? lastCompletedSegmentReplicationState.getTimer().time() : 0; + private long getLastCompletedReplicationLag(SegmentReplicationState completedSegmentReplicationState) { + return completedSegmentReplicationState != null ? completedSegmentReplicationState.getTimer().time() : 0; } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 9e712b981d30a..29130b18ffc7b 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -89,7 +89,7 @@ public static Stage fromId(byte id) { private String sourceDescription; private DiscoveryNode targetNode; - private ReplicationCheckpoint latestReplicationCheckpoint; + private ReplicationCheckpoint replicationCheckpoint; public ShardRouting getShardRouting() { return shardRouting; @@ -151,8 +151,8 @@ public TimeValue getFinalizeReplicationStageTime() { return new TimeValue(time); } - public ReplicationCheckpoint getLatestReplicationCheckpoint() { - return this.latestReplicationCheckpoint; + public ReplicationCheckpoint getReplicationCheckpoint() { + return this.replicationCheckpoint; } public SegmentReplicationState( @@ -259,8 +259,8 @@ public void setStage(Stage stage) { } } - public void setLatestReplicationCheckpoint(ReplicationCheckpoint latestReplicationCheckpoint) { - this.latestReplicationCheckpoint = latestReplicationCheckpoint; + public void setReplicationCheckpoint(ReplicationCheckpoint replicationCheckpoint) { + this.replicationCheckpoint = replicationCheckpoint; } @Override diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index bf86e316db8ec..8223f5d504700 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -177,7 +177,7 @@ public void startReplication(ActionListener listener) { source.getCheckpointMetadata(getId(), checkpoint, checkpointInfoListener); checkpointInfoListener.whenComplete(checkpointInfo -> { - state.setLatestReplicationCheckpoint(checkpointInfo.getCheckpoint()); + state.setReplicationCheckpoint(checkpointInfo.getCheckpoint()); final List filesToFetch = getFiles(checkpointInfo); state.setStage(SegmentReplicationState.Stage.GET_FILES); cancellableThreads.checkForCancel(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java new file mode 100644 index 0000000000000..cbb737d92972f --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java @@ -0,0 +1,495 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.replication; + +import org.junit.Before; +import org.opensearch.Version; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlock; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.AllocationId; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.ShardIterator; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.ShardsIterator; +import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexService; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.SegmentReplicationPerGroupStats; +import org.opensearch.index.SegmentReplicationPressureService; +import org.opensearch.index.SegmentReplicationShardStats; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.SegmentReplicationState; +import org.opensearch.indices.replication.SegmentReplicationTargetService; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.indices.replication.common.ReplicationTimer; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.TransportService; + +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.EMPTY_LIST; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportSegmentReplicationStatsActionTests extends OpenSearchTestCase { + + private static final String TEST_INDEX = "test-index"; + + private SegmentReplicationPerGroupStats segmentReplicationPerGroupStats; + + private IndexShard indexShard; + + private SegmentReplicationState completedSegmentReplicationState; + private SegmentReplicationState onGoingSegmentReplicationState; + + private SegmentReplicationTargetService targetService; + + private ShardId shardId; + + + TransportSegmentReplicationStatsAction action; + + private final ClusterBlock writeClusterBlock = new ClusterBlock( + 1, + "uuid", + "", + true, + true, + true, + RestStatus.OK, + EnumSet.of(ClusterBlockLevel.METADATA_WRITE) + ); + + private final ClusterBlock readClusterBlock = new ClusterBlock( + 1, + "uuid", + "", + true, + true, + true, + RestStatus.OK, + EnumSet.of(ClusterBlockLevel.METADATA_READ) + ); + + @Before + public void setUp() throws Exception { + super.setUp(); + Index index = new Index(TEST_INDEX, "_na_"); + shardId = new ShardId(TEST_INDEX, "_na_", 0); + + IndicesService indicesService = mock(IndicesService.class); + IndexService indexService = mock(IndexService.class); + + indexShard = mock(IndexShard.class); + SegmentReplicationPressureService pressureService = mock(SegmentReplicationPressureService.class); + segmentReplicationPerGroupStats = mock(SegmentReplicationPerGroupStats.class); + targetService = mock(SegmentReplicationTargetService.class); + completedSegmentReplicationState = mock(SegmentReplicationState.class); + onGoingSegmentReplicationState = mock(SegmentReplicationState.class); + ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); + ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); + + ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); + ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + IndexMetadata indexMetadata = new IndexMetadata.Builder(TEST_INDEX).settings(settings).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + + when(indicesService.indexServiceSafe(index)).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(pressureService.getStatsForShard(indexShard)).thenReturn(segmentReplicationPerGroupStats); + when(indexShard.indexSettings()).thenReturn(indexSettings); + + when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); + when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); + when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); + when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); + + long segmentInfoCompleted = 5; + long segmentInfoOngoing = 9; + when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); + when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); + + final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); + final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); + + when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); + when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); + + long time1 = 10; + long time2 = 15; + when(replicationTimerOngoing.time()).thenReturn(time1); + when(replicationTimerCompleted.time()).thenReturn(time2); + + action = new TransportSegmentReplicationStatsAction( + mock(ClusterService.class), + mock(TransportService.class), + indicesService, + targetService, + new ActionFilters(new HashSet<>()), + mock(IndexNameExpressionResolver.class), + pressureService + ); + } + + DiscoveryNode newNode(int nodeId) { + return new DiscoveryNode("node_" + nodeId, buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + } + + public void testShardReturnsAllTheShardsForTheIndex1() { + SegmentReplicationStatsRequest segmentReplicationStatsRequest = mock(SegmentReplicationStatsRequest.class); + String[] concreteIndices = new String[] { TEST_INDEX }; + ClusterState clusterState = mock(ClusterState.class); + RoutingTable routingTables = mock(RoutingTable.class); + ShardsIterator shardsIterator = mock(ShardIterator.class); + + when(clusterState.routingTable()).thenReturn(routingTables); + when(routingTables.allShardsIncludingRelocationTargets(any())).thenReturn(shardsIterator); + assertEquals(shardsIterator, action.shards(clusterState, segmentReplicationStatsRequest, concreteIndices)); + } + + public void testGlobalBlockCheck() { + ClusterBlocks.Builder builder = ClusterBlocks.builder(); + builder.addGlobalBlock(writeClusterBlock); + ClusterState metadataWriteBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNull(action.checkGlobalBlock(metadataWriteBlockedState, new SegmentReplicationStatsRequest())); + + builder = ClusterBlocks.builder(); + builder.addGlobalBlock(readClusterBlock); + ClusterState metadataReadBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNotNull(action.checkGlobalBlock(metadataReadBlockedState, new SegmentReplicationStatsRequest())); + } + + public void testIndexBlockCheck() { + String indexName = "test"; + ClusterBlocks.Builder builder = ClusterBlocks.builder(); + builder.addIndexBlock(indexName, writeClusterBlock); + ClusterState metadataWriteBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNull(action.checkRequestBlock(metadataWriteBlockedState, new SegmentReplicationStatsRequest(), new String[] { indexName })); + + builder = ClusterBlocks.builder(); + builder.addIndexBlock(indexName, readClusterBlock); + ClusterState metadataReadBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNotNull(action.checkRequestBlock(metadataReadBlockedState, new SegmentReplicationStatsRequest(), new String[] { indexName })); + } + + public void testShardOperationWhenReplicationIsNotSegRep() { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + IndexMetadata indexMetadata = new IndexMetadata.Builder(TEST_INDEX).settings(settings).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + + when(indexShard.indexSettings()).thenReturn(indexSettings); + + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + TEST_INDEX, + shardId.getId(), + node.getId(), + true, + ShardRoutingState.STARTED + ); + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); + assertNull(response); + } + + public void testShardOperationOnPrimaryShard() { + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + TEST_INDEX, + shardId.getId(), + node.getId(), + true, + ShardRoutingState.STARTED + ); + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); + + assertEquals(segmentReplicationPerGroupStats, response.getPrimaryStats()); + assertNull(response.getReplicaStats()); + assertNull(response.getSegmentReplicationShardStats()); + } + + public void testShardOperationOnReplicaShard() { + when(targetService.getSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); + + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + TEST_INDEX, + shardId.getId(), + node.getId(), + false, + ShardRoutingState.STARTED + ); + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + segmentReplicationStatsRequest.activeOnly(false); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); + + assertEquals(completedSegmentReplicationState, response.getReplicaStats()); + assertNull(response.getPrimaryStats()); + assertNull(response.getSegmentReplicationShardStats()); + } + + public void testShardOperationOnReplicaShardWhenActiveOnlyIsSet() { + when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + TEST_INDEX, + shardId.getId(), + node.getId(), + false, + ShardRoutingState.STARTED + ); + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + segmentReplicationStatsRequest.activeOnly(true); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); + + assertEquals(onGoingSegmentReplicationState, response.getReplicaStats()); + assertNull(response.getPrimaryStats()); + assertNull(response.getSegmentReplicationShardStats()); + } + + public void testShardOperationOnSearchReplicaWhenCompletedAndOngoingSegRepStateNotNull() { + when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); + when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + + ShardRouting searchShardRouting = TestShardRouting.newShardRouting( + shardId, + node.getId(), + null, + false, + true, + ShardRoutingState.STARTED, + null + ); + + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + + assertNull(response.getPrimaryStats()); + assertNull(response.getReplicaStats()); + assertNotNull(response.getSegmentReplicationShardStats()); + } + + public void testShardOperationOnSearchReplicaWhenCompletedSegRepStateIsNull() { + when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + + ShardRouting searchShardRouting = TestShardRouting.newShardRouting( + shardId, + node.getId(), + null, + false, + true, + ShardRoutingState.STARTED, + null + ); + + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + + assertNull(response.getPrimaryStats()); + assertNull(response.getReplicaStats()); + assertNotNull(response.getSegmentReplicationShardStats()); + } + + public void testShardOperationOnSearchReplicaWhenOngoingSegRepStateIsNull() { + when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); + when(targetService.getSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); + + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + + ShardRouting searchShardRouting = TestShardRouting.newShardRouting( + shardId, + node.getId(), + null, + false, + true, + ShardRoutingState.STARTED, + null + ); + + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + + assertNull(response.getPrimaryStats()); + assertNull(response.getReplicaStats()); + assertNotNull(response.getSegmentReplicationShardStats()); + } + + public void testShardOperationOnSearchReplicaWhenCompletedAndOngoingSegRepStateIsNull() { + final DiscoveryNode node = newNode(0); + final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + + ShardRouting searchShardRouting = TestShardRouting.newShardRouting( + shardId, + node.getId(), + null, + false, + true, + ShardRoutingState.STARTED, + null + ); + + SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); + SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + + assertNull(response.getPrimaryStats()); + assertNull(response.getReplicaStats()); + assertNotNull(response.getSegmentReplicationShardStats()); + } + + public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + String[] shards = {"1", "2", "3"}; + request.shards(shards); + + int totalShards = 3; + int successfulShards = 3; + int failedShard = 0; + String allocIdOne = "allocIdOne"; + String allocIdTwo = "allocIdTwo"; + ShardId shardIdOne = mock(ShardId.class); + ShardId shardIdTwo = mock(ShardId.class); + ShardId shardIdThree = mock(ShardId.class); + ShardRouting shardRoutingOne = mock(ShardRouting.class); + ShardRouting shardRoutingTwo = mock(ShardRouting.class); + ShardRouting shardRoutingThree = mock(ShardRouting.class); + when(shardIdOne.getId()).thenReturn(1); + when(shardIdTwo.getId()).thenReturn(2); + when(shardIdThree.getId()).thenReturn(3); + when(shardRoutingOne.shardId()).thenReturn(shardIdOne); + when(shardRoutingTwo.shardId()).thenReturn(shardIdTwo); + when(shardRoutingThree.shardId()).thenReturn(shardIdThree); + AllocationId allocationId = mock(AllocationId.class); + when(allocationId.getId()).thenReturn(allocIdOne); + when(shardRoutingTwo.allocationId()).thenReturn(allocationId); + when(shardIdOne.getIndexName()).thenReturn(TEST_INDEX); + + Set segmentReplicationShardStats = new HashSet<>(); + SegmentReplicationShardStats segmentReplicationShardStatsOfReplica = new SegmentReplicationShardStats(allocIdOne, 0, 0, 0, 0, 0); + segmentReplicationShardStats.add(segmentReplicationShardStatsOfReplica); + SegmentReplicationPerGroupStats segmentReplicationPerGroupStats = new SegmentReplicationPerGroupStats(shardIdOne, segmentReplicationShardStats, 0); + + SegmentReplicationState segmentReplicationState = mock(SegmentReplicationState.class); + SegmentReplicationShardStats segmentReplicationShardStatsFromSearchReplica = mock(SegmentReplicationShardStats.class); + when(segmentReplicationShardStatsFromSearchReplica.getAllocationId()).thenReturn("alloc2"); + when(segmentReplicationState.getShardRouting()).thenReturn(shardRoutingTwo); + + List responses = List.of( + new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStats), + new SegmentReplicationShardStatsResponse(segmentReplicationState), + new SegmentReplicationShardStatsResponse(segmentReplicationShardStatsFromSearchReplica) + ); + + SegmentReplicationStatsResponse response = action.newResponse( + request, totalShards, successfulShards, failedShard, responses, EMPTY_LIST, ClusterState.EMPTY_STATE); + + List responseStats = response.getReplicationStats().get(TEST_INDEX); + SegmentReplicationPerGroupStats primStats = responseStats.get(0); + Set segRpShardStatsSet = primStats.getReplicaStats(); + + for (SegmentReplicationShardStats segRpShardStats: segRpShardStatsSet) { + if(segRpShardStats.getAllocationId().equals(allocIdOne)) { + assertEquals(segmentReplicationState, segRpShardStats.getCurrentReplicationState()); + } + + if (segRpShardStats.getAllocationId().equals(allocIdTwo)) { + assertEquals(segmentReplicationShardStatsFromSearchReplica, segRpShardStats); + } + } + } + + public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + String[] shards = {"1", "2"}; + request.shards(shards); + int totalShards = 3; + int successfulShards = 3; + int failedShard = 0; + + SegmentReplicationPerGroupStats segmentReplicationPerGroupStatsOne = mock(SegmentReplicationPerGroupStats.class); + SegmentReplicationPerGroupStats segmentReplicationPerGroupStatsTwo = mock(SegmentReplicationPerGroupStats.class); + + ShardId shardIdOne = mock(ShardId.class); + ShardId shardIdTwo = mock(ShardId.class); + when(segmentReplicationPerGroupStatsOne.getShardId()).thenReturn(shardIdOne); + when(segmentReplicationPerGroupStatsTwo.getShardId()).thenReturn(shardIdTwo); + when(shardIdOne.getIndexName()).thenReturn(TEST_INDEX); + when(shardIdTwo.getIndexName()).thenReturn(TEST_INDEX); + when(shardIdOne.getId()).thenReturn(1); + when(shardIdTwo.getId()).thenReturn(2); + + List responses = List.of( + new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStatsOne), + new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStatsTwo) + ); + + SegmentReplicationStatsResponse response = action.newResponse( + request, totalShards, successfulShards, failedShard, responses, EMPTY_LIST, ClusterState.EMPTY_STATE); + + List responseStats = response.getReplicationStats().get(TEST_INDEX); + + for (SegmentReplicationPerGroupStats primStat: responseStats) { + if(primStat.getShardId().equals(shardIdOne)) { + assertEquals(segmentReplicationPerGroupStatsOne, primStat); + } + + if(primStat.getShardId().equals(shardIdTwo)) { + assertEquals(segmentReplicationPerGroupStatsTwo, primStat); + } + } + } +} From c6e2c808c185c138b63a48353346af132cc48817 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Fri, 22 Nov 2024 13:57:49 -0800 Subject: [PATCH 04/15] fixed java style after running precommit locally Signed-off-by: Vinay Krishna Pudyodu --- ...ransportSegmentReplicationStatsAction.java | 22 ++++----- ...ortSegmentReplicationStatsActionTests.java | 46 +++++++++++++------ 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index 38a5b1c05c8cf..ca408e5433837 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -166,9 +166,7 @@ protected SegmentReplicationStatsRequest readRequestFrom(StreamInput in) throws @Override protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplicationStatsRequest request, ShardRouting shardRouting) { ShardId shardId = shardRouting.shardId(); - IndexShard indexShard = indicesService - .indexServiceSafe(shardId.getIndex()) - .getShard(shardId.id()); + IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); if (indexShard.indexSettings().isSegRepEnabledOrRemoteNode() == false) { return null; @@ -237,28 +235,24 @@ private long calculateCheckpointsBehind( return 0; } - if(completedSegmentReplicationState == null || - completedSegmentReplicationState.getReplicationCheckpoint() == null) { - return ongoingSegmentReplicationState - .getReplicationCheckpoint() - .getSegmentInfosVersion(); + if (completedSegmentReplicationState == null || completedSegmentReplicationState.getReplicationCheckpoint() == null) { + return ongoingSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion(); } - return ongoingSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion() - - completedSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion(); + return ongoingSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion() - completedSegmentReplicationState + .getReplicationCheckpoint() + .getSegmentInfosVersion(); } private long calculateBytesBehind( SegmentReplicationState completedSegmentReplicationState, SegmentReplicationState ongoingSegmentReplicationState ) { - if (ongoingSegmentReplicationState == null || - ongoingSegmentReplicationState.getReplicationCheckpoint() == null) { + if (ongoingSegmentReplicationState == null || ongoingSegmentReplicationState.getReplicationCheckpoint() == null) { return 0; } - if (completedSegmentReplicationState == null || - completedSegmentReplicationState.getReplicationCheckpoint() == null) { + if (completedSegmentReplicationState == null || completedSegmentReplicationState.getReplicationCheckpoint() == null) { Store.RecoveryDiff diff = Store.segmentReplicationDiff( ongoingSegmentReplicationState.getReplicationCheckpoint().getMetadataMap(), Collections.emptyMap() diff --git a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java index cbb737d92972f..a69f5a9c67537 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java @@ -8,7 +8,6 @@ package org.opensearch.action.admin.indices.replication; -import org.junit.Before; import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; @@ -27,6 +26,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.rest.RestStatus; @@ -45,14 +45,15 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportService; +import org.junit.Before; +import java.util.ArrayList; import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import static java.util.Collections.EMPTY_LIST; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.mockito.ArgumentMatchers.any; @@ -74,7 +75,6 @@ public class TransportSegmentReplicationStatsActionTests extends OpenSearchTestC private ShardId shardId; - TransportSegmentReplicationStatsAction action; private final ClusterBlock writeClusterBlock = new ClusterBlock( @@ -393,7 +393,8 @@ public void testShardOperationOnSearchReplicaWhenCompletedAndOngoingSegRepStateI public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - String[] shards = {"1", "2", "3"}; + List shardFailures = new ArrayList<>(); + String[] shards = { "1", "2", "3" }; request.shards(shards); int totalShards = 3; @@ -421,7 +422,11 @@ public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { Set segmentReplicationShardStats = new HashSet<>(); SegmentReplicationShardStats segmentReplicationShardStatsOfReplica = new SegmentReplicationShardStats(allocIdOne, 0, 0, 0, 0, 0); segmentReplicationShardStats.add(segmentReplicationShardStatsOfReplica); - SegmentReplicationPerGroupStats segmentReplicationPerGroupStats = new SegmentReplicationPerGroupStats(shardIdOne, segmentReplicationShardStats, 0); + SegmentReplicationPerGroupStats segmentReplicationPerGroupStats = new SegmentReplicationPerGroupStats( + shardIdOne, + segmentReplicationShardStats, + 0 + ); SegmentReplicationState segmentReplicationState = mock(SegmentReplicationState.class); SegmentReplicationShardStats segmentReplicationShardStatsFromSearchReplica = mock(SegmentReplicationShardStats.class); @@ -435,14 +440,21 @@ public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { ); SegmentReplicationStatsResponse response = action.newResponse( - request, totalShards, successfulShards, failedShard, responses, EMPTY_LIST, ClusterState.EMPTY_STATE); + request, + totalShards, + successfulShards, + failedShard, + responses, + shardFailures, + ClusterState.EMPTY_STATE + ); List responseStats = response.getReplicationStats().get(TEST_INDEX); SegmentReplicationPerGroupStats primStats = responseStats.get(0); Set segRpShardStatsSet = primStats.getReplicaStats(); - for (SegmentReplicationShardStats segRpShardStats: segRpShardStatsSet) { - if(segRpShardStats.getAllocationId().equals(allocIdOne)) { + for (SegmentReplicationShardStats segRpShardStats : segRpShardStatsSet) { + if (segRpShardStats.getAllocationId().equals(allocIdOne)) { assertEquals(segmentReplicationState, segRpShardStats.getCurrentReplicationState()); } @@ -454,7 +466,8 @@ public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - String[] shards = {"1", "2"}; + List shardFailures = new ArrayList<>(); + String[] shards = { "1", "2" }; request.shards(shards); int totalShards = 3; int successfulShards = 3; @@ -478,16 +491,23 @@ public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { ); SegmentReplicationStatsResponse response = action.newResponse( - request, totalShards, successfulShards, failedShard, responses, EMPTY_LIST, ClusterState.EMPTY_STATE); + request, + totalShards, + successfulShards, + failedShard, + responses, + shardFailures, + ClusterState.EMPTY_STATE + ); List responseStats = response.getReplicationStats().get(TEST_INDEX); - for (SegmentReplicationPerGroupStats primStat: responseStats) { - if(primStat.getShardId().equals(shardIdOne)) { + for (SegmentReplicationPerGroupStats primStat : responseStats) { + if (primStat.getShardId().equals(shardIdOne)) { assertEquals(segmentReplicationPerGroupStatsOne, primStat); } - if(primStat.getShardId().equals(shardIdTwo)) { + if (primStat.getShardId().equals(shardIdTwo)) { assertEquals(segmentReplicationPerGroupStatsTwo, primStat); } } From fcdcdda2da14a5dfe6481abb3644368b3ec34a9d Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Mon, 25 Nov 2024 14:15:45 -0800 Subject: [PATCH 05/15] refined the test cases Signed-off-by: Vinay Krishna Pudyodu --- ...ransportSegmentReplicationStatsAction.java | 1 + ...ortSegmentReplicationStatsActionTests.java | 705 +++++++++++------- 2 files changed, 448 insertions(+), 258 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index ca408e5433837..714b4990a1479 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -168,6 +168,7 @@ protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplication ShardId shardId = shardRouting.shardId(); IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); + System.out.println(indexShard.indexSettings().isSegRepEnabledOrRemoteNode()); if (indexShard.indexSettings().isSegRepEnabledOrRemoteNode() == false) { return null; } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java index a69f5a9c67537..6e0ce969348bf 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java @@ -8,6 +8,9 @@ package org.opensearch.action.admin.indices.replication; +import org.junit.Before; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; @@ -16,14 +19,11 @@ import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.ShardsIterator; -import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.support.DefaultShardOperationFailedException; @@ -45,7 +45,6 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportService; -import org.junit.Before; import java.util.ArrayList; import java.util.EnumSet; @@ -54,125 +53,52 @@ import java.util.Map; import java.util.Set; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TransportSegmentReplicationStatsActionTests extends OpenSearchTestCase { - - private static final String TEST_INDEX = "test-index"; - - private SegmentReplicationPerGroupStats segmentReplicationPerGroupStats; - + @Mock + private ClusterService clusterService; + @Mock + private TransportService transportService; + @Mock + private IndicesService indicesService; + @Mock + private SegmentReplicationTargetService targetService; + @Mock + private ActionFilters actionFilters; + @Mock + private IndexNameExpressionResolver indexNameExpressionResolver; + @Mock + private SegmentReplicationPressureService pressureService; + @Mock private IndexShard indexShard; - private SegmentReplicationState completedSegmentReplicationState; - private SegmentReplicationState onGoingSegmentReplicationState; + @Mock + private IndexService indexService; - private SegmentReplicationTargetService targetService; - - private ShardId shardId; - - TransportSegmentReplicationStatsAction action; - - private final ClusterBlock writeClusterBlock = new ClusterBlock( - 1, - "uuid", - "", - true, - true, - true, - RestStatus.OK, - EnumSet.of(ClusterBlockLevel.METADATA_WRITE) - ); - - private final ClusterBlock readClusterBlock = new ClusterBlock( - 1, - "uuid", - "", - true, - true, - true, - RestStatus.OK, - EnumSet.of(ClusterBlockLevel.METADATA_READ) - ); + private TransportSegmentReplicationStatsAction action; @Before public void setUp() throws Exception { + MockitoAnnotations.openMocks(this); super.setUp(); - Index index = new Index(TEST_INDEX, "_na_"); - shardId = new ShardId(TEST_INDEX, "_na_", 0); - - IndicesService indicesService = mock(IndicesService.class); - IndexService indexService = mock(IndexService.class); - - indexShard = mock(IndexShard.class); - SegmentReplicationPressureService pressureService = mock(SegmentReplicationPressureService.class); - segmentReplicationPerGroupStats = mock(SegmentReplicationPerGroupStats.class); - targetService = mock(SegmentReplicationTargetService.class); - completedSegmentReplicationState = mock(SegmentReplicationState.class); - onGoingSegmentReplicationState = mock(SegmentReplicationState.class); - ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); - ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); - - ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); - ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); - - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .build(); - IndexMetadata indexMetadata = new IndexMetadata.Builder(TEST_INDEX).settings(settings).build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - - when(indicesService.indexServiceSafe(index)).thenReturn(indexService); - when(indexService.getShard(shardId.id())).thenReturn(indexShard); - when(pressureService.getStatsForShard(indexShard)).thenReturn(segmentReplicationPerGroupStats); - when(indexShard.indexSettings()).thenReturn(indexSettings); - - when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); - when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); - when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); - when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); - - long segmentInfoCompleted = 5; - long segmentInfoOngoing = 9; - when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); - when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); - - final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); - final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); - - when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); - when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); - - long time1 = 10; - long time2 = 15; - when(replicationTimerOngoing.time()).thenReturn(time1); - when(replicationTimerCompleted.time()).thenReturn(time2); - action = new TransportSegmentReplicationStatsAction( - mock(ClusterService.class), - mock(TransportService.class), + clusterService, + transportService, indicesService, targetService, - new ActionFilters(new HashSet<>()), - mock(IndexNameExpressionResolver.class), + actionFilters, + indexNameExpressionResolver, pressureService ); } - DiscoveryNode newNode(int nodeId) { - return new DiscoveryNode("node_" + nodeId, buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); - } - - public void testShardReturnsAllTheShardsForTheIndex1() { + public void testShardReturnsAllTheShardsForTheIndex() { SegmentReplicationStatsRequest segmentReplicationStatsRequest = mock(SegmentReplicationStatsRequest.class); - String[] concreteIndices = new String[] { TEST_INDEX }; + String[] concreteIndices = new String[] { "test-index" }; ClusterState clusterState = mock(ClusterState.class); RoutingTable routingTables = mock(RoutingTable.class); ShardsIterator shardsIterator = mock(ShardIterator.class); @@ -182,213 +108,297 @@ public void testShardReturnsAllTheShardsForTheIndex1() { assertEquals(shardsIterator, action.shards(clusterState, segmentReplicationStatsRequest, concreteIndices)); } - public void testGlobalBlockCheck() { - ClusterBlocks.Builder builder = ClusterBlocks.builder(); - builder.addGlobalBlock(writeClusterBlock); - ClusterState metadataWriteBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); - assertNull(action.checkGlobalBlock(metadataWriteBlockedState, new SegmentReplicationStatsRequest())); - - builder = ClusterBlocks.builder(); - builder.addGlobalBlock(readClusterBlock); - ClusterState metadataReadBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); - assertNotNull(action.checkGlobalBlock(metadataReadBlockedState, new SegmentReplicationStatsRequest())); - } - - public void testIndexBlockCheck() { - String indexName = "test"; - ClusterBlocks.Builder builder = ClusterBlocks.builder(); - builder.addIndexBlock(indexName, writeClusterBlock); - ClusterState metadataWriteBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); - assertNull(action.checkRequestBlock(metadataWriteBlockedState, new SegmentReplicationStatsRequest(), new String[] { indexName })); - - builder = ClusterBlocks.builder(); - builder.addIndexBlock(indexName, readClusterBlock); - ClusterState metadataReadBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); - assertNotNull(action.checkRequestBlock(metadataReadBlockedState, new SegmentReplicationStatsRequest(), new String[] { indexName })); - } + public void testShardOperationWithPrimaryShard() { + ShardRouting shardRouting = mock(ShardRouting.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - public void testShardOperationWhenReplicationIsNotSegRep() { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .build(); - IndexMetadata indexMetadata = new IndexMetadata.Builder(TEST_INDEX).settings(settings).build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(true); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); - when(indexShard.indexSettings()).thenReturn(indexSettings); + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); - ShardRouting shardRouting = TestShardRouting.newShardRouting( - TEST_INDEX, - shardId.getId(), - node.getId(), - true, - ShardRoutingState.STARTED - ); - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); - assertNull(response); + assertNotNull(response); + verify(pressureService).getStatsForShard(any()); } - public void testShardOperationOnPrimaryShard() { - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); - ShardRouting shardRouting = TestShardRouting.newShardRouting( - TEST_INDEX, - shardId.getId(), - node.getId(), - true, - ShardRoutingState.STARTED - ); - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); - - assertEquals(segmentReplicationPerGroupStats, response.getPrimaryStats()); - assertNull(response.getReplicaStats()); - assertNull(response.getSegmentReplicationShardStats()); - } + public void testShardOperationWithReplicaShard() { + ShardRouting shardRouting = mock(ShardRouting.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + request.activeOnly(false); + SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); - public void testShardOperationOnReplicaShard() { + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); - ShardRouting shardRouting = TestShardRouting.newShardRouting( - TEST_INDEX, - shardId.getId(), - node.getId(), - false, - ShardRoutingState.STARTED - ); - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - segmentReplicationStatsRequest.activeOnly(false); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); - assertEquals(completedSegmentReplicationState, response.getReplicaStats()); + assertNotNull(response); assertNull(response.getPrimaryStats()); + assertNotNull(response.getReplicaStats()); assertNull(response.getSegmentReplicationShardStats()); + verify(targetService).getSegmentReplicationState(shardId); } - public void testShardOperationOnReplicaShardWhenActiveOnlyIsSet() { + public void testShardOperationWithReplicaShardActiveOnly() { + ShardRouting shardRouting = mock(ShardRouting.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + request.activeOnly(true); + SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); + + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); - ShardRouting shardRouting = TestShardRouting.newShardRouting( - TEST_INDEX, - shardId.getId(), - node.getId(), - false, - ShardRoutingState.STARTED - ); - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - segmentReplicationStatsRequest.activeOnly(true); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, shardRouting); + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); - assertEquals(onGoingSegmentReplicationState, response.getReplicaStats()); + assertNotNull(response); assertNull(response.getPrimaryStats()); + assertNotNull(response.getReplicaStats()); assertNull(response.getSegmentReplicationShardStats()); + verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationOnSearchReplicaWhenCompletedAndOngoingSegRepStateNotNull() { + public void testShardOperationWithSearchOnlyReplicaWhenCompletedAndOngoingStateNotNull() { + ShardRouting shardRouting = mock(ShardRouting.class); + SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); + SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + AllocationId allocationId = AllocationId.newInitializing(); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); + ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); + ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); + ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); + long time1 = 10; + long time2 = 15; + final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); + final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); + long segmentInfoCompleted = 5; + long segmentInfoOngoing = 9; + + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(shardRouting.isSearchOnly()).thenReturn(true); + when(shardRouting.allocationId()).thenReturn(allocationId); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); + when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); + when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); + when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); + when(replicationTimerOngoing.time()).thenReturn(time1); + when(replicationTimerCompleted.time()).thenReturn(time2); + when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); + when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); + when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); + when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); - ShardRouting searchShardRouting = TestShardRouting.newShardRouting( - shardId, - node.getId(), - null, - false, - true, - ShardRoutingState.STARTED, - null - ); + assertNotNull(response); + verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); + verify(targetService).getOngoingEventSegmentReplicationState(shardId); + } - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedState() { + ShardRouting shardRouting = mock(ShardRouting.class); + SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + AllocationId allocationId = AllocationId.newInitializing(); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); + ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); + long time1 = 10; + final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); + final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); + long segmentInfoOngoing = 9; + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(shardRouting.isSearchOnly()).thenReturn(true); + when(shardRouting.allocationId()).thenReturn(allocationId); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); + when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); + when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); + when(replicationTimerOngoing.time()).thenReturn(time1); + when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); + when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); + + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + + assertNotNull(response); assertNull(response.getPrimaryStats()); assertNull(response.getReplicaStats()); assertNotNull(response.getSegmentReplicationShardStats()); + verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); + verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationOnSearchReplicaWhenCompletedSegRepStateIsNull() { + public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedCheckpoint() { + ShardRouting shardRouting = mock(ShardRouting.class); + SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); + SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + AllocationId allocationId = AllocationId.newInitializing(); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); + ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); + ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); + long time1 = 10; + long time2 = 15; + final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); + final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); + long segmentInfoOngoing = 9; + + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(shardRouting.isSearchOnly()).thenReturn(true); + when(shardRouting.allocationId()).thenReturn(allocationId); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); + when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); + when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); + when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); + when(replicationTimerOngoing.time()).thenReturn(time1); + when(replicationTimerCompleted.time()).thenReturn(time2); + when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); + when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); - - ShardRouting searchShardRouting = TestShardRouting.newShardRouting( - shardId, - node.getId(), - null, - false, - true, - ShardRoutingState.STARTED, - null - ); - - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + assertNotNull(response); assertNull(response.getPrimaryStats()); assertNull(response.getReplicaStats()); assertNotNull(response.getSegmentReplicationShardStats()); + verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); + verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationOnSearchReplicaWhenOngoingSegRepStateIsNull() { + public void testShardOperationWithSearchOnlyReplicaWhenNoOngoingState() { + ShardRouting shardRouting = mock(ShardRouting.class); + SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + AllocationId allocationId = AllocationId.newInitializing(); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); + ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); + long time2 = 15; + final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); + long segmentInfoCompleted = 5; + + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(shardRouting.isSearchOnly()).thenReturn(true); + when(shardRouting.allocationId()).thenReturn(allocationId); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); - when(targetService.getSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); + when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); + when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); + when(replicationTimerCompleted.time()).thenReturn(time2); + when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); + when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); - ShardRouting searchShardRouting = TestShardRouting.newShardRouting( - shardId, - node.getId(), - null, - false, - true, - ShardRoutingState.STARTED, - null - ); + assertNotNull(response); + assertNull(response.getPrimaryStats()); + assertNull(response.getReplicaStats()); + assertNotNull(response.getSegmentReplicationShardStats()); + verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); + verify(targetService).getOngoingEventSegmentReplicationState(shardId); + } - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + public void testShardOperationWithSearchOnlyReplicaWhenNoOngoingCheckpoint() { + ShardRouting shardRouting = mock(ShardRouting.class); + SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); + SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + AllocationId allocationId = AllocationId.newInitializing(); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); + ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); + ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); + long time1 = 10; + long time2 = 15; + final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); + long segmentInfoCompleted = 5; + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(shardRouting.isSearchOnly()).thenReturn(true); + when(shardRouting.allocationId()).thenReturn(allocationId); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); + when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); + when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); + when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); + when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); + when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); + when(replicationTimerCompleted.time()).thenReturn(time2); + when(replicationTimerOngoing.time()).thenReturn(time1); + when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); + when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); + + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + + assertNotNull(response); assertNull(response.getPrimaryStats()); assertNull(response.getReplicaStats()); assertNotNull(response.getSegmentReplicationShardStats()); + verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); + verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationOnSearchReplicaWhenCompletedAndOngoingSegRepStateIsNull() { - final DiscoveryNode node = newNode(0); - final ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0); + public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedAndOngoingState() { + ShardRouting shardRouting = mock(ShardRouting.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + AllocationId allocationId = AllocationId.newInitializing(); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - ShardRouting searchShardRouting = TestShardRouting.newShardRouting( - shardId, - node.getId(), - null, - false, - true, - ShardRoutingState.STARTED, - null - ); + when(shardRouting.shardId()).thenReturn(shardId); + when(shardRouting.primary()).thenReturn(false); + when(shardRouting.isSearchOnly()).thenReturn(true); + when(shardRouting.allocationId()).thenReturn(allocationId); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); - SegmentReplicationStatsRequest segmentReplicationStatsRequest = new SegmentReplicationStatsRequest(); - SegmentReplicationShardStatsResponse response = action.shardOperation(segmentReplicationStatsRequest, searchShardRouting); + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + assertNotNull(response); assertNull(response.getPrimaryStats()); assertNull(response.getReplicaStats()); assertNotNull(response.getSegmentReplicationShardStats()); + verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); + verify(targetService).getOngoingEventSegmentReplicationState(shardId); } public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { @@ -417,7 +427,7 @@ public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { AllocationId allocationId = mock(AllocationId.class); when(allocationId.getId()).thenReturn(allocIdOne); when(shardRoutingTwo.allocationId()).thenReturn(allocationId); - when(shardIdOne.getIndexName()).thenReturn(TEST_INDEX); + when(shardIdOne.getIndexName()).thenReturn("test-index"); Set segmentReplicationShardStats = new HashSet<>(); SegmentReplicationShardStats segmentReplicationShardStatsOfReplica = new SegmentReplicationShardStats(allocIdOne, 0, 0, 0, 0, 0); @@ -449,7 +459,7 @@ public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { ClusterState.EMPTY_STATE ); - List responseStats = response.getReplicationStats().get(TEST_INDEX); + List responseStats = response.getReplicationStats().get("test-index"); SegmentReplicationPerGroupStats primStats = responseStats.get(0); Set segRpShardStatsSet = primStats.getReplicaStats(); @@ -480,8 +490,8 @@ public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { ShardId shardIdTwo = mock(ShardId.class); when(segmentReplicationPerGroupStatsOne.getShardId()).thenReturn(shardIdOne); when(segmentReplicationPerGroupStatsTwo.getShardId()).thenReturn(shardIdTwo); - when(shardIdOne.getIndexName()).thenReturn(TEST_INDEX); - when(shardIdTwo.getIndexName()).thenReturn(TEST_INDEX); + when(shardIdOne.getIndexName()).thenReturn("test-index"); + when(shardIdTwo.getIndexName()).thenReturn("test-index"); when(shardIdOne.getId()).thenReturn(1); when(shardIdTwo.getId()).thenReturn(2); @@ -500,7 +510,7 @@ public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { ClusterState.EMPTY_STATE ); - List responseStats = response.getReplicationStats().get(TEST_INDEX); + List responseStats = response.getReplicationStats().get("test-index"); for (SegmentReplicationPerGroupStats primStat : responseStats) { if (primStat.getShardId().equals(shardIdOne)) { @@ -512,4 +522,183 @@ public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { } } } + + public void testNewResponseWhenShardsToFetchEmptyAndResponsesContainsNull() { + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + List shardFailures = new ArrayList<>(); + String[] shards = { }; + request.shards(shards); + + int totalShards = 3; + int successfulShards = 3; + int failedShard = 0; + String allocIdOne = "allocIdOne"; + String allocIdTwo = "allocIdTwo"; + ShardId shardIdOne = mock(ShardId.class); + ShardId shardIdTwo = mock(ShardId.class); + ShardId shardIdThree = mock(ShardId.class); + ShardRouting shardRoutingOne = mock(ShardRouting.class); + ShardRouting shardRoutingTwo = mock(ShardRouting.class); + ShardRouting shardRoutingThree = mock(ShardRouting.class); + when(shardIdOne.getId()).thenReturn(1); + when(shardIdTwo.getId()).thenReturn(2); + when(shardIdThree.getId()).thenReturn(3); + when(shardRoutingOne.shardId()).thenReturn(shardIdOne); + when(shardRoutingTwo.shardId()).thenReturn(shardIdTwo); + when(shardRoutingThree.shardId()).thenReturn(shardIdThree); + AllocationId allocationId = mock(AllocationId.class); + when(allocationId.getId()).thenReturn(allocIdOne); + when(shardRoutingTwo.allocationId()).thenReturn(allocationId); + when(shardIdOne.getIndexName()).thenReturn("test-index"); + + Set segmentReplicationShardStats = new HashSet<>(); + SegmentReplicationShardStats segmentReplicationShardStatsOfReplica = new SegmentReplicationShardStats(allocIdOne, 0, 0, 0, 0, 0); + segmentReplicationShardStats.add(segmentReplicationShardStatsOfReplica); + SegmentReplicationPerGroupStats segmentReplicationPerGroupStats = new SegmentReplicationPerGroupStats( + shardIdOne, + segmentReplicationShardStats, + 0 + ); + + SegmentReplicationState segmentReplicationState = mock(SegmentReplicationState.class); + SegmentReplicationShardStats segmentReplicationShardStatsFromSearchReplica = mock(SegmentReplicationShardStats.class); + when(segmentReplicationShardStatsFromSearchReplica.getAllocationId()).thenReturn("alloc2"); + when(segmentReplicationState.getShardRouting()).thenReturn(shardRoutingTwo); + + List responses = new ArrayList<>(); + responses.add(null); + responses.add(new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStats)); + responses.add(new SegmentReplicationShardStatsResponse(segmentReplicationState)); + responses.add(new SegmentReplicationShardStatsResponse(segmentReplicationShardStatsFromSearchReplica)); + + SegmentReplicationStatsResponse response = action.newResponse( + request, + totalShards, + successfulShards, + failedShard, + responses, + shardFailures, + ClusterState.EMPTY_STATE + ); + + List responseStats = response.getReplicationStats().get("test-index"); + SegmentReplicationPerGroupStats primStats = responseStats.get(0); + Set segRpShardStatsSet = primStats.getReplicaStats(); + + for (SegmentReplicationShardStats segRpShardStats : segRpShardStatsSet) { + if (segRpShardStats.getAllocationId().equals(allocIdOne)) { + assertEquals(segmentReplicationState, segRpShardStats.getCurrentReplicationState()); + } + + if (segRpShardStats.getAllocationId().equals(allocIdTwo)) { + assertEquals(segmentReplicationShardStatsFromSearchReplica, segRpShardStats); + } + } + } + + + public void testShardOperationWithSegRepDisabled() { + ShardRouting shardRouting = mock(ShardRouting.class); + ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); + SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); + + when(shardRouting.shardId()).thenReturn(shardId); + when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); + when(indexService.getShard(shardId.id())).thenReturn(indexShard); + when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepDisabled()); + + SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + + assertNull(response); + } + + public void testGlobalBlockCheck() { + ClusterBlock writeClusterBlock = new ClusterBlock( + 1, + "uuid", + "", + true, + true, + true, + RestStatus.OK, + EnumSet.of(ClusterBlockLevel.METADATA_WRITE) + ); + + ClusterBlock readClusterBlock = new ClusterBlock( + 1, + "uuid", + "", + true, + true, + true, + RestStatus.OK, + EnumSet.of(ClusterBlockLevel.METADATA_READ) + ); + + ClusterBlocks.Builder builder = ClusterBlocks.builder(); + builder.addGlobalBlock(writeClusterBlock); + ClusterState metadataWriteBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNull(action.checkGlobalBlock(metadataWriteBlockedState, new SegmentReplicationStatsRequest())); + + builder = ClusterBlocks.builder(); + builder.addGlobalBlock(readClusterBlock); + ClusterState metadataReadBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNotNull(action.checkGlobalBlock(metadataReadBlockedState, new SegmentReplicationStatsRequest())); + } + + public void testIndexBlockCheck() { + ClusterBlock writeClusterBlock = new ClusterBlock( + 1, + "uuid", + "", + true, + true, + true, + RestStatus.OK, + EnumSet.of(ClusterBlockLevel.METADATA_WRITE) + ); + + ClusterBlock readClusterBlock = new ClusterBlock( + 1, + "uuid", + "", + true, + true, + true, + RestStatus.OK, + EnumSet.of(ClusterBlockLevel.METADATA_READ) + ); + + String indexName = "test"; + ClusterBlocks.Builder builder = ClusterBlocks.builder(); + builder.addIndexBlock(indexName, writeClusterBlock); + ClusterState metadataWriteBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNull(action.checkRequestBlock(metadataWriteBlockedState, new SegmentReplicationStatsRequest(), new String[] { indexName })); + + builder = ClusterBlocks.builder(); + builder.addIndexBlock(indexName, readClusterBlock); + ClusterState metadataReadBlockedState = ClusterState.builder(ClusterState.EMPTY_STATE).blocks(builder).build(); + assertNotNull(action.checkRequestBlock(metadataReadBlockedState, new SegmentReplicationStatsRequest(), new String[] { indexName })); + } + + private IndexSettings createIndexSettingsWithSegRepEnabled() { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + + return new IndexSettings(IndexMetadata.builder("test").settings(settings).build(), settings); + } + + private IndexSettings createIndexSettingsWithSegRepDisabled() { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + return new IndexSettings(IndexMetadata.builder("test").settings(settings).build(), settings); + } } From d3a8a897b35f58df589d32222d0dc998131325b4 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Mon, 25 Nov 2024 14:25:58 -0800 Subject: [PATCH 06/15] fixed style issues Signed-off-by: Vinay Krishna Pudyodu --- .../TransportSegmentReplicationStatsAction.java | 1 - .../TransportSegmentReplicationStatsActionTests.java | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index 714b4990a1479..ca408e5433837 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -168,7 +168,6 @@ protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplication ShardId shardId = shardRouting.shardId(); IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); - System.out.println(indexShard.indexSettings().isSegRepEnabledOrRemoteNode()); if (indexShard.indexSettings().isSegRepEnabledOrRemoteNode() == false) { return null; } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java index 6e0ce969348bf..31ab27956cacd 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java @@ -8,9 +8,6 @@ package org.opensearch.action.admin.indices.replication; -import org.junit.Before; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; @@ -45,6 +42,7 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportService; +import org.junit.Before; import java.util.ArrayList; import java.util.EnumSet; @@ -53,6 +51,9 @@ import java.util.Map; import java.util.Set; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -526,7 +527,7 @@ public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { public void testNewResponseWhenShardsToFetchEmptyAndResponsesContainsNull() { SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); List shardFailures = new ArrayList<>(); - String[] shards = { }; + String[] shards = {}; request.shards(shards); int totalShards = 3; @@ -596,7 +597,6 @@ public void testNewResponseWhenShardsToFetchEmptyAndResponsesContainsNull() { } } - public void testShardOperationWithSegRepDisabled() { ShardRouting shardRouting = mock(ShardRouting.class); ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); From dc8db3c14ca4de5ad1beafc09bb55cd3a33e1668 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Fri, 6 Dec 2024 11:10:28 -0800 Subject: [PATCH 07/15] Made changes in the bytes to download calculation based on comments Signed-off-by: Vinay Krishna Pudyodu --- CHANGELOG.md | 2 +- .../SegmentReplicationStatsIT.java | 43 +++--- .../SegmentReplicationShardStatsResponse.java | 22 +-- ...ransportSegmentReplicationStatsAction.java | 55 ++----- .../index/SegmentReplicationShardStats.java | 2 +- .../index/seqno/ReplicationTracker.java | 2 +- .../replication/SegmentReplicationState.java | 11 -- .../replication/SegmentReplicationTarget.java | 1 - ...ortSegmentReplicationStatsActionTests.java | 141 +++--------------- 9 files changed, 63 insertions(+), 216 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0ed8758a253d..8e4239be0810c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489)) - Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271)) - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). -- Added implementation for the stats calculation for search and regular replica in shards ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678)) +- Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678)) ### Dependencies - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java index 11f00fbbb97ab..c7a76b72dd317 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java @@ -441,9 +441,7 @@ protected Settings featureFlagSettings() { public void testSegmentReplicationStatsResponseWithSearchReplica() throws Exception { internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); - internalCluster().startDataOnlyNode(); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNodes(3); int numShards = 2; assertAcked( @@ -472,25 +470,22 @@ public void testSegmentReplicationStatsResponseWithSearchReplica() throws Except .setDetailed(true) .execute() .actionGet(); + assertEquals(1, segmentReplicationStatsResponse.getReplicationStats().size()); + assertEquals(numShards * 3, segmentReplicationStatsResponse.getTotalShards()); + assertEquals(numShards * 3, segmentReplicationStatsResponse.getSuccessfulShards()); + SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); - final SegmentReplicationState currentReplicationState = perGroupStats.getReplicaStats() - .stream() - .findFirst() - .get() - .getCurrentReplicationState(); - assertEquals(segmentReplicationStatsResponse.getReplicationStats().size(), 1); - assertEquals(segmentReplicationStatsResponse.getTotalShards(), numShards * 3); - assertEquals(segmentReplicationStatsResponse.getSuccessfulShards(), numShards * 3); - assertNotNull(currentReplicationState); - assertEquals(currentReplicationState.getStage(), SegmentReplicationState.Stage.DONE); - assertTrue(currentReplicationState.getIndex().recoveredFileCount() > 0); + Set replicaStats = perGroupStats.getReplicaStats(); + for (SegmentReplicationShardStats replica : replicaStats) { + assertNotNull(replica.getCurrentReplicationState()); + } + assertEquals(3, replicaStats.size()); }, 1, TimeUnit.MINUTES); } public void testSegmentReplicationStatsResponseWithOnlySearchReplica() throws Exception { internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNodes(2); int numShards = 1; assertAcked( @@ -519,18 +514,16 @@ public void testSegmentReplicationStatsResponseWithOnlySearchReplica() throws Ex .setDetailed(true) .execute() .actionGet(); - SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); - final SegmentReplicationState currentReplicationState = perGroupStats.getReplicaStats() - .stream() - .findFirst() - .get() - .getCurrentReplicationState(); assertEquals(segmentReplicationStatsResponse.getReplicationStats().size(), 1); assertEquals(segmentReplicationStatsResponse.getTotalShards(), 2); assertEquals(segmentReplicationStatsResponse.getSuccessfulShards(), 2); - assertNotNull(currentReplicationState); - assertEquals(currentReplicationState.getStage(), SegmentReplicationState.Stage.DONE); - assertTrue(currentReplicationState.getIndex().recoveredFileCount() > 0); + + SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); + Set replicaStats = perGroupStats.getReplicaStats(); + for (SegmentReplicationShardStats replica : replicaStats) { + assertNotNull(replica.getCurrentReplicationState()); + } + assertEquals(1, replicaStats.size()); }, 1, TimeUnit.MINUTES); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java index d5a05a60e8849..b093afcf25980 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java @@ -33,30 +33,30 @@ public class SegmentReplicationShardStatsResponse implements Writeable { private final SegmentReplicationState replicaStats; @Nullable - private final SegmentReplicationShardStats segmentReplicationShardStats; + private final SegmentReplicationShardStats searchReplicaReplicationStats; public SegmentReplicationShardStatsResponse(StreamInput in) throws IOException { this.primaryStats = in.readOptionalWriteable(SegmentReplicationPerGroupStats::new); this.replicaStats = in.readOptionalWriteable(SegmentReplicationState::new); - this.segmentReplicationShardStats = in.readOptionalWriteable(SegmentReplicationShardStats::new); + this.searchReplicaReplicationStats = in.readOptionalWriteable(SegmentReplicationShardStats::new); } public SegmentReplicationShardStatsResponse(SegmentReplicationPerGroupStats primaryStats) { this.primaryStats = primaryStats; this.replicaStats = null; - this.segmentReplicationShardStats = null; + this.searchReplicaReplicationStats = null; } public SegmentReplicationShardStatsResponse(SegmentReplicationState replicaStats) { this.replicaStats = replicaStats; this.primaryStats = null; - this.segmentReplicationShardStats = null; + this.searchReplicaReplicationStats = null; } public SegmentReplicationShardStatsResponse(SegmentReplicationShardStats segmentReplicationShardStats) { this.primaryStats = null; this.replicaStats = null; - this.segmentReplicationShardStats = segmentReplicationShardStats; + this.searchReplicaReplicationStats = segmentReplicationShardStats; } public SegmentReplicationPerGroupStats getPrimaryStats() { @@ -67,19 +67,23 @@ public SegmentReplicationState getReplicaStats() { return replicaStats; } - public SegmentReplicationShardStats getSegmentReplicationShardStats() { - return segmentReplicationShardStats; + public SegmentReplicationShardStats getSearchReplicaReplicationStats() { + return searchReplicaReplicationStats; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(primaryStats); out.writeOptionalWriteable(replicaStats); - out.writeOptionalWriteable(segmentReplicationShardStats); + out.writeOptionalWriteable(searchReplicaReplicationStats); } @Override public String toString() { - return "SegmentReplicationShardStatsResponse{" + "primaryStats=" + primaryStats + ", replicaStats=" + replicaStats + '}'; + return "SegmentReplicationShardStatsResponse{" + + "primaryStats=" + primaryStats + + ", replicaStats=" + replicaStats + + ", searchReplicaReplicationStats=" + searchReplicaReplicationStats + + '}'; } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index ca408e5433837..71545a29cfcff 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -25,8 +25,6 @@ import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.store.Store; -import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; @@ -36,7 +34,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -117,8 +114,8 @@ protected SegmentReplicationStatsResponse newResponse( } } - if (response.getSegmentReplicationShardStats() != null) { - searchReplicaSegRepShardStats.add(response.getSegmentReplicationShardStats()); + if (response.getSearchReplicaReplicationStats() != null) { + searchReplicaSegRepShardStats.add(response.getSearchReplicaReplicationStats()); } if (response.getPrimaryStats() != null) { @@ -175,7 +172,7 @@ protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplication if (shardRouting.primary()) { return new SegmentReplicationShardStatsResponse(pressureService.getStatsForShard(indexShard)); } else if (shardRouting.isSearchOnly()) { - SegmentReplicationShardStats segmentReplicationShardStats = calcualteSegmentReplicationShardStats(shardRouting); + SegmentReplicationShardStats segmentReplicationShardStats = calculateSegmentReplicationShardStats(shardRouting); return new SegmentReplicationShardStatsResponse(segmentReplicationShardStats); } else { return new SegmentReplicationShardStatsResponse(getSegmentReplicationState(shardId, request.activeOnly())); @@ -201,15 +198,15 @@ protected ClusterBlockException checkRequestBlock( return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); } - private SegmentReplicationShardStats calcualteSegmentReplicationShardStats(ShardRouting shardRouting) { + private SegmentReplicationShardStats calculateSegmentReplicationShardStats(ShardRouting shardRouting) { ShardId shardId = shardRouting.shardId(); SegmentReplicationState completedSegmentReplicationState = targetService.getlatestCompletedEventSegmentReplicationState(shardId); SegmentReplicationState ongoingSegmentReplicationState = targetService.getOngoingEventSegmentReplicationState(shardId); SegmentReplicationShardStats segmentReplicationShardStats = new SegmentReplicationShardStats( shardRouting.allocationId().getId(), - calculateCheckpointsBehind(completedSegmentReplicationState, ongoingSegmentReplicationState), - calculateBytesBehind(completedSegmentReplicationState, ongoingSegmentReplicationState), + 0, + calculateBytesRemainingToReplicate(ongoingSegmentReplicationState), 0, getCurrentReplicationLag(ongoingSegmentReplicationState), getLastCompletedReplicationLag(completedSegmentReplicationState) @@ -227,44 +224,16 @@ private SegmentReplicationState getSegmentReplicationState(ShardId shardId, bool } } - private long calculateCheckpointsBehind( - SegmentReplicationState completedSegmentReplicationState, - SegmentReplicationState ongoingSegmentReplicationState - ) { - if (ongoingSegmentReplicationState == null || ongoingSegmentReplicationState.getReplicationCheckpoint() == null) { - return 0; - } - - if (completedSegmentReplicationState == null || completedSegmentReplicationState.getReplicationCheckpoint() == null) { - return ongoingSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion(); - } - - return ongoingSegmentReplicationState.getReplicationCheckpoint().getSegmentInfosVersion() - completedSegmentReplicationState - .getReplicationCheckpoint() - .getSegmentInfosVersion(); - } - - private long calculateBytesBehind( - SegmentReplicationState completedSegmentReplicationState, + private long calculateBytesRemainingToReplicate( SegmentReplicationState ongoingSegmentReplicationState ) { - if (ongoingSegmentReplicationState == null || ongoingSegmentReplicationState.getReplicationCheckpoint() == null) { + if (ongoingSegmentReplicationState == null) { return 0; } - - if (completedSegmentReplicationState == null || completedSegmentReplicationState.getReplicationCheckpoint() == null) { - Store.RecoveryDiff diff = Store.segmentReplicationDiff( - ongoingSegmentReplicationState.getReplicationCheckpoint().getMetadataMap(), - Collections.emptyMap() - ); - return diff.missing.stream().mapToLong(StoreFileMetadata::length).sum(); - } - - Store.RecoveryDiff diff = Store.segmentReplicationDiff( - ongoingSegmentReplicationState.getReplicationCheckpoint().getMetadataMap(), - completedSegmentReplicationState.getReplicationCheckpoint().getMetadataMap() - ); - return diff.missing.stream().mapToLong(StoreFileMetadata::length).sum(); + return ongoingSegmentReplicationState.getIndex() + .fileDetails().stream() + .mapToLong(index -> index.length() - index.recovered()) + .sum(); } private long getCurrentReplicationLag(SegmentReplicationState ongoingSegmentReplicationState) { diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java index 40fac40f3ce54..3845dedfb8fd0 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java @@ -119,7 +119,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("current_replication_lag", new TimeValue(currentReplicationLagMillis)); builder.field("last_completed_replication_time", new TimeValue(lastCompletedReplicationTimeMillis)); if (currentReplicationState != null) { - builder.startObject("current_replication_state"); + builder.startObject(); currentReplicationState.toXContent(builder, params); builder.endObject(); } diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java index 1e43827afeb47..abb52ff1a5169 100644 --- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java @@ -1345,7 +1345,7 @@ && shouldSkipReplicationTimer(entry.getKey()) == false && isShardOnRemoteEnabledNode.apply(routingTable.getByAllocationId(entry.getKey()).currentNodeId()))) ) .map(entry -> buildShardStats(entry.getKey(), entry.getValue())) - .collect(Collectors.toUnmodifiableSet()); + .collect(Collectors.toSet()); } return Collections.emptySet(); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 29130b18ffc7b..5fa123948c5ac 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -18,7 +18,6 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationTimer; @@ -89,8 +88,6 @@ public static Stage fromId(byte id) { private String sourceDescription; private DiscoveryNode targetNode; - private ReplicationCheckpoint replicationCheckpoint; - public ShardRouting getShardRouting() { return shardRouting; } @@ -151,10 +148,6 @@ public TimeValue getFinalizeReplicationStageTime() { return new TimeValue(time); } - public ReplicationCheckpoint getReplicationCheckpoint() { - return this.replicationCheckpoint; - } - public SegmentReplicationState( ShardRouting shardRouting, ReplicationLuceneIndex index, @@ -259,10 +252,6 @@ public void setStage(Stage stage) { } } - public void setReplicationCheckpoint(ReplicationCheckpoint replicationCheckpoint) { - this.replicationCheckpoint = replicationCheckpoint; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 8223f5d504700..7131b49a41834 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -177,7 +177,6 @@ public void startReplication(ActionListener listener) { source.getCheckpointMetadata(getId(), checkpoint, checkpointInfoListener); checkpointInfoListener.whenComplete(checkpointInfo -> { - state.setReplicationCheckpoint(checkpointInfo.getCheckpoint()); final List filesToFetch = getFiles(checkpointInfo); state.setStage(SegmentReplicationState.Stage.GET_FILES); cancellableThreads.checkForCancel(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java index 31ab27956cacd..aeea9a5b02a0f 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java @@ -8,6 +8,9 @@ package org.opensearch.action.admin.indices.replication; +import org.junit.Before; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; @@ -33,27 +36,21 @@ import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; import org.opensearch.index.shard.IndexShard; -import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; -import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationTimer; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportService; -import org.junit.Before; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -76,7 +73,6 @@ public class TransportSegmentReplicationStatsActionTests extends OpenSearchTestC private SegmentReplicationPressureService pressureService; @Mock private IndexShard indexShard; - @Mock private IndexService indexService; @@ -145,7 +141,7 @@ public void testShardOperationWithReplicaShard() { assertNotNull(response); assertNull(response.getPrimaryStats()); assertNotNull(response.getReplicaStats()); - assertNull(response.getSegmentReplicationShardStats()); + assertNull(response.getSearchReplicaReplicationStats()); verify(targetService).getSegmentReplicationState(shardId); } @@ -168,7 +164,7 @@ public void testShardOperationWithReplicaShardActiveOnly() { assertNotNull(response); assertNull(response.getPrimaryStats()); assertNotNull(response.getReplicaStats()); - assertNull(response.getSegmentReplicationShardStats()); + assertNull(response.getSearchReplicaReplicationStats()); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } @@ -181,14 +177,11 @@ public void testShardOperationWithSearchOnlyReplicaWhenCompletedAndOngoingStateN SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); - ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); - ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); long time1 = 10; long time2 = 15; - final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); - final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); - long segmentInfoCompleted = 5; - long segmentInfoOngoing = 9; + ReplicationLuceneIndex replicationLuceneIndex = new ReplicationLuceneIndex(); + replicationLuceneIndex.addFileDetail("name1", 10, false); + replicationLuceneIndex.addFileDetail("name2", 5, false); when(shardRouting.shardId()).thenReturn(shardId); when(shardRouting.primary()).thenReturn(false); @@ -201,15 +194,10 @@ public void testShardOperationWithSearchOnlyReplicaWhenCompletedAndOngoingStateN when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); - when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); - when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); when(replicationTimerOngoing.time()).thenReturn(time1); when(replicationTimerCompleted.time()).thenReturn(time2); - when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); - when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); - when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); - when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); + when(onGoingSegmentReplicationState.getIndex()).thenReturn(replicationLuceneIndex); SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); @@ -225,11 +213,10 @@ public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedState() { AllocationId allocationId = AllocationId.newInitializing(); SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); - ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); long time1 = 10; - final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); - final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); - long segmentInfoOngoing = 9; + ReplicationLuceneIndex replicationLuceneIndex = new ReplicationLuceneIndex(); + replicationLuceneIndex.addFileDetail("name1", 10, false); + replicationLuceneIndex.addFileDetail("name2", 5, false); when(shardRouting.shardId()).thenReturn(shardId); when(shardRouting.primary()).thenReturn(false); @@ -241,61 +228,15 @@ public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedState() { when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); - when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); when(replicationTimerOngoing.time()).thenReturn(time1); - when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); - when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); + when(onGoingSegmentReplicationState.getIndex()).thenReturn(replicationLuceneIndex); SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); assertNotNull(response); assertNull(response.getPrimaryStats()); assertNull(response.getReplicaStats()); - assertNotNull(response.getSegmentReplicationShardStats()); - verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); - verify(targetService).getOngoingEventSegmentReplicationState(shardId); - } - - public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedCheckpoint() { - ShardRouting shardRouting = mock(ShardRouting.class); - SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); - SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); - ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); - AllocationId allocationId = AllocationId.newInitializing(); - SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); - ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); - ReplicationCheckpoint onGoingCheckpoint = mock(ReplicationCheckpoint.class); - long time1 = 10; - long time2 = 15; - final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); - final StoreFileMetadata segment_2 = new StoreFileMetadata("segment_2", 50L, "abcd", org.apache.lucene.util.Version.LATEST); - long segmentInfoOngoing = 9; - - when(shardRouting.shardId()).thenReturn(shardId); - when(shardRouting.primary()).thenReturn(false); - when(shardRouting.isSearchOnly()).thenReturn(true); - when(shardRouting.allocationId()).thenReturn(allocationId); - when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); - when(indexService.getShard(shardId.id())).thenReturn(indexShard); - when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); - when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); - when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); - when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); - when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); - when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); - when(onGoingSegmentReplicationState.getReplicationCheckpoint()).thenReturn(onGoingCheckpoint); - when(replicationTimerOngoing.time()).thenReturn(time1); - when(replicationTimerCompleted.time()).thenReturn(time2); - when(onGoingCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1, "segment_2", segment_2)); - when(onGoingCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoOngoing); - - SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); - - assertNotNull(response); - assertNull(response.getPrimaryStats()); - assertNull(response.getReplicaStats()); - assertNotNull(response.getSegmentReplicationShardStats()); + assertNotNull(response.getSearchReplicaReplicationStats()); verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } @@ -307,49 +248,7 @@ public void testShardOperationWithSearchOnlyReplicaWhenNoOngoingState() { AllocationId allocationId = AllocationId.newInitializing(); SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); - ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); - long time2 = 15; - final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); - long segmentInfoCompleted = 5; - - when(shardRouting.shardId()).thenReturn(shardId); - when(shardRouting.primary()).thenReturn(false); - when(shardRouting.isSearchOnly()).thenReturn(true); - when(shardRouting.allocationId()).thenReturn(allocationId); - when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); - when(indexService.getShard(shardId.id())).thenReturn(indexShard); - when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); - when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); - when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); - when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); - when(replicationTimerCompleted.time()).thenReturn(time2); - when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); - when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); - - SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); - - assertNotNull(response); - assertNull(response.getPrimaryStats()); - assertNull(response.getReplicaStats()); - assertNotNull(response.getSegmentReplicationShardStats()); - verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); - verify(targetService).getOngoingEventSegmentReplicationState(shardId); - } - - public void testShardOperationWithSearchOnlyReplicaWhenNoOngoingCheckpoint() { - ShardRouting shardRouting = mock(ShardRouting.class); - SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); - SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); - ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); - AllocationId allocationId = AllocationId.newInitializing(); - SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); - ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); - ReplicationCheckpoint completedCheckpoint = mock(ReplicationCheckpoint.class); - long time1 = 10; long time2 = 15; - final StoreFileMetadata segment_1 = new StoreFileMetadata("segment_1", 1L, "abcd", org.apache.lucene.util.Version.LATEST); - long segmentInfoCompleted = 5; when(shardRouting.shardId()).thenReturn(shardId); when(shardRouting.primary()).thenReturn(false); @@ -359,21 +258,15 @@ public void testShardOperationWithSearchOnlyReplicaWhenNoOngoingCheckpoint() { when(indexService.getShard(shardId.id())).thenReturn(indexShard); when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); - when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); - when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); - when(completedSegmentReplicationState.getReplicationCheckpoint()).thenReturn(completedCheckpoint); when(replicationTimerCompleted.time()).thenReturn(time2); - when(replicationTimerOngoing.time()).thenReturn(time1); - when(completedCheckpoint.getMetadataMap()).thenReturn(Map.of("segment_1", segment_1)); - when(completedCheckpoint.getSegmentInfosVersion()).thenReturn(segmentInfoCompleted); SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); assertNotNull(response); assertNull(response.getPrimaryStats()); assertNull(response.getReplicaStats()); - assertNotNull(response.getSegmentReplicationShardStats()); + assertNotNull(response.getSearchReplicaReplicationStats()); verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } @@ -397,7 +290,7 @@ public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedAndOngoingStat assertNotNull(response); assertNull(response.getPrimaryStats()); assertNull(response.getReplicaStats()); - assertNotNull(response.getSegmentReplicationShardStats()); + assertNotNull(response.getSearchReplicaReplicationStats()); verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } From 4e21d4cea2012f05300919b6fcd96b729be5be2f Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Fri, 6 Dec 2024 11:52:25 -0800 Subject: [PATCH 08/15] added addReplicaStats method to SegmentReplicationPerGroupStats Signed-off-by: Vinay Krishna Pudyodu --- .../replication/TransportSegmentReplicationStatsAction.java | 4 +--- .../opensearch/index/SegmentReplicationPerGroupStats.java | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index 71545a29cfcff..8dae5fb9c67ca 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -146,9 +146,7 @@ protected SegmentReplicationStatsResponse newResponse( // combine the search replica stats with the stats of other replicas for (Map.Entry> entry : primaryStats.entrySet()) { for (SegmentReplicationPerGroupStats group : entry.getValue()) { - Set updatedSet = new HashSet<>(group.getReplicaStats()); - updatedSet.addAll(searchReplicaSegRepShardStats); - group.setReplicaStats(updatedSet); + group.addReplicaStats(searchReplicaSegRepShardStats); } } diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java index 98d8bfc7d66d8..55d13c42530f3 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java @@ -28,7 +28,7 @@ public class SegmentReplicationPerGroupStats implements Writeable, ToXContentFragment { private final ShardId shardId; - private Set replicaStats; + private final Set replicaStats; private final long rejectedRequestCount; public SegmentReplicationPerGroupStats(ShardId shardId, Set replicaStats, long rejectedRequestCount) { @@ -55,8 +55,8 @@ public ShardId getShardId() { return shardId; } - public void setReplicaStats(Set replicaStats) { - this.replicaStats = replicaStats; + public void addReplicaStats(Set replicaStats) { + this.replicaStats.addAll(replicaStats); } @Override From bed071241188eaef438aa5d11d21e15f0db6133c Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Fri, 6 Dec 2024 12:09:02 -0800 Subject: [PATCH 09/15] fixed style issues Signed-off-by: Vinay Krishna Pudyodu --- .../SegmentReplicationShardStatsResponse.java | 13 ++++++++----- .../TransportSegmentReplicationStatsAction.java | 7 +++---- .../index/SegmentReplicationPerGroupStats.java | 2 +- ...TransportSegmentReplicationStatsActionTests.java | 7 ++++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java index b093afcf25980..cf51431731cdf 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java @@ -80,10 +80,13 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String toString() { - return "SegmentReplicationShardStatsResponse{" + - "primaryStats=" + primaryStats + - ", replicaStats=" + replicaStats + - ", searchReplicaReplicationStats=" + searchReplicaReplicationStats + - '}'; + return "SegmentReplicationShardStatsResponse{" + + "primaryStats=" + + primaryStats + + ", replicaStats=" + + replicaStats + + ", searchReplicaReplicationStats=" + + searchReplicaReplicationStats + + '}'; } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index 8dae5fb9c67ca..218d32098eae5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -222,14 +222,13 @@ private SegmentReplicationState getSegmentReplicationState(ShardId shardId, bool } } - private long calculateBytesRemainingToReplicate( - SegmentReplicationState ongoingSegmentReplicationState - ) { + private long calculateBytesRemainingToReplicate(SegmentReplicationState ongoingSegmentReplicationState) { if (ongoingSegmentReplicationState == null) { return 0; } return ongoingSegmentReplicationState.getIndex() - .fileDetails().stream() + .fileDetails() + .stream() .mapToLong(index -> index.length() - index.recovered()) .sum(); } diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java index 55d13c42530f3..8d5e06e95af01 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java @@ -56,7 +56,7 @@ public ShardId getShardId() { } public void addReplicaStats(Set replicaStats) { - this.replicaStats.addAll(replicaStats); + this.replicaStats.addAll(replicaStats); } @Override diff --git a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java index aeea9a5b02a0f..6d4527a9007c3 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java @@ -8,9 +8,6 @@ package org.opensearch.action.admin.indices.replication; -import org.junit.Before; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import org.opensearch.Version; import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; @@ -44,6 +41,7 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.TransportService; +import org.junit.Before; import java.util.ArrayList; import java.util.EnumSet; @@ -51,6 +49,9 @@ import java.util.List; import java.util.Set; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; From 4d03379143c350b69c4e5275e9674dada15b3037 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Fri, 6 Dec 2024 14:38:24 -0800 Subject: [PATCH 10/15] Fixed issue with immutable set Signed-off-by: Vinay Krishna Pudyodu --- .../indices/replication/SegmentReplicationStatsIT.java | 8 ++++---- .../index/SegmentReplicationPerGroupStats.java | 9 +++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java index c7a76b72dd317..14be9b1cb3980 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java @@ -520,10 +520,10 @@ public void testSegmentReplicationStatsResponseWithOnlySearchReplica() throws Ex SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); Set replicaStats = perGroupStats.getReplicaStats(); - for (SegmentReplicationShardStats replica : replicaStats) { - assertNotNull(replica.getCurrentReplicationState()); - } - assertEquals(1, replicaStats.size()); + // for (SegmentReplicationShardStats replica : replicaStats) { + // assertNotNull(replica.getCurrentReplicationState()); + // } + // assertEquals(1, replicaStats.size()); }, 1, TimeUnit.MINUTES); } } diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java index 8d5e06e95af01..6b12211bfaa09 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java @@ -28,7 +28,7 @@ public class SegmentReplicationPerGroupStats implements Writeable, ToXContentFragment { private final ShardId shardId; - private final Set replicaStats; + private Set replicaStats; private final long rejectedRequestCount; public SegmentReplicationPerGroupStats(ShardId shardId, Set replicaStats, long rejectedRequestCount) { @@ -56,7 +56,12 @@ public ShardId getShardId() { } public void addReplicaStats(Set replicaStats) { - this.replicaStats.addAll(replicaStats); + if (this.replicaStats.isEmpty()) { + // When there is only search replica, replicaStats is empty. EmptySet is immutable and doesn't support adding item + this.replicaStats = replicaStats; + } else { + this.replicaStats.addAll(replicaStats); + } } @Override From 6944ce6a01bac67238776c71ef8968b78a5464d8 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Tue, 10 Dec 2024 14:27:23 -0800 Subject: [PATCH 11/15] Fixed PR comments and moved the integration tests to separate module Signed-off-by: Vinay Krishna Pudyodu --- .../SearchReplicaReplicationIT.java | 97 +++++++ .../SegmentReplicationStatsIT.java | 94 ------ .../SegmentReplicationShardStatsResponse.java | 27 +- ...ransportSegmentReplicationStatsAction.java | 88 +++--- .../SegmentReplicationPerGroupStats.java | 11 +- .../index/seqno/ReplicationTracker.java | 2 +- ...ortSegmentReplicationStatsActionTests.java | 273 +++++++++--------- 7 files changed, 290 insertions(+), 302 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java index a1b512c326ac5..a3a8e7d1db0ae 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java @@ -8,14 +8,20 @@ package org.opensearch.indices.replication; +import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.SegmentReplicationPerGroupStats; +import org.opensearch.index.SegmentReplicationShardStats; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; import org.junit.Before; import java.nio.file.Path; +import java.util.List; +import java.util.Set; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class SearchReplicaReplicationIT extends SegmentReplicationBaseIT { @@ -82,4 +88,95 @@ public void testReplication() throws Exception { waitForSearchableDocs(docCount, primary, replica); } + public void testSegmentReplicationStatsResponseWithSearchReplica() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + final List nodes = internalCluster().startDataOnlyNodes(3); + int numOfPrimaryShards = 2; + createIndex( + INDEX_NAME, + Settings.builder() + .put("number_of_shards", numOfPrimaryShards) + .put("number_of_replicas", 1) + .put("number_of_search_only_replicas", 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build() + ); + ensureGreen(INDEX_NAME); + + final int docCount = 5; + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + refresh(INDEX_NAME); + waitForSearchableDocs(docCount, nodes); + + SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() + .indices() + .prepareSegmentReplicationStats(INDEX_NAME) + .setDetailed(true) + .execute() + .actionGet(); + + // Verify the number of indices + assertEquals(1, segmentReplicationStatsResponse.getReplicationStats().size()); + // Verify total shards + assertEquals(numOfPrimaryShards * 3, segmentReplicationStatsResponse.getTotalShards()); + // Verify the number of primary shards + assertEquals(numOfPrimaryShards, segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).size()); + + List segmentReplicationPerGroupStats = segmentReplicationStatsResponse.getReplicationStats() + .get(INDEX_NAME); + for (SegmentReplicationPerGroupStats perGroupStats : segmentReplicationPerGroupStats) { + Set replicaStats = perGroupStats.getReplicaStats(); + // Verify the number of replica stats + assertEquals(2, replicaStats.size()); + for (SegmentReplicationShardStats replicaStat : replicaStats) { + assertNotNull(replicaStat.getCurrentReplicationState()); + } + } + } + + public void testSegmentReplicationStatsResponseWithOnlySearchReplica() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + final List nodes = internalCluster().startDataOnlyNodes(2); + createIndex( + INDEX_NAME, + Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .put("number_of_search_only_replicas", 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build() + ); + ensureGreen(INDEX_NAME); + + final int docCount = 5; + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + refresh(INDEX_NAME); + waitForSearchableDocs(docCount, nodes); + + SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() + .indices() + .prepareSegmentReplicationStats(INDEX_NAME) + .setDetailed(true) + .execute() + .actionGet(); + + // Verify the number of indices + assertEquals(1, segmentReplicationStatsResponse.getReplicationStats().size()); + // Verify total shards + assertEquals(2, segmentReplicationStatsResponse.getTotalShards()); + // Verify the number of primary shards + assertEquals(1, segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).size()); + + SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); + Set replicaStats = perGroupStats.getReplicaStats(); + // Verify the number of replica stats + assertEquals(1, replicaStats.size()); + for (SegmentReplicationShardStats replicaStat : replicaStats) { + assertNotNull(replicaStat.getCurrentReplicationState()); + } + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java index 14be9b1cb3980..b2161577f7040 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java @@ -16,7 +16,6 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.lease.Releasable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.ReplicationStats; import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationShardStats; @@ -433,97 +432,4 @@ public void testSegmentReplicationNodeAndIndexStats() throws Exception { } } - - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, true).build(); - } - - public void testSegmentReplicationStatsResponseWithSearchReplica() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(3); - - int numShards = 2; - assertAcked( - prepareCreate( - INDEX_NAME, - 0, - Settings.builder() - .put("number_of_shards", numShards) - .put("number_of_replicas", 1) - .put("number_of_search_only_replicas", 1) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - ) - ); - ensureGreen(); - final long numDocs = scaledRandomIntBetween(50, 100); - for (int i = 0; i < numDocs; i++) { - index(INDEX_NAME, "doc", Integer.toString(i)); - } - refresh(INDEX_NAME); - ensureSearchable(INDEX_NAME); - - assertBusy(() -> { - SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() - .indices() - .prepareSegmentReplicationStats(INDEX_NAME) - .setDetailed(true) - .execute() - .actionGet(); - assertEquals(1, segmentReplicationStatsResponse.getReplicationStats().size()); - assertEquals(numShards * 3, segmentReplicationStatsResponse.getTotalShards()); - assertEquals(numShards * 3, segmentReplicationStatsResponse.getSuccessfulShards()); - - SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); - Set replicaStats = perGroupStats.getReplicaStats(); - for (SegmentReplicationShardStats replica : replicaStats) { - assertNotNull(replica.getCurrentReplicationState()); - } - assertEquals(3, replicaStats.size()); - }, 1, TimeUnit.MINUTES); - } - - public void testSegmentReplicationStatsResponseWithOnlySearchReplica() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - - int numShards = 1; - assertAcked( - prepareCreate( - INDEX_NAME, - 0, - Settings.builder() - .put("number_of_shards", numShards) - .put("number_of_replicas", 0) - .put("number_of_search_only_replicas", 1) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - ) - ); - ensureGreen(); - final long numDocs = scaledRandomIntBetween(50, 100); - for (int i = 0; i < numDocs; i++) { - index(INDEX_NAME, "doc", Integer.toString(i)); - } - refresh(INDEX_NAME); - ensureSearchable(INDEX_NAME); - - assertBusy(() -> { - SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() - .indices() - .prepareSegmentReplicationStats(INDEX_NAME) - .setDetailed(true) - .execute() - .actionGet(); - assertEquals(segmentReplicationStatsResponse.getReplicationStats().size(), 1); - assertEquals(segmentReplicationStatsResponse.getTotalShards(), 2); - assertEquals(segmentReplicationStatsResponse.getSuccessfulShards(), 2); - - SegmentReplicationPerGroupStats perGroupStats = segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0); - Set replicaStats = perGroupStats.getReplicaStats(); - // for (SegmentReplicationShardStats replica : replicaStats) { - // assertNotNull(replica.getCurrentReplicationState()); - // } - // assertEquals(1, replicaStats.size()); - }, 1, TimeUnit.MINUTES); - } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java index cf51431731cdf..ce17176a220ae 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/SegmentReplicationShardStatsResponse.java @@ -13,7 +13,6 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.index.SegmentReplicationPerGroupStats; -import org.opensearch.index.SegmentReplicationShardStats; import org.opensearch.indices.replication.SegmentReplicationState; import java.io.IOException; @@ -32,31 +31,19 @@ public class SegmentReplicationShardStatsResponse implements Writeable { @Nullable private final SegmentReplicationState replicaStats; - @Nullable - private final SegmentReplicationShardStats searchReplicaReplicationStats; - public SegmentReplicationShardStatsResponse(StreamInput in) throws IOException { this.primaryStats = in.readOptionalWriteable(SegmentReplicationPerGroupStats::new); this.replicaStats = in.readOptionalWriteable(SegmentReplicationState::new); - this.searchReplicaReplicationStats = in.readOptionalWriteable(SegmentReplicationShardStats::new); } public SegmentReplicationShardStatsResponse(SegmentReplicationPerGroupStats primaryStats) { this.primaryStats = primaryStats; this.replicaStats = null; - this.searchReplicaReplicationStats = null; } public SegmentReplicationShardStatsResponse(SegmentReplicationState replicaStats) { this.replicaStats = replicaStats; this.primaryStats = null; - this.searchReplicaReplicationStats = null; - } - - public SegmentReplicationShardStatsResponse(SegmentReplicationShardStats segmentReplicationShardStats) { - this.primaryStats = null; - this.replicaStats = null; - this.searchReplicaReplicationStats = segmentReplicationShardStats; } public SegmentReplicationPerGroupStats getPrimaryStats() { @@ -67,26 +54,14 @@ public SegmentReplicationState getReplicaStats() { return replicaStats; } - public SegmentReplicationShardStats getSearchReplicaReplicationStats() { - return searchReplicaReplicationStats; - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(primaryStats); out.writeOptionalWriteable(replicaStats); - out.writeOptionalWriteable(searchReplicaReplicationStats); } @Override public String toString() { - return "SegmentReplicationShardStatsResponse{" - + "primaryStats=" - + primaryStats - + ", replicaStats=" - + replicaStats - + ", searchReplicaReplicationStats=" - + searchReplicaReplicationStats - + '}'; + return "SegmentReplicationShardStatsResponse{" + "primaryStats=" + primaryStats + ", replicaStats=" + replicaStats + '}'; } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index 218d32098eae5..1a4f073941494 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -35,11 +35,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Transport action for shard segment replication operation. This transport action does not actually @@ -97,13 +97,10 @@ protected SegmentReplicationStatsResponse newResponse( ) { String[] shards = request.shards(); final List shardsToFetch = Arrays.stream(shards).map(Integer::valueOf).collect(Collectors.toList()); - // organize replica responses by allocationId. final Map replicaStats = new HashMap<>(); // map of index name to list of replication group stats. final Map> primaryStats = new HashMap<>(); - // search replica responses - final Set searchReplicaSegRepShardStats = new HashSet<>(); for (SegmentReplicationShardStatsResponse response : responses) { if (response != null) { @@ -114,10 +111,6 @@ protected SegmentReplicationStatsResponse newResponse( } } - if (response.getSearchReplicaReplicationStats() != null) { - searchReplicaSegRepShardStats.add(response.getSearchReplicaReplicationStats()); - } - if (response.getPrimaryStats() != null) { final ShardId shardId = response.getPrimaryStats().getShardId(); if (shardsToFetch.isEmpty() || shardsToFetch.contains(shardId.getId())) { @@ -135,22 +128,20 @@ protected SegmentReplicationStatsResponse newResponse( } } } - // combine the replica stats to the shard stat entry in each group. - for (Map.Entry> entry : primaryStats.entrySet()) { - for (SegmentReplicationPerGroupStats group : entry.getValue()) { - for (SegmentReplicationShardStats replicaStat : group.getReplicaStats()) { - replicaStat.setCurrentReplicationState(replicaStats.getOrDefault(replicaStat.getAllocationId(), null)); - } - } - } - // combine the search replica stats with the stats of other replicas - for (Map.Entry> entry : primaryStats.entrySet()) { - for (SegmentReplicationPerGroupStats group : entry.getValue()) { - group.addReplicaStats(searchReplicaSegRepShardStats); - } - } - return new SegmentReplicationStatsResponse(totalShards, successfulShards, failedShards, primaryStats, shardFailures); + Map> replicationStats = primaryStats.entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue() + .stream() + .map(groupStats -> updateGroupStats(groupStats, replicaStats)) + .collect(Collectors.toList()) + ) + ); + + return new SegmentReplicationStatsResponse(totalShards, successfulShards, failedShards, replicationStats, shardFailures); } @Override @@ -169,12 +160,9 @@ protected SegmentReplicationShardStatsResponse shardOperation(SegmentReplication if (shardRouting.primary()) { return new SegmentReplicationShardStatsResponse(pressureService.getStatsForShard(indexShard)); - } else if (shardRouting.isSearchOnly()) { - SegmentReplicationShardStats segmentReplicationShardStats = calculateSegmentReplicationShardStats(shardRouting); - return new SegmentReplicationShardStatsResponse(segmentReplicationShardStats); - } else { - return new SegmentReplicationShardStatsResponse(getSegmentReplicationState(shardId, request.activeOnly())); } + + return new SegmentReplicationShardStatsResponse(getSegmentReplicationState(shardId, request.activeOnly())); } @Override @@ -196,12 +184,49 @@ protected ClusterBlockException checkRequestBlock( return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); } - private SegmentReplicationShardStats calculateSegmentReplicationShardStats(ShardRouting shardRouting) { + private SegmentReplicationPerGroupStats updateGroupStats( + SegmentReplicationPerGroupStats groupStats, + Map replicaStats + ) { + // Update the SegmentReplicationState for each of the replicas + Set updatedReplicaStats = groupStats.getReplicaStats() + .stream() + .peek(replicaStat -> replicaStat.setCurrentReplicationState(replicaStats.getOrDefault(replicaStat.getAllocationId(), null))) + .collect(Collectors.toSet()); + + // Compute search replica stats + Set searchReplicaStats = computeSearchReplicaStats(groupStats.getShardId(), replicaStats); + + // Combine ReplicaStats and SearchReplicaStats + Set combinedStats = Stream.concat(updatedReplicaStats.stream(), searchReplicaStats.stream()) + .collect(Collectors.toSet()); + + return new SegmentReplicationPerGroupStats(groupStats.getShardId(), combinedStats, 0); + } + + private Set computeSearchReplicaStats( + ShardId shardId, + Map replicaStats + ) { + return replicaStats.values() + .stream() + .filter(segmentReplicationState -> segmentReplicationState.getShardRouting().shardId().equals(shardId)) + .filter(segmentReplicationState -> segmentReplicationState.getShardRouting().isSearchOnly()) + .map(segmentReplicationState -> { + ShardRouting shardRouting = segmentReplicationState.getShardRouting(); + SegmentReplicationShardStats segmentReplicationStats = computeSegmentReplicationShardStats(shardRouting); + segmentReplicationStats.setCurrentReplicationState(segmentReplicationState); + return segmentReplicationStats; + }) + .collect(Collectors.toSet()); + } + + SegmentReplicationShardStats computeSegmentReplicationShardStats(ShardRouting shardRouting) { ShardId shardId = shardRouting.shardId(); SegmentReplicationState completedSegmentReplicationState = targetService.getlatestCompletedEventSegmentReplicationState(shardId); SegmentReplicationState ongoingSegmentReplicationState = targetService.getOngoingEventSegmentReplicationState(shardId); - SegmentReplicationShardStats segmentReplicationShardStats = new SegmentReplicationShardStats( + return new SegmentReplicationShardStats( shardRouting.allocationId().getId(), 0, calculateBytesRemainingToReplicate(ongoingSegmentReplicationState), @@ -209,9 +234,6 @@ private SegmentReplicationShardStats calculateSegmentReplicationShardStats(Shard getCurrentReplicationLag(ongoingSegmentReplicationState), getLastCompletedReplicationLag(completedSegmentReplicationState) ); - - segmentReplicationShardStats.setCurrentReplicationState(targetService.getSegmentReplicationState(shardId)); - return segmentReplicationShardStats; } private SegmentReplicationState getSegmentReplicationState(ShardId shardId, boolean isActiveOnly) { diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java index 6b12211bfaa09..884686ee48fa1 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPerGroupStats.java @@ -28,7 +28,7 @@ public class SegmentReplicationPerGroupStats implements Writeable, ToXContentFragment { private final ShardId shardId; - private Set replicaStats; + private final Set replicaStats; private final long rejectedRequestCount; public SegmentReplicationPerGroupStats(ShardId shardId, Set replicaStats, long rejectedRequestCount) { @@ -55,15 +55,6 @@ public ShardId getShardId() { return shardId; } - public void addReplicaStats(Set replicaStats) { - if (this.replicaStats.isEmpty()) { - // When there is only search replica, replicaStats is empty. EmptySet is immutable and doesn't support adding item - this.replicaStats = replicaStats; - } else { - this.replicaStats.addAll(replicaStats); - } - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("rejected_requests", rejectedRequestCount); diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java index abb52ff1a5169..1e43827afeb47 100644 --- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java @@ -1345,7 +1345,7 @@ && shouldSkipReplicationTimer(entry.getKey()) == false && isShardOnRemoteEnabledNode.apply(routingTable.getByAllocationId(entry.getKey()).currentNodeId()))) ) .map(entry -> buildShardStats(entry.getKey(), entry.getValue())) - .collect(Collectors.toSet()); + .collect(Collectors.toUnmodifiableSet()); } return Collections.emptySet(); } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java index 6d4527a9007c3..ea455d607f058 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsActionTests.java @@ -142,7 +142,6 @@ public void testShardOperationWithReplicaShard() { assertNotNull(response); assertNull(response.getPrimaryStats()); assertNotNull(response.getReplicaStats()); - assertNull(response.getSearchReplicaReplicationStats()); verify(targetService).getSegmentReplicationState(shardId); } @@ -165,133 +164,112 @@ public void testShardOperationWithReplicaShardActiveOnly() { assertNotNull(response); assertNull(response.getPrimaryStats()); assertNotNull(response.getReplicaStats()); - assertNull(response.getSearchReplicaReplicationStats()); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationWithSearchOnlyReplicaWhenCompletedAndOngoingStateNotNull() { + public void testComputeBytesRemainingToReplicateWhenCompletedAndOngoingStateNotNull() { ShardRouting shardRouting = mock(ShardRouting.class); SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); AllocationId allocationId = AllocationId.newInitializing(); - SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); long time1 = 10; long time2 = 15; ReplicationLuceneIndex replicationLuceneIndex = new ReplicationLuceneIndex(); replicationLuceneIndex.addFileDetail("name1", 10, false); - replicationLuceneIndex.addFileDetail("name2", 5, false); + replicationLuceneIndex.addFileDetail("name2", 15, false); when(shardRouting.shardId()).thenReturn(shardId); - when(shardRouting.primary()).thenReturn(false); - when(shardRouting.isSearchOnly()).thenReturn(true); when(shardRouting.allocationId()).thenReturn(allocationId); - when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); - when(indexService.getShard(shardId.id())).thenReturn(indexShard); - when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); - when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); when(replicationTimerOngoing.time()).thenReturn(time1); when(replicationTimerCompleted.time()).thenReturn(time2); when(onGoingSegmentReplicationState.getIndex()).thenReturn(replicationLuceneIndex); - SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + SegmentReplicationShardStats segmentReplicationShardStats = action.computeSegmentReplicationShardStats(shardRouting); + + assertNotNull(segmentReplicationShardStats); + assertEquals(25, segmentReplicationShardStats.getBytesBehindCount()); + assertEquals(10, segmentReplicationShardStats.getCurrentReplicationLagMillis()); + assertEquals(15, segmentReplicationShardStats.getLastCompletedReplicationTimeMillis()); - assertNotNull(response); verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedState() { + public void testCalculateBytesRemainingToReplicateWhenNoCompletedState() { ShardRouting shardRouting = mock(ShardRouting.class); SegmentReplicationState onGoingSegmentReplicationState = mock(SegmentReplicationState.class); ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); AllocationId allocationId = AllocationId.newInitializing(); - SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); ReplicationTimer replicationTimerOngoing = mock(ReplicationTimer.class); long time1 = 10; ReplicationLuceneIndex replicationLuceneIndex = new ReplicationLuceneIndex(); replicationLuceneIndex.addFileDetail("name1", 10, false); - replicationLuceneIndex.addFileDetail("name2", 5, false); + replicationLuceneIndex.addFileDetail("name2", 15, false); when(shardRouting.shardId()).thenReturn(shardId); - when(shardRouting.primary()).thenReturn(false); - when(shardRouting.isSearchOnly()).thenReturn(true); when(shardRouting.allocationId()).thenReturn(allocationId); - when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); - when(indexService.getShard(shardId.id())).thenReturn(indexShard); - when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getOngoingEventSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); - when(targetService.getSegmentReplicationState(shardId)).thenReturn(onGoingSegmentReplicationState); when(onGoingSegmentReplicationState.getTimer()).thenReturn(replicationTimerOngoing); when(replicationTimerOngoing.time()).thenReturn(time1); when(onGoingSegmentReplicationState.getIndex()).thenReturn(replicationLuceneIndex); - SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + SegmentReplicationShardStats segmentReplicationShardStats = action.computeSegmentReplicationShardStats(shardRouting); + + assertNotNull(segmentReplicationShardStats); + assertEquals(25, segmentReplicationShardStats.getBytesBehindCount()); + assertEquals(10, segmentReplicationShardStats.getCurrentReplicationLagMillis()); + assertEquals(0, segmentReplicationShardStats.getLastCompletedReplicationTimeMillis()); - assertNotNull(response); - assertNull(response.getPrimaryStats()); - assertNull(response.getReplicaStats()); - assertNotNull(response.getSearchReplicaReplicationStats()); verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationWithSearchOnlyReplicaWhenNoOngoingState() { + public void testCalculateBytesRemainingToReplicateWhenNoOnGoingState() { ShardRouting shardRouting = mock(ShardRouting.class); SegmentReplicationState completedSegmentReplicationState = mock(SegmentReplicationState.class); ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); AllocationId allocationId = AllocationId.newInitializing(); - SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); ReplicationTimer replicationTimerCompleted = mock(ReplicationTimer.class); long time2 = 15; when(shardRouting.shardId()).thenReturn(shardId); - when(shardRouting.primary()).thenReturn(false); - when(shardRouting.isSearchOnly()).thenReturn(true); when(shardRouting.allocationId()).thenReturn(allocationId); - when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); - when(indexService.getShard(shardId.id())).thenReturn(indexShard); - when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); when(targetService.getlatestCompletedEventSegmentReplicationState(shardId)).thenReturn(completedSegmentReplicationState); when(completedSegmentReplicationState.getTimer()).thenReturn(replicationTimerCompleted); when(replicationTimerCompleted.time()).thenReturn(time2); - SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + SegmentReplicationShardStats segmentReplicationShardStats = action.computeSegmentReplicationShardStats(shardRouting); + + assertNotNull(segmentReplicationShardStats); + assertEquals(0, segmentReplicationShardStats.getBytesBehindCount()); + assertEquals(0, segmentReplicationShardStats.getCurrentReplicationLagMillis()); + assertEquals(15, segmentReplicationShardStats.getLastCompletedReplicationTimeMillis()); - assertNotNull(response); - assertNull(response.getPrimaryStats()); - assertNull(response.getReplicaStats()); - assertNotNull(response.getSearchReplicaReplicationStats()); verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } - public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedAndOngoingState() { + public void testCalculateBytesRemainingToReplicateWhenNoCompletedAndOngoingState() { ShardRouting shardRouting = mock(ShardRouting.class); ShardId shardId = new ShardId(new Index("test-index", "test-uuid"), 0); AllocationId allocationId = AllocationId.newInitializing(); - SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - when(shardRouting.shardId()).thenReturn(shardId); - when(shardRouting.primary()).thenReturn(false); - when(shardRouting.isSearchOnly()).thenReturn(true); when(shardRouting.allocationId()).thenReturn(allocationId); - when(indicesService.indexServiceSafe(shardId.getIndex())).thenReturn(indexService); - when(indexService.getShard(shardId.id())).thenReturn(indexShard); - when(indexShard.indexSettings()).thenReturn(createIndexSettingsWithSegRepEnabled()); - SegmentReplicationShardStatsResponse response = action.shardOperation(request, shardRouting); + SegmentReplicationShardStats segmentReplicationShardStats = action.computeSegmentReplicationShardStats(shardRouting); + + assertNotNull(segmentReplicationShardStats); + assertEquals(0, segmentReplicationShardStats.getBytesBehindCount()); + assertEquals(0, segmentReplicationShardStats.getCurrentReplicationLagMillis()); + assertEquals(0, segmentReplicationShardStats.getLastCompletedReplicationTimeMillis()); - assertNotNull(response); - assertNull(response.getPrimaryStats()); - assertNull(response.getReplicaStats()); - assertNotNull(response.getSearchReplicaReplicationStats()); verify(targetService).getlatestCompletedEventSegmentReplicationState(shardId); verify(targetService).getOngoingEventSegmentReplicationState(shardId); } @@ -299,49 +277,105 @@ public void testShardOperationWithSearchOnlyReplicaWhenNoCompletedAndOngoingStat public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); List shardFailures = new ArrayList<>(); - String[] shards = { "1", "2", "3" }; + String[] shards = { "0", "1" }; request.shards(shards); - int totalShards = 3; - int successfulShards = 3; + int totalShards = 6; + int successfulShards = 6; int failedShard = 0; String allocIdOne = "allocIdOne"; String allocIdTwo = "allocIdTwo"; - ShardId shardIdOne = mock(ShardId.class); - ShardId shardIdTwo = mock(ShardId.class); - ShardId shardIdThree = mock(ShardId.class); - ShardRouting shardRoutingOne = mock(ShardRouting.class); - ShardRouting shardRoutingTwo = mock(ShardRouting.class); - ShardRouting shardRoutingThree = mock(ShardRouting.class); - when(shardIdOne.getId()).thenReturn(1); - when(shardIdTwo.getId()).thenReturn(2); - when(shardIdThree.getId()).thenReturn(3); - when(shardRoutingOne.shardId()).thenReturn(shardIdOne); - when(shardRoutingTwo.shardId()).thenReturn(shardIdTwo); - when(shardRoutingThree.shardId()).thenReturn(shardIdThree); - AllocationId allocationId = mock(AllocationId.class); - when(allocationId.getId()).thenReturn(allocIdOne); - when(shardRoutingTwo.allocationId()).thenReturn(allocationId); - when(shardIdOne.getIndexName()).thenReturn("test-index"); + String allocIdThree = "allocIdThree"; + String allocIdFour = "allocIdFour"; + String allocIdFive = "allocIdFive"; + String allocIdSix = "allocIdSix"; + + ShardId shardId0 = mock(ShardId.class); + ShardRouting primary0 = mock(ShardRouting.class); + ShardRouting replica0 = mock(ShardRouting.class); + ShardRouting searchReplica0 = mock(ShardRouting.class); + + ShardId shardId1 = mock(ShardId.class); + ShardRouting primary1 = mock(ShardRouting.class); + ShardRouting replica1 = mock(ShardRouting.class); + ShardRouting searchReplica1 = mock(ShardRouting.class); + + when(shardId0.getId()).thenReturn(0); + when(shardId0.getIndexName()).thenReturn("test-index-1"); + when(primary0.shardId()).thenReturn(shardId0); + when(replica0.shardId()).thenReturn(shardId0); + when(searchReplica0.shardId()).thenReturn(shardId0); + + when(shardId1.getId()).thenReturn(1); + when(shardId1.getIndexName()).thenReturn("test-index-1"); + when(primary1.shardId()).thenReturn(shardId1); + when(replica1.shardId()).thenReturn(shardId1); + when(searchReplica1.shardId()).thenReturn(shardId1); + + AllocationId allocationIdOne = mock(AllocationId.class); + AllocationId allocationIdTwo = mock(AllocationId.class); + AllocationId allocationIdThree = mock(AllocationId.class); + AllocationId allocationIdFour = mock(AllocationId.class); + AllocationId allocationIdFive = mock(AllocationId.class); + AllocationId allocationIdSix = mock(AllocationId.class); + + when(allocationIdOne.getId()).thenReturn(allocIdOne); + when(allocationIdTwo.getId()).thenReturn(allocIdTwo); + when(allocationIdThree.getId()).thenReturn(allocIdThree); + when(allocationIdFour.getId()).thenReturn(allocIdFour); + when(allocationIdFive.getId()).thenReturn(allocIdFive); + when(allocationIdSix.getId()).thenReturn(allocIdSix); + when(primary0.allocationId()).thenReturn(allocationIdOne); + when(replica0.allocationId()).thenReturn(allocationIdTwo); + when(searchReplica0.allocationId()).thenReturn(allocationIdThree); + when(primary1.allocationId()).thenReturn(allocationIdFour); + when(replica1.allocationId()).thenReturn(allocationIdFive); + when(searchReplica1.allocationId()).thenReturn(allocationIdSix); + + when(primary0.isSearchOnly()).thenReturn(false); + when(replica0.isSearchOnly()).thenReturn(false); + when(searchReplica0.isSearchOnly()).thenReturn(true); + when(primary1.isSearchOnly()).thenReturn(false); + when(replica1.isSearchOnly()).thenReturn(false); + when(searchReplica1.isSearchOnly()).thenReturn(true); + + Set segmentReplicationShardStats0 = new HashSet<>(); + SegmentReplicationShardStats segmentReplicationShardStatsOfReplica0 = new SegmentReplicationShardStats(allocIdTwo, 0, 0, 0, 0, 0); + segmentReplicationShardStats0.add(segmentReplicationShardStatsOfReplica0); + + Set segmentReplicationShardStats1 = new HashSet<>(); + SegmentReplicationShardStats segmentReplicationShardStatsOfReplica1 = new SegmentReplicationShardStats(allocIdFive, 0, 0, 0, 0, 0); + segmentReplicationShardStats1.add(segmentReplicationShardStatsOfReplica1); + + SegmentReplicationPerGroupStats segmentReplicationPerGroupStats0 = new SegmentReplicationPerGroupStats( + shardId0, + segmentReplicationShardStats0, + 0 + ); - Set segmentReplicationShardStats = new HashSet<>(); - SegmentReplicationShardStats segmentReplicationShardStatsOfReplica = new SegmentReplicationShardStats(allocIdOne, 0, 0, 0, 0, 0); - segmentReplicationShardStats.add(segmentReplicationShardStatsOfReplica); - SegmentReplicationPerGroupStats segmentReplicationPerGroupStats = new SegmentReplicationPerGroupStats( - shardIdOne, - segmentReplicationShardStats, + SegmentReplicationPerGroupStats segmentReplicationPerGroupStats1 = new SegmentReplicationPerGroupStats( + shardId1, + segmentReplicationShardStats1, 0 ); - SegmentReplicationState segmentReplicationState = mock(SegmentReplicationState.class); - SegmentReplicationShardStats segmentReplicationShardStatsFromSearchReplica = mock(SegmentReplicationShardStats.class); - when(segmentReplicationShardStatsFromSearchReplica.getAllocationId()).thenReturn("alloc2"); - when(segmentReplicationState.getShardRouting()).thenReturn(shardRoutingTwo); + SegmentReplicationState segmentReplicationState0 = mock(SegmentReplicationState.class); + SegmentReplicationState searchReplicaSegmentReplicationState0 = mock(SegmentReplicationState.class); + SegmentReplicationState segmentReplicationState1 = mock(SegmentReplicationState.class); + SegmentReplicationState searchReplicaSegmentReplicationState1 = mock(SegmentReplicationState.class); + + when(segmentReplicationState0.getShardRouting()).thenReturn(replica0); + when(searchReplicaSegmentReplicationState0.getShardRouting()).thenReturn(searchReplica0); + when(segmentReplicationState1.getShardRouting()).thenReturn(replica1); + when(searchReplicaSegmentReplicationState1.getShardRouting()).thenReturn(searchReplica1); List responses = List.of( - new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStats), - new SegmentReplicationShardStatsResponse(segmentReplicationState), - new SegmentReplicationShardStatsResponse(segmentReplicationShardStatsFromSearchReplica) + new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStats0), + new SegmentReplicationShardStatsResponse(segmentReplicationState0), + new SegmentReplicationShardStatsResponse(searchReplicaSegmentReplicationState0), + new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStats1), + new SegmentReplicationShardStatsResponse(segmentReplicationState1), + new SegmentReplicationShardStatsResponse(searchReplicaSegmentReplicationState1) ); SegmentReplicationStatsResponse response = action.newResponse( @@ -354,66 +388,30 @@ public void testNewResponseWhenAllReplicasReturnResponseCombinesTheResults() { ClusterState.EMPTY_STATE ); - List responseStats = response.getReplicationStats().get("test-index"); - SegmentReplicationPerGroupStats primStats = responseStats.get(0); - Set segRpShardStatsSet = primStats.getReplicaStats(); - - for (SegmentReplicationShardStats segRpShardStats : segRpShardStatsSet) { - if (segRpShardStats.getAllocationId().equals(allocIdOne)) { - assertEquals(segmentReplicationState, segRpShardStats.getCurrentReplicationState()); + List responseStats = response.getReplicationStats().get("test-index-1"); + SegmentReplicationPerGroupStats primStats0 = responseStats.get(0); + Set replicaStats0 = primStats0.getReplicaStats(); + assertEquals(2, replicaStats0.size()); + for (SegmentReplicationShardStats replicaStat : replicaStats0) { + if (replicaStat.getAllocationId().equals(allocIdTwo)) { + assertEquals(segmentReplicationState0, replicaStat.getCurrentReplicationState()); } - if (segRpShardStats.getAllocationId().equals(allocIdTwo)) { - assertEquals(segmentReplicationShardStatsFromSearchReplica, segRpShardStats); + if (replicaStat.getAllocationId().equals(allocIdThree)) { + assertEquals(searchReplicaSegmentReplicationState0, replicaStat.getCurrentReplicationState()); } } - } - - public void testNewResponseWhenTwoPrimaryShardsForSameIndex() { - SegmentReplicationStatsRequest request = new SegmentReplicationStatsRequest(); - List shardFailures = new ArrayList<>(); - String[] shards = { "1", "2" }; - request.shards(shards); - int totalShards = 3; - int successfulShards = 3; - int failedShard = 0; - - SegmentReplicationPerGroupStats segmentReplicationPerGroupStatsOne = mock(SegmentReplicationPerGroupStats.class); - SegmentReplicationPerGroupStats segmentReplicationPerGroupStatsTwo = mock(SegmentReplicationPerGroupStats.class); - - ShardId shardIdOne = mock(ShardId.class); - ShardId shardIdTwo = mock(ShardId.class); - when(segmentReplicationPerGroupStatsOne.getShardId()).thenReturn(shardIdOne); - when(segmentReplicationPerGroupStatsTwo.getShardId()).thenReturn(shardIdTwo); - when(shardIdOne.getIndexName()).thenReturn("test-index"); - when(shardIdTwo.getIndexName()).thenReturn("test-index"); - when(shardIdOne.getId()).thenReturn(1); - when(shardIdTwo.getId()).thenReturn(2); - - List responses = List.of( - new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStatsOne), - new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStatsTwo) - ); - - SegmentReplicationStatsResponse response = action.newResponse( - request, - totalShards, - successfulShards, - failedShard, - responses, - shardFailures, - ClusterState.EMPTY_STATE - ); - - List responseStats = response.getReplicationStats().get("test-index"); - for (SegmentReplicationPerGroupStats primStat : responseStats) { - if (primStat.getShardId().equals(shardIdOne)) { - assertEquals(segmentReplicationPerGroupStatsOne, primStat); + SegmentReplicationPerGroupStats primStats1 = responseStats.get(1); + Set replicaStats1 = primStats1.getReplicaStats(); + assertEquals(2, replicaStats1.size()); + for (SegmentReplicationShardStats replicaStat : replicaStats1) { + if (replicaStat.getAllocationId().equals(allocIdFive)) { + assertEquals(segmentReplicationState1, replicaStat.getCurrentReplicationState()); } - if (primStat.getShardId().equals(shardIdTwo)) { - assertEquals(segmentReplicationPerGroupStatsTwo, primStat); + if (replicaStat.getAllocationId().equals(allocIdSix)) { + assertEquals(searchReplicaSegmentReplicationState1, replicaStat.getCurrentReplicationState()); } } } @@ -464,7 +462,6 @@ public void testNewResponseWhenShardsToFetchEmptyAndResponsesContainsNull() { responses.add(null); responses.add(new SegmentReplicationShardStatsResponse(segmentReplicationPerGroupStats)); responses.add(new SegmentReplicationShardStatsResponse(segmentReplicationState)); - responses.add(new SegmentReplicationShardStatsResponse(segmentReplicationShardStatsFromSearchReplica)); SegmentReplicationStatsResponse response = action.newResponse( request, From 046e26cee4312fd5018b3a4d968fb8acc5afc6c1 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Tue, 10 Dec 2024 21:22:32 -0800 Subject: [PATCH 12/15] Fixed failing integ tests Signed-off-by: Vinay Krishna Pudyodu --- .../indices/replication/SegmentReplicationStatsIT.java | 1 + .../replication/TransportSegmentReplicationStatsAction.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java index b2161577f7040..89aef6f0be1a6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationStatsIT.java @@ -432,4 +432,5 @@ public void testSegmentReplicationNodeAndIndexStats() throws Exception { } } + } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java index 1a4f073941494..44408c5043fcf 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/replication/TransportSegmentReplicationStatsAction.java @@ -201,7 +201,7 @@ private SegmentReplicationPerGroupStats updateGroupStats( Set combinedStats = Stream.concat(updatedReplicaStats.stream(), searchReplicaStats.stream()) .collect(Collectors.toSet()); - return new SegmentReplicationPerGroupStats(groupStats.getShardId(), combinedStats, 0); + return new SegmentReplicationPerGroupStats(groupStats.getShardId(), combinedStats, groupStats.getRejectedRequestCount()); } private Set computeSearchReplicaStats( From e09dde888ab3617d66347122029bb031f72f774c Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Wed, 11 Dec 2024 15:39:50 -0800 Subject: [PATCH 13/15] Fixed failing integ test Signed-off-by: Vinay Krishna Pudyodu --- .../SearchReplicaReplicationIT.java | 52 +------------------ 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java index a3a8e7d1db0ae..d303e02b8dad6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java @@ -33,7 +33,7 @@ public class SearchReplicaReplicationIT extends SegmentReplicationBaseIT { @Before public void randomizeRemoteStoreEnabled() { - useRemoteStore = randomBoolean(); + useRemoteStore = true; } @Override @@ -61,7 +61,7 @@ public void teardown() { public Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) .build(); @@ -89,54 +89,6 @@ public void testReplication() throws Exception { } public void testSegmentReplicationStatsResponseWithSearchReplica() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - final List nodes = internalCluster().startDataOnlyNodes(3); - int numOfPrimaryShards = 2; - createIndex( - INDEX_NAME, - Settings.builder() - .put("number_of_shards", numOfPrimaryShards) - .put("number_of_replicas", 1) - .put("number_of_search_only_replicas", 1) - .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .build() - ); - ensureGreen(INDEX_NAME); - - final int docCount = 5; - for (int i = 0; i < docCount; i++) { - client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); - } - refresh(INDEX_NAME); - waitForSearchableDocs(docCount, nodes); - - SegmentReplicationStatsResponse segmentReplicationStatsResponse = dataNodeClient().admin() - .indices() - .prepareSegmentReplicationStats(INDEX_NAME) - .setDetailed(true) - .execute() - .actionGet(); - - // Verify the number of indices - assertEquals(1, segmentReplicationStatsResponse.getReplicationStats().size()); - // Verify total shards - assertEquals(numOfPrimaryShards * 3, segmentReplicationStatsResponse.getTotalShards()); - // Verify the number of primary shards - assertEquals(numOfPrimaryShards, segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).size()); - - List segmentReplicationPerGroupStats = segmentReplicationStatsResponse.getReplicationStats() - .get(INDEX_NAME); - for (SegmentReplicationPerGroupStats perGroupStats : segmentReplicationPerGroupStats) { - Set replicaStats = perGroupStats.getReplicaStats(); - // Verify the number of replica stats - assertEquals(2, replicaStats.size()); - for (SegmentReplicationShardStats replicaStat : replicaStats) { - assertNotNull(replicaStat.getCurrentReplicationState()); - } - } - } - - public void testSegmentReplicationStatsResponseWithOnlySearchReplica() throws Exception { internalCluster().startClusterManagerOnlyNode(); final List nodes = internalCluster().startDataOnlyNodes(2); createIndex( From 43521143113c053493ecd4414a1b2dd953691025 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Thu, 12 Dec 2024 12:44:54 -0800 Subject: [PATCH 14/15] fixed some comments for PR Signed-off-by: Vinay Krishna Pudyodu --- .../indices/replication/SearchReplicaReplicationIT.java | 2 +- .../java/org/opensearch/index/SegmentReplicationShardStats.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java index d303e02b8dad6..8ec6e7bf1ac17 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java @@ -33,7 +33,7 @@ public class SearchReplicaReplicationIT extends SegmentReplicationBaseIT { @Before public void randomizeRemoteStoreEnabled() { - useRemoteStore = true; + useRemoteStore = randomBoolean(); } @Override diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java index 3845dedfb8fd0..c493ac90b4b51 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java @@ -135,7 +135,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(currentReplicationTimeMillis); out.writeVLong(lastCompletedReplicationTimeMillis); out.writeVLong(currentReplicationLagMillis); - out.writeOptionalWriteable(currentReplicationState); } @Override From 70cb875bd9bda9cfcb3e94e019e0fc5e6a198b1a Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Thu, 12 Dec 2024 14:37:31 -0800 Subject: [PATCH 15/15] fixed failing tests Signed-off-by: Vinay Krishna Pudyodu --- .../indices/replication/SearchReplicaReplicationIT.java | 2 +- .../java/org/opensearch/index/SegmentReplicationShardStats.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java index 8ec6e7bf1ac17..f660695af9965 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java @@ -61,7 +61,7 @@ public void teardown() { public Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) .build(); diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java index c493ac90b4b51..e381ade253422 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java @@ -64,7 +64,6 @@ public SegmentReplicationShardStats(StreamInput in) throws IOException { this.currentReplicationTimeMillis = in.readVLong(); this.lastCompletedReplicationTimeMillis = in.readVLong(); this.currentReplicationLagMillis = in.readVLong(); - this.currentReplicationState = in.readOptionalWriteable(SegmentReplicationState::new); } public String getAllocationId() {