Skip to content

Commit

Permalink
Add remote-routing table diff
Browse files Browse the repository at this point in the history 
Signed-off-by: Arpit Bandejiya <[email protected]>
  • Loading branch information
@Arpit-Bandejiya
Arpit-Bandejiya committed May 16, 2024
1 parent bd45e85 commit 54f1f3d
Showing 1 changed file with 94 additions and 4 deletions.
98 changes: 94 additions & 4 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.cluster.routing.IndexRoutingTable;
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.core.ParseField;
import org.opensearch.core.common.Strings;
import org.opensearch.core.common.io.stream.StreamInput;
Expand Down Expand Up @@ -1152,6 +1154,9 @@ public static class ClusterDiffManifest implements ToXContentObject {
private static final ParseField DELETES_FIELD = new ParseField("deletes");
private static final ParseField CLUSTER_BLOCKS_UPDATED_FIELD = new ParseField("cluster_blocks_diff");
private static final ParseField DISCOVERY_NODES_UPDATED_FIELD = new ParseField("discovery_nodes_diff");
private static final ParseField ROUTING_TABLE_DIFF = new ParseField("routing_table_diff");
private static final ParseField ROUTING_TABLE_UPSERT_FIELD = new ParseField("routing_table_upsert");
private static final ParseField ROUTING_TABLE_DELETE_FIELD = new ParseField("routing_table_delete");
private static final ObjectParser.NamedObjectParser<ClusterDiffManifest, Void> PARSER;
static {
ConstructingObjectParser<ClusterDiffManifest, Void> innerParser = new ConstructingObjectParser<ClusterDiffManifest, Void>(
Expand All @@ -1166,6 +1171,8 @@ public static class ClusterDiffManifest implements ToXContentObject {
.indicesDeleted((List<String>) fields[6])
.clusterBlocksUpdated((Boolean) fields[7])
.discoveryNodesUpdated((Boolean) fields[8])
.indicesRoutingUpdated((List<String>) fields[9])
.indicesRoutingDeleted((List<String>) fields[10])
.build()
);
innerParser.declareString(ConstructingObjectParser.constructorArg(), FROM_STATE_UUID_FIELD);
Expand All @@ -1177,6 +1184,9 @@ public static class ClusterDiffManifest implements ToXContentObject {
innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), DELETES_FIELD);
innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), DISCOVERY_NODES_UPDATED_FIELD);
innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_BLOCKS_UPDATED_FIELD);
innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_UPSERT_FIELD);
innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_DELETE_FIELD);

PARSER = ((p, c, name) -> innerParser.parse(p, null));
}
private final String fromStateUUID;
Expand All @@ -1189,6 +1199,8 @@ public static class ClusterDiffManifest implements ToXContentObject {
private final List<String> indicesDeleted;
private final boolean clusterBlocksUpdated;
private final boolean discoveryNodesUpdated;
private final List<String> indicesRoutingUpdated;
private final List<String> indicesRoutingDeleted;

ClusterDiffManifest(ClusterState state, ClusterState previousState) {
fromStateUUID = previousState.stateUUID();
Expand All @@ -1207,9 +1219,22 @@ public static class ClusterDiffManifest implements ToXContentObject {
state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))
);
}
}

public ClusterDiffManifest(String fromStateUUID, String toStateUUID, boolean coordinationMetadataUpdated, boolean settingsMetadataUpdated, boolean templatesMetadataUpdated, Map<String, Boolean> customMetadataUpdated, List<String> indicesUpdated, List<String> indicesDeleted, boolean clusterBlocksUpdated, boolean discoveryNodesUpdated) {
indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable());
indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable());
}

public ClusterDiffManifest(String fromStateUUID,
String toStateUUID,
boolean coordinationMetadataUpdated,
boolean settingsMetadataUpdated,
boolean templatesMetadataUpdated,
Map<String, Boolean> customMetadataUpdated,
List<String> indicesUpdated,
List<String> indicesDeleted,
boolean clusterBlocksUpdated,
boolean discoveryNodesUpdated,
List<String>indicesRoutingUpdated,
List<String>indicesRoutingDeleted) {
this.fromStateUUID = fromStateUUID;
this.toStateUUID = toStateUUID;
this.coordinationMetadataUpdated = coordinationMetadataUpdated;
Expand All @@ -1220,6 +1245,8 @@ public ClusterDiffManifest(String fromStateUUID, String toStateUUID, boolean coo
this.indicesDeleted = indicesDeleted;
this.clusterBlocksUpdated = clusterBlocksUpdated;
this.discoveryNodesUpdated = discoveryNodesUpdated;
this.indicesRoutingUpdated = indicesRoutingUpdated;
this.indicesRoutingDeleted = indicesRoutingDeleted;
}

@Override
Expand Down Expand Up @@ -1252,6 +1279,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.endObject();
builder.field(CLUSTER_BLOCKS_UPDATED_FIELD.getPreferredName(), clusterBlocksUpdated);
builder.field(DISCOVERY_NODES_UPDATED_FIELD.getPreferredName(), discoveryNodesUpdated);

builder.startObject(ROUTING_TABLE_DIFF.getPreferredName());
builder.startArray(ROUTING_TABLE_UPSERT_FIELD.getPreferredName());
for (String index : indicesRoutingUpdated) {
builder.value(index);
}
builder.endArray();
builder.startArray(ROUTING_TABLE_DELETE_FIELD.getPreferredName());
for (String index : indicesRoutingDeleted) {
builder.value(index);
}
builder.endArray();
builder.endObject();
}
return builder;
}
Expand Down Expand Up @@ -1283,6 +1323,34 @@ public List<String> findUpdatedIndices(Map<String, IndexMetadata> indices, Map<S
return updatedIndices;
}

public List<String> getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) {
List<String> deletedIndices = new ArrayList<>();
for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) {
if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) {
// Latest Routing Table does not have entry for the index which means the index is deleted
deletedIndices.add(previousIndexRouting.getIndex().getName());
}
}
return deletedIndices;
}

public List<String> getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) {
List<String> updatedIndicesRouting = new ArrayList<>();
for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) {
if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) {
// Latest Routing Table does not have entry for the index which means the index is created
updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName());
} else {
if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) {
// if the latest routing table has the same routing table as the previous routing table, then the index is not updated
continue;
}
updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName());
}
}
return updatedIndicesRouting;
}

public String getFromStateUUID() {
return fromStateUUID;
}
Expand Down Expand Up @@ -1323,6 +1391,14 @@ public boolean isDiscoveryNodesUpdated() {
return discoveryNodesUpdated;
}

public List<String> getIndicesRoutingUpdated() {
return indicesRoutingUpdated;
}

public List<String> getIndicesRoutingDeleted() {
return indicesRoutingDeleted;
}

public static ClusterDiffManifest.Builder builder() {
return new Builder();
}
Expand All @@ -1338,6 +1414,8 @@ public static class Builder {
private List<String> indicesDeleted;
private boolean clusterBlocksUpdated;
private boolean discoveryNodesUpdated;
private List<String> indicesRoutingUpdated;
private List<String> indicesRoutingDeleted;
public Builder() {}

public Builder fromStateUUID(String fromStateUUID) {
Expand Down Expand Up @@ -1390,6 +1468,16 @@ public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) {
return this;
}

public Builder indicesRoutingUpdated(List<String> indicesRoutingUpdated) {
this.indicesRoutingUpdated = indicesRoutingUpdated;
return this;
}

public Builder indicesRoutingDeleted(List<String> indicesRoutingDeleted) {
this.indicesRoutingDeleted = indicesRoutingDeleted;
return this;
}

public ClusterDiffManifest build() {
return new ClusterDiffManifest(
fromStateUUID,
Expand All @@ -1401,7 +1489,9 @@ public ClusterDiffManifest build() {
indicesUpdated,
indicesDeleted,
clusterBlocksUpdated,
discoveryNodesUpdated
discoveryNodesUpdated,
indicesRoutingUpdated,
indicesRoutingDeleted
);
}
}
Expand Down

0 comments on commit 54f1f3d

Please sign in to comment.