-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add rest, transport, service layer changes for Tiering
Signed-off-by: Neetika Singhal <[email protected]>
- Loading branch information
1 parent
fbe048f
commit 2a315ef
Showing
15 changed files
with
1,940 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
49 changes: 49 additions & 0 deletions
49
server/src/main/java/org/opensearch/action/tiering/RestWarmTieringAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.tiering; | ||
|
|
||
| import org.opensearch.client.node.NodeClient; | ||
| import org.opensearch.rest.BaseRestHandler; | ||
| import org.opensearch.rest.RestHandler; | ||
| import org.opensearch.rest.RestRequest; | ||
| import org.opensearch.rest.action.RestToXContentListener; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| import static java.util.Collections.singletonList; | ||
| import static org.opensearch.rest.RestRequest.Method.POST; | ||
|
|
||
| /** | ||
| * Rest Tiering API class to move index from hot to warm | ||
| * | ||
| * @opensearch.experimental | ||
| */ | ||
| public class RestWarmTieringAction extends BaseRestHandler { | ||
|
|
||
| @Override | ||
| public List<RestHandler.Route> routes() { | ||
| return singletonList(new RestHandler.Route(POST, "/{index}/_tier/_warm")); | ||
| } | ||
|
|
||
| @Override | ||
| public String getName() { | ||
| return "tiering_warm"; | ||
| } | ||
|
|
||
| @Override | ||
| protected BaseRestHandler.RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { | ||
| final TieringIndexRequest tieringIndexRequest = new TieringIndexRequest(request.param("index")); | ||
| tieringIndexRequest.timeout(request.paramAsTime("timeout", tieringIndexRequest.timeout())); | ||
| tieringIndexRequest.clusterManagerNodeTimeout( | ||
| request.paramAsTime("cluster_manager_timeout", tieringIndexRequest.clusterManagerNodeTimeout()) | ||
| ); | ||
| tieringIndexRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false)); | ||
| return channel -> client.admin().cluster().execute(WarmTieringAction.INSTANCE, tieringIndexRequest, new RestToXContentListener<>(channel)); | ||
| } | ||
| } |
129 changes: 129 additions & 0 deletions
129
server/src/main/java/org/opensearch/action/tiering/TieringIndexRequest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.tiering; | ||
|
|
||
| import org.opensearch.action.ActionRequestValidationException; | ||
| import org.opensearch.action.IndicesRequest; | ||
| import org.opensearch.action.support.IndicesOptions; | ||
| import org.opensearch.action.support.master.AcknowledgedRequest; | ||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Arrays; | ||
| import java.util.Objects; | ||
|
|
||
| import static org.opensearch.action.ValidateActions.addValidationError; | ||
|
|
||
| /** | ||
| * Represents the tiering request for indices | ||
| * to move to a different tier | ||
| * | ||
| * @opensearch.experimental | ||
| */ | ||
| @ExperimentalApi | ||
| public class TieringIndexRequest extends AcknowledgedRequest<TieringIndexRequest> implements IndicesRequest.Replaceable { | ||
|
|
||
| private String[] indices; | ||
| private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true); | ||
| private boolean waitForCompletion; | ||
| public TieringIndexRequest() { | ||
| } | ||
|
|
||
| @Override | ||
| public ActionRequestValidationException validate() { | ||
| ActionRequestValidationException validationException = null; | ||
| if (indices == null || indices.length == 0) { | ||
| validationException = addValidationError("Mandatory parameter - indices is missing from the request", validationException); | ||
| } | ||
| return validationException; | ||
| } | ||
|
|
||
| public TieringIndexRequest(String... indices) { | ||
| this.indices = indices; | ||
| } | ||
|
|
||
| public TieringIndexRequest(StreamInput in) throws IOException { | ||
| super(in); | ||
| indices = in.readStringArray(); | ||
| indicesOptions = IndicesOptions.readIndicesOptions(in); | ||
| waitForCompletion = in.readBoolean(); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| out.writeStringArray(indices); | ||
| indicesOptions.writeIndicesOptions(out); | ||
| out.writeBoolean(waitForCompletion); | ||
| } | ||
|
|
||
| @Override | ||
| public String[] indices() { | ||
| return indices; | ||
| } | ||
|
|
||
| @Override | ||
| public IndicesOptions indicesOptions() { | ||
| return indicesOptions; | ||
| } | ||
|
|
||
| @Override | ||
| public TieringIndexRequest indices(String... indices) { | ||
| this.indices = indices; | ||
| return this; | ||
| } | ||
|
|
||
| public TieringIndexRequest indicesOptions(IndicesOptions indicesOptions) { | ||
| this.indicesOptions = indicesOptions; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * If this parameter is set to true the operation will wait for completion of tiering process before returning. | ||
| * | ||
| * @param waitForCompletion if true the operation will wait for completion | ||
| * @return this request | ||
| */ | ||
| public TieringIndexRequest waitForCompletion(boolean waitForCompletion) { | ||
| this.waitForCompletion = waitForCompletion; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Returns wait for completion setting | ||
| * | ||
| * @return true if the operation will wait for completion | ||
| */ | ||
| public boolean waitForCompletion() { | ||
| return waitForCompletion; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (this == o) { | ||
| return true; | ||
| } | ||
| if (o == null || getClass() != o.getClass()) { | ||
| return false; | ||
| } | ||
| TieringIndexRequest that = (TieringIndexRequest) o; | ||
| return clusterManagerNodeTimeout.equals(that.clusterManagerNodeTimeout) | ||
| && timeout.equals(that.timeout) | ||
| && Objects.equals(indicesOptions, that.indicesOptions) | ||
| && Arrays.equals(indices, that.indices) | ||
| && waitForCompletion == that.waitForCompletion; | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(clusterManagerNodeTimeout, timeout, indicesOptions, waitForCompletion, Arrays.hashCode(indices)); | ||
| } | ||
| } |
120 changes: 120 additions & 0 deletions
120
server/src/main/java/org/opensearch/action/tiering/TransportWarmTieringAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.tiering; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; | ||
| import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; | ||
| import org.opensearch.action.support.ActionFilters; | ||
| import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; | ||
| import org.opensearch.action.support.master.AcknowledgedResponse; | ||
| import org.opensearch.client.Client; | ||
| import org.opensearch.cluster.ClusterState; | ||
| import org.opensearch.cluster.block.ClusterBlockException; | ||
| import org.opensearch.cluster.block.ClusterBlockLevel; | ||
| import org.opensearch.cluster.metadata.IndexNameExpressionResolver; | ||
| import org.opensearch.cluster.service.ClusterService; | ||
| import org.opensearch.common.inject.Inject; | ||
| import org.opensearch.core.action.ActionListener; | ||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.threadpool.ThreadPool; | ||
| import org.opensearch.tiering.TieringClusterStateListener; | ||
| import org.opensearch.tiering.TieringService; | ||
| import org.opensearch.transport.TransportService; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS; | ||
|
|
||
| /** | ||
| * Transport Tiering API class to move index from hot to warm | ||
| * | ||
| * @opensearch.experimental | ||
| */ | ||
| public class TransportWarmTieringAction extends TransportClusterManagerNodeAction<TieringIndexRequest, AcknowledgedResponse> { | ||
|
|
||
| private static final Logger logger = LogManager.getLogger(TransportWarmTieringAction.class); | ||
| private final TieringService tieringService; | ||
| private final Client client; | ||
| @Inject | ||
| public TransportWarmTieringAction(TransportService transportService, ClusterService clusterService, | ||
| ThreadPool threadPool, TieringService tieringService, | ||
| ActionFilters actionFilters, | ||
| IndexNameExpressionResolver indexNameExpressionResolver, Client client) { | ||
| super(WarmTieringAction.NAME, transportService, clusterService, threadPool, actionFilters, | ||
| TieringIndexRequest::new, indexNameExpressionResolver); | ||
| this.client = client; | ||
| this.tieringService = tieringService; | ||
| } | ||
|
|
||
| @Override | ||
| protected String executor() { | ||
| return ThreadPool.Names.SAME; | ||
| } | ||
|
|
||
| @Override | ||
| protected AcknowledgedResponse read(StreamInput in) throws IOException { | ||
| return new AcknowledgedResponse(in); | ||
| } | ||
|
|
||
| @Override | ||
| protected ClusterBlockException checkBlock(TieringIndexRequest request, ClusterState state) { | ||
| ClusterBlockException blockException = | ||
| state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); | ||
| if (blockException == null) { | ||
| // Check indices level block | ||
| blockException = state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, | ||
| indexNameExpressionResolver.concreteIndexNames(clusterService.state(), request)); | ||
| } | ||
| return blockException; | ||
| } | ||
|
|
||
| @Override | ||
| protected void clusterManagerOperation(TieringIndexRequest request, ClusterState state, | ||
| ActionListener<AcknowledgedResponse> listener) throws Exception { | ||
| // Collect node stats to get node level filesystem info. The response will be used for performing some | ||
| // validations. | ||
| client.admin().cluster().prepareNodesStats().clear().addMetric(FS.metricName()).execute( | ||
| new ActionListener<NodesStatsResponse>() { | ||
| @Override | ||
| public void onResponse(NodesStatsResponse nodesStatsResponse) { | ||
| // Collect index level stats. This response is also used for validations. | ||
| client.admin().indices().prepareStats().clear().setStore(true).setIndices(request.indices()).execute( | ||
| new ActionListener<IndicesStatsResponse>() { | ||
| @Override | ||
| public void onResponse(IndicesStatsResponse indicesStatsResponse) { | ||
| tieringService.tier(request, nodesStatsResponse, indicesStatsResponse, | ||
| ActionListener.delegateFailure(listener, (delegatedListener, acknowledgedResponse) -> { | ||
| if (request.waitForCompletion()) { | ||
| TieringClusterStateListener.createAndRegisterListener( | ||
| clusterService, | ||
| new AcknowledgedResponse(acknowledgedResponse.isAcknowledged()), | ||
| delegatedListener | ||
| ); | ||
| } else { | ||
| delegatedListener.onResponse(new AcknowledgedResponse(true)); | ||
| } | ||
| })); | ||
| } | ||
| @Override | ||
| public void onFailure(Exception e) { | ||
| logger.debug("Indices stats call failed with exception", e); | ||
| listener.onFailure(e); | ||
| } | ||
| }); | ||
| } | ||
| @Override | ||
| public void onFailure(Exception e) { | ||
| logger.debug("Node stats call failed with exception", e); | ||
| listener.onFailure(e); | ||
| } | ||
| }); | ||
| } | ||
| } |
27 changes: 27 additions & 0 deletions
27
server/src/main/java/org/opensearch/action/tiering/WarmTieringAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.tiering; | ||
|
|
||
| import org.opensearch.action.ActionType; | ||
| import org.opensearch.action.support.master.AcknowledgedResponse; | ||
|
|
||
| /** | ||
| * Tiering action class to move index from hot to warm | ||
| * | ||
| * @opensearch.experimental | ||
| */ | ||
| public class WarmTieringAction extends ActionType<AcknowledgedResponse> { | ||
|
|
||
| public static final WarmTieringAction INSTANCE = new WarmTieringAction(); | ||
| public static final String NAME = "indices:admin/tiering/warm"; | ||
|
|
||
| public WarmTieringAction() { | ||
| super(NAME, AcknowledgedResponse::new); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.