From a12dd35a949de2e57f5ac2935c96141f2119f45a Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Mon, 15 Apr 2024 15:18:08 -0700 Subject: [PATCH 1/2] Adding support for ignore_unavailable_shards in search request Signed-off-by: Ankit Jain --- .../search/AbstractSearchAsyncAction.java | 6 +++-- .../action/search/SearchRequest.java | 22 +++++++++++++++++++ .../rest/action/search/RestSearchAction.java | 5 +++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index 0520a4a7aecec..c665cbdb30a59 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -425,8 +425,10 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha currentPhase.getName() ); } - onPhaseFailure(currentPhase, "Partial shards failure (" + discrepancy + " shards unavailable)", null); - return; + if (!request.getIgnoreUnavailableShards()) { + onPhaseFailure(currentPhase, "Partial shards failure (" + discrepancy + " shards unavailable)", null); + return; + } } } if (logger.isTraceEnabled()) { diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java index 3b8a6937815aa..c6fea74b2e041 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java @@ -102,6 +102,8 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla private Boolean allowPartialSearchResults; + private Boolean ignoreUnavailableShards; + private Scroll scroll; private int batchedReduceSize = DEFAULT_BATCHED_REDUCE_SIZE; @@ -198,6 +200,7 @@ private SearchRequest( boolean finalReduce ) { this.allowPartialSearchResults = searchRequest.allowPartialSearchResults; + this.ignoreUnavailableShards = searchRequest.ignoreUnavailableShards; this.batchedReduceSize = searchRequest.batchedReduceSize; this.ccsMinimizeRoundtrips = searchRequest.ccsMinimizeRoundtrips; this.indices = indices; @@ -246,6 +249,9 @@ public SearchRequest(StreamInput in) throws IOException { maxConcurrentShardRequests = in.readVInt(); preFilterShardSize = in.readOptionalVInt(); allowPartialSearchResults = in.readOptionalBoolean(); + if (in.getVersion().onOrAfter(Version.V_2_14_0)) { + ignoreUnavailableShards = in.readOptionalBoolean(); + } localClusterAlias = in.readOptionalString(); if (localClusterAlias != null) { absoluteStartMillis = in.readVLong(); @@ -283,6 +289,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(maxConcurrentShardRequests); out.writeOptionalVInt(preFilterShardSize); out.writeOptionalBoolean(allowPartialSearchResults); + if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + out.writeOptionalBoolean(ignoreUnavailableShards); + } out.writeOptionalString(localClusterAlias); if (localClusterAlias != null) { out.writeVLong(absoluteStartMillis); @@ -567,6 +576,15 @@ public Boolean allowPartialSearchResults() { return this.allowPartialSearchResults; } + public SearchRequest ignoreUnavailableShards(boolean ignoreUnavailableShards) { + this.ignoreUnavailableShards = ignoreUnavailableShards; + return this; + } + + public Boolean getIgnoreUnavailableShards() { + return this.ignoreUnavailableShards; + } + /** * Sets the number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection * mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large. @@ -747,6 +765,7 @@ public boolean equals(Object o) { && Objects.equals(preFilterShardSize, that.preFilterShardSize) && Objects.equals(indicesOptions, that.indicesOptions) && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults) + && Objects.equals(ignoreUnavailableShards, that.ignoreUnavailableShards) && Objects.equals(localClusterAlias, that.localClusterAlias) && absoluteStartMillis == that.absoluteStartMillis && ccsMinimizeRoundtrips == that.ccsMinimizeRoundtrips @@ -770,6 +789,7 @@ public int hashCode() { maxConcurrentShardRequests, preFilterShardSize, allowPartialSearchResults, + ignoreUnavailableShards, localClusterAlias, absoluteStartMillis, ccsMinimizeRoundtrips, @@ -805,6 +825,8 @@ public String toString() { + preFilterShardSize + ", allowPartialSearchResults=" + allowPartialSearchResults + + ", ignoreUnavailableShards=" + + ignoreUnavailableShards + ", localClusterAlias=" + localClusterAlias + ", getOrCreateAbsoluteStartMillis=" diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java index 80dc34c4d5d68..2f2095c610da7 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java @@ -183,6 +183,11 @@ public static void parseSearchRequest( searchRequest.allowPartialSearchResults(request.paramAsBoolean("allow_partial_search_results", null)); } + if (request.hasParam("ignore_unavailable_shards")) { + // only set if we have the parameter passed to override the cluster-level default + searchRequest.ignoreUnavailableShards(request.paramAsBoolean("ignore_unavailable_shards", null)); + } + if (request.hasParam("phase_took")) { // only set if we have the parameter passed to override the cluster-level default // else phaseTook = null From 3d9b1c34951818e30c5a52e87dfd14bbd789eb7e Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 16 Apr 2024 18:00:25 -0700 Subject: [PATCH 2/2] Including changes to javadoc, CHANGELOG.md and request builders Signed-off-by: Ankit Jain --- CHANGELOG.md | 3 +- .../opensearch/client/RequestConverters.java | 3 ++ .../search/AbstractSearchAsyncAction.java | 2 +- .../action/search/SearchRequest.java | 29 +++++++++++-------- .../action/search/SearchRequestBuilder.java | 9 ++++++ .../rest/action/search/RestSearchAction.java | 4 +-- .../search/RandomSearchRequestGenerator.java | 1 + 7 files changed, 35 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50b38450aa724..e2c4c4075fc6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954)) - Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569)) - Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044)) +- Adding support for ignore_unavailable_shards in search request ([#13209](https://github.com/opensearch-project/OpenSearch/pull/13209)) - Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656)) - [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704)) - Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686)) @@ -140,7 +141,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix bulk API ignores ingest pipeline for upsert ([#12883](https://github.com/opensearch-project/OpenSearch/pull/12883)) - Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849)) - Fix UOE While building Exists query for nested search_as_you_type field ([#12048](https://github.com/opensearch-project/OpenSearch/pull/12048)) -- Client with Java 8 runtime and Apache HttpClient 5 Transport fails with java.lang.NoSuchMethodError: java.nio.ByteBuffer.flip()Ljava/nio/ByteBuffer ([#13100](https://github.com/opensearch-project/opensearch-java/pull/13100)) +- Client with Java 8 runtime and Apache HttpClient 5 Transport fails with java.lang.NoSuchMethodError: java.nio.ByteBuffer.flip()Ljava/nio/ByteBuffer ([#13100](https://github.com/opensearch-project/opensearch-java/pull/13100)) ### Security diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java index 35d9929a649ff..975d92cb27547 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java @@ -470,6 +470,9 @@ static void addSearchRequestParams(Params params, SearchRequest searchRequest) { if (searchRequest.allowPartialSearchResults() != null) { params.withAllowPartialResults(searchRequest.allowPartialSearchResults()); } + if (searchRequest.ignoreUnavailable() != null) { + params.withIgnoreUnavailable(searchRequest.ignoreUnavailable()); + } params.withBatchedReduceSize(searchRequest.getBatchedReduceSize()); if (searchRequest.scroll() != null) { params.putParam("scroll", searchRequest.scroll().keepAlive()); diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index c665cbdb30a59..ae2f424fadcfa 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -425,7 +425,7 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha currentPhase.getName() ); } - if (!request.getIgnoreUnavailableShards()) { + if (!request.ignoreUnavailable()) { onPhaseFailure(currentPhase, "Partial shards failure (" + discrepancy + " shards unavailable)", null); return; } diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java index c6fea74b2e041..c7b803c22e671 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java @@ -102,7 +102,7 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla private Boolean allowPartialSearchResults; - private Boolean ignoreUnavailableShards; + private Boolean ignoreUnavailable; private Scroll scroll; @@ -200,7 +200,7 @@ private SearchRequest( boolean finalReduce ) { this.allowPartialSearchResults = searchRequest.allowPartialSearchResults; - this.ignoreUnavailableShards = searchRequest.ignoreUnavailableShards; + this.ignoreUnavailable = searchRequest.ignoreUnavailable; this.batchedReduceSize = searchRequest.batchedReduceSize; this.ccsMinimizeRoundtrips = searchRequest.ccsMinimizeRoundtrips; this.indices = indices; @@ -250,7 +250,7 @@ public SearchRequest(StreamInput in) throws IOException { preFilterShardSize = in.readOptionalVInt(); allowPartialSearchResults = in.readOptionalBoolean(); if (in.getVersion().onOrAfter(Version.V_2_14_0)) { - ignoreUnavailableShards = in.readOptionalBoolean(); + ignoreUnavailable = in.readOptionalBoolean(); } localClusterAlias = in.readOptionalString(); if (localClusterAlias != null) { @@ -290,7 +290,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(preFilterShardSize); out.writeOptionalBoolean(allowPartialSearchResults); if (out.getVersion().onOrAfter(Version.V_2_14_0)) { - out.writeOptionalBoolean(ignoreUnavailableShards); + out.writeOptionalBoolean(ignoreUnavailable); } out.writeOptionalString(localClusterAlias); if (localClusterAlias != null) { @@ -576,13 +576,18 @@ public Boolean allowPartialSearchResults() { return this.allowPartialSearchResults; } - public SearchRequest ignoreUnavailableShards(boolean ignoreUnavailableShards) { - this.ignoreUnavailableShards = ignoreUnavailableShards; + /** + * Sets if this request should ignore unavailable shards. This option is superseded + * by allowPartialSearchResults (ignores any type of shard failures) instead of just + * unavailable shards. (If method is not called, will default to the cluster level setting). + */ + public SearchRequest ignoreUnavailable(boolean ignoreUnavailable) { + this.ignoreUnavailable = ignoreUnavailable; return this; } - public Boolean getIgnoreUnavailableShards() { - return this.ignoreUnavailableShards; + public Boolean ignoreUnavailable() { + return this.ignoreUnavailable; } /** @@ -765,7 +770,7 @@ public boolean equals(Object o) { && Objects.equals(preFilterShardSize, that.preFilterShardSize) && Objects.equals(indicesOptions, that.indicesOptions) && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults) - && Objects.equals(ignoreUnavailableShards, that.ignoreUnavailableShards) + && Objects.equals(ignoreUnavailable, that.ignoreUnavailable) && Objects.equals(localClusterAlias, that.localClusterAlias) && absoluteStartMillis == that.absoluteStartMillis && ccsMinimizeRoundtrips == that.ccsMinimizeRoundtrips @@ -789,7 +794,7 @@ public int hashCode() { maxConcurrentShardRequests, preFilterShardSize, allowPartialSearchResults, - ignoreUnavailableShards, + ignoreUnavailable, localClusterAlias, absoluteStartMillis, ccsMinimizeRoundtrips, @@ -825,8 +830,8 @@ public String toString() { + preFilterShardSize + ", allowPartialSearchResults=" + allowPartialSearchResults - + ", ignoreUnavailableShards=" - + ignoreUnavailableShards + + ", ignoreUnavailable=" + + ignoreUnavailable + ", localClusterAlias=" + localClusterAlias + ", getOrCreateAbsoluteStartMillis=" diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java index 9dac827e7d518..f4b97d7b72e42 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java @@ -556,6 +556,15 @@ public SearchRequestBuilder setAllowPartialSearchResults(boolean allowPartialSea return this; } + /** + * Sets if this request should ignore unavailable shards. (If method is not called, + * will default to the cluster level setting). + */ + public SearchRequestBuilder setIgnoreUnavailable(boolean ignoreUnavailable) { + request.ignoreUnavailable(ignoreUnavailable); + return this; + } + /** * Should the query be profiled. Defaults to false */ diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java index 2f2095c610da7..429766433a005 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java @@ -183,9 +183,9 @@ public static void parseSearchRequest( searchRequest.allowPartialSearchResults(request.paramAsBoolean("allow_partial_search_results", null)); } - if (request.hasParam("ignore_unavailable_shards")) { + if (request.hasParam("ignore_unavailable")) { // only set if we have the parameter passed to override the cluster-level default - searchRequest.ignoreUnavailableShards(request.paramAsBoolean("ignore_unavailable_shards", null)); + searchRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", null)); } if (request.hasParam("phase_took")) { diff --git a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java index 74de1e6d96d93..70f75874a4817 100644 --- a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java @@ -101,6 +101,7 @@ private RandomSearchRequestGenerator() {} public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { SearchRequest searchRequest = new SearchRequest(); searchRequest.allowPartialSearchResults(true); + searchRequest.ignoreUnavailable(randomBoolean()); if (randomBoolean()) { searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); }