diff --git a/.codecov.yml b/.codecov.yml index 7e4eb060..b36ce72b 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -11,4 +11,3 @@ coverage: target: 80% # the required coverage value threshold: 1% # the leniency in hitting the target patch: off - diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index a8199a10..67f6e2a3 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -4,4 +4,4 @@ contact_links: about: Please ask and answer questions here. - name: AWS/Amazon Security url: https://aws.amazon.com/security/vulnerability-reporting/ - about: Please report security vulnerabilities here. \ No newline at end of file + about: Please report security vulnerabilities here. diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index a598da2c..f50a4cf6 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -26,4 +26,4 @@ jobs: - uses: ncipollo/release-action@v1 with: github_token: ${{ steps.github_app_token.outputs.token }} - bodyFile: release-notes/opensearch-asynchronous-search.release-notes-${{steps.tag.outputs.tag}}.md \ No newline at end of file + bodyFile: release-notes/opensearch-asynchronous-search.release-notes-${{steps.tag.outputs.tag}}.md diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 3e924caf..bb81fabf 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -31,4 +31,4 @@ jobs: export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) echo "::add-mask::$SONATYPE_USERNAME" echo "::add-mask::$SONATYPE_PASSWORD" - ./gradlew publishPluginZipPublicationToSnapshotsRepository \ No newline at end of file + ./gradlew publishPluginZipPublicationToSnapshotsRepository diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml index e96fb5e4..fed90ef4 100644 --- a/.github/workflows/release-workflow.yml +++ b/.github/workflows/release-workflow.yml @@ -120,4 +120,3 @@ jobs: with: name: asynchronous-search-plugin path: asynchronous-search-artifacts - diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 4c4f8898..7d6d5129 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -17,4 +17,3 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Maintainer | GitHub ID | Affiliation | |-------------| ------------------------------------------ | ----------- | | Anshul Agarwal | [anshul291995](https://github.com/anshul291995) | Amazon | - diff --git a/README.md b/README.md index 933c27ff..f19b6c35 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![codecov](https://codecov.io/gh/opensearch-project/asynchronous-search/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/asynchronous-search) # Asynchronous search -Asynchronous search makes it possible for users to run such queries without worrying about the query timing out. +Asynchronous search makes it possible for users to run such queries without worrying about the query timing out. These queries run in the background, and users can track the progress, and retrieve partial results as they become available. The asynchronous search plugin supports the below operations @@ -48,12 +48,12 @@ GET /_plugins/_asynchronous_search/stats ## Setup 1. Check out this package from version control. -2. Launch Intellij IDEA, choose **Import Project**, and select the `settings.gradle` file in the root of this package. +2. Launch Intellij IDEA, choose **Import Project**, and select the `settings.gradle` file in the root of this package. 3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 8 before running `./gradlew`. - Unix System 1. `export JAVA_HOME=jdk-install-dir`: Replace `jdk-install-dir` with the JAVA_HOME directory of your system. 2. `export PATH=$JAVA_HOME/bin:$PATH` - + - Windows System 1. Find **My Computers** from file directory, right click and select **properties**. 2. Select the **Advanced** tab, select **Environment variables**. @@ -83,7 +83,7 @@ When launching a cluster using one of the above commands, logs are placed in `bu ### Debugging -Sometimes it is useful to attach a debugger to either the OpenSearch cluster or the integ tests to see what's going on. When running unit tests, hit **Debug** from the IDE's gutter to debug the tests. For the OpenSearch cluster or the integ tests, first, make sure start a debugger listening on port `5005`. +Sometimes it is useful to attach a debugger to either the OpenSearch cluster or the integ tests to see what's going on. When running unit tests, hit **Debug** from the IDE's gutter to debug the tests. For the OpenSearch cluster or the integ tests, first, make sure start a debugger listening on port `5005`. To debug the server code, run: diff --git a/RELEASING.md b/RELEASING.md index 6903e716..91263ac4 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1 +1 @@ -This project follows the [OpenSearch release process](https://github.com/opensearch-project/.github/blob/main/RELEASING.md). \ No newline at end of file +This project follows the [OpenSearch release process](https://github.com/opensearch-project/.github/blob/main/RELEASING.md). diff --git a/build.gradle b/build.gradle index f5eb6428..9590d721 100644 --- a/build.gradle +++ b/build.gradle @@ -45,6 +45,7 @@ plugins { id "de.undercouch.download" version "5.3.0" id 'com.netflix.nebula.ospackage' version "11.10.0" id 'checkstyle' + id "com.diffplug.spotless" version "6.25.0" } repositories { @@ -60,6 +61,7 @@ apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' +apply from: 'gradle/formatting.gradle' checkstyle { diff --git a/formatter/formatterConfig.xml b/formatter/formatterConfig.xml new file mode 100644 index 00000000..b6051435 --- /dev/null +++ b/formatter/formatterConfig.xml @@ -0,0 +1,373 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/formatter/license-header.txt b/formatter/license-header.txt new file mode 100644 index 00000000..098d3cb0 --- /dev/null +++ b/formatter/license-header.txt @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ \ No newline at end of file diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle new file mode 100644 index 00000000..4b9714dc --- /dev/null +++ b/gradle/formatting.gradle @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +allprojects { + project.apply plugin: "com.diffplug.spotless" + spotless { + java { + // Normally this isn't necessary, but we have Java sources in + // non-standard places + target '**/*.java' + + removeUnusedImports() + eclipse().configFile rootProject.file('formatter/formatterConfig.xml') + trimTrailingWhitespace() + endWithNewline(); + + custom 'Refuse wildcard imports', { + // Wildcard imports can't be resolved; fail the build + if (it =~ /\s+import .*\*;/) { + throw new AssertionError("Do not use wildcard imports. 'spotlessApply' cannot resolve this issue.") + } + } + + // See DEVELOPER_GUIDE.md for details of when to enable this. + if (System.getProperty('spotless.paddedcell') != null) { + paddedCell() + } + } + format 'misc', { + target '*.md', '*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg' + + trimTrailingWhitespace() + endWithNewline() + } + format("license", { + licenseHeaderFile("${rootProject.file("formatter/license-header.txt")}", "package "); + target("src/*/java/**/*.java") + }) + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/search/asynchronous/action/AsynchronousSearchStatsAction.java b/src/main/java/org/opensearch/search/asynchronous/action/AsynchronousSearchStatsAction.java index 8a9bc1d0..38b85868 100644 --- a/src/main/java/org/opensearch/search/asynchronous/action/AsynchronousSearchStatsAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/action/AsynchronousSearchStatsAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.action; import org.opensearch.search.asynchronous.response.AsynchronousSearchStatsResponse; @@ -11,7 +14,6 @@ public class AsynchronousSearchStatsAction extends ActionType { - public static final AsynchronousSearchStatsAction INSTANCE = new AsynchronousSearchStatsAction(); public static final String NAME = "cluster:admin/opendistro/asynchronous_search/stats"; diff --git a/src/main/java/org/opensearch/search/asynchronous/action/DeleteAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/action/DeleteAsynchronousSearchAction.java index c788462e..a59b09f0 100644 --- a/src/main/java/org/opensearch/search/asynchronous/action/DeleteAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/action/DeleteAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.action; import org.opensearch.search.asynchronous.response.AcknowledgedResponse; diff --git a/src/main/java/org/opensearch/search/asynchronous/action/GetAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/action/GetAsynchronousSearchAction.java index f654747e..f9fa42c9 100644 --- a/src/main/java/org/opensearch/search/asynchronous/action/GetAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/action/GetAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.action; import org.opensearch.search.asynchronous.response.AsynchronousSearchResponse; diff --git a/src/main/java/org/opensearch/search/asynchronous/action/SubmitAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/action/SubmitAsynchronousSearchAction.java index 46d282a6..1f84040f 100644 --- a/src/main/java/org/opensearch/search/asynchronous/action/SubmitAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/action/SubmitAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.action; import org.opensearch.search.asynchronous.response.AsynchronousSearchResponse; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContext.java b/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContext.java index 26342e50..2a268ebd 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContext.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContext.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context; import org.opensearch.commons.authuser.User; @@ -15,7 +18,6 @@ import java.util.function.LongSupplier; - /** * Wrapper around information that needs to stay around when an asynchronous search has been submitted. * This class encapsulates the details of the various elements pertaining to an asynchronous search, including the @@ -35,8 +37,7 @@ public AsynchronousSearchContext(AsynchronousSearchContextId asynchronousSearchC this.currentTimeSupplier = currentTimeSupplier; } - public @Nullable - AsynchronousSearchProgressListener getAsynchronousSearchProgressListener() { + public @Nullable AsynchronousSearchProgressListener getAsynchronousSearchProgressListener() { return asynchronousSearchProgressListener; } @@ -58,22 +59,25 @@ public AsynchronousSearchContextId getContextId() { public abstract long getStartTimeMillis(); - public abstract @Nullable - SearchResponse getSearchResponse(); + public abstract @Nullable SearchResponse getSearchResponse(); - public abstract @Nullable - Exception getSearchError(); + public abstract @Nullable Exception getSearchError(); - public abstract @Nullable - User getUser(); + public abstract @Nullable User getUser(); public boolean isExpired() { return getExpirationTimeMillis() < currentTimeSupplier.getAsLong(); } public AsynchronousSearchResponse getAsynchronousSearchResponse() { - return new AsynchronousSearchResponse(getAsynchronousSearchId(), getAsynchronousSearchState(), getStartTimeMillis(), - getExpirationTimeMillis(), getSearchResponse(), getSearchError()); + return new AsynchronousSearchResponse( + getAsynchronousSearchId(), + getAsynchronousSearchState(), + getStartTimeMillis(), + getExpirationTimeMillis(), + getSearchResponse(), + getSearchError() + ); } public void setState(AsynchronousSearchState targetState) { diff --git a/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContextId.java b/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContextId.java index 3669f5c3..369674cd 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContextId.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/AsynchronousSearchContextId.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContext.java b/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContext.java index aee08a7b..776cefea 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContext.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContext.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.active; import org.opensearch.commons.authuser.User; @@ -56,11 +59,17 @@ public class AsynchronousSearchActiveContext extends AsynchronousSearchContext i @Nullable private final User user; - public AsynchronousSearchActiveContext(AsynchronousSearchContextId asynchronousSearchContextId, String nodeId, - TimeValue keepAlive, boolean keepOnCompletion, ThreadPool threadPool, - LongSupplier currentTimeSupplier, - AsynchronousSearchProgressListener asynchronousSearchProgressListener, @Nullable User user, - Supplier persistSearchFailureSupplier) { + public AsynchronousSearchActiveContext( + AsynchronousSearchContextId asynchronousSearchContextId, + String nodeId, + TimeValue keepAlive, + boolean keepOnCompletion, + ThreadPool threadPool, + LongSupplier currentTimeSupplier, + AsynchronousSearchProgressListener asynchronousSearchProgressListener, + @Nullable User user, + Supplier persistSearchFailureSupplier + ) { super(asynchronousSearchContextId, currentTimeSupplier); this.keepOnCompletion = keepOnCompletion; this.error = new SetOnce<>(); @@ -73,8 +82,9 @@ public AsynchronousSearchActiveContext(AsynchronousSearchContextId asynchronousS this.asynchronousSearchId = new SetOnce<>(); this.completed = new AtomicBoolean(false); this.closed = new AtomicBoolean(false); - this.asynchronousSearchContextPermits = keepOnCompletion ? new AsynchronousSearchContextPermits(asynchronousSearchContextId, - threadPool) : new NoopAsynchronousSearchContextPermits(asynchronousSearchContextId); + this.asynchronousSearchContextPermits = keepOnCompletion + ? new AsynchronousSearchContextPermits(asynchronousSearchContextId, threadPool) + : new NoopAsynchronousSearchContextPermits(asynchronousSearchContextId); this.user = user; this.persistSearchFailureSupplier = persistSearchFailureSupplier; } @@ -87,8 +97,9 @@ public void setTask(SearchTask searchTask) { this.searchTask.set(searchTask); this.startTimeMillis = searchTask.getStartTime(); this.expirationTimeMillis = startTimeMillis + keepAlive.getMillis(); - this.asynchronousSearchId.set(AsynchronousSearchIdConverter.buildAsyncId(new AsynchronousSearchId(nodeId, searchTask.getId(), - getContextId()))); + this.asynchronousSearchId.set( + AsynchronousSearchIdConverter.buildAsyncId(new AsynchronousSearchId(nodeId, searchTask.getId(), getContextId())) + ); } public void processSearchFailure(Exception e) { diff --git a/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStore.java b/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStore.java index 2d6655de..598adaeb 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStore.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStore.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.active; import org.apache.logging.log4j.LogManager; @@ -25,38 +28,46 @@ import static org.opensearch.common.util.concurrent.ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency; - public class AsynchronousSearchActiveStore { private static Logger logger = LogManager.getLogger(AsynchronousSearchActiveStore.class); private volatile int nodeConcurrentRunningSearches; public static final int NODE_CONCURRENT_RUNNING_SEARCHES = 20; public static final Setting NODE_CONCURRENT_RUNNING_SEARCHES_SETTING = Setting.intSetting( - "plugins.asynchronous_search.node_concurrent_running_searches", - LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, 0, - Setting.Property.Dynamic, Setting.Property.NodeScope); + "plugins.asynchronous_search.node_concurrent_running_searches", + LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + 0, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); private final ConcurrentMapLong activeContexts = newConcurrentMapLongWithAggressiveConcurrency(); public AsynchronousSearchActiveStore(ClusterService clusterService) { Settings settings = clusterService.getSettings(); nodeConcurrentRunningSearches = NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(settings); - clusterService.getClusterSettings().addSettingsUpdateConsumer(NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - this::setNodeConcurrentRunningSearches); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, this::setNodeConcurrentRunningSearches); } private void setNodeConcurrentRunningSearches(int nodeConcurrentRunningSearches) { this.nodeConcurrentRunningSearches = nodeConcurrentRunningSearches; } - public synchronized void putContext(AsynchronousSearchContextId asynchronousSearchContextId, - AsynchronousSearchActiveContext asynchronousSearchContext, - Consumer contextRejectionEventConsumer) { + public synchronized void putContext( + AsynchronousSearchContextId asynchronousSearchContextId, + AsynchronousSearchActiveContext asynchronousSearchContext, + Consumer contextRejectionEventConsumer + ) { if (activeContexts.size() >= nodeConcurrentRunningSearches) { contextRejectionEventConsumer.accept(asynchronousSearchContextId); - throw new OpenSearchRejectedExecutionException("Trying to create too many concurrent searches. Must be less than or equal to: [" - + nodeConcurrentRunningSearches + "]. This limit can be set by changing the [" - + NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey() + "] settings."); + throw new OpenSearchRejectedExecutionException( + "Trying to create too many concurrent searches. Must be less than or equal to: [" + + nodeConcurrentRunningSearches + + "]. This limit can be set by changing the [" + + NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey() + + "] settings." + ); } activeContexts.put(asynchronousSearchContextId.getId(), asynchronousSearchContext); } @@ -72,7 +83,6 @@ public Optional getContext(AsynchronousSearchCo return Optional.empty(); } - public Map getAllContexts() { return CollectionUtils.copyMap(activeContexts); } @@ -96,20 +106,18 @@ public boolean freeContext(AsynchronousSearchContextId asynchronousSearchContext } private static boolean calledFromAsynchronousSearchStateMachine() { - return Stream.of(Thread.currentThread().getStackTrace()). - skip(1). //skip getStackTrace - limit(10). //limit depth of analysis to 10 frames, it should be enough - anyMatch(f -> - { - try { - boolean isTestMethodInvocation = f.getClassName().contains("AsynchronousSearchActiveStoreTests"); - boolean isStateMachineTriggerMethodInvocation = AsynchronousSearchStateMachine.class - .isAssignableFrom(Class.forName(f.getClassName())) && f.getMethodName().equals("trigger"); - return isTestMethodInvocation || isStateMachineTriggerMethodInvocation; - } catch (Exception ignored) { - return false; - } + return Stream.of(Thread.currentThread().getStackTrace()).skip(1). // skip getStackTrace + limit(10). // limit depth of analysis to 10 frames, it should be enough + anyMatch(f -> { + try { + boolean isTestMethodInvocation = f.getClassName().contains("AsynchronousSearchActiveStoreTests"); + boolean isStateMachineTriggerMethodInvocation = AsynchronousSearchStateMachine.class.isAssignableFrom( + Class.forName(f.getClassName()) + ) && f.getMethodName().equals("trigger"); + return isTestMethodInvocation || isStateMachineTriggerMethodInvocation; + } catch (Exception ignored) { + return false; } - ); + }); } } diff --git a/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchContextClosedException.java b/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchContextClosedException.java index 2a060a95..203f5cd8 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchContextClosedException.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchContextClosedException.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.active; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermits.java b/src/main/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermits.java index 66acadc8..786a1549 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermits.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermits.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.permits; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; @@ -48,15 +51,18 @@ public AsynchronousSearchContextPermits(AsynchronousSearchContextId asynchronous this.semaphore = new Semaphore(TOTAL_PERMITS, true); } - public AsynchronousSearchContextPermits(AsynchronousSearchContextId asynchronousSearchContextId, ThreadPool threadPool, - Semaphore semaphore) { + public AsynchronousSearchContextPermits( + AsynchronousSearchContextId asynchronousSearchContextId, + ThreadPool threadPool, + Semaphore semaphore + ) { this.asynchronousSearchContextId = asynchronousSearchContextId; this.threadPool = threadPool; this.semaphore = semaphore; } private Releasable acquirePermits(int permits, TimeValue timeout, final String details) throws AsynchronousSearchContextClosedException, - TimeoutException { + TimeoutException { RunOnce release = new RunOnce(() -> {}); if (closed) { logger.debug("Trying to acquire permit for closed context [{}]", asynchronousSearchContextId); @@ -67,18 +73,28 @@ private Releasable acquirePermits(int permits, TimeValue timeout, final String d this.lockDetails = details; release = new RunOnce(() -> { logger.debug("Releasing permit(s) [{}] with reason [{}]", permits, lockDetails); - semaphore.release(permits);}); + semaphore.release(permits); + }); if (closed) { release.run(); logger.debug("Trying to acquire permit for closed context [{}]", asynchronousSearchContextId); - throw new AsynchronousSearchContextClosedException( asynchronousSearchContextId); + throw new AsynchronousSearchContextClosedException(asynchronousSearchContextId); } return release::run; } else { - throw new TimeoutException("obtaining context lock" + asynchronousSearchContextId + "timed out after " + - timeout.getMillis() + "ms, previous lock details: [" + lockDetails + "] trying to lock for [" + details + "]"); + throw new TimeoutException( + "obtaining context lock" + + asynchronousSearchContextId + + "timed out after " + + timeout.getMillis() + + "ms, previous lock details: [" + + lockDetails + + "] trying to lock for [" + + details + + "]" + ); } - } catch (InterruptedException e ) { + } catch (InterruptedException e) { Thread.currentThread().interrupt(); release.run(); throw new RuntimeException("thread interrupted while trying to obtain context lock", e); @@ -89,8 +105,7 @@ private void asyncAcquirePermit(int permits, final ActionListener on threadPool.executor(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME).execute(new AbstractRunnable() { @Override public void onFailure(final Exception e) { - logger.debug(() -> new ParameterizedMessage("Failed to acquire permit [{}] for [{}]", - permits, reason), e); + logger.debug(() -> new ParameterizedMessage("Failed to acquire permit [{}] for [{}]", permits, reason), e); onAcquired.onFailure(e); } diff --git a/src/main/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermits.java b/src/main/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermits.java index 40c004c4..f7d90a2a 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermits.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermits.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.permits; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContext.java b/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContext.java index de4ae17d..0229f1d0 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContext.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContext.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.persistence; import org.opensearch.commons.authuser.User; @@ -34,10 +37,13 @@ public class AsynchronousSearchPersistenceContext extends AsynchronousSearchCont private final AsynchronousSearchPersistenceModel asynchronousSearchPersistenceModel; private final NamedWriteableRegistry namedWriteableRegistry; - public AsynchronousSearchPersistenceContext(String asynchronousSearchId, AsynchronousSearchContextId asynchronousSearchContextId, - AsynchronousSearchPersistenceModel asynchronousSearchPersistenceModel, - LongSupplier currentTimeSupplier, - NamedWriteableRegistry namedWriteableRegistry) { + public AsynchronousSearchPersistenceContext( + String asynchronousSearchId, + AsynchronousSearchContextId asynchronousSearchContextId, + AsynchronousSearchPersistenceModel asynchronousSearchPersistenceModel, + LongSupplier currentTimeSupplier, + NamedWriteableRegistry namedWriteableRegistry + ) { super(asynchronousSearchContextId, currentTimeSupplier); Objects.requireNonNull(asynchronousSearchId); Objects.requireNonNull(asynchronousSearchContextId); @@ -76,16 +82,26 @@ public SearchResponse getSearchResponse() { if (asynchronousSearchPersistenceModel.getResponse() == null) { return null; } else { - BytesReference bytesReference = - BytesReference.fromByteBuffer(ByteBuffer.wrap(Base64.getUrlDecoder().decode( - asynchronousSearchPersistenceModel.getResponse()))); - try (NamedWriteableAwareStreamInput wrapperStreamInput = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), - namedWriteableRegistry)) { + BytesReference bytesReference = BytesReference.fromByteBuffer( + ByteBuffer.wrap(Base64.getUrlDecoder().decode(asynchronousSearchPersistenceModel.getResponse())) + ); + try ( + NamedWriteableAwareStreamInput wrapperStreamInput = new NamedWriteableAwareStreamInput( + bytesReference.streamInput(), + namedWriteableRegistry + ) + ) { wrapperStreamInput.setVersion(wrapperStreamInput.readVersion()); return new SearchResponse(wrapperStreamInput); } catch (IOException e) { - logger.error(() -> new ParameterizedMessage("Failed to parse search response for asynchronous search [{}] Response : [{}] ", - asynchronousSearchId, asynchronousSearchPersistenceModel.getResponse()), e); + logger.error( + () -> new ParameterizedMessage( + "Failed to parse search response for asynchronous search [{}] Response : [{}] ", + asynchronousSearchId, + asynchronousSearchPersistenceModel.getResponse() + ), + e + ); return null; } } @@ -96,16 +112,26 @@ public Exception getSearchError() { if (asynchronousSearchPersistenceModel.getError() == null) { return null; } - BytesReference bytesReference = - BytesReference.fromByteBuffer(ByteBuffer.wrap(Base64.getUrlDecoder() - .decode(asynchronousSearchPersistenceModel.getError()))); - try (NamedWriteableAwareStreamInput wrapperStreamInput = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), - namedWriteableRegistry)) { + BytesReference bytesReference = BytesReference.fromByteBuffer( + ByteBuffer.wrap(Base64.getUrlDecoder().decode(asynchronousSearchPersistenceModel.getError())) + ); + try ( + NamedWriteableAwareStreamInput wrapperStreamInput = new NamedWriteableAwareStreamInput( + bytesReference.streamInput(), + namedWriteableRegistry + ) + ) { wrapperStreamInput.setVersion(wrapperStreamInput.readVersion()); return wrapperStreamInput.readException(); } catch (IOException e) { - logger.error(() -> new ParameterizedMessage("Failed to parse search error for asynchronous search [{}] Error : [{}] ", - asynchronousSearchId, asynchronousSearchPersistenceModel.getResponse()), e); + logger.error( + () -> new ParameterizedMessage( + "Failed to parse search error for asynchronous search [{}] Error : [{}] ", + asynchronousSearchId, + asynchronousSearchPersistenceModel.getResponse() + ), + e + ); return null; } } @@ -127,14 +153,11 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; AsynchronousSearchPersistenceContext asynchronousSearchPersistenceContext = (AsynchronousSearchPersistenceContext) o; - return asynchronousSearchPersistenceContext.getAsynchronousSearchId() - .equals(this.asynchronousSearchId) && asynchronousSearchPersistenceContext.getAsynchronousSearchPersistenceModel() - .equals(this.asynchronousSearchPersistenceModel); + return asynchronousSearchPersistenceContext.getAsynchronousSearchId().equals(this.asynchronousSearchId) + && asynchronousSearchPersistenceContext.getAsynchronousSearchPersistenceModel().equals(this.asynchronousSearchPersistenceModel); } } diff --git a/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceModel.java b/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceModel.java index 1b4f7158..d46e7c7c 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceModel.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceModel.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.persistence; import org.opensearch.commons.authuser.User; @@ -34,8 +37,13 @@ public AsynchronousSearchPersistenceModel(long startTimeMillis, long expirationT this.user = user; } - public AsynchronousSearchPersistenceModel(long startTimeMillis, long expirationTimeMillis, SearchResponse response, - Exception error, User user) throws IOException { + public AsynchronousSearchPersistenceModel( + long startTimeMillis, + long expirationTimeMillis, + SearchResponse response, + Exception error, + User user + ) throws IOException { this.startTimeMillis = startTimeMillis; this.expirationTimeMillis = expirationTimeMillis; this.response = serializeResponse(response); @@ -109,14 +117,12 @@ public boolean equals(Object o) { return false; } AsynchronousSearchPersistenceModel other = (AsynchronousSearchPersistenceModel) o; - return - startTimeMillis == other.startTimeMillis && expirationTimeMillis == other.expirationTimeMillis - && ((response == null && other.response == null) || - (response != null && other.response != null && response.equals(other.response))) - && ((error == null && other.error == null) || - (error != null && other.error != null && error.equals(other.error))) - && ((user == null && other.user == null) || - (user != null && other.user != null && user.equals(other.user))); + return startTimeMillis == other.startTimeMillis + && expirationTimeMillis == other.expirationTimeMillis + && ((response == null && other.response == null) + || (response != null && other.response != null && response.equals(other.response))) + && ((error == null && other.error == null) || (error != null && other.error != null && error.equals(other.error))) + && ((user == null && other.user == null) || (user != null && other.user != null && user.equals(other.user))); } } diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchContextEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchContextEvent.java index cd78991f..5e8fa8ac 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchContextEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchContextEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; @@ -26,4 +29,3 @@ public AsynchronousSearchContext asynchronousSearchContext() { } } - diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchState.java b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchState.java index 17443bc4..f0f4acc4 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchState.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchState.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state; import org.opensearch.action.search.SearchTask; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachine.java b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachine.java index e248d22c..dce7b427 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachine.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachine.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; @@ -33,8 +36,11 @@ public class AsynchronousSearchStateMachine implements StateMachine states; private final AsynchronousSearchContextEventListener asynchronousSearchContextEventListener; - public AsynchronousSearchStateMachine(final Set states, final AsynchronousSearchState initialState, - AsynchronousSearchContextEventListener asynchronousSearchContextEventListener) { + public AsynchronousSearchStateMachine( + final Set states, + final AsynchronousSearchState initialState, + AsynchronousSearchContextEventListener asynchronousSearchContextEventListener + ) { super(); this.transitionsMap = new HashMap<>(); this.states = states; @@ -92,31 +98,45 @@ public AsynchronousSearchState trigger(AsynchronousSearchContextEvent event) thr AsynchronousSearchTransition transition = transitionsMap.get(transitionId); execute(transition.onEvent(), event, currentState); asynchronousSearchContext.setState(transition.targetState()); - logger.debug("Executed event [{}] for asynchronous search id [{}] ", event.getClass().getName(), - event.asynchronousSearchContext.getAsynchronousSearchId()); + logger.debug( + "Executed event [{}] for asynchronous search id [{}] ", + event.getClass().getName(), + event.asynchronousSearchContext.getAsynchronousSearchId() + ); BiConsumer eventListener = transition.eventListener(); try { eventListener.accept(event.asynchronousSearchContext().getContextId(), asynchronousSearchContextEventListener); } catch (Exception ex) { - logger.error(() -> new ParameterizedMessage("Failed to execute listener for asynchronous search id : [{}]", - event.asynchronousSearchContext.getAsynchronousSearchId()), ex); + logger.error( + () -> new ParameterizedMessage( + "Failed to execute listener for asynchronous search id : [{}]", + event.asynchronousSearchContext.getAsynchronousSearchId() + ), + ex + ); } return asynchronousSearchContext.getAsynchronousSearchState(); } else { - String message = String.format(Locale.ROOT, "Invalid transition for " + - "asynchronous search context [%s] from source state [%s] on event [%s]", - asynchronousSearchContext.getAsynchronousSearchId(), currentState, event.getClass().getName()); + String message = String.format( + Locale.ROOT, + "Invalid transition for " + "asynchronous search context [%s] from source state [%s] on event [%s]", + asynchronousSearchContext.getAsynchronousSearchId(), + currentState, + event.getClass().getName() + ); logger.error(message); throw new IllegalStateException(message); } } } - @SuppressWarnings("unchecked") - //Suppress the warning since we know the type of the event and transition based on the validation - private void execute(BiConsumer onEvent, AsynchronousSearchContextEvent event, - AsynchronousSearchState state) { + // Suppress the warning since we know the type of the event and transition based on the validation + private void execute( + BiConsumer onEvent, + AsynchronousSearchContextEvent event, + AsynchronousSearchState state + ) { onEvent.accept(state, (T) event); } @@ -139,4 +159,3 @@ private String getTransitionId(AsynchronousSearchState sourceState, Class eve } } - diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachineClosedException.java b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachineClosedException.java index c5b0423b..8e68fb62 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachineClosedException.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchStateMachineClosedException.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state; import java.util.Locale; @@ -12,10 +15,19 @@ public class AsynchronousSearchStateMachineClosedException extends Exception { private final AsynchronousSearchState currentState; private final AsynchronousSearchContextEvent contextEvent; - public AsynchronousSearchStateMachineClosedException(AsynchronousSearchState currentState, - AsynchronousSearchContextEvent contextEvent) { - super(String.format(Locale.ROOT, "Invalid transition for CLOSED context [%s] from source state [%s] on event [%s]", - contextEvent.asynchronousSearchContext.getAsynchronousSearchId(), currentState, contextEvent.getClass().getName())); + public AsynchronousSearchStateMachineClosedException( + AsynchronousSearchState currentState, + AsynchronousSearchContextEvent contextEvent + ) { + super( + String.format( + Locale.ROOT, + "Invalid transition for CLOSED context [%s] from source state [%s] on event [%s]", + contextEvent.asynchronousSearchContext.getAsynchronousSearchId(), + currentState, + contextEvent.getClass().getName() + ) + ); this.currentState = currentState; this.contextEvent = contextEvent; } diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchTransition.java b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchTransition.java index f38381a3..e857fb47 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchTransition.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/AsynchronousSearchTransition.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -11,7 +14,8 @@ import java.util.function.BiConsumer; public class AsynchronousSearchTransition - implements Transition { + implements + Transition { private final AsynchronousSearchState sourceState; private final AsynchronousSearchState targetState; @@ -19,10 +23,13 @@ public class AsynchronousSearchTransition eventListener; private final Class eventType; - public AsynchronousSearchTransition(AsynchronousSearchState sourceState, AsynchronousSearchState targetState, - BiConsumer onEvent, - BiConsumer eventListener, - Class eventName) { + public AsynchronousSearchTransition( + AsynchronousSearchState sourceState, + AsynchronousSearchState targetState, + BiConsumer onEvent, + BiConsumer eventListener, + Class eventName + ) { this.sourceState = sourceState; this.targetState = targetState; this.onEvent = onEvent; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/StateMachine.java b/src/main/java/org/opensearch/search/asynchronous/context/state/StateMachine.java index 24a2811d..4caf980e 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/StateMachine.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/StateMachine.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state; import java.util.Map; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/Transition.java b/src/main/java/org/opensearch/search/asynchronous/context/state/Transition.java index 0504858e..e7c957d8 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/Transition.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/Transition.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state; import java.util.function.BiConsumer; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/event/BeginPersistEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/event/BeginPersistEvent.java index 33fee24b..1758e465 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/event/BeginPersistEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/event/BeginPersistEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state.event; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; @@ -24,12 +27,21 @@ public BeginPersistEvent(AsynchronousSearchContext asynchronousSearchContext) { public AsynchronousSearchPersistenceModel getAsynchronousSearchPersistenceModel() { try { - return new AsynchronousSearchPersistenceModel(asynchronousSearchContext.getStartTimeMillis(), - asynchronousSearchContext.getExpirationTimeMillis(), asynchronousSearchContext.getSearchResponse(), - asynchronousSearchContext.getSearchError(), asynchronousSearchContext.getUser()); + return new AsynchronousSearchPersistenceModel( + asynchronousSearchContext.getStartTimeMillis(), + asynchronousSearchContext.getExpirationTimeMillis(), + asynchronousSearchContext.getSearchResponse(), + asynchronousSearchContext.getSearchError(), + asynchronousSearchContext.getUser() + ); } catch (IOException e) { - logger.error(() -> new ParameterizedMessage("Failed to create asynchronous search persistence model" + - " for asynchronous search [{}]", asynchronousSearchContext.getAsynchronousSearchId()), e); + logger.error( + () -> new ParameterizedMessage( + "Failed to create asynchronous search persistence model" + " for asynchronous search [{}]", + asynchronousSearchContext.getAsynchronousSearchId() + ), + e + ); return null; } } diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchDeletedEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchDeletedEvent.java index acbdf8a8..f5c06c80 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchDeletedEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchDeletedEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state.event; import org.opensearch.search.asynchronous.context.active.AsynchronousSearchActiveContext; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchFailureEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchFailureEvent.java index 74f68138..b33debc3 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchFailureEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchFailureEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state.event; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistFailedEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistFailedEvent.java index b2d84528..78b45cdd 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistFailedEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistFailedEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state.event; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistedEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistedEvent.java index 590bffa3..276deed4 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistedEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchResponsePersistedEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state.event; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; @@ -17,4 +20,3 @@ public SearchResponsePersistedEvent(AsynchronousSearchContext asynchronousSearch super(asynchronousSearchContext); } } - diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchStartedEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchStartedEvent.java index eafdeaa0..ef18d78e 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchStartedEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchStartedEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state.event; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; diff --git a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchSuccessfulEvent.java b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchSuccessfulEvent.java index d8a92013..5be8ac8e 100644 --- a/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchSuccessfulEvent.java +++ b/src/main/java/org/opensearch/search/asynchronous/context/state/event/SearchSuccessfulEvent.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.state.event; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; diff --git a/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchId.java b/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchId.java index e083512b..e4aae22c 100644 --- a/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchId.java +++ b/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchId.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.id; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -47,7 +50,6 @@ public String toString() { return "[" + node + "][" + taskId + "][" + asynchronousSearchContextId + "]"; } - @Override public int hashCode() { return Objects.hash(this.asynchronousSearchContextId, this.node, this.taskId); @@ -55,13 +57,11 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; AsynchronousSearchId asynchronousSearchId = (AsynchronousSearchId) o; return asynchronousSearchId.asynchronousSearchContextId.equals(this.asynchronousSearchContextId) - && asynchronousSearchId.node.equals(this.node) - && asynchronousSearchId.taskId == this.taskId; + && asynchronousSearchId.node.equals(this.node) + && asynchronousSearchId.taskId == this.taskId; } } diff --git a/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdConverter.java b/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdConverter.java index 43a1a1fb..06bf7667 100644 --- a/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdConverter.java +++ b/src/main/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdConverter.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.id; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; diff --git a/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchContextEventListener.java b/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchContextEventListener.java index 303a35bd..95fc1b6f 100644 --- a/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchContextEventListener.java +++ b/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchContextEventListener.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -15,66 +18,55 @@ public interface AsynchronousSearchContextEventListener { /** * @param contextId Executed when a new asynchronous search context was created */ - default void onNewContext(AsynchronousSearchContextId contextId) { - } + default void onNewContext(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when a previously created asynchronous search context completes. */ - default void onContextCompleted(AsynchronousSearchContextId contextId) { - } + default void onContextCompleted(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when a previously created asynchronous search context fails. */ - default void onContextFailed(AsynchronousSearchContextId contextId) { - } + default void onContextFailed(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when a previously created asynchronous search context is persisted. */ - default void onContextPersisted(AsynchronousSearchContextId contextId) { - } + default void onContextPersisted(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when a previously created asynchronous search context fails persisting. */ - default void onContextPersistFailed(AsynchronousSearchContextId contextId) { - } + default void onContextPersistFailed(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when a previously created asynchronous search context is deleted. */ - default void onContextDeleted(AsynchronousSearchContextId contextId) { - } + default void onContextDeleted(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when a previously created asynchronous search context is running. */ - default void onContextRunning(AsynchronousSearchContextId contextId) { - } + default void onContextRunning(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when asynchronous search context creation is rejected */ - default void onContextRejected(AsynchronousSearchContextId contextId) { - } + default void onContextRejected(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when a running asynchronous search context is deleted and has bypassed succeeded/failed state */ - default void onRunningContextDeleted(AsynchronousSearchContextId contextId) { - } + default void onRunningContextDeleted(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when an asynchronous search context is cancelled */ - default void onContextCancelled(AsynchronousSearchContextId contextId) { - } + default void onContextCancelled(AsynchronousSearchContextId contextId) {} /** * @param contextId Executed when an asynchronous search context is initialized */ - default void onContextInitialized(AsynchronousSearchContextId contextId) { - } + default void onContextInitialized(AsynchronousSearchContextId contextId) {} } diff --git a/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListener.java b/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListener.java index c62b9b86..a5e75f96 100644 --- a/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListener.java +++ b/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListener.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.search.asynchronous.response.AsynchronousSearchResponse; @@ -29,7 +32,6 @@ import java.util.function.LongSupplier; import java.util.function.Supplier; - /*** * The implementation of {@link CompositeSearchProgressActionListener} responsible for updating the partial results of a single asynchronous * search request. All partial results are updated atomically. @@ -42,11 +44,14 @@ public class AsynchronousSearchProgressListener extends SearchProgressActionList private final Function failureFunction; private final ExecutorService executor; - public AsynchronousSearchProgressListener(long relativeStartMillis, Function successFunction, - Function failureFunction, - ExecutorService executor, LongSupplier relativeTimeSupplier, - Supplier reduceContextBuilder) { + public AsynchronousSearchProgressListener( + long relativeStartMillis, + Function successFunction, + Function failureFunction, + ExecutorService executor, + LongSupplier relativeTimeSupplier, + Supplier reduceContextBuilder + ) { this.successFunction = successFunction; this.failureFunction = failureFunction; this.executor = executor; @@ -54,7 +59,6 @@ public AsynchronousSearchProgressListener(long relativeStartMillis, Function(); } - /*** * Returns the partial response for the search response. * @return the partial search response @@ -65,8 +69,12 @@ public SearchResponse partialResponse() { } @Override - protected void onListShards(List shards, List skippedShards, SearchResponse.Clusters clusters, - boolean fetchPhase) { + protected void onListShards( + List shards, + List skippedShards, + SearchResponse.Clusters clusters, + boolean fetchPhase + ) { partialResultsHolder.hasFetchPhase.set(fetchPhase); partialResultsHolder.totalShards.set(shards.size() + skippedShards.size()); partialResultsHolder.skippedShards.set(skippedShards.size()); @@ -77,8 +85,10 @@ protected void onListShards(List shards, List skippedS @Override protected void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { - assert reducePhase > partialResultsHolder.reducePhase.get() : "reduce phase " + reducePhase + "less than previous phase" - + partialResultsHolder.reducePhase.get(); + assert reducePhase > partialResultsHolder.reducePhase.get() : "reduce phase " + + reducePhase + + "less than previous phase" + + partialResultsHolder.reducePhase.get(); partialResultsHolder.partialInternalAggregations.set(aggs); partialResultsHolder.reducePhase.set(reducePhase); partialResultsHolder.totalHits.set(totalHits); @@ -86,10 +96,12 @@ protected void onPartialReduce(List shards, TotalHits totalHits, In @Override protected void onFinalReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { - assert reducePhase > partialResultsHolder.reducePhase.get() : "reduce phase " + reducePhase + "less than previous phase" - + partialResultsHolder.reducePhase.get(); + assert reducePhase > partialResultsHolder.reducePhase.get() : "reduce phase " + + reducePhase + + "less than previous phase" + + partialResultsHolder.reducePhase.get(); partialResultsHolder.internalAggregations.set(aggs); - //we don't need to hold its reference beyond this point + // we don't need to hold its reference beyond this point partialResultsHolder.partialInternalAggregations.set(null); partialResultsHolder.reducePhase.set(reducePhase); partialResultsHolder.totalHits.set(totalHits); @@ -127,11 +139,11 @@ private synchronized void onShardResult(int shardIndex) { } private synchronized void onSearchFailure(int shardIndex, SearchShardTarget shardTarget, Exception e) { - //It's hard to build partial search failures since the elasticsearch doesn't consider shard not available exceptions as failures - //while internally it has exceptions from all shards of a particular shard group, it exposes only the exception on the - //final shard of the group, the exception for which could be shard not available while a previous failure on a shard of the same - //group could be outside this category. Since the final exception overrides the exception for the group, it causes inconsistency - //between the partial search failure and failures post completion. + // It's hard to build partial search failures since the elasticsearch doesn't consider shard not available exceptions as failures + // while internally it has exceptions from all shards of a particular shard group, it exposes only the exception on the + // final shard of the group, the exception for which could be shard not available while a previous failure on a shard of the same + // group could be outside this category. Since the final exception overrides the exception for the group, it causes inconsistency + // between the partial search failure and failures post completion. if (partialResultsHolder.successfulShardIds.contains(shardIndex)) { partialResultsHolder.successfulShardIds.remove(shardIndex); partialResultsHolder.successfulShards.decrementAndGet(); @@ -196,9 +208,11 @@ static class PartialResultsHolder { final LongSupplier relativeTimeSupplier; final Supplier reduceContextBuilder; - - PartialResultsHolder(long relativeStartMillis, LongSupplier relativeTimeSupplier, - Supplier reduceContextBuilder) { + PartialResultsHolder( + long relativeStartMillis, + LongSupplier relativeTimeSupplier, + Supplier reduceContextBuilder + ) { this.internalAggregations = new AtomicReference<>(); this.totalShards = new SetOnce<>(); this.successfulShards = new AtomicInteger(); @@ -219,20 +233,37 @@ public SearchResponse partialResponse() { if (isInitialized) { SearchHits searchHits = new SearchHits(SearchHits.EMPTY, totalHits.get(), Float.NaN); InternalAggregations finalAggregation = null; - //after final reduce phase this should be present + // after final reduce phase this should be present if (internalAggregations.get() != null) { finalAggregation = internalAggregations.get(); - //before final reduce phase ensure we do a top-level final reduce to get reduced aggregation results - //else we might be returning back all the partial results aggregated so far + // before final reduce phase ensure we do a top-level final reduce to get reduced aggregation results + // else we might be returning back all the partial results aggregated so far } else if (partialInternalAggregations.get() != null) { - finalAggregation = InternalAggregations.topLevelReduce(Collections.singletonList(partialInternalAggregations.get()), - reduceContextBuilder.get().forFinalReduction()); + finalAggregation = InternalAggregations.topLevelReduce( + Collections.singletonList(partialInternalAggregations.get()), + reduceContextBuilder.get().forFinalReduction() + ); } - InternalSearchResponse internalSearchResponse = new InternalSearchResponse(searchHits, - finalAggregation, null, null, false, null, reducePhase.get()); + InternalSearchResponse internalSearchResponse = new InternalSearchResponse( + searchHits, + finalAggregation, + null, + null, + false, + null, + reducePhase.get() + ); long tookInMillis = relativeTimeSupplier.getAsLong() - relativeStartMillis; - return new SearchResponse(internalSearchResponse, null, totalShards.get(), - successfulShards.get(), skippedShards.get(), tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters.get()); + return new SearchResponse( + internalSearchResponse, + null, + totalShards.get(), + successfulShards.get(), + skippedShards.get(), + tookInMillis, + ShardSearchFailure.EMPTY_ARRAY, + clusters.get() + ); } else { return null; } diff --git a/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapper.java b/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapper.java index 5efbfaa2..b38a3f00 100644 --- a/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapper.java +++ b/src/main/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapper.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.apache.logging.log4j.LogManager; @@ -34,10 +37,13 @@ public class AsynchronousSearchTimeoutWrapper { * @param the response * @return PrioritizedListener */ - public static PrioritizedActionListener wrapScheduledTimeout(ThreadPool threadPool, TimeValue timeout, - String executor, - ActionListener actionListener, - Consumer> timeoutConsumer) { + public static PrioritizedActionListener wrapScheduledTimeout( + ThreadPool threadPool, + TimeValue timeout, + String executor, + ActionListener actionListener, + Consumer> timeoutConsumer + ) { return scheduleTimeout(threadPool, timeout, executor, initListener(actionListener, timeoutConsumer)); } @@ -48,10 +54,14 @@ public static PrioritizedActionListener wrapScheduledTimeou * @param Response * @return PrioritizedListener */ - public static PrioritizedActionListener initListener(ActionListener actionListener, - Consumer> timeoutConsumer) { - CompletionPrioritizedActionListener completionTimeoutListener = - new CompletionPrioritizedActionListener<>(actionListener, timeoutConsumer); + public static PrioritizedActionListener initListener( + ActionListener actionListener, + Consumer> timeoutConsumer + ) { + CompletionPrioritizedActionListener completionTimeoutListener = new CompletionPrioritizedActionListener<>( + actionListener, + timeoutConsumer + ); return completionTimeoutListener; } @@ -64,10 +74,17 @@ public static PrioritizedActionListener initListener(Action * @param Response * @return PrioritizedListener */ - public static PrioritizedActionListener scheduleTimeout(ThreadPool threadPool, TimeValue timeout, String executor, - PrioritizedActionListener completionTimeoutListener) { - ((CompletionPrioritizedActionListener)completionTimeoutListener).cancellable = threadPool.schedule( - (Runnable) completionTimeoutListener, timeout, executor); + public static PrioritizedActionListener scheduleTimeout( + ThreadPool threadPool, + TimeValue timeout, + String executor, + PrioritizedActionListener completionTimeoutListener + ) { + ((CompletionPrioritizedActionListener) completionTimeoutListener).cancellable = threadPool.schedule( + (Runnable) completionTimeoutListener, + timeout, + executor + ); return completionTimeoutListener; } diff --git a/src/main/java/org/opensearch/search/asynchronous/listener/CompositeSearchProgressActionListener.java b/src/main/java/org/opensearch/search/asynchronous/listener/CompositeSearchProgressActionListener.java index b4b8ea4e..8d352134 100644 --- a/src/main/java/org/opensearch/search/asynchronous/listener/CompositeSearchProgressActionListener.java +++ b/src/main/java/org/opensearch/search/asynchronous/listener/CompositeSearchProgressActionListener.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.core.action.ActionListener; @@ -49,7 +52,6 @@ private synchronized boolean addListener(ActionListener listener) { return false; } - @Override public void onResponse(T response) { Iterable> actionListenersToBeInvoked = finalizeListeners(); diff --git a/src/main/java/org/opensearch/search/asynchronous/listener/PartialResponseProvider.java b/src/main/java/org/opensearch/search/asynchronous/listener/PartialResponseProvider.java index 01f7630f..5df4f593 100644 --- a/src/main/java/org/opensearch/search/asynchronous/listener/PartialResponseProvider.java +++ b/src/main/java/org/opensearch/search/asynchronous/listener/PartialResponseProvider.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.action.search.SearchResponse; diff --git a/src/main/java/org/opensearch/search/asynchronous/listener/PrioritizedActionListener.java b/src/main/java/org/opensearch/search/asynchronous/listener/PrioritizedActionListener.java index 2c6954b1..11077275 100644 --- a/src/main/java/org/opensearch/search/asynchronous/listener/PrioritizedActionListener.java +++ b/src/main/java/org/opensearch/search/asynchronous/listener/PrioritizedActionListener.java @@ -1,11 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; - import org.opensearch.core.action.ActionListener; public interface PrioritizedActionListener extends ActionListener { diff --git a/src/main/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementService.java b/src/main/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementService.java index 8d16c912..ab8c9d5a 100644 --- a/src/main/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementService.java +++ b/src/main/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementService.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.management; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; @@ -72,22 +75,30 @@ public class AsynchronousSearchManagementService extends AbstractLifecycleCompon private TimeValue activeContextReaperInterval; private TimeValue persistedResponseCleanUpInterval; - public static final String PERSISTED_RESPONSE_CLEANUP_ACTION_NAME = - "indices:data/read/opendistro/asynchronous_search/response_cleanup"; - - public static final Setting ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING = - Setting.timeSetting("plugins.asynchronous_search.active.context.reaper_interval", - LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, - TimeValue.timeValueSeconds(5), Setting.Property.NodeScope); - public static final Setting PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING = - Setting.timeSetting("plugins.asynchronous_search.expired.persisted_response.cleanup_interval", - LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, TimeValue.timeValueSeconds(5), - Setting.Property.NodeScope); + public static final String PERSISTED_RESPONSE_CLEANUP_ACTION_NAME = "indices:data/read/opendistro/asynchronous_search/response_cleanup"; + + public static final Setting ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING = Setting.timeSetting( + "plugins.asynchronous_search.active.context.reaper_interval", + LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, + TimeValue.timeValueSeconds(5), + Setting.Property.NodeScope + ); + public static final Setting PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING = Setting.timeSetting( + "plugins.asynchronous_search.expired.persisted_response.cleanup_interval", + LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, + TimeValue.timeValueSeconds(5), + Setting.Property.NodeScope + ); @Inject - public AsynchronousSearchManagementService(Settings settings, ClusterService clusterService, ThreadPool threadPool, - AsynchronousSearchService asynchronousSearchService, TransportService transportService, - AsynchronousSearchPersistenceService asynchronousSearchPersistenceService) { + public AsynchronousSearchManagementService( + Settings settings, + ClusterService clusterService, + ThreadPool threadPool, + AsynchronousSearchService asynchronousSearchService, + TransportService transportService, + AsynchronousSearchPersistenceService asynchronousSearchPersistenceService + ) { this.clusterService = clusterService; this.threadPool = threadPool; this.clusterService.addListener(this); @@ -97,8 +108,14 @@ public AsynchronousSearchManagementService(Settings settings, ClusterService clu this.activeContextReaperInterval = ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.get(settings); this.persistedResponseCleanUpInterval = PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.get(settings); - transportService.registerRequestHandler(PERSISTED_RESPONSE_CLEANUP_ACTION_NAME, ThreadPool.Names.SAME, false, false, - AsynchronousSearchCleanUpRequest::new, new PersistedResponseCleanUpTransportHandler()); + transportService.registerRequestHandler( + PERSISTED_RESPONSE_CLEANUP_ACTION_NAME, + ThreadPool.Names.SAME, + false, + false, + AsynchronousSearchCleanUpRequest::new, + new PersistedResponseCleanUpTransportHandler() + ); } class PersistedResponseCleanUpTransportHandler implements TransportRequestHandler { @@ -109,20 +126,22 @@ public void messageReceived(AsynchronousSearchCleanUpRequest request, TransportC try { channel.sendResponse(e); } catch (IOException ex) { - logger.warn(() -> new ParameterizedMessage( - "Failed to send cleanup error response for request [{}]", request), ex); + logger.warn(() -> new ParameterizedMessage("Failed to send cleanup error response for request [{}]", request), ex); } })); } } private void asyncCleanUpOperation(AsynchronousSearchCleanUpRequest request, Task task, ActionListener listener) { - transportService.getThreadPool().executor(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME) - .execute(() -> performPersistedResponseCleanUpAction(request, listener)); + transportService.getThreadPool() + .executor(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME) + .execute(() -> performPersistedResponseCleanUpAction(request, listener)); } - private void performPersistedResponseCleanUpAction(AsynchronousSearchCleanUpRequest request, - ActionListener listener) { + private void performPersistedResponseCleanUpAction( + AsynchronousSearchCleanUpRequest request, + ActionListener listener + ) { asynchronousSearchPersistenceService.deleteExpiredResponses(listener, request.absoluteTimeInMillis); } @@ -150,8 +169,11 @@ private void triggerCleanUp(ClusterState clusterState, String reason) { @Override protected void doStart() { - activeContextReaperScheduledFuture = threadPool.scheduleWithFixedDelay(new ActiveContextReaper(), activeContextReaperInterval, - AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME); + activeContextReaperScheduledFuture = threadPool.scheduleWithFixedDelay( + new ActiveContextReaper(), + activeContextReaperInterval, + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ); } @Override @@ -174,14 +196,24 @@ public void run() { Set toFree = asynchronousSearchService.getContextsToReap(); // don't block on response toFree.forEach( - context -> asynchronousSearchService.freeContext(context.getAsynchronousSearchId(), context.getContextId(), - null, ActionListener.wrap( - (response) -> logger.debug("Successfully freed up context [{}] running duration [{}]", - context.getAsynchronousSearchId(), context.getExpirationTimeMillis() - context.getStartTimeMillis()), - (exception) -> logger.debug(() -> new ParameterizedMessage( - "Failed to cleanup asynchronous search context [{}] running duration [{}] due to ", - context.getAsynchronousSearchId(),context.getExpirationTimeMillis() - - context.getStartTimeMillis()), exception) + context -> asynchronousSearchService.freeContext( + context.getAsynchronousSearchId(), + context.getContextId(), + null, + ActionListener.wrap( + (response) -> logger.debug( + "Successfully freed up context [{}] running duration [{}]", + context.getAsynchronousSearchId(), + context.getExpirationTimeMillis() - context.getStartTimeMillis() + ), + (exception) -> logger.debug( + () -> new ParameterizedMessage( + "Failed to cleanup asynchronous search context [{}] running duration [{}] due to ", + context.getAsynchronousSearchId(), + context.getExpirationTimeMillis() - context.getStartTimeMillis() + ), + exception + ) ) ) ); @@ -195,47 +227,53 @@ public final void performCleanUp() { final ThreadContext threadContext = threadPool.getThreadContext(); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { final Map dataNodes = clusterService.state().nodes().getDataNodes(); - List nodes = Stream.of(dataNodes.values().toArray(new DiscoveryNode[0])) - .collect(Collectors.toList()); + List nodes = Stream.of(dataNodes.values().toArray(new DiscoveryNode[0])).collect(Collectors.toList()); if (nodes == null || nodes.isEmpty()) { logger.debug("Found empty data nodes with asynchronous search enabled attribute [{}] for response clean up", dataNodes); return; } int pos = Randomness.get().nextInt(nodes.size()); DiscoveryNode randomNode = nodes.get(pos); - transportService.sendRequest(randomNode, PERSISTED_RESPONSE_CLEANUP_ACTION_NAME, - new AsynchronousSearchCleanUpRequest(threadPool.absoluteTimeInMillis()), - new TransportResponseHandler() { - - @Override - public AcknowledgedResponse read(StreamInput in) throws IOException { - return new AcknowledgedResponse(in); - } - - @Override - public void handleResponse(AcknowledgedResponse response) { - logger.debug("Successfully executed clean up action on node [{}] with response [{}]", randomNode, - response.isAcknowledged()); - } - - @Override - public void handleException(TransportException e) { - logger.error(() -> new ParameterizedMessage("Exception executing action [{}]", - PERSISTED_RESPONSE_CLEANUP_ACTION_NAME), e); - } - - @Override - public String executor() { - return AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME; - } - }); + transportService.sendRequest( + randomNode, + PERSISTED_RESPONSE_CLEANUP_ACTION_NAME, + new AsynchronousSearchCleanUpRequest(threadPool.absoluteTimeInMillis()), + new TransportResponseHandler() { + + @Override + public AcknowledgedResponse read(StreamInput in) throws IOException { + return new AcknowledgedResponse(in); + } + + @Override + public void handleResponse(AcknowledgedResponse response) { + logger.debug( + "Successfully executed clean up action on node [{}] with response [{}]", + randomNode, + response.isAcknowledged() + ); + } + + @Override + public void handleException(TransportException e) { + logger.error( + () -> new ParameterizedMessage("Exception executing action [{}]", PERSISTED_RESPONSE_CLEANUP_ACTION_NAME), + e + ); + } + + @Override + public String executor() { + return AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME; + } + } + ); } catch (Exception ex) { logger.error("Failed to schedule asynchronous search cleanup", ex); } } - private class ResponseCleanUpRunnable extends AbstractRunnable { private final String reason; @@ -253,16 +291,14 @@ public void onFailure(Exception e) { logger.warn(new ParameterizedMessage("sync search clean up job failed [{}]", reason), e); } - @Override public void onRejection(Exception e) { - final boolean shutDown = e instanceof OpenSearchRejectedExecutionException && ((OpenSearchRejectedExecutionException) e) - .isExecutorShutdown(); + final boolean shutDown = e instanceof OpenSearchRejectedExecutionException + && ((OpenSearchRejectedExecutionException) e).isExecutorShutdown(); logger.log(shutDown ? Level.DEBUG : Level.WARN, "asynchronous search clean up job rejected [{}]", reason, e); } } - private class PersistedResponseCleanUpAndRescheduleRunnable extends ResponseCleanUpRunnable { PersistedResponseCleanUpAndRescheduleRunnable() { super("scheduled"); @@ -286,7 +322,6 @@ public void onAfter() { } } - static class AsynchronousSearchCleanUpRequest extends ActionRequest { private final long absoluteTimeInMillis; @@ -318,7 +353,6 @@ public long getAbsoluteTimeInMillis() { return absoluteTimeInMillis; } - @Override public int hashCode() { return Objects.hash(absoluteTimeInMillis); @@ -326,10 +360,8 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; AsynchronousSearchCleanUpRequest asynchronousSearchCleanUpRequest = (AsynchronousSearchCleanUpRequest) o; return absoluteTimeInMillis == asynchronousSearchCleanUpRequest.absoluteTimeInMillis; } diff --git a/src/main/java/org/opensearch/search/asynchronous/plugin/AsynchronousSearchPlugin.java b/src/main/java/org/opensearch/search/asynchronous/plugin/AsynchronousSearchPlugin.java index 9590c734..9504b06b 100644 --- a/src/main/java/org/opensearch/search/asynchronous/plugin/AsynchronousSearchPlugin.java +++ b/src/main/java/org/opensearch/search/asynchronous/plugin/AsynchronousSearchPlugin.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.plugin; import org.opensearch.search.asynchronous.action.AsynchronousSearchStatsAction; @@ -61,7 +64,6 @@ import java.util.List; import java.util.function.Supplier; - public class AsynchronousSearchPlugin extends Plugin implements ActionPlugin, SystemIndexPlugin { public static final String OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME = "opensearch_asynchronous_search_generic"; @@ -74,8 +76,12 @@ public class AsynchronousSearchPlugin extends Plugin implements ActionPlugin, Sy @Override public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.singletonList(new SystemIndexDescriptor(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, - "Stores the response for asynchronous search")); + return Collections.singletonList( + new SystemIndexDescriptor( + AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, + "Stores the response for asynchronous search" + ) + ); } @Override @@ -83,70 +89,95 @@ public Collection> getGuiceServiceClasses() return Collections.singletonList(AsynchronousSearchManagementService.class); } - - //TODO Revisit these once we performance test the feature + // TODO Revisit these once we performance test the feature @Override public List> getExecutorBuilders(Settings settings) { final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); List> executorBuilders = new ArrayList<>(); - executorBuilders.add(new ScalingExecutorBuilder(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30))); + executorBuilders.add( + new ScalingExecutorBuilder( + OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ) + ); return executorBuilders; } @Override - public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, ScriptService scriptService, - NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier) { + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { this.persistenceService = new AsynchronousSearchPersistenceService(client, clusterService, threadPool); this.asynchronousSearchActiveStore = new AsynchronousSearchActiveStore(clusterService); - this.asynchronousSearchService = new AsynchronousSearchService(persistenceService, asynchronousSearchActiveStore, client, - clusterService, - threadPool, new InternalAsynchronousSearchStats(), namedWriteableRegistry); + this.asynchronousSearchService = new AsynchronousSearchService( + persistenceService, + asynchronousSearchActiveStore, + client, + clusterService, + threadPool, + new InternalAsynchronousSearchStats(), + namedWriteableRegistry + ); return Arrays.asList(persistenceService, asynchronousSearchService); } @Override public List> getActions() { return Arrays.asList( - new ActionHandler<>(SubmitAsynchronousSearchAction.INSTANCE, TransportSubmitAsynchronousSearchAction.class), - new ActionHandler<>(AsynchronousSearchStatsAction.INSTANCE, TransportAsynchronousSearchStatsAction.class), - new ActionHandler<>(GetAsynchronousSearchAction.INSTANCE, TransportGetAsynchronousSearchAction.class), - new ActionHandler<>(DeleteAsynchronousSearchAction.INSTANCE, TransportDeleteAsynchronousSearchAction.class)); + new ActionHandler<>(SubmitAsynchronousSearchAction.INSTANCE, TransportSubmitAsynchronousSearchAction.class), + new ActionHandler<>(AsynchronousSearchStatsAction.INSTANCE, TransportAsynchronousSearchStatsAction.class), + new ActionHandler<>(GetAsynchronousSearchAction.INSTANCE, TransportGetAsynchronousSearchAction.class), + new ActionHandler<>(DeleteAsynchronousSearchAction.INSTANCE, TransportDeleteAsynchronousSearchAction.class) + ); } @Override public List> getSettings() { return Arrays.asList( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, - AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, - AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, - LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING, - LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING, - LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, - LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, - LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, - LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, + AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, + AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, + LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING, + LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING, + LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, + LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, + LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, + LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING ); } @Override - public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, - IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster) { + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { return Arrays.asList( - new RestSubmitAsynchronousSearchAction(), - new RestGetAsynchronousSearchAction(), - new RestDeleteAsynchronousSearchAction(), - new RestAsynchronousSearchStatsAction()); + new RestSubmitAsynchronousSearchAction(), + new RestGetAsynchronousSearchAction(), + new RestDeleteAsynchronousSearchAction(), + new RestAsynchronousSearchStatsAction() + ); } } diff --git a/src/main/java/org/opensearch/search/asynchronous/processor/AsynchronousSearchPostProcessor.java b/src/main/java/org/opensearch/search/asynchronous/processor/AsynchronousSearchPostProcessor.java index 7ac320b4..7c466653 100644 --- a/src/main/java/org/opensearch/search/asynchronous/processor/AsynchronousSearchPostProcessor.java +++ b/src/main/java/org/opensearch/search/asynchronous/processor/AsynchronousSearchPostProcessor.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.processor; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -36,7 +39,6 @@ import java.util.concurrent.TimeoutException; import java.util.function.Consumer; - /** * Performs the processing after a search completes. */ @@ -50,11 +52,14 @@ public class AsynchronousSearchPostProcessor { private final Consumer freeActiveContextConsumer; private final ThreadPool threadPool; - public AsynchronousSearchPostProcessor(AsynchronousSearchPersistenceService asynchronousSearchPersistenceService, - AsynchronousSearchActiveStore asynchronousSearchActiveStore, - AsynchronousSearchStateMachine stateMachine, - Consumer freeActiveContextConsumer, - ThreadPool threadPool, ClusterService clusterService) { + public AsynchronousSearchPostProcessor( + AsynchronousSearchPersistenceService asynchronousSearchPersistenceService, + AsynchronousSearchActiveStore asynchronousSearchActiveStore, + AsynchronousSearchStateMachine stateMachine, + Consumer freeActiveContextConsumer, + ThreadPool threadPool, + ClusterService clusterService + ) { this.asynchronousSearchActiveStore = asynchronousSearchActiveStore; this.asynchronousSearchPersistenceService = asynchronousSearchPersistenceService; this.asynchronousSearchStateMachine = stateMachine; @@ -63,8 +68,9 @@ public AsynchronousSearchPostProcessor(AsynchronousSearchPersistenceService asyn } public AsynchronousSearchResponse processSearchFailure(Exception exception, AsynchronousSearchContextId asynchronousSearchContextId) { - final Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore - .getContext(asynchronousSearchContextId); + final Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore.getContext( + asynchronousSearchContextId + ); try { if (asynchronousSearchContextOptional.isPresent()) { AsynchronousSearchActiveContext asynchronousSearchContext = asynchronousSearchContextOptional.get(); @@ -73,19 +79,32 @@ public AsynchronousSearchResponse processSearchFailure(Exception exception, Asyn return asynchronousSearchContext.getAsynchronousSearchResponse(); } // Best effort to return the response. - return new AsynchronousSearchResponse(AsynchronousSearchState.FAILED, -1L, -1L, null, - ExceptionsHelper.convertToOpenSearchException(exception)); + return new AsynchronousSearchResponse( + AsynchronousSearchState.FAILED, + -1L, + -1L, + null, + ExceptionsHelper.convertToOpenSearchException(exception) + ); } catch (AsynchronousSearchStateMachineClosedException ex) { // Best effort to return the response. - return new AsynchronousSearchResponse(AsynchronousSearchState.FAILED, -1L, -1L, null, - ExceptionsHelper.convertToOpenSearchException(exception)); + return new AsynchronousSearchResponse( + AsynchronousSearchState.FAILED, + -1L, + -1L, + null, + ExceptionsHelper.convertToOpenSearchException(exception) + ); } } - public AsynchronousSearchResponse processSearchResponse(SearchResponse searchResponse, - AsynchronousSearchContextId asynchronousSearchContextId) { - final Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore - .getContext(asynchronousSearchContextId); + public AsynchronousSearchResponse processSearchResponse( + SearchResponse searchResponse, + AsynchronousSearchContextId asynchronousSearchContextId + ) { + final Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore.getContext( + asynchronousSearchContextId + ); try { if (asynchronousSearchContextOptional.isPresent()) { AsynchronousSearchActiveContext asynchronousSearchContext = asynchronousSearchContextOptional.get(); @@ -101,76 +120,94 @@ public AsynchronousSearchResponse processSearchResponse(SearchResponse searchRes } } - public void persistResponse(AsynchronousSearchActiveContext asynchronousSearchContext, - AsynchronousSearchPersistenceModel persistenceModel) { + public void persistResponse( + AsynchronousSearchActiveContext asynchronousSearchContext, + AsynchronousSearchPersistenceModel persistenceModel + ) { // acquire all permits non-blocking asynchronousSearchContext.acquireAllContextPermits(ActionListener.wrap(releasable -> { - // check again after acquiring permit if the context has been deleted mean while - if (asynchronousSearchContext.shouldPersist() == false) { + // check again after acquiring permit if the context has been deleted mean while + if (asynchronousSearchContext.shouldPersist() == false) { + logger.debug( + "Async search context [{}] has been closed while waiting to acquire permits for post processing", + asynchronousSearchContext.getAsynchronousSearchId() + ); + releasable.close(); + return; + } + logger.debug("Persisting response for asynchronous search id [{}]", asynchronousSearchContext.getAsynchronousSearchId()); + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { + asynchronousSearchPersistenceService.storeResponse( + asynchronousSearchContext.getAsynchronousSearchId(), + persistenceModel, + ActionListener.runAfter(ActionListener.wrap((indexResponse) -> { + // Mark any dangling reference as PERSISTED and cleaning it up from the IN_MEMORY context logger.debug( - "Async search context [{}] has been closed while waiting to acquire permits for post processing", - asynchronousSearchContext.getAsynchronousSearchId()); - releasable.close(); - return; - } - logger.debug("Persisting response for asynchronous search id [{}]", - asynchronousSearchContext.getAsynchronousSearchId()); - try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { - asynchronousSearchPersistenceService.storeResponse(asynchronousSearchContext.getAsynchronousSearchId(), - persistenceModel, ActionListener.runAfter(ActionListener.wrap( - (indexResponse) -> { - //Mark any dangling reference as PERSISTED and cleaning it up from the IN_MEMORY context - logger.debug("Successfully persisted response for asynchronous search id [{}]", - asynchronousSearchContext.getAsynchronousSearchId()); - try { - asynchronousSearchStateMachine.trigger(new SearchResponsePersistedEvent( - asynchronousSearchContext)); - } catch (AsynchronousSearchStateMachineClosedException ex) { - // this should never happen since we had checked after acquiring the all permits so a - // concurrent delete is not expected here, however an external task cancellation - // can cause this - logger.warn("Unexpected state, possibly caused by external task cancellation," + - " context with id [{}] closed while triggering event [{}]", - asynchronousSearchContext.getAsynchronousSearchId(), - SearchResponsePersistedEvent.class.getName()); - } finally { - freeActiveContextConsumer.accept(asynchronousSearchContext); - } - }, + "Successfully persisted response for asynchronous search id [{}]", + asynchronousSearchContext.getAsynchronousSearchId() + ); + try { + asynchronousSearchStateMachine.trigger(new SearchResponsePersistedEvent(asynchronousSearchContext)); + } catch (AsynchronousSearchStateMachineClosedException ex) { + // this should never happen since we had checked after acquiring the all permits so a + // concurrent delete is not expected here, however an external task cancellation + // can cause this + logger.warn( + "Unexpected state, possibly caused by external task cancellation," + + " context with id [{}] closed while triggering event [{}]", + asynchronousSearchContext.getAsynchronousSearchId(), + SearchResponsePersistedEvent.class.getName() + ); + } finally { + freeActiveContextConsumer.accept(asynchronousSearchContext); + } + }, - (e) -> { - try { - asynchronousSearchStateMachine.trigger(new SearchResponsePersistFailedEvent( - asynchronousSearchContext)); - } catch (AsynchronousSearchStateMachineClosedException ex) { - //this should never happen since we had checked after acquiring the all permits so a - // concurrent delete is not expected here, however an external task cancellation - // can cause this - logger.warn("Unexpected state, possibly caused by external task cancellation," + - " context with id [{}] closed while triggering event [{}]", - asynchronousSearchContext.getAsynchronousSearchId(), - SearchResponsePersistFailedEvent.class.getName()); - } finally { - freeActiveContextConsumer.accept(asynchronousSearchContext); - } - logger.error(() -> new ParameterizedMessage( - "Failed to persist final response for [{}] due to [{}]", - asynchronousSearchContext.getAsynchronousSearchId(), e)); - } - ), releasable::close)); - } + (e) -> { + try { + asynchronousSearchStateMachine.trigger(new SearchResponsePersistFailedEvent(asynchronousSearchContext)); + } catch (AsynchronousSearchStateMachineClosedException ex) { + // this should never happen since we had checked after acquiring the all permits so a + // concurrent delete is not expected here, however an external task cancellation + // can cause this + logger.warn( + "Unexpected state, possibly caused by external task cancellation," + + " context with id [{}] closed while triggering event [{}]", + asynchronousSearchContext.getAsynchronousSearchId(), + SearchResponsePersistFailedEvent.class.getName() + ); + } finally { + freeActiveContextConsumer.accept(asynchronousSearchContext); + } + logger.error( + () -> new ParameterizedMessage( + "Failed to persist final response for [{}] due to [{}]", + asynchronousSearchContext.getAsynchronousSearchId(), + e + ) + ); + } + ), releasable::close) + ); + } - }, (e) -> { - // Failure to acquire context can happen either due to a TimeoutException or AsynchronousSearchAlreadyClosedException - // If we weren't able to acquire permits we clean up the context to release heap. - Throwable cause = ExceptionsHelper.unwrapCause(e); - Level level = cause instanceof AsynchronousSearchContextClosedException || cause instanceof TimeoutException - ? Level.DEBUG : Level.WARN; - logger.log(level, () -> new ParameterizedMessage("Exception occured while acquiring the permit for " + - "asynchronousSearchContext [{}]", asynchronousSearchContext.getAsynchronousSearchId()), e); - freeActiveContextConsumer.accept(asynchronousSearchContext); - }), - TimeValue.timeValueSeconds(120), "persisting response"); + }, (e) -> { + // Failure to acquire context can happen either due to a TimeoutException or AsynchronousSearchAlreadyClosedException + // If we weren't able to acquire permits we clean up the context to release heap. + Throwable cause = ExceptionsHelper.unwrapCause(e); + Level level = cause instanceof AsynchronousSearchContextClosedException || cause instanceof TimeoutException + ? Level.DEBUG + : Level.WARN; + logger.log( + level, + () -> new ParameterizedMessage( + "Exception occured while acquiring the permit for " + "asynchronousSearchContext [{}]", + asynchronousSearchContext.getAsynchronousSearchId() + ), + e + ); + freeActiveContextConsumer.accept(asynchronousSearchContext); + }), TimeValue.timeValueSeconds(120), "persisting response"); } private void handlePersist(AsynchronousSearchActiveContext asynchronousSearchContext) { @@ -178,8 +215,8 @@ private void handlePersist(AsynchronousSearchActiveContext asynchronousSearchCon try { asynchronousSearchStateMachine.trigger(new BeginPersistEvent(asynchronousSearchContext)); } catch (AsynchronousSearchStateMachineClosedException e) { - //very rare since we checked if the context is alive before firing this event - //anyways clean it, it's idempotent + // very rare since we checked if the context is alive before firing this event + // anyways clean it, it's idempotent freeActiveContextConsumer.accept(asynchronousSearchContext); } } else { diff --git a/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRoutingRequest.java b/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRoutingRequest.java index 1c2825a2..ff084f72 100644 --- a/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRoutingRequest.java +++ b/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRoutingRequest.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.request; import org.opensearch.action.ActionRequest; diff --git a/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchStatsRequest.java b/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchStatsRequest.java index e63ab012..e09e0487 100644 --- a/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchStatsRequest.java +++ b/src/main/java/org/opensearch/search/asynchronous/request/AsynchronousSearchStatsRequest.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.request; import org.opensearch.action.support.nodes.BaseNodesRequest; diff --git a/src/main/java/org/opensearch/search/asynchronous/request/DeleteAsynchronousSearchRequest.java b/src/main/java/org/opensearch/search/asynchronous/request/DeleteAsynchronousSearchRequest.java index 497780f1..4b20d929 100644 --- a/src/main/java/org/opensearch/search/asynchronous/request/DeleteAsynchronousSearchRequest.java +++ b/src/main/java/org/opensearch/search/asynchronous/request/DeleteAsynchronousSearchRequest.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.request; import org.opensearch.action.ActionRequestValidationException; diff --git a/src/main/java/org/opensearch/search/asynchronous/request/GetAsynchronousSearchRequest.java b/src/main/java/org/opensearch/search/asynchronous/request/GetAsynchronousSearchRequest.java index 5c352917..26ce45a3 100644 --- a/src/main/java/org/opensearch/search/asynchronous/request/GetAsynchronousSearchRequest.java +++ b/src/main/java/org/opensearch/search/asynchronous/request/GetAsynchronousSearchRequest.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.request; import org.opensearch.action.ActionRequestValidationException; @@ -44,7 +47,6 @@ public void setKeepAlive(TimeValue keepAlive) { this.keepAlive = keepAlive; } - public GetAsynchronousSearchRequest(StreamInput streamInput) throws IOException { super(streamInput); keepAlive = streamInput.readOptionalTimeValue(); diff --git a/src/main/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequest.java b/src/main/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequest.java index b17d423e..b158f2a3 100644 --- a/src/main/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequest.java +++ b/src/main/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequest.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.request; import org.opensearch.search.asynchronous.task.SubmitAsynchronousSearchTask; @@ -34,7 +37,6 @@ public class SubmitAsynchronousSearchRequest extends ActionRequest { public static final Boolean DEFAULT_CCS_MINIMIZE_ROUNDTRIPS = Boolean.FALSE; public static final Boolean DEFAULT_REQUEST_CACHE = Boolean.TRUE; - /** * The minimum time that the request should wait before returning a partial result (defaults to 1 second). */ @@ -58,7 +60,6 @@ public class SubmitAsynchronousSearchRequest extends ActionRequest { */ private final SearchRequest searchRequest; - /** * Creates a new request from a {@linkplain SearchRequest} * @@ -117,7 +118,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(keepOnCompletion); } - @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -129,20 +129,25 @@ public ActionRequestValidationException validate() { } if (searchRequest.isCcsMinimizeRoundtrips()) { validationException = addValidationError( - "[ccs_minimize_roundtrips] must be false, got: " + searchRequest.isCcsMinimizeRoundtrips(), validationException); + "[ccs_minimize_roundtrips] must be false, got: " + searchRequest.isCcsMinimizeRoundtrips(), + validationException + ); } if (keepAlive != null && keepAlive.getMillis() < MIN_KEEP_ALIVE) { validationException = addValidationError( - "[keep_alive] must be greater than 1 minute, got: " + keepAlive.toString(), validationException); + "[keep_alive] must be greater than 1 minute, got: " + keepAlive.toString(), + validationException + ); } if (waitForCompletionTimeout != null && waitForCompletionTimeout.getMillis() < MIN_WAIT_FOR_COMPLETION_TIMEOUT) { - validationException = addValidationError("[wait_for_completion_timeout] must be greater than 0 milliseconds, got: " - + waitForCompletionTimeout.toString(), validationException); + validationException = addValidationError( + "[wait_for_completion_timeout] must be greater than 0 milliseconds, got: " + waitForCompletionTimeout.toString(), + validationException + ); } return validationException != null ? validationException : searchRequest.validate(); } - @Override public boolean equals(Object o) { if (this == o) { @@ -153,9 +158,9 @@ public boolean equals(Object o) { } SubmitAsynchronousSearchRequest request = (SubmitAsynchronousSearchRequest) o; return Objects.equals(searchRequest, request.searchRequest) - && Objects.equals(keepAlive, request.getKeepAlive()) - && Objects.equals(waitForCompletionTimeout, request.getWaitForCompletionTimeout()) - && Objects.equals(keepOnCompletion, request.getKeepOnCompletion()); + && Objects.equals(keepAlive, request.getKeepAlive()) + && Objects.equals(waitForCompletionTimeout, request.getWaitForCompletionTimeout()) + && Objects.equals(keepOnCompletion, request.getKeepOnCompletion()); } @Override @@ -166,8 +171,14 @@ public int hashCode() { @Override public SubmitAsynchronousSearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { // generating description in a lazy way since source can be quite big - SubmitAsynchronousSearchTask submitAsynchronousSearchTask = new SubmitAsynchronousSearchTask(id, type, action, null, - parentTaskId, headers) { + SubmitAsynchronousSearchTask submitAsynchronousSearchTask = new SubmitAsynchronousSearchTask( + id, + type, + action, + null, + parentTaskId, + headers + ) { @Override public String getDescription() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/opensearch/search/asynchronous/response/AcknowledgedResponse.java b/src/main/java/org/opensearch/search/asynchronous/response/AcknowledgedResponse.java index dad0c7db..b8f1f777 100644 --- a/src/main/java/org/opensearch/search/asynchronous/response/AcknowledgedResponse.java +++ b/src/main/java/org/opensearch/search/asynchronous/response/AcknowledgedResponse.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.response; import org.opensearch.core.action.ActionResponse; @@ -78,11 +81,18 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO * A generic parser that simply parses the acknowledged flag */ private static final ConstructingObjectParser ACKNOWLEDGED_FLAG_PARSER = new ConstructingObjectParser<>( - "acknowledged_flag", true, args -> (Boolean) args[0]); + "acknowledged_flag", + true, + args -> (Boolean) args[0] + ); static { - ACKNOWLEDGED_FLAG_PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), ACKNOWLEDGED, - ObjectParser.ValueType.BOOLEAN); + ACKNOWLEDGED_FLAG_PARSER.declareField( + constructorArg(), + (parser, context) -> parser.booleanValue(), + ACKNOWLEDGED, + ObjectParser.ValueType.BOOLEAN + ); } @Override diff --git a/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponse.java b/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponse.java index 5c29834a..f18d9341 100644 --- a/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponse.java +++ b/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponse.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.response; import org.opensearch.common.xcontent.StatusToXContentObject; @@ -42,7 +45,7 @@ public class AsynchronousSearchResponse extends ActionResponse implements Status private static final ParseField RESPONSE = new ParseField("response"); private static final ParseField ERROR = new ParseField("error"); @Nullable - //when the search is cancelled we don't have the id + // when the search is cancelled we don't have the id private final String id; private final AsynchronousSearchState state; private final long startTimeMillis; @@ -52,8 +55,14 @@ public class AsynchronousSearchResponse extends ActionResponse implements Status @Nullable private OpenSearchException error; - public AsynchronousSearchResponse(String id, AsynchronousSearchState state, long startTimeMillis, long expirationTimeMillis, - SearchResponse searchResponse, Exception error) { + public AsynchronousSearchResponse( + String id, + AsynchronousSearchState state, + long startTimeMillis, + long expirationTimeMillis, + SearchResponse searchResponse, + Exception error + ) { this.id = id; this.state = state; this.startTimeMillis = startTimeMillis; @@ -62,8 +71,14 @@ public AsynchronousSearchResponse(String id, AsynchronousSearchState state, long this.error = error == null ? null : ExceptionsHelper.convertToOpenSearchException(error); } - public AsynchronousSearchResponse(String id, AsynchronousSearchState state, long startTimeMillis, long expirationTimeMillis, - SearchResponse searchResponse, OpenSearchException error) { + public AsynchronousSearchResponse( + String id, + AsynchronousSearchState state, + long startTimeMillis, + long expirationTimeMillis, + SearchResponse searchResponse, + OpenSearchException error + ) { this.id = id; this.state = state; this.startTimeMillis = startTimeMillis; @@ -72,8 +87,13 @@ public AsynchronousSearchResponse(String id, AsynchronousSearchState state, long this.error = error; } - public AsynchronousSearchResponse(AsynchronousSearchState state, long startTimeMillis, long expirationTimeMillis, - SearchResponse searchResponse, OpenSearchException error) { + public AsynchronousSearchResponse( + AsynchronousSearchState state, + long startTimeMillis, + long expirationTimeMillis, + SearchResponse searchResponse, + OpenSearchException error + ) { this.state = state; this.startTimeMillis = startTimeMillis; this.expirationTimeMillis = expirationTimeMillis; @@ -163,7 +183,6 @@ public String toString() { return Strings.toString(XContentType.JSON, this); } - /** * {@linkplain SearchResponse} and {@linkplain OpenSearchException} don't override hashcode, hence cannot be included in * the hashcode calculation for {@linkplain AsynchronousSearchResponse}. Given that we are using these methods only in tests; on the @@ -189,12 +208,12 @@ public boolean equals(Object o) { } AsynchronousSearchResponse other = (AsynchronousSearchResponse) o; try { - return ((id == null && other.id == null) || (id != null && id.equals(other.id))) && - state.equals(other.state) && - startTimeMillis == other.startTimeMillis && - expirationTimeMillis == other.expirationTimeMillis - && Objects.equals(getErrorAsMap(error), getErrorAsMap(other.error)) - && Objects.equals(getResponseAsMap(searchResponse), getResponseAsMap(other.searchResponse)); + return ((id == null && other.id == null) || (id != null && id.equals(other.id))) + && state.equals(other.state) + && startTimeMillis == other.startTimeMillis + && expirationTimeMillis == other.expirationTimeMillis + && Objects.equals(getErrorAsMap(error), getErrorAsMap(other.error)) + && Objects.equals(getResponseAsMap(searchResponse), getResponseAsMap(other.searchResponse)); } catch (IOException e) { return false; } @@ -218,7 +237,10 @@ private Map getErrorAsMap(OpenSearchException exception) throws private Map getResponseAsMap(SearchResponse searchResponse) throws IOException { if (searchResponse != null) { BytesReference response = org.opensearch.core.xcontent.XContentHelper.toXContent( - searchResponse, Requests.INDEX_CONTENT_TYPE,true); + searchResponse, + Requests.INDEX_CONTENT_TYPE, + true + ); if (response == null) { return emptyMap(); } @@ -271,7 +293,7 @@ public static AsynchronousSearchResponse innerFromXContent(XContentParser parser return new AsynchronousSearchResponse(id, status, startTimeMillis, expirationTimeMillis, searchResponse, error); } - //visible for testing + // visible for testing public static AsynchronousSearchResponse empty(String id, SearchResponse searchResponse, Exception exception) { return new AsynchronousSearchResponse(id, null, -1, -1, searchResponse, exception); } diff --git a/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchStatsResponse.java b/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchStatsResponse.java index ae85acaa..3818616b 100644 --- a/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchStatsResponse.java +++ b/src/main/java/org/opensearch/search/asynchronous/response/AsynchronousSearchStatsResponse.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.response; import org.opensearch.common.xcontent.XContentFactory; @@ -26,8 +29,11 @@ public AsynchronousSearchStatsResponse(StreamInput in) throws IOException { super(in); } - public AsynchronousSearchStatsResponse(ClusterName clusterName, List nodes, - List failures) { + public AsynchronousSearchStatsResponse( + ClusterName clusterName, + List nodes, + List failures + ) { super(clusterName, nodes, failures); } diff --git a/src/main/java/org/opensearch/search/asynchronous/rest/RestAsynchronousSearchStatsAction.java b/src/main/java/org/opensearch/search/asynchronous/rest/RestAsynchronousSearchStatsAction.java index cdd31420..d060cadb 100644 --- a/src/main/java/org/opensearch/search/asynchronous/rest/RestAsynchronousSearchStatsAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/rest/RestAsynchronousSearchStatsAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.rest; import org.opensearch.search.asynchronous.action.AsynchronousSearchStatsAction; @@ -29,15 +32,13 @@ public class RestAsynchronousSearchStatsAction extends BaseRestHandler { private static final String NAME = "asynchronous_search_stats_action"; - public RestAsynchronousSearchStatsAction() { - } + public RestAsynchronousSearchStatsAction() {} @Override public String getName() { return NAME; } - @Override public List routes() { return Collections.emptyList(); @@ -47,15 +48,18 @@ public List routes() { protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { AsynchronousSearchStatsRequest asynchronousSearchStatsRequest = getRequest(request); - return channel -> client.execute(AsynchronousSearchStatsAction.INSTANCE, asynchronousSearchStatsRequest, - new RestActions.NodesResponseRestListener<>(channel)); + return channel -> client.execute( + AsynchronousSearchStatsAction.INSTANCE, + asynchronousSearchStatsRequest, + new RestActions.NodesResponseRestListener<>(channel) + ); } @Override public List replacedRoutes() { - return Arrays.asList(new ReplacedRoute(GET, BASE_URI + "/_nodes/{nodeId}/stats", - GET, LEGACY_OPENDISTRO_BASE_URI + "/_nodes/{nodeId}/stats"), - new ReplacedRoute(GET, BASE_URI + "/stats", GET, LEGACY_OPENDISTRO_BASE_URI + "/stats") + return Arrays.asList( + new ReplacedRoute(GET, BASE_URI + "/_nodes/{nodeId}/stats", GET, LEGACY_OPENDISTRO_BASE_URI + "/_nodes/{nodeId}/stats"), + new ReplacedRoute(GET, BASE_URI + "/stats", GET, LEGACY_OPENDISTRO_BASE_URI + "/stats") ); } @@ -68,4 +72,3 @@ private AsynchronousSearchStatsRequest getRequest(RestRequest request) { return asynchronousSearchStatsRequest; } } - diff --git a/src/main/java/org/opensearch/search/asynchronous/rest/RestDeleteAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/rest/RestDeleteAsynchronousSearchAction.java index 6b25bd64..4e9e8859 100644 --- a/src/main/java/org/opensearch/search/asynchronous/rest/RestDeleteAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/rest/RestDeleteAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.rest; import org.opensearch.search.asynchronous.action.DeleteAsynchronousSearchAction; @@ -32,8 +35,13 @@ public List routes() { @Override public List replacedRoutes() { - return Collections.singletonList(new ReplacedRoute(DELETE, AsynchronousSearchPlugin.BASE_URI + "/{id}", - DELETE, AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/{id}") + return Collections.singletonList( + new ReplacedRoute( + DELETE, + AsynchronousSearchPlugin.BASE_URI + "/{id}", + DELETE, + AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/{id}" + ) ); } diff --git a/src/main/java/org/opensearch/search/asynchronous/rest/RestGetAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/rest/RestGetAsynchronousSearchAction.java index 98777853..1f613479 100644 --- a/src/main/java/org/opensearch/search/asynchronous/rest/RestGetAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/rest/RestGetAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.rest; import org.opensearch.search.asynchronous.action.GetAsynchronousSearchAction; @@ -33,8 +36,13 @@ public List routes() { @Override public List replacedRoutes() { - return Collections.singletonList(new ReplacedRoute(GET, AsynchronousSearchPlugin.BASE_URI + "/{id}", - GET, AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/{id}") + return Collections.singletonList( + new ReplacedRoute( + GET, + AsynchronousSearchPlugin.BASE_URI + "/{id}", + GET, + AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/{id}" + ) ); } diff --git a/src/main/java/org/opensearch/search/asynchronous/rest/RestSubmitAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/rest/RestSubmitAsynchronousSearchAction.java index c3da1ad4..9329aa34 100644 --- a/src/main/java/org/opensearch/search/asynchronous/rest/RestSubmitAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/rest/RestSubmitAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.rest; import org.opensearch.search.asynchronous.action.SubmitAsynchronousSearchAction; @@ -54,8 +57,8 @@ public List routes() { @Override public List replacedRoutes() { return Arrays.asList( - new ReplacedRoute(POST, "/{index}" + BASE_URI, POST, "/{index}" + LEGACY_OPENDISTRO_BASE_URI), - new ReplacedRoute(POST, BASE_URI, POST, LEGACY_OPENDISTRO_BASE_URI) + new ReplacedRoute(POST, "/{index}" + BASE_URI, POST, "/{index}" + LEGACY_OPENDISTRO_BASE_URI), + new ReplacedRoute(POST, BASE_URI, POST, LEGACY_OPENDISTRO_BASE_URI) ); } @@ -64,27 +67,34 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC SearchRequest searchRequest = new SearchRequest(); IntConsumer setSize = size -> searchRequest.source().size(size); - request.withContentOrSourceParamParserOrNull(parser -> - RestSearchAction.parseSearchRequest(searchRequest, request, parser, client.getNamedWriteableRegistry(), setSize)); + request.withContentOrSourceParamParserOrNull( + parser -> RestSearchAction.parseSearchRequest(searchRequest, request, parser, client.getNamedWriteableRegistry(), setSize) + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); - submitAsynchronousSearchRequest.waitForCompletionTimeout(request.paramAsTime("wait_for_completion_timeout", - SubmitAsynchronousSearchRequest.DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT)); + submitAsynchronousSearchRequest.waitForCompletionTimeout( + request.paramAsTime("wait_for_completion_timeout", SubmitAsynchronousSearchRequest.DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT) + ); submitAsynchronousSearchRequest.keepAlive(request.paramAsTime("keep_alive", SubmitAsynchronousSearchRequest.DEFAULT_KEEP_ALIVE)); - submitAsynchronousSearchRequest.keepOnCompletion(request.paramAsBoolean("keep_on_completion", - SubmitAsynchronousSearchRequest.DEFAULT_KEEP_ON_COMPLETION)); + submitAsynchronousSearchRequest.keepOnCompletion( + request.paramAsBoolean("keep_on_completion", SubmitAsynchronousSearchRequest.DEFAULT_KEEP_ON_COMPLETION) + ); searchRequest.requestCache(request.paramAsBoolean("request_cache", SubmitAsynchronousSearchRequest.DEFAULT_REQUEST_CACHE)); - searchRequest.setBatchedReduceSize(request.paramAsInt("batched_reduce_size", - SubmitAsynchronousSearchRequest.DEFAULT_BATCHED_REDUCE_SIZE)); + searchRequest.setBatchedReduceSize( + request.paramAsInt("batched_reduce_size", SubmitAsynchronousSearchRequest.DEFAULT_BATCHED_REDUCE_SIZE) + ); return channel -> { RestCancellableNodeClient cancellableClient = new RestCancellableNodeClient(client, request.getHttpChannel()); - cancellableClient.execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, - new RestStatusToXContentListener<>(channel)); + cancellableClient.execute( + SubmitAsynchronousSearchAction.INSTANCE, + submitAsynchronousSearchRequest, + new RestStatusToXContentListener<>(channel) + ); }; } diff --git a/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPersistenceService.java b/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPersistenceService.java index 86c79292..1cbd2d5f 100644 --- a/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPersistenceService.java +++ b/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPersistenceService.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service; import org.opensearch.common.xcontent.XContentType; @@ -71,8 +74,7 @@ public class AsynchronousSearchPersistenceService { * The backoff policy to use when saving a asynchronous search response fails. The total wait * time is 600000 milliseconds, ten minutes. */ - public static final BackoffPolicy STORE_BACKOFF_POLICY = - BackoffPolicy.exponentialBackoff(timeValueMillis(250), 14); + public static final BackoffPolicy STORE_BACKOFF_POLICY = BackoffPolicy.exponentialBackoff(timeValueMillis(250), 14); public static final String BACKEND_ROLES = "backend_roles"; public static final String SETTING_INDEX_CODEC = "index.codec"; public static final String BEST_COMPRESSION_CODEC = "best_compression"; @@ -87,7 +89,6 @@ public AsynchronousSearchPersistenceService(Client client, ClusterService cluste this.threadPool = threadPool; } - /** * Creates asynchronous search response as document in index. Creates index if necessary, before creating document. Retries response * creation on failure with exponential backoff @@ -118,37 +119,37 @@ public void getResponse(String id, User user, ActionListener - { - if (getResponse.isExists()) { - Map source = getResponse.getSource(); - AsynchronousSearchPersistenceModel asynchronousSearchPersistenceModel = new AsynchronousSearchPersistenceModel( - (long) source.get(START_TIME_MILLIS), - (long) source.get(EXPIRATION_TIME_MILLIS), - source.containsKey(RESPONSE) ? (String) source.get(RESPONSE) : null, - source.containsKey(ERROR) ? (String) source.get(ERROR) : null, - parseUser((Map) source.get(USER))); - if (isUserValid(user, asynchronousSearchPersistenceModel.getUser())) { - listener.onResponse(asynchronousSearchPersistenceModel); - } else { - logger.debug("Invalid user requesting GET persisted context for asynchronous search [{}]", id); - listener.onFailure(new OpenSearchSecurityException( - "User doesn't have necessary roles to access the asynchronous search [" + id + "]", - RestStatus.FORBIDDEN)); - } - } else { - listener.onFailure(new ResourceNotFoundException(id)); - } - }, - exception -> { - logger.error(() -> new ParameterizedMessage("Failed to get response for asynchronous search [{}]", id), - exception); - final Throwable cause = ExceptionsHelper.unwrapCause(exception); - listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause)); - })); + client.get(request, ActionListener.wrap(getResponse -> { + if (getResponse.isExists()) { + Map source = getResponse.getSource(); + AsynchronousSearchPersistenceModel asynchronousSearchPersistenceModel = new AsynchronousSearchPersistenceModel( + (long) source.get(START_TIME_MILLIS), + (long) source.get(EXPIRATION_TIME_MILLIS), + source.containsKey(RESPONSE) ? (String) source.get(RESPONSE) : null, + source.containsKey(ERROR) ? (String) source.get(ERROR) : null, + parseUser((Map) source.get(USER)) + ); + if (isUserValid(user, asynchronousSearchPersistenceModel.getUser())) { + listener.onResponse(asynchronousSearchPersistenceModel); + } else { + logger.debug("Invalid user requesting GET persisted context for asynchronous search [{}]", id); + listener.onFailure( + new OpenSearchSecurityException( + "User doesn't have necessary roles to access the asynchronous search [" + id + "]", + RestStatus.FORBIDDEN + ) + ); + } + } else { + listener.onFailure(new ResourceNotFoundException(id)); + } + }, exception -> { + logger.error(() -> new ParameterizedMessage("Failed to get response for asynchronous search [{}]", id), exception); + final Throwable cause = ExceptionsHelper.unwrapCause(exception); + listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause)); + })); } - /** * This method should be safe to call even if there isn't a prior document that exists. If the doc was actually deleted, the listener * returns true @@ -186,9 +187,9 @@ public void deleteResponse(String id, User user, ActionListener listene }, onFailure)); } else { UpdateRequest updateRequest = new UpdateRequest(ASYNC_SEARCH_RESPONSE_INDEX, id); - String scriptCode = "if (ctx._source.user == null || ctx._source.user.backend_roles == null || " + - "( params.backend_roles!=null && params.backend_roles.containsAll(ctx._source.user.backend_roles))) " + - "{ ctx.op = 'delete' } else { ctx.op = 'none' }"; + String scriptCode = "if (ctx._source.user == null || ctx._source.user.backend_roles == null || " + + "( params.backend_roles!=null && params.backend_roles.containsAll(ctx._source.user.backend_roles))) " + + "{ ctx.op = 'delete' } else { ctx.op = 'none' }"; Map params = new HashMap<>(); params.put("backend_roles", user.getBackendRoles()); Script deleteConditionallyScript = new Script(ScriptType.INLINE, "painless", scriptCode, params); @@ -196,12 +197,17 @@ public void deleteResponse(String id, User user, ActionListener listene client.update(updateRequest, ActionListener.wrap(deleteResponse -> { switch (deleteResponse.getResult()) { case UPDATED: - listener.onFailure(new IllegalStateException("Document updated when requesting delete for asynchronous search id " - + id)); + listener.onFailure( + new IllegalStateException("Document updated when requesting delete for asynchronous search id " + id) + ); break; case NOOP: - listener.onFailure(new OpenSearchSecurityException( - "User doesn't have necessary roles to access the asynchronous search with id " + id, RestStatus.FORBIDDEN)); + listener.onFailure( + new OpenSearchSecurityException( + "User doesn't have necessary roles to access the asynchronous search with id " + id, + RestStatus.FORBIDDEN + ) + ); break; case NOT_FOUND: listener.onFailure(new ResourceNotFoundException(id)); @@ -223,8 +229,12 @@ public void deleteResponse(String id, User user, ActionListener listene * @param listener listener invoked with the response on completion of update request */ @SuppressWarnings("unchecked") - public void updateExpirationTime(String id, long expirationTimeMillis, - User user, ActionListener listener) { + public void updateExpirationTime( + String id, + long expirationTimeMillis, + User user, + ActionListener listener + ) { if (indexExists() == false) { listener.onFailure(new ResourceNotFoundException(id)); return; @@ -236,9 +246,9 @@ public void updateExpirationTime(String id, long expirationTimeMillis, source.put(EXPIRATION_TIME_MILLIS, expirationTimeMillis); updateRequest.doc(source, XContentType.JSON); } else { - String scriptCode = "if (ctx._source.user == null || ctx._source.user.backend_roles == null || " + - "(params.backend_roles != null && params.backend_roles.containsAll(ctx._source.user.backend_roles))) " + - "{ ctx._source.expiration_time_millis = params.expiration_time_millis } else { ctx.op = 'none' }"; + String scriptCode = "if (ctx._source.user == null || ctx._source.user.backend_roles == null || " + + "(params.backend_roles != null && params.backend_roles.containsAll(ctx._source.user.backend_roles))) " + + "{ ctx._source.expiration_time_millis = params.expiration_time_millis } else { ctx.op = 'none' }"; Map params = new HashMap<>(); params.put(BACKEND_ROLES, user.getBackendRoles()); params.put(EXPIRATION_TIME_MILLIS, expirationTimeMillis); @@ -250,27 +260,45 @@ public void updateExpirationTime(String id, long expirationTimeMillis, switch (updateResponse.getResult()) { case NOOP: if (user != null) { - listener.onFailure(new OpenSearchSecurityException( - "User doesn't have necessary roles to access the asynchronous search with id " + id, RestStatus.FORBIDDEN)); + listener.onFailure( + new OpenSearchSecurityException( + "User doesn't have necessary roles to access the asynchronous search with id " + id, + RestStatus.FORBIDDEN + ) + ); } else { Map updatedSource = updateResponse.getGetResult().getSource(); - listener.onResponse(new AsynchronousSearchPersistenceModel((long) updatedSource.get(START_TIME_MILLIS), + listener.onResponse( + new AsynchronousSearchPersistenceModel( + (long) updatedSource.get(START_TIME_MILLIS), (long) updatedSource.get(EXPIRATION_TIME_MILLIS), - (String) updatedSource.get(RESPONSE), (String) updatedSource.get(ERROR), - parseUser((Map) updatedSource.get(USER)))); + (String) updatedSource.get(RESPONSE), + (String) updatedSource.get(ERROR), + parseUser((Map) updatedSource.get(USER)) + ) + ); } break; case UPDATED: Map updatedSource = updateResponse.getGetResult().getSource(); - listener.onResponse(new AsynchronousSearchPersistenceModel((long) updatedSource.get(START_TIME_MILLIS), + listener.onResponse( + new AsynchronousSearchPersistenceModel( + (long) updatedSource.get(START_TIME_MILLIS), (long) updatedSource.get(EXPIRATION_TIME_MILLIS), - (String) updatedSource.get(RESPONSE), (String) updatedSource.get(ERROR), - parseUser((Map) updatedSource.get(USER)))); + (String) updatedSource.get(RESPONSE), + (String) updatedSource.get(ERROR), + parseUser((Map) updatedSource.get(USER)) + ) + ); break; case NOT_FOUND: case DELETED: - logger.debug("Update Result [{}] for id [{}], expiration time requested, [{}]", - updateResponse.getResult(), id, expirationTimeMillis); + logger.debug( + "Update Result [{}] for id [{}], expiration time requested, [{}]", + updateResponse.getResult(), + id, + expirationTimeMillis + ); listener.onFailure(new ResourceNotFoundException(id)); break; } @@ -279,8 +307,10 @@ public void updateExpirationTime(String id, long expirationTimeMillis, if (cause instanceof DocumentMissingException) { listener.onFailure(new ResourceNotFoundException(id)); } else { - logger.error(() -> new ParameterizedMessage("Exception occurred updating expiration time for asynchronous search [{}]", - id), exception); + logger.error( + () -> new ParameterizedMessage("Exception occurred updating expiration time for asynchronous search [{}]", id), + exception + ); listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause)); } })); @@ -298,48 +328,56 @@ public void deleteExpiredResponses(ActionListener listener logger.debug("Async search index not yet created! Nothing to delete."); listener.onResponse(new AcknowledgedResponse(true)); } else { - DeleteByQueryRequest request = new DeleteByQueryRequest(ASYNC_SEARCH_RESPONSE_INDEX) - .setQuery(QueryBuilders.rangeQuery(EXPIRATION_TIME_MILLIS).lte(expirationTimeInMillis)); - client.execute(DeleteByQueryAction.INSTANCE, request, ActionListener.wrap( - deleteResponse -> { - if ((deleteResponse.getBulkFailures() != null && deleteResponse.getBulkFailures().size() > 0) || - (deleteResponse.getSearchFailures() != null && deleteResponse.getSearchFailures().size() > 0)) { - logger.error("Failed to delete expired asynchronous search responses with bulk failures[{}] / search " + - "failures [{}]", deleteResponse.getBulkFailures(), deleteResponse.getSearchFailures()); - listener.onResponse(new AcknowledgedResponse(false)); - - } else { - logger.debug("Successfully deleted expired responses"); - listener.onResponse(new AcknowledgedResponse(true)); - } - }, - (e) -> { - logger.error(() -> new ParameterizedMessage("Failed to delete expired response for expiration time {}", - expirationTimeInMillis), e); - final Throwable cause = ExceptionsHelper.unwrapCause(e); - listener.onFailure(cause instanceof Exception ? (Exception) cause : - new NotSerializableExceptionWrapper(cause)); - }) + DeleteByQueryRequest request = new DeleteByQueryRequest(ASYNC_SEARCH_RESPONSE_INDEX).setQuery( + QueryBuilders.rangeQuery(EXPIRATION_TIME_MILLIS).lte(expirationTimeInMillis) ); + client.execute(DeleteByQueryAction.INSTANCE, request, ActionListener.wrap(deleteResponse -> { + if ((deleteResponse.getBulkFailures() != null && deleteResponse.getBulkFailures().size() > 0) + || (deleteResponse.getSearchFailures() != null && deleteResponse.getSearchFailures().size() > 0)) { + logger.error( + "Failed to delete expired asynchronous search responses with bulk failures[{}] / search " + "failures [{}]", + deleteResponse.getBulkFailures(), + deleteResponse.getSearchFailures() + ); + listener.onResponse(new AcknowledgedResponse(false)); + + } else { + logger.debug("Successfully deleted expired responses"); + listener.onResponse(new AcknowledgedResponse(true)); + } + }, (e) -> { + logger.error( + () -> new ParameterizedMessage("Failed to delete expired response for expiration time {}", expirationTimeInMillis), + e + ); + final Throwable cause = ExceptionsHelper.unwrapCause(e); + listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause)); + })); } } - private void createIndexAndDoStoreResult(String id, AsynchronousSearchPersistenceModel persistenceModel, - ActionListener listener) { - client.admin().indices().prepareCreate(ASYNC_SEARCH_RESPONSE_INDEX).setMapping(mapping()) - .setSettings(indexSettings()).execute(ActionListener.wrap(createIndexResponse -> doStoreResult(id, persistenceModel, - listener), exception -> { - if (ExceptionsHelper.unwrapCause(exception) instanceof ResourceAlreadyExistsException) { - try { - doStoreResult(id, persistenceModel, listener); - } catch (Exception inner) { - inner.addSuppressed(exception); - listener.onFailure(inner); + private void createIndexAndDoStoreResult( + String id, + AsynchronousSearchPersistenceModel persistenceModel, + ActionListener listener + ) { + client.admin() + .indices() + .prepareCreate(ASYNC_SEARCH_RESPONSE_INDEX) + .setMapping(mapping()) + .setSettings(indexSettings()) + .execute(ActionListener.wrap(createIndexResponse -> doStoreResult(id, persistenceModel, listener), exception -> { + if (ExceptionsHelper.unwrapCause(exception) instanceof ResourceAlreadyExistsException) { + try { + doStoreResult(id, persistenceModel, listener); + } catch (Exception inner) { + inner.addSuppressed(exception); + listener.onFailure(inner); + } + } else { + listener.onFailure(exception); } - } else { - listener.onFailure(exception); - } - })); + })); } private void doStoreResult(String id, AsynchronousSearchPersistenceModel model, ActionListener listener) { @@ -350,12 +388,16 @@ private void doStoreResult(String id, AsynchronousSearchPersistenceModel model, source.put(START_TIME_MILLIS, model.getStartTimeMillis()); source.put(USER, model.getUser()); IndexRequestBuilder indexRequestBuilder = client.prepareIndex(ASYNC_SEARCH_RESPONSE_INDEX) - .setId(id).setSource(source, XContentType.JSON); + .setId(id) + .setSource(source, XContentType.JSON); doStoreResult(STORE_BACKOFF_POLICY.iterator(), indexRequestBuilder, listener); } - private void doStoreResult(Iterator backoff, IndexRequestBuilder indexRequestBuilder, - ActionListener listener) { + private void doStoreResult( + Iterator backoff, + IndexRequestBuilder indexRequestBuilder, + ActionListener listener + ) { indexRequestBuilder.execute(new ActionListener() { @Override public void onResponse(IndexResponse indexResponse) { @@ -367,12 +409,23 @@ public void onFailure(Exception e) { final Throwable cause = ExceptionsHelper.unwrapCause(e); if ((cause instanceof OpenSearchRejectedExecutionException) && backoff.hasNext()) { TimeValue wait = backoff.next(); - logger.warn(() -> new ParameterizedMessage("failed to store asynchronous search response [{}], retrying in [{}]", - indexRequestBuilder.request().id(), wait), e); + logger.warn( + () -> new ParameterizedMessage( + "failed to store asynchronous search response [{}], retrying in [{}]", + indexRequestBuilder.request().id(), + wait + ), + e + ); threadPool.schedule(() -> doStoreResult(backoff, indexRequestBuilder, listener), wait, ThreadPool.Names.SAME); } else { - logger.error(() -> new ParameterizedMessage("failed to store asynchronous search response [{}], not retrying", - indexRequestBuilder.request().id()), e); + logger.error( + () -> new ParameterizedMessage( + "failed to store asynchronous search response [{}], not retrying", + indexRequestBuilder.request().id() + ), + e + ); listener.onFailure(e); } } @@ -381,35 +434,35 @@ public void onFailure(Exception e) { private Settings indexSettings() { return Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 5) - .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-1") - .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .put(SETTING_INDEX_CODEC, BEST_COMPRESSION_CODEC) - .build(); + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 5) + .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-1") + .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) + .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) + .put(SETTING_INDEX_CODEC, BEST_COMPRESSION_CODEC) + .build(); } private XContentBuilder mapping() { try { XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject() - .startObject("properties") - .startObject(START_TIME_MILLIS) - .field("type", "date") - .field("format", "epoch_millis") - .endObject() - .startObject(EXPIRATION_TIME_MILLIS) - .field("type", "date") - .field("format", "epoch_millis") - .endObject() - .startObject(RESPONSE) - .field("type", "binary") - .endObject() - .startObject(ERROR) - .field("type", "binary") - .endObject() - .endObject() - .endObject(); + .startObject("properties") + .startObject(START_TIME_MILLIS) + .field("type", "date") + .field("format", "epoch_millis") + .endObject() + .startObject(EXPIRATION_TIME_MILLIS) + .field("type", "date") + .field("format", "epoch_millis") + .endObject() + .startObject(RESPONSE) + .field("type", "binary") + .endObject() + .startObject(ERROR) + .field("type", "binary") + .endObject() + .endObject() + .endObject(); return builder; } catch (IOException e) { throw new IllegalArgumentException("Async search persistence mapping cannot be read correctly.", e); diff --git a/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchService.java b/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchService.java index 516fec3a..2a4871b0 100644 --- a/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchService.java +++ b/src/main/java/org/opensearch/search/asynchronous/service/AsynchronousSearchService.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service; import org.apache.logging.log4j.LogManager; @@ -95,23 +98,31 @@ public class AsynchronousSearchService extends AbstractLifecycleComponent implem private static final Logger logger = LogManager.getLogger(AsynchronousSearchService.class); - public static final Setting MAX_KEEP_ALIVE_SETTING = - Setting.positiveTimeSetting("plugins.asynchronous_search.max_keep_alive", - LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING, - Setting.Property.NodeScope, Setting.Property.Dynamic); - public static final Setting MAX_SEARCH_RUNNING_TIME_SETTING = - Setting.positiveTimeSetting("plugins.asynchronous_search.max_search_running_time", - LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING, - Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting MAX_KEEP_ALIVE_SETTING = Setting.positiveTimeSetting( + "plugins.asynchronous_search.max_keep_alive", + LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final Setting MAX_SEARCH_RUNNING_TIME_SETTING = Setting.positiveTimeSetting( + "plugins.asynchronous_search.max_search_running_time", + LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); public static final Setting MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING = Setting.positiveTimeSetting( - "plugins.asynchronous_search.max_wait_for_completion_timeout", - LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, - Setting.Property.NodeScope, Setting.Property.Dynamic); - - public static final Setting PERSIST_SEARCH_FAILURES_SETTING = - Setting.boolSetting("plugins.asynchronous_search.persist_search_failures", - LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING, - Setting.Property.NodeScope, Setting.Property.Dynamic); + "plugins.asynchronous_search.max_wait_for_completion_timeout", + LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final Setting PERSIST_SEARCH_FAILURES_SETTING = Setting.boolSetting( + "plugins.asynchronous_search.persist_search_failures", + LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); private volatile long maxKeepAlive; private volatile long maxWaitForCompletionTimeout; @@ -129,18 +140,22 @@ public class AsynchronousSearchService extends AbstractLifecycleComponent implem private final AsynchronousSearchContextEventListener contextEventListener; private volatile boolean persistSearchFailure; - public AsynchronousSearchService(AsynchronousSearchPersistenceService asynchronousSearchPersistenceService, - AsynchronousSearchActiveStore asynchronousSearchActiveStore, Client client, - ClusterService clusterService, ThreadPool threadPool, - AsynchronousSearchContextEventListener contextEventListener, - NamedWriteableRegistry namedWriteableRegistry) { + public AsynchronousSearchService( + AsynchronousSearchPersistenceService asynchronousSearchPersistenceService, + AsynchronousSearchActiveStore asynchronousSearchActiveStore, + Client client, + ClusterService clusterService, + ThreadPool threadPool, + AsynchronousSearchContextEventListener contextEventListener, + NamedWriteableRegistry namedWriteableRegistry + ) { this.contextEventListener = contextEventListener; this.client = client; Settings settings = clusterService.getSettings(); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_KEEP_ALIVE_SETTING, this::setKeepAlive); setKeepAlive(MAX_KEEP_ALIVE_SETTING.get(settings)); - clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, - this::setMaxWaitForCompletionTimeout); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, this::setMaxWaitForCompletionTimeout); setMaxWaitForCompletionTimeout(MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(settings)); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_SEARCH_RUNNING_TIME_SETTING, this::setMaxSearchRunningTime); setMaxSearchRunningTime(MAX_SEARCH_RUNNING_TIME_SETTING.get(settings)); @@ -152,8 +167,14 @@ public AsynchronousSearchService(AsynchronousSearchPersistenceService asynchrono this.currentTimeSupplier = System::currentTimeMillis; this.asynchronousSearchActiveStore = asynchronousSearchActiveStore; this.asynchronousSearchStateMachine = initStateMachine(); - this.asynchronousSearchPostProcessor = new AsynchronousSearchPostProcessor(persistenceService, asynchronousSearchActiveStore, - asynchronousSearchStateMachine, this::freeActiveContext, threadPool, clusterService); + this.asynchronousSearchPostProcessor = new AsynchronousSearchPostProcessor( + persistenceService, + asynchronousSearchActiveStore, + asynchronousSearchStateMachine, + this::freeActiveContext, + threadPool, + clusterService + ); this.namedWriteableRegistry = namedWriteableRegistry; } @@ -178,23 +199,42 @@ private void setKeepAlive(TimeValue maxKeepAlive) { * @param reduceContextBuilder the reference for the reduceContextBuilder * @return the AsynchronousSearchContext for the submitted request */ - public AsynchronousSearchContext createAndStoreContext(SubmitAsynchronousSearchRequest request, long relativeStartTimeMillis, - Supplier reduceContextBuilder, User user) { + public AsynchronousSearchContext createAndStoreContext( + SubmitAsynchronousSearchRequest request, + long relativeStartTimeMillis, + Supplier reduceContextBuilder, + User user + ) { validateRequest(request); - AsynchronousSearchContextId asynchronousSearchContextId = new AsynchronousSearchContextId(UUIDs.base64UUID(), - idGenerator.incrementAndGet()); + AsynchronousSearchContextId asynchronousSearchContextId = new AsynchronousSearchContextId( + UUIDs.base64UUID(), + idGenerator.incrementAndGet() + ); contextEventListener.onNewContext(asynchronousSearchContextId); - AsynchronousSearchProgressListener progressActionListener = new AsynchronousSearchProgressListener(relativeStartTimeMillis, - (response) -> asynchronousSearchPostProcessor.processSearchResponse(response, asynchronousSearchContextId), - (e) -> asynchronousSearchPostProcessor.processSearchFailure(e, asynchronousSearchContextId), - threadPool.executor(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME), - threadPool::relativeTimeInMillis, - reduceContextBuilder); - AsynchronousSearchActiveContext asynchronousSearchContext = new AsynchronousSearchActiveContext(asynchronousSearchContextId, - clusterService.localNode().getId(), request.getKeepAlive(), request.getKeepOnCompletion(), threadPool, currentTimeSupplier, - progressActionListener, user, () -> persistSearchFailure); - asynchronousSearchActiveStore.putContext(asynchronousSearchContextId, asynchronousSearchContext, - contextEventListener::onContextRejected); + AsynchronousSearchProgressListener progressActionListener = new AsynchronousSearchProgressListener( + relativeStartTimeMillis, + (response) -> asynchronousSearchPostProcessor.processSearchResponse(response, asynchronousSearchContextId), + (e) -> asynchronousSearchPostProcessor.processSearchFailure(e, asynchronousSearchContextId), + threadPool.executor(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME), + threadPool::relativeTimeInMillis, + reduceContextBuilder + ); + AsynchronousSearchActiveContext asynchronousSearchContext = new AsynchronousSearchActiveContext( + asynchronousSearchContextId, + clusterService.localNode().getId(), + request.getKeepAlive(), + request.getKeepOnCompletion(), + threadPool, + currentTimeSupplier, + progressActionListener, + user, + () -> persistSearchFailure + ); + asynchronousSearchActiveStore.putContext( + asynchronousSearchContextId, + asynchronousSearchContext, + contextEventListener::onContextRejected + ); contextEventListener.onContextInitialized(asynchronousSearchContextId); return asynchronousSearchContext; } @@ -207,15 +247,22 @@ public AsynchronousSearchContext createAndStoreContext(SubmitAsynchronousSearchR * @param asynchronousSearchContextId the id of the active asyncsearch context */ public void bootstrapSearch(SearchTask searchTask, AsynchronousSearchContextId asynchronousSearchContextId) { - Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore - .getContext(asynchronousSearchContextId); + Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore.getContext( + asynchronousSearchContextId + ); if (asynchronousSearchContextOptional.isPresent()) { AsynchronousSearchActiveContext context = asynchronousSearchContextOptional.get(); try { asynchronousSearchStateMachine.trigger(new SearchStartedEvent(context, searchTask)); } catch (AsynchronousSearchStateMachineClosedException e) { - throw new IllegalStateException(String.format(Locale.ROOT, "Unexpected! State machine already closed for " + - "context [%s] while triggering event [%s]", context.getAsynchronousSearchId(), SearchStartedEvent.class.getName())); + throw new IllegalStateException( + String.format( + Locale.ROOT, + "Unexpected! State machine already closed for " + "context [%s] while triggering event [%s]", + context.getAsynchronousSearchId(), + SearchStartedEvent.class.getName() + ) + ); } } } @@ -231,12 +278,17 @@ public void bootstrapSearch(SearchTask searchTask, AsynchronousSearchContextId a * @param user current user * @param listener to be invoked on finding an {@linkplain AsynchronousSearchContext} */ - public void findContext(String id, AsynchronousSearchContextId asynchronousSearchContextId, User user, - ActionListener listener) { + public void findContext( + String id, + AsynchronousSearchContextId asynchronousSearchContextId, + User user, + ActionListener listener + ) { ActionListener exceptionTranslationListener = getExceptionTranslationWrapper(id, listener); - Optional optionalAsynchronousSearchActiveContext = asynchronousSearchActiveStore - .getContext(asynchronousSearchContextId); + Optional optionalAsynchronousSearchActiveContext = asynchronousSearchActiveStore.getContext( + asynchronousSearchContextId + ); // If context is CLOSED we can't acquire permits and hence can't update active context // so most likely a CLOSED context is stale if (optionalAsynchronousSearchActiveContext.isPresent() && optionalAsynchronousSearchActiveContext.get().isAlive()) { @@ -244,22 +296,36 @@ public void findContext(String id, AsynchronousSearchContextId asynchronousSearc AsynchronousSearchActiveContext asynchronousSearchActiveContext = optionalAsynchronousSearchActiveContext.get(); if (isUserValid(user, asynchronousSearchActiveContext.getUser()) == false) { logger.debug("Invalid user requesting GET active context for asynchronous search id {}", id); - exceptionTranslationListener.onFailure(new OpenSearchSecurityException( - "User doesn't have necessary roles to access the asynchronous search with id " + id, RestStatus.FORBIDDEN)); + exceptionTranslationListener.onFailure( + new OpenSearchSecurityException( + "User doesn't have necessary roles to access the asynchronous search with id " + id, + RestStatus.FORBIDDEN + ) + ); } else { exceptionTranslationListener.onResponse(asynchronousSearchActiveContext); } } else { logger.debug("Active context is not present for asynchronous search ID [{}]", id); - persistenceService.getResponse(id, user, wrap( - (persistenceModel) -> - exceptionTranslationListener.onResponse(new AsynchronousSearchPersistenceContext(id, - asynchronousSearchContextId, persistenceModel, currentTimeSupplier, namedWriteableRegistry)), + persistenceService.getResponse( + id, + user, + wrap( + (persistenceModel) -> exceptionTranslationListener.onResponse( + new AsynchronousSearchPersistenceContext( + id, + asynchronousSearchContextId, + persistenceModel, + currentTimeSupplier, + namedWriteableRegistry + ) + ), ex -> { logger.debug(() -> new ParameterizedMessage("Context not found for ID in the system index {}", id), ex); exceptionTranslationListener.onFailure(ex); } - )); + ) + ); } } @@ -269,14 +335,17 @@ public Map getAllActiveContexts() { public Set getContextsToReap() { Map allContexts = asynchronousSearchActiveStore.getAllContexts(); - return Collections.unmodifiableSet(allContexts.values().stream() + return Collections.unmodifiableSet( + allContexts.values() + .stream() .filter(Objects::nonNull) - .filter((c) -> EnumSet.of(CLOSED, PERSIST_FAILED).contains(c.getAsynchronousSearchState()) || - isOverRunning(c) || c.isExpired()) - .collect(Collectors.toSet())); + .filter( + (c) -> EnumSet.of(CLOSED, PERSIST_FAILED).contains(c.getAsynchronousSearchState()) || isOverRunning(c) || c.isExpired() + ) + .collect(Collectors.toSet()) + ); } - /** * Attempts to find both an {@linkplain AsynchronousSearchActiveContext} and an {@linkplain AsynchronousSearchPersistenceContext} * and delete them. If at least one of the aforementioned objects are found and deleted successfully, the listener is invoked with @@ -287,130 +356,167 @@ public Set getContextsToReap() { * @param user current user * @param listener listener to invoke on deletion or failure to do so */ - public void freeContext(String id, AsynchronousSearchContextId asynchronousSearchContextId, User user, - ActionListener listener) { + public void freeContext( + String id, + AsynchronousSearchContextId asynchronousSearchContextId, + User user, + ActionListener listener + ) { ActionListener exceptionTranslationWrapper = getExceptionTranslationWrapper(id, listener); - Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore - .getContext(asynchronousSearchContextId); + Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore.getContext( + asynchronousSearchContextId + ); if (asynchronousSearchContextOptional.isPresent()) { logger.debug("Active context present for asynchronous search id [{}]", id); AsynchronousSearchActiveContext asynchronousSearchContext = asynchronousSearchContextOptional.get(); if (isUserValid(user, asynchronousSearchContext.getUser())) { cancelAndFreeActiveAndPersistedContext(asynchronousSearchContext, exceptionTranslationWrapper, user); } else { - exceptionTranslationWrapper.onFailure(new OpenSearchSecurityException( - "User doesn't have necessary roles to access the asynchronous search with id " + id, RestStatus.FORBIDDEN)); + exceptionTranslationWrapper.onFailure( + new OpenSearchSecurityException( + "User doesn't have necessary roles to access the asynchronous search with id " + id, + RestStatus.FORBIDDEN + ) + ); } } else { logger.debug("Active context NOT present for asynchronous search [{}]", id); - //asynchronous search context didn't exist so obviously we didn't delete - //deleted persisted context if one exists. If not the listener returns acknowledged as false - //we don't need to acquire lock if the in-memory context doesn't exist. For persistence context we have a distributed view - //with the last writer wins policy + // asynchronous search context didn't exist so obviously we didn't delete + // deleted persisted context if one exists. If not the listener returns acknowledged as false + // we don't need to acquire lock if the in-memory context doesn't exist. For persistence context we have a distributed view + // with the last writer wins policy logger.debug("Deleting asynchronous search [{}] from system index ", id); persistenceService.deleteResponse(id, user, exceptionTranslationWrapper); } } - private void cancelTask(AsynchronousSearchActiveContext asynchronousSearchContext, String reason, - ActionListener listener) { - CancelTasksRequest cancelTasksRequest = new CancelTasksRequest() - .setTaskId(new TaskId(clusterService.localNode().getId(), asynchronousSearchContext.getTask().getId())) - .setReason(reason); + private void cancelTask( + AsynchronousSearchActiveContext asynchronousSearchContext, + String reason, + ActionListener listener + ) { + CancelTasksRequest cancelTasksRequest = new CancelTasksRequest().setTaskId( + new TaskId(clusterService.localNode().getId(), asynchronousSearchContext.getTask().getId()) + ).setReason(reason); client.admin().cluster().cancelTasks(cancelTasksRequest, listener); } + private boolean shouldCancel(AsynchronousSearchActiveContext asynchronousSearchContext) { - return asynchronousSearchContext.getTask() != null && asynchronousSearchContext.getTask().isCancelled() == false - && asynchronousSearchContext.isCompleted() == false; + return asynchronousSearchContext.getTask() != null + && asynchronousSearchContext.getTask().isCancelled() == false + && asynchronousSearchContext.isCompleted() == false; } // We are skipping user check in this while deleting from the persisted layer // as we have already checked for user in the present active context. - private void cancelAndFreeActiveAndPersistedContext(AsynchronousSearchActiveContext asynchronousSearchContext, - ActionListener listener, User user) { + private void cancelAndFreeActiveAndPersistedContext( + AsynchronousSearchActiveContext asynchronousSearchContext, + ActionListener listener, + User user + ) { // if there are no context found to be cleaned up we throw a ResourceNotFoundException AtomicReference releasableReference = new AtomicReference<>(() -> {}); ActionListener releasableListener = runAfter(listener, releasableReference.get()::close); - GroupedActionListener groupedDeletionListener = new GroupedActionListener<>( - wrap((responses) -> { - if (responses.stream().anyMatch(r -> r)) { - logger.debug("Free context for asynchronous search [{}] successful ", - asynchronousSearchContext.getAsynchronousSearchId()); - releasableListener.onResponse(true); - } else { - logger.debug("Freeing context, asynchronous search [{}] not found ", - asynchronousSearchContext.getAsynchronousSearchId()); - releasableListener.onFailure(new ResourceNotFoundException(asynchronousSearchContext.getAsynchronousSearchId())); - } - }, releasableListener::onFailure), 2); - - //We get a true or a ResourceNotFound from persistence layer. We want to translate it to either a true/false or any other exception - //that should be surfaced up - ActionListener translatedListener = wrap( - groupedDeletionListener::onResponse, (ex) -> { - if (ex instanceof ResourceNotFoundException) { - groupedDeletionListener.onResponse(false); - } else { - logger.debug(() -> new ParameterizedMessage("Translating exception, received for asynchronous search [{}]", - asynchronousSearchContext.getAsynchronousSearchId()), ex); - groupedDeletionListener.onFailure(ex); - } - }); + GroupedActionListener groupedDeletionListener = new GroupedActionListener<>(wrap((responses) -> { + if (responses.stream().anyMatch(r -> r)) { + logger.debug("Free context for asynchronous search [{}] successful ", asynchronousSearchContext.getAsynchronousSearchId()); + releasableListener.onResponse(true); + } else { + logger.debug("Freeing context, asynchronous search [{}] not found ", asynchronousSearchContext.getAsynchronousSearchId()); + releasableListener.onFailure(new ResourceNotFoundException(asynchronousSearchContext.getAsynchronousSearchId())); + } + }, releasableListener::onFailure), 2); + + // We get a true or a ResourceNotFound from persistence layer. We want to translate it to either a true/false or any other exception + // that should be surfaced up + ActionListener translatedListener = wrap(groupedDeletionListener::onResponse, (ex) -> { + if (ex instanceof ResourceNotFoundException) { + groupedDeletionListener.onResponse(false); + } else { + logger.debug( + () -> new ParameterizedMessage( + "Translating exception, received for asynchronous search [{}]", + asynchronousSearchContext.getAsynchronousSearchId() + ), + ex + ); + groupedDeletionListener.onFailure(ex); + } + }); String triggeredBy = user != null ? (" by user [" + user + "]") : ""; - String cancelTaskReason = "Delete asynchronous search [" + asynchronousSearchContext.getAsynchronousSearchId() - + "] has been triggered" + triggeredBy + ". Attempting to cancel in-progress search task"; - //Intent of the lock here is to disallow ongoing migration to system index + String cancelTaskReason = "Delete asynchronous search [" + + asynchronousSearchContext.getAsynchronousSearchId() + + "] has been triggered" + + triggeredBy + + ". Attempting to cancel in-progress search task"; + // Intent of the lock here is to disallow ongoing migration to system index // as if that is underway we might end up creating a new document post a DELETE was executed - asynchronousSearchContext.acquireContextPermitIfRequired(wrap( - releasable -> { - releasableReference.set(releasable); - if (asynchronousSearchContext.keepOnCompletion()) { - handleCancelTaskPermitAcquired(asynchronousSearchContext, groupedDeletionListener, cancelTaskReason); - logger.debug("Deleting asynchronous search id [{}] from system index ", - asynchronousSearchContext.getAsynchronousSearchId()); - persistenceService.deleteResponse(asynchronousSearchContext.getAsynchronousSearchId(), user, translatedListener); - } else { //keep on completion is false. simply cancel task and clean up active context - handleCancelTaskPermitAcquired(asynchronousSearchContext, wrap(r -> { - if (r) { - releasableListener.onResponse(true); - } else { - releasableListener.onFailure( - new ResourceNotFoundException(asynchronousSearchContext.getAsynchronousSearchId())); - } - }, releasableListener::onFailure), cancelTaskReason); - } - }, exception -> { - Throwable cause = ExceptionsHelper.unwrapCause(exception); - if (cause instanceof TimeoutException) { - // this should ideally not happen. This would mean we couldn't acquire permits within the timeout - logger.debug(() -> new ParameterizedMessage("Failed to acquire permits for " + - "asynchronous search id [{}] for updating context within timeout 5s", - asynchronousSearchContext.getAsynchronousSearchId()), exception); - listener.onFailure(new OpenSearchTimeoutException(asynchronousSearchContext.getAsynchronousSearchId())); + asynchronousSearchContext.acquireContextPermitIfRequired(wrap(releasable -> { + releasableReference.set(releasable); + if (asynchronousSearchContext.keepOnCompletion()) { + handleCancelTaskPermitAcquired(asynchronousSearchContext, groupedDeletionListener, cancelTaskReason); + logger.debug( + "Deleting asynchronous search id [{}] from system index ", + asynchronousSearchContext.getAsynchronousSearchId() + ); + persistenceService.deleteResponse(asynchronousSearchContext.getAsynchronousSearchId(), user, translatedListener); + } else { // keep on completion is false. simply cancel task and clean up active context + handleCancelTaskPermitAcquired(asynchronousSearchContext, wrap(r -> { + if (r) { + releasableListener.onResponse(true); } else { - // best effort clean up with acknowledged as false - if (asynchronousSearchContext.keepOnCompletion()) { - - handleCancelTaskPermitAcquisitionFailed(asynchronousSearchContext, groupedDeletionListener, cancelTaskReason, - exception); - logger.debug("Deleting asynchronous search id [{}] from system index ", - asynchronousSearchContext.getAsynchronousSearchId()); - persistenceService.deleteResponse(asynchronousSearchContext.getAsynchronousSearchId(), - user, translatedListener); - } else { - handleCancelTaskPermitAcquisitionFailed(asynchronousSearchContext, releasableListener, cancelTaskReason, - exception); - } + releasableListener.onFailure(new ResourceNotFoundException(asynchronousSearchContext.getAsynchronousSearchId())); } + }, releasableListener::onFailure), cancelTaskReason); + } + }, exception -> { + Throwable cause = ExceptionsHelper.unwrapCause(exception); + if (cause instanceof TimeoutException) { + // this should ideally not happen. This would mean we couldn't acquire permits within the timeout + logger.debug( + () -> new ParameterizedMessage( + "Failed to acquire permits for " + "asynchronous search id [{}] for updating context within timeout 5s", + asynchronousSearchContext.getAsynchronousSearchId() + ), + exception + ); + listener.onFailure(new OpenSearchTimeoutException(asynchronousSearchContext.getAsynchronousSearchId())); + } else { + // best effort clean up with acknowledged as false + if (asynchronousSearchContext.keepOnCompletion()) { + + handleCancelTaskPermitAcquisitionFailed( + asynchronousSearchContext, + groupedDeletionListener, + cancelTaskReason, + exception + ); + logger.debug( + "Deleting asynchronous search id [{}] from system index ", + asynchronousSearchContext.getAsynchronousSearchId() + ); + persistenceService.deleteResponse(asynchronousSearchContext.getAsynchronousSearchId(), user, translatedListener); + } else { + handleCancelTaskPermitAcquisitionFailed(asynchronousSearchContext, releasableListener, cancelTaskReason, exception); } - ), TimeValue.timeValueSeconds(5), "free context"); - } - - private void handleCancelTaskPermitAcquisitionFailed(AsynchronousSearchActiveContext asynchronousSearchContext, - ActionListener listener, String cancelTaskReason, Exception e) { - logger.debug(() -> new ParameterizedMessage("Failed to acquire permits for asynchronous search id " + - "[{}] for freeing context", asynchronousSearchContext.getAsynchronousSearchId()), e); + } + }), TimeValue.timeValueSeconds(5), "free context"); + } + + private void handleCancelTaskPermitAcquisitionFailed( + AsynchronousSearchActiveContext asynchronousSearchContext, + ActionListener listener, + String cancelTaskReason, + Exception e + ) { + logger.debug( + () -> new ParameterizedMessage( + "Failed to acquire permits for asynchronous search id " + "[{}] for freeing context", + asynchronousSearchContext.getAsynchronousSearchId() + ), + e + ); if (shouldCancel(asynchronousSearchContext)) { cancelTask(asynchronousSearchContext, cancelTaskReason, wrap(() -> listener.onResponse(false))); } else { @@ -418,25 +524,34 @@ private void handleCancelTaskPermitAcquisitionFailed(AsynchronousSearchActiveCon } } - private void handleCancelTaskPermitAcquired(AsynchronousSearchActiveContext asynchronousSearchContext, - ActionListener listener, String cancelTaskReason) { + private void handleCancelTaskPermitAcquired( + AsynchronousSearchActiveContext asynchronousSearchContext, + ActionListener listener, + String cancelTaskReason + ) { if (shouldCancel(asynchronousSearchContext)) { cancelTask(asynchronousSearchContext, cancelTaskReason, wrap(cancelTasksResponse -> { - logger.debug("Successfully cancelled tasks [{}] with asynchronous search [{}] with response [{}]", - asynchronousSearchContext.getTask(), asynchronousSearchContext.getAsynchronousSearchId(), - cancelTasksResponse); - /* we don't free active context here as AsynchronousSearchTask#onCancelled() takes care of that. This ensures that - freeActiveContext() is invoked only after task is cancelled or completed */ - listener.onResponse(true); - }, - e -> { - logger.error(() -> new ParameterizedMessage( - "Failed to cancel task [{}] with asynchronous search [{}] with exception", - asynchronousSearchContext.getTask(), asynchronousSearchContext.getAsynchronousSearchId()), - e); - //no onCancelled hook due to failure, so we invoke free active context - listener.onResponse(freeActiveContext(asynchronousSearchContext)); - })); + logger.debug( + "Successfully cancelled tasks [{}] with asynchronous search [{}] with response [{}]", + asynchronousSearchContext.getTask(), + asynchronousSearchContext.getAsynchronousSearchId(), + cancelTasksResponse + ); + /* we don't free active context here as AsynchronousSearchTask#onCancelled() takes care of that. This ensures that + freeActiveContext() is invoked only after task is cancelled or completed */ + listener.onResponse(true); + }, e -> { + logger.error( + () -> new ParameterizedMessage( + "Failed to cancel task [{}] with asynchronous search [{}] with exception", + asynchronousSearchContext.getTask(), + asynchronousSearchContext.getAsynchronousSearchId() + ), + e + ); + // no onCancelled hook due to failure, so we invoke free active context + listener.onResponse(freeActiveContext(asynchronousSearchContext)); + })); } else { listener.onResponse(freeActiveContext(asynchronousSearchContext)); } @@ -452,9 +567,10 @@ private void handleCancelTaskPermitAcquired(AsynchronousSearchActiveContext asyn boolean freeActiveContext(AsynchronousSearchActiveContext asynchronousSearchContext) { try { // asserts that task is cancelled/completed/removed so that we don't leave orphan tasks - assert asynchronousSearchContext.getTask() == null || asynchronousSearchContext.getTask().isCancelled() || - asynchronousSearchContext.isCompleted() : "Either the asynchronous search task should have been cancelled or " + - "completed "; + assert asynchronousSearchContext.getTask() == null + || asynchronousSearchContext.getTask().isCancelled() + || asynchronousSearchContext.isCompleted() : "Either the asynchronous search task should have been cancelled or " + + "completed "; asynchronousSearchStateMachine.trigger(new SearchDeletedEvent(asynchronousSearchContext)); return true; } catch (AsynchronousSearchStateMachineClosedException ex) { @@ -486,79 +602,137 @@ public boolean onCancelledFreeActiveContext(AsynchronousSearchActiveContext asyn * @param user current user * @param listener listener to invoke after updating expiration. */ - public void updateKeepAliveAndGetContext(String id, TimeValue keepAlive, AsynchronousSearchContextId asynchronousSearchContextId, - User user, ActionListener listener) { + public void updateKeepAliveAndGetContext( + String id, + TimeValue keepAlive, + AsynchronousSearchContextId asynchronousSearchContextId, + User user, + ActionListener listener + ) { ActionListener exceptionTranslationWrapper = getExceptionTranslationWrapper(id, listener); validateKeepAlive(keepAlive); long requestedExpirationTime = currentTimeSupplier.getAsLong() + keepAlive.getMillis(); // find an active context on this node if one exists - Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore - .getContext(asynchronousSearchContextId); + Optional asynchronousSearchContextOptional = asynchronousSearchActiveStore.getContext( + asynchronousSearchContextId + ); // for all other stages we don't really care much as those contexts are destined to be discarded if (asynchronousSearchContextOptional.isPresent()) { AsynchronousSearchActiveContext asynchronousSearchActiveContext = asynchronousSearchContextOptional.get(); - asynchronousSearchActiveContext.acquireContextPermitIfRequired(wrap( - releasable -> { - ActionListener releasableActionListener = runAfter(exceptionTranslationWrapper, - releasable::close); - // At this point it's possible that the response would have been persisted to system index - if (asynchronousSearchActiveContext.isAlive() == false && asynchronousSearchActiveContext.keepOnCompletion()) { - logger.debug("Updating persistence store after state is PERSISTED asynchronous search id [{}] " + - "for updating context", asynchronousSearchActiveContext.getAsynchronousSearchId()); - persistenceService.updateExpirationTime(id, requestedExpirationTime, user, wrap( - (actionResponse) -> - releasableActionListener.onResponse(new AsynchronousSearchPersistenceContext(id, - asynchronousSearchContextId, - actionResponse, currentTimeSupplier, namedWriteableRegistry)), - releasableActionListener::onFailure)); - } else { - if (isUserValid(user, asynchronousSearchActiveContext.getUser())) { - logger.debug("Updating persistence store: NO as state is NOT PERSISTED yet asynchronous search id [{}] " + - "for updating context", asynchronousSearchActiveContext.getAsynchronousSearchId()); - asynchronousSearchActiveContext.setExpirationTimeMillis(requestedExpirationTime); - releasableActionListener.onResponse(asynchronousSearchActiveContext); - } else { - releasableActionListener.onFailure( - new OpenSearchSecurityException("User doesn't have necessary roles to access the " + - "asynchronous search with id " + id, RestStatus.FORBIDDEN)); - } - } - }, - exception -> { - Throwable cause = ExceptionsHelper.unwrapCause(exception); - if (cause instanceof TimeoutException) { - // this should ideally not happen. This would mean we couldn't acquire permits within the timeout - logger.debug(() -> new ParameterizedMessage("Failed to acquire permits for " + - "asynchronous search id [{}] for updating context within timeout 5s", - asynchronousSearchActiveContext.getAsynchronousSearchId()), exception); - listener.onFailure(new OpenSearchTimeoutException(id)); - } else { - // best effort we try an update the doc if one exists - if (asynchronousSearchActiveContext.keepOnCompletion()) { - logger.debug( - "Updating persistence store after failing to acquire permits for asynchronous search id [{}] for " + - "updating context with expiration time [{}]", asynchronousSearchActiveContext - .getAsynchronousSearchId(), - requestedExpirationTime); - persistenceService.updateExpirationTime(id, requestedExpirationTime, user, - wrap((actionResponse) -> exceptionTranslationWrapper.onResponse( - new AsynchronousSearchPersistenceContext(id, asynchronousSearchContextId, - actionResponse, currentTimeSupplier, namedWriteableRegistry)), - exceptionTranslationWrapper::onFailure)); - } else { - exceptionTranslationWrapper.onFailure(new ResourceNotFoundException( - asynchronousSearchActiveContext.getAsynchronousSearchId())); - } - } - }), TimeValue.timeValueSeconds(5), "update keep alive"); + asynchronousSearchActiveContext.acquireContextPermitIfRequired(wrap(releasable -> { + ActionListener releasableActionListener = runAfter( + exceptionTranslationWrapper, + releasable::close + ); + // At this point it's possible that the response would have been persisted to system index + if (asynchronousSearchActiveContext.isAlive() == false && asynchronousSearchActiveContext.keepOnCompletion()) { + logger.debug( + "Updating persistence store after state is PERSISTED asynchronous search id [{}] " + "for updating context", + asynchronousSearchActiveContext.getAsynchronousSearchId() + ); + persistenceService.updateExpirationTime( + id, + requestedExpirationTime, + user, + wrap( + (actionResponse) -> releasableActionListener.onResponse( + new AsynchronousSearchPersistenceContext( + id, + asynchronousSearchContextId, + actionResponse, + currentTimeSupplier, + namedWriteableRegistry + ) + ), + releasableActionListener::onFailure + ) + ); + } else { + if (isUserValid(user, asynchronousSearchActiveContext.getUser())) { + logger.debug( + "Updating persistence store: NO as state is NOT PERSISTED yet asynchronous search id [{}] " + + "for updating context", + asynchronousSearchActiveContext.getAsynchronousSearchId() + ); + asynchronousSearchActiveContext.setExpirationTimeMillis(requestedExpirationTime); + releasableActionListener.onResponse(asynchronousSearchActiveContext); + } else { + releasableActionListener.onFailure( + new OpenSearchSecurityException( + "User doesn't have necessary roles to access the " + "asynchronous search with id " + id, + RestStatus.FORBIDDEN + ) + ); + } + } + }, exception -> { + Throwable cause = ExceptionsHelper.unwrapCause(exception); + if (cause instanceof TimeoutException) { + // this should ideally not happen. This would mean we couldn't acquire permits within the timeout + logger.debug( + () -> new ParameterizedMessage( + "Failed to acquire permits for " + "asynchronous search id [{}] for updating context within timeout 5s", + asynchronousSearchActiveContext.getAsynchronousSearchId() + ), + exception + ); + listener.onFailure(new OpenSearchTimeoutException(id)); + } else { + // best effort we try an update the doc if one exists + if (asynchronousSearchActiveContext.keepOnCompletion()) { + logger.debug( + "Updating persistence store after failing to acquire permits for asynchronous search id [{}] for " + + "updating context with expiration time [{}]", + asynchronousSearchActiveContext.getAsynchronousSearchId(), + requestedExpirationTime + ); + persistenceService.updateExpirationTime( + id, + requestedExpirationTime, + user, + wrap( + (actionResponse) -> exceptionTranslationWrapper.onResponse( + new AsynchronousSearchPersistenceContext( + id, + asynchronousSearchContextId, + actionResponse, + currentTimeSupplier, + namedWriteableRegistry + ) + ), + exceptionTranslationWrapper::onFailure + ) + ); + } else { + exceptionTranslationWrapper.onFailure( + new ResourceNotFoundException(asynchronousSearchActiveContext.getAsynchronousSearchId()) + ); + } + } + }), TimeValue.timeValueSeconds(5), "update keep alive"); } else { // try update the doc on the index assuming there exists one. - logger.debug("Updating persistence store after active context evicted for asynchronous search id [{}] " + - "for updating context", id); - persistenceService.updateExpirationTime(id, requestedExpirationTime, user, - wrap((actionResponse) -> exceptionTranslationWrapper.onResponse(new AsynchronousSearchPersistenceContext( - id, asynchronousSearchContextId, actionResponse, currentTimeSupplier, namedWriteableRegistry)), - exceptionTranslationWrapper::onFailure)); + logger.debug( + "Updating persistence store after active context evicted for asynchronous search id [{}] " + "for updating context", + id + ); + persistenceService.updateExpirationTime( + id, + requestedExpirationTime, + user, + wrap( + (actionResponse) -> exceptionTranslationWrapper.onResponse( + new AsynchronousSearchPersistenceContext( + id, + asynchronousSearchContextId, + actionResponse, + currentTimeSupplier, + namedWriteableRegistry + ) + ), + exceptionTranslationWrapper::onFailure + ) + ); } } @@ -568,48 +742,109 @@ AsynchronousSearchStateMachine getStateMachine() { private AsynchronousSearchStateMachine initStateMachine() { AsynchronousSearchStateMachine stateMachine = new AsynchronousSearchStateMachine( - EnumSet.allOf(AsynchronousSearchState.class), INIT, contextEventListener); + EnumSet.allOf(AsynchronousSearchState.class), + INIT, + contextEventListener + ); stateMachine.markTerminalStates(EnumSet.of(CLOSED)); - stateMachine.registerTransition(new AsynchronousSearchTransition<>(INIT, RUNNING, + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + INIT, + RUNNING, (s, e) -> ((AsynchronousSearchActiveContext) e.asynchronousSearchContext()).setTask(e.getSearchTask()), - (contextId, listener) -> listener.onContextRunning(contextId), SearchStartedEvent.class)); - - stateMachine.registerTransition(new AsynchronousSearchTransition<>(RUNNING, SUCCEEDED, + (contextId, listener) -> listener.onContextRunning(contextId), + SearchStartedEvent.class + ) + ); + + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + RUNNING, + SUCCEEDED, (s, e) -> ((AsynchronousSearchActiveContext) e.asynchronousSearchContext()).processSearchResponse(e.getSearchResponse()), - (contextId, listener) -> listener.onContextCompleted(contextId), SearchSuccessfulEvent.class)); - - stateMachine.registerTransition(new AsynchronousSearchTransition<>(RUNNING, FAILED, + (contextId, listener) -> listener.onContextCompleted(contextId), + SearchSuccessfulEvent.class + ) + ); + + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + RUNNING, + FAILED, (s, e) -> ((AsynchronousSearchActiveContext) e.asynchronousSearchContext()).processSearchFailure(e.getException()), - (contextId, listener) -> listener.onContextFailed(contextId), SearchFailureEvent.class)); - - stateMachine.registerTransition(new AsynchronousSearchTransition<>(SUCCEEDED, PERSISTING, - (s, e) -> asynchronousSearchPostProcessor.persistResponse((AsynchronousSearchActiveContext) e.asynchronousSearchContext(), - e.getAsynchronousSearchPersistenceModel()), - (contextId, listener) -> {}, BeginPersistEvent.class)); - - stateMachine.registerTransition(new AsynchronousSearchTransition<>(FAILED, PERSISTING, - (s, e) -> asynchronousSearchPostProcessor.persistResponse((AsynchronousSearchActiveContext) e.asynchronousSearchContext(), - e.getAsynchronousSearchPersistenceModel()), - (contextId, listener) -> {}, BeginPersistEvent.class)); - - stateMachine.registerTransition(new AsynchronousSearchTransition<>(PERSISTING, PERSIST_SUCCEEDED, + (contextId, listener) -> listener.onContextFailed(contextId), + SearchFailureEvent.class + ) + ); + + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + SUCCEEDED, + PERSISTING, + (s, e) -> asynchronousSearchPostProcessor.persistResponse( + (AsynchronousSearchActiveContext) e.asynchronousSearchContext(), + e.getAsynchronousSearchPersistenceModel() + ), + (contextId, listener) -> {}, + BeginPersistEvent.class + ) + ); + + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + FAILED, + PERSISTING, + (s, e) -> asynchronousSearchPostProcessor.persistResponse( + (AsynchronousSearchActiveContext) e.asynchronousSearchContext(), + e.getAsynchronousSearchPersistenceModel() + ), + (contextId, listener) -> {}, + BeginPersistEvent.class + ) + ); + + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + PERSISTING, + PERSIST_SUCCEEDED, (s, e) -> {}, - (contextId, listener) -> listener.onContextPersisted(contextId), SearchResponsePersistedEvent.class)); - - stateMachine.registerTransition(new AsynchronousSearchTransition<>(PERSISTING, PERSIST_FAILED, + (contextId, listener) -> listener.onContextPersisted(contextId), + SearchResponsePersistedEvent.class + ) + ); + + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + PERSISTING, + PERSIST_FAILED, (s, e) -> {}, - (contextId, listener) -> listener.onContextPersistFailed(contextId), SearchResponsePersistFailedEvent.class)); - - stateMachine.registerTransition(new AsynchronousSearchTransition<>(RUNNING, CLOSED, + (contextId, listener) -> listener.onContextPersistFailed(contextId), + SearchResponsePersistFailedEvent.class + ) + ); + + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + RUNNING, + CLOSED, (s, e) -> asynchronousSearchActiveStore.freeContext(e.asynchronousSearchContext().getContextId()), - (contextId, listener) -> listener.onRunningContextDeleted(contextId), SearchDeletedEvent.class)); + (contextId, listener) -> listener.onRunningContextDeleted(contextId), + SearchDeletedEvent.class + ) + ); for (AsynchronousSearchState state : EnumSet.of(PERSISTING, PERSIST_SUCCEEDED, PERSIST_FAILED, SUCCEEDED, FAILED, INIT)) { - stateMachine.registerTransition(new AsynchronousSearchTransition<>(state, CLOSED, + stateMachine.registerTransition( + new AsynchronousSearchTransition<>( + state, + CLOSED, (s, e) -> asynchronousSearchActiveStore.freeContext(e.asynchronousSearchContext().getContextId()), - (contextId, listener) -> listener.onContextDeleted(contextId), SearchDeletedEvent.class)); + (contextId, listener) -> listener.onContextDeleted(contextId), + SearchDeletedEvent.class + ) + ); } return stateMachine; } @@ -657,19 +892,28 @@ private void validateRequest(SubmitAsynchronousSearchRequest request) { private void validateWaitForCompletionTimeout(TimeValue waitForCompletionTimeout) { if (waitForCompletionTimeout.getMillis() > maxWaitForCompletionTimeout) { throw new IllegalArgumentException( - "Wait for completion timeout for asynchronous search (" + waitForCompletionTimeout.getMillis() - + ") is too large. It must be less than (" + TimeValue.timeValueMillis(maxWaitForCompletionTimeout) - + ").This limit can be set by changing the [" + MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey() - + "] cluster level setting."); + "Wait for completion timeout for asynchronous search (" + + waitForCompletionTimeout.getMillis() + + ") is too large. It must be less than (" + + TimeValue.timeValueMillis(maxWaitForCompletionTimeout) + + ").This limit can be set by changing the [" + + MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey() + + "] cluster level setting." + ); } } private void validateKeepAlive(TimeValue keepAlive) { if (keepAlive.getMillis() > maxKeepAlive) { throw new IllegalArgumentException( - "Keep alive for asynchronous search (" + keepAlive.getMillis() + ") is too large. It must be less than (" + - TimeValue.timeValueMillis(maxKeepAlive) + ").This limit can be set by changing the [" - + MAX_KEEP_ALIVE_SETTING.getKey() + "] cluster level setting."); + "Keep alive for asynchronous search (" + + keepAlive.getMillis() + + ") is too large. It must be less than (" + + TimeValue.timeValueMillis(maxKeepAlive) + + ").This limit can be set by changing the [" + + MAX_KEEP_ALIVE_SETTING.getKey() + + "] cluster level setting." + ); } } @@ -678,8 +922,8 @@ private void validateKeepAlive(TimeValue keepAlive) { * @return Where the search has been running beyond the max search running time. */ private boolean isOverRunning(AsynchronousSearchActiveContext asynchronousSearchActiveContext) { - return EnumSet.of(RUNNING, INIT).contains(asynchronousSearchActiveContext.getAsynchronousSearchState()) && - asynchronousSearchActiveContext.getStartTimeMillis() + maxSearchRunningTime < threadPool.absoluteTimeInMillis(); + return EnumSet.of(RUNNING, INIT).contains(asynchronousSearchActiveContext.getAsynchronousSearchState()) + && asynchronousSearchActiveContext.getStartTimeMillis() + maxSearchRunningTime < threadPool.absoluteTimeInMillis(); } private ActionListener getExceptionTranslationWrapper(String id, ActionListener listener) { diff --git a/src/main/java/org/opensearch/search/asynchronous/settings/LegacyOpendistroAsynchronousSearchSettings.java b/src/main/java/org/opensearch/search/asynchronous/settings/LegacyOpendistroAsynchronousSearchSettings.java index 832ba65b..c3659522 100644 --- a/src/main/java/org/opensearch/search/asynchronous/settings/LegacyOpendistroAsynchronousSearchSettings.java +++ b/src/main/java/org/opensearch/search/asynchronous/settings/LegacyOpendistroAsynchronousSearchSettings.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.settings; import org.opensearch.common.settings.Setting; @@ -16,29 +19,56 @@ public class LegacyOpendistroAsynchronousSearchSettings { public static final Setting NODE_CONCURRENT_RUNNING_SEARCHES_SETTING = Setting.intSetting( - "opendistro.asynchronous_search.node_concurrent_running_searches", NODE_CONCURRENT_RUNNING_SEARCHES, 0, - Setting.Property.Dynamic, Setting.Property.NodeScope, Setting.Property.Deprecated); + "opendistro.asynchronous_search.node_concurrent_running_searches", + NODE_CONCURRENT_RUNNING_SEARCHES, + 0, + Setting.Property.Dynamic, + Setting.Property.NodeScope, + Setting.Property.Deprecated + ); - public static final Setting MAX_KEEP_ALIVE_SETTING = - Setting.positiveTimeSetting("opendistro.asynchronous_search.max_keep_alive", timeValueDays(5), - Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); - public static final Setting MAX_SEARCH_RUNNING_TIME_SETTING = - Setting.positiveTimeSetting("opendistro.asynchronous_search.max_search_running_time", timeValueHours(12), - Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); + public static final Setting MAX_KEEP_ALIVE_SETTING = Setting.positiveTimeSetting( + "opendistro.asynchronous_search.max_keep_alive", + timeValueDays(5), + Setting.Property.NodeScope, + Setting.Property.Dynamic, + Setting.Property.Deprecated + ); + public static final Setting MAX_SEARCH_RUNNING_TIME_SETTING = Setting.positiveTimeSetting( + "opendistro.asynchronous_search.max_search_running_time", + timeValueHours(12), + Setting.Property.NodeScope, + Setting.Property.Dynamic, + Setting.Property.Deprecated + ); public static final Setting MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING = Setting.positiveTimeSetting( - "opendistro.asynchronous_search.max_wait_for_completion_timeout", timeValueMinutes(1), Setting.Property.NodeScope, - Setting.Property.Dynamic, Setting.Property.Deprecated); + "opendistro.asynchronous_search.max_wait_for_completion_timeout", + timeValueMinutes(1), + Setting.Property.NodeScope, + Setting.Property.Dynamic, + Setting.Property.Deprecated + ); - public static final Setting PERSIST_SEARCH_FAILURES_SETTING = - Setting.boolSetting("opendistro.asynchronous_search.persist_search_failures", false, - Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated); + public static final Setting PERSIST_SEARCH_FAILURES_SETTING = Setting.boolSetting( + "opendistro.asynchronous_search.persist_search_failures", + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic, + Setting.Property.Deprecated + ); - public static final Setting ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING = - Setting.timeSetting("opendistro.asynchronous_search.active.context.reaper_interval", TimeValue.timeValueMinutes(5), - TimeValue.timeValueSeconds(5), - Setting.Property.NodeScope, Setting.Property.Deprecated); - public static final Setting PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING = - Setting.timeSetting("opendistro.asynchronous_search.expired.persisted_response.cleanup_interval", - TimeValue.timeValueMinutes(30), TimeValue.timeValueSeconds(5), - Setting.Property.NodeScope, Setting.Property.Deprecated); + public static final Setting ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING = Setting.timeSetting( + "opendistro.asynchronous_search.active.context.reaper_interval", + TimeValue.timeValueMinutes(5), + TimeValue.timeValueSeconds(5), + Setting.Property.NodeScope, + Setting.Property.Deprecated + ); + public static final Setting PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING = Setting.timeSetting( + "opendistro.asynchronous_search.expired.persisted_response.cleanup_interval", + TimeValue.timeValueMinutes(30), + TimeValue.timeValueSeconds(5), + Setting.Property.NodeScope, + Setting.Property.Deprecated + ); } diff --git a/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchCountStats.java b/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchCountStats.java index 55047cea..373f58ba 100644 --- a/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchCountStats.java +++ b/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchCountStats.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.stats; import org.opensearch.core.common.io.stream.StreamInput; @@ -28,8 +31,17 @@ public class AsynchronousSearchCountStats implements Writeable, ToXContentFragme private final long cancelledCount; private final long submittedCount; - public AsynchronousSearchCountStats(long runningCount, long persistedCount, long completedCount, long failedCount, long throttledCount, - long persistFailedCount, long initializedCount, long submittedCount, long cancelledCount) { + public AsynchronousSearchCountStats( + long runningCount, + long persistedCount, + long completedCount, + long failedCount, + long throttledCount, + long persistFailedCount, + long initializedCount, + long submittedCount, + long cancelledCount + ) { this.runningCount = runningCount; this.persistedCount = persistedCount; this.persistFailedCount = persistFailedCount; diff --git a/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchStats.java b/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchStats.java index a07a0180..2024f8cc 100644 --- a/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchStats.java +++ b/src/main/java/org/opensearch/search/asynchronous/stats/AsynchronousSearchStats.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.stats; import org.opensearch.action.support.nodes.BaseNodeResponse; diff --git a/src/main/java/org/opensearch/search/asynchronous/stats/InternalAsynchronousSearchStats.java b/src/main/java/org/opensearch/search/asynchronous/stats/InternalAsynchronousSearchStats.java index 23aed103..b57e6b54 100644 --- a/src/main/java/org/opensearch/search/asynchronous/stats/InternalAsynchronousSearchStats.java +++ b/src/main/java/org/opensearch/search/asynchronous/stats/InternalAsynchronousSearchStats.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.stats; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -40,7 +43,6 @@ public void onContextRejected(AsynchronousSearchContextId contextId) { countStatsHolder.rejectedAsynchronousSearchCount.inc(); } - @Override public void onNewContext(AsynchronousSearchContextId contextId) { countStatsHolder.submittedAsynchronousSearchCount.inc(); @@ -82,13 +84,18 @@ static final class CountStatsHolder { final CounterMetric cancelledAsynchronousSearchCount = new CounterMetric(); final CounterMetric initializedAsynchronousSearchCount = new CounterMetric(); - public AsynchronousSearchCountStats countStats() { - return new AsynchronousSearchCountStats(runningAsynchronousSearchCount.count(), persistedAsynchronousSearchCount.count(), - completedAsynchronousSearchCount.count(), failedAsynchronousSearchCount.count(), - rejectedAsynchronousSearchCount.count(), persistFailedAsynchronousSearchCount.count(), - initializedAsynchronousSearchCount.count(), submittedAsynchronousSearchCount.count(), - cancelledAsynchronousSearchCount.count()); + return new AsynchronousSearchCountStats( + runningAsynchronousSearchCount.count(), + persistedAsynchronousSearchCount.count(), + completedAsynchronousSearchCount.count(), + failedAsynchronousSearchCount.count(), + rejectedAsynchronousSearchCount.count(), + persistFailedAsynchronousSearchCount.count(), + initializedAsynchronousSearchCount.count(), + submittedAsynchronousSearchCount.count(), + cancelledAsynchronousSearchCount.count() + ); } } } diff --git a/src/main/java/org/opensearch/search/asynchronous/task/AsynchronousSearchTask.java b/src/main/java/org/opensearch/search/asynchronous/task/AsynchronousSearchTask.java index fd6d0dff..e2833fab 100644 --- a/src/main/java/org/opensearch/search/asynchronous/task/AsynchronousSearchTask.java +++ b/src/main/java/org/opensearch/search/asynchronous/task/AsynchronousSearchTask.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.task; import org.opensearch.search.asynchronous.context.active.AsynchronousSearchActiveContext; @@ -31,9 +34,16 @@ public class AsynchronousSearchTask extends SearchTask { public static final String NAME = "indices:data/read/opendistro/asynchronous_search"; - public AsynchronousSearchTask(long id, String type, String action, TaskId parentTaskId, Map headers, - AsynchronousSearchActiveContext asynchronousSearchContext, SubmitAsynchronousSearchRequest request, - Consumer freeActiveContextConsumer) { + public AsynchronousSearchTask( + long id, + String type, + String action, + TaskId parentTaskId, + Map headers, + AsynchronousSearchActiveContext asynchronousSearchContext, + SubmitAsynchronousSearchRequest request, + Consumer freeActiveContextConsumer + ) { super(id, type, action, () -> description(request), parentTaskId, headers); Objects.requireNonNull(asynchronousSearchContext); Objects.requireNonNull(freeActiveContextConsumer); @@ -44,8 +54,11 @@ public AsynchronousSearchTask(long id, String type, String action, TaskId parent @Override protected void onCancelled() { - logger.debug("On Cancelled event received for asynchronous search context [{}] due to [{}]", - asynchronousSearchActiveContext.getAsynchronousSearchId(), getReasonCancelled()); + logger.debug( + "On Cancelled event received for asynchronous search context [{}] due to [{}]", + asynchronousSearchActiveContext.getAsynchronousSearchId(), + getReasonCancelled() + ); freeActiveContextConsumer.accept(asynchronousSearchActiveContext); } @@ -58,8 +71,7 @@ private static String description(SubmitAsynchronousSearchRequest request) { sb.append("keep_on_completion[").append(request.getKeepOnCompletion()).append("], "); sb.append("keep_alive[").append(request.getKeepAlive()).append("], "); if (request.getSearchRequest().source() != null) { - sb.append("source[").append(request.getSearchRequest().source() - .toString(SearchRequest.FORMAT_PARAMS)).append("]"); + sb.append("source[").append(request.getSearchRequest().source().toString(SearchRequest.FORMAT_PARAMS)).append("]"); } else { sb.append("source[]"); } diff --git a/src/main/java/org/opensearch/search/asynchronous/task/SubmitAsynchronousSearchTask.java b/src/main/java/org/opensearch/search/asynchronous/task/SubmitAsynchronousSearchTask.java index 57daf100..f513691c 100644 --- a/src/main/java/org/opensearch/search/asynchronous/task/SubmitAsynchronousSearchTask.java +++ b/src/main/java/org/opensearch/search/asynchronous/task/SubmitAsynchronousSearchTask.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.task; import org.opensearch.search.asynchronous.request.SubmitAsynchronousSearchRequest; @@ -17,7 +20,13 @@ public class SubmitAsynchronousSearchTask extends CancellableTask { public SubmitAsynchronousSearchTask( - long id, String type, String action, String description, TaskId parentTaskId, Map headers) { + long id, + String type, + String action, + String description, + TaskId parentTaskId, + Map headers + ) { super(id, type, action, description, parentTaskId, headers); } @@ -27,4 +36,3 @@ public boolean shouldCancelChildrenOnCancellation() { } } - diff --git a/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchRoutingAction.java b/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchRoutingAction.java index 3aa10379..12f71f52 100644 --- a/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchRoutingAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchRoutingAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.transport; import org.opensearch.commons.ConfigConstants; @@ -42,8 +45,9 @@ * {@link TransportSubmitAsynchronousSearchAction}. The class forwards the request to the coordinator and executes the * {@link TransportGetAsynchronousSearchAction} or the {@link TransportDeleteAsynchronousSearchAction} */ -public abstract class TransportAsynchronousSearchRoutingAction, - Response extends ActionResponse> extends HandledTransportAction { +public abstract class TransportAsynchronousSearchRoutingAction< + Request extends AsynchronousSearchRoutingRequest, + Response extends ActionResponse> extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportAsynchronousSearchRoutingAction.class); @@ -55,10 +59,17 @@ public abstract class TransportAsynchronousSearchRoutingAction requestReader, - Writeable.Reader responseReader) { + public TransportAsynchronousSearchRoutingAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + String actionName, + ActionFilters actionFilters, + AsynchronousSearchService asynchronousSearchService, + Writeable.Reader requestReader, + Writeable.Reader responseReader + ) { super(actionName, transportService, actionFilters, requestReader); this.transportService = transportService; this.clusterService = clusterService; @@ -78,8 +89,12 @@ protected void doExecute(Task task, Request request, ActionListener li } } - public abstract void handleRequest(AsynchronousSearchId asynchronousSearchId, Request request, ActionListener listener, - User user); + public abstract void handleRequest( + AsynchronousSearchId asynchronousSearchId, + Request request, + ActionListener listener, + User user + ); final class AsyncForwardAction extends AbstractRunnable { @@ -104,8 +119,14 @@ final class AsyncForwardAction extends AbstractRunnable { @Override public void onFailure(Exception e) { - logger.error(() -> new ParameterizedMessage( - "Failed to dispatch request for action [{}] for asynchronous search [{}]", actionName, request.getId()), e); + logger.error( + () -> new ParameterizedMessage( + "Failed to dispatch request for action [{}] for asynchronous search [{}]", + actionName, + request.getId() + ), + e + ); sendLocalRequest(asynchronousSearchId, request, listener); } @@ -114,38 +135,55 @@ protected void doRun() { ClusterState state = clusterService.state(); // forward request only if the local node isn't the node coordinating the search and the node coordinating // the search exists in the cluster - TransportRequestOptions requestOptions = TransportRequestOptions.builder().withTimeout( - asynchronousSearchService.getMaxWaitForCompletionTimeout()).build(); + TransportRequestOptions requestOptions = TransportRequestOptions.builder() + .withTimeout(asynchronousSearchService.getMaxWaitForCompletionTimeout()) + .build(); if (targetNode != null && state.nodes().getLocalNode().equals(targetNode) == false && state.nodes().nodeExists(targetNode)) { logger.debug("Forwarding asynchronous search id [{}] request to target node [{}]", request.getId(), targetNode); - transportService.sendRequest(targetNode, actionName, request, requestOptions, - new ActionListenerResponseHandler(listener, responseReader) { - @Override - public void handleException(final TransportException exp) { - Throwable cause = exp.unwrapCause(); - if (cause instanceof ConnectTransportException || - (exp instanceof RemoteTransportException && cause instanceof NodeClosedException)) { - // we want to retry here a bit to see if the node connects backs - logger.debug("Connection exception while trying to forward request with id[{}] to " + - "target node [{}] Error: [{}]", - request.getId(), targetNode, exp.getDetailedMessage()); - //try on local node since we weren't able to forward - sendLocalRequest(asynchronousSearchId, request, listener); - } else { - logger.debug("Exception received for request with id[{}] to from target node [{}], Error: [{}]", - request.getId(), targetNode, exp.getDetailedMessage()); - listener.onFailure(cause instanceof Exception ? (Exception) cause - : new NotSerializableExceptionWrapper(cause)); - } - } - - @Override - public void handleResponse(Response response) { - logger.debug("Received the response for asynchronous search id [{}] from target node [{}]", request.getId(), - targetNode); - listener.onResponse(response); + transportService.sendRequest( + targetNode, + actionName, + request, + requestOptions, + new ActionListenerResponseHandler(listener, responseReader) { + @Override + public void handleException(final TransportException exp) { + Throwable cause = exp.unwrapCause(); + if (cause instanceof ConnectTransportException + || (exp instanceof RemoteTransportException && cause instanceof NodeClosedException)) { + // we want to retry here a bit to see if the node connects backs + logger.debug( + "Connection exception while trying to forward request with id[{}] to " + "target node [{}] Error: [{}]", + request.getId(), + targetNode, + exp.getDetailedMessage() + ); + // try on local node since we weren't able to forward + sendLocalRequest(asynchronousSearchId, request, listener); + } else { + logger.debug( + "Exception received for request with id[{}] to from target node [{}], Error: [{}]", + request.getId(), + targetNode, + exp.getDetailedMessage() + ); + listener.onFailure( + cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause) + ); } - }); + } + + @Override + public void handleResponse(Response response) { + logger.debug( + "Received the response for asynchronous search id [{}] from target node [{}]", + request.getId(), + targetNode + ); + listener.onResponse(response); + } + } + ); } else { sendLocalRequest(asynchronousSearchId, request, listener); } diff --git a/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchStatsAction.java b/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchStatsAction.java index 29708441..07a8a39c 100644 --- a/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchStatsAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/transport/TransportAsynchronousSearchStatsAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.transport; import org.opensearch.search.asynchronous.action.AsynchronousSearchStatsAction; @@ -24,26 +27,44 @@ import java.io.IOException; import java.util.List; -public class TransportAsynchronousSearchStatsAction - extends TransportNodesAction { +public class TransportAsynchronousSearchStatsAction extends TransportNodesAction< + AsynchronousSearchStatsRequest, + AsynchronousSearchStatsResponse, + TransportAsynchronousSearchStatsAction.AsynchronousSearchStatsNodeRequest, + AsynchronousSearchStats> { private final AsynchronousSearchService asynchronousSearchService; @Inject - public TransportAsynchronousSearchStatsAction(ThreadPool threadPool, ClusterService clusterService, TransportService transportService, - ActionFilters actionFilters, AsynchronousSearchService asynchronousSearchService) { - - super(AsynchronousSearchStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, - AsynchronousSearchStatsRequest::new, AsynchronousSearchStatsNodeRequest::new, ThreadPool.Names.MANAGEMENT, - AsynchronousSearchStats.class); + public TransportAsynchronousSearchStatsAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + AsynchronousSearchService asynchronousSearchService + ) { + + super( + AsynchronousSearchStatsAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + AsynchronousSearchStatsRequest::new, + AsynchronousSearchStatsNodeRequest::new, + ThreadPool.Names.MANAGEMENT, + AsynchronousSearchStats.class + ); this.asynchronousSearchService = asynchronousSearchService; } @Override - protected AsynchronousSearchStatsResponse newResponse(AsynchronousSearchStatsRequest request, List responses, - List failures) { + protected AsynchronousSearchStatsResponse newResponse( + AsynchronousSearchStatsRequest request, + List responses, + List failures + ) { return new AsynchronousSearchStatsResponse(clusterService.getClusterName(), responses, failures); } diff --git a/src/main/java/org/opensearch/search/asynchronous/transport/TransportDeleteAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/transport/TransportDeleteAsynchronousSearchAction.java index 14e70f69..1cff9ace 100644 --- a/src/main/java/org/opensearch/search/asynchronous/transport/TransportDeleteAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/transport/TransportDeleteAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.transport; import org.opensearch.commons.authuser.User; @@ -25,28 +28,51 @@ /** * Transport action for deleting an asynchronous search request */ -public class TransportDeleteAsynchronousSearchAction extends TransportAsynchronousSearchRoutingAction { +public class TransportDeleteAsynchronousSearchAction extends TransportAsynchronousSearchRoutingAction< + DeleteAsynchronousSearchRequest, + AcknowledgedResponse> { private static final Logger logger = LogManager.getLogger(TransportAsynchronousSearchRoutingAction.class); private final AsynchronousSearchService asynchronousSearchService; @Inject - public TransportDeleteAsynchronousSearchAction(ThreadPool threadPool, TransportService transportService, ClusterService clusterService, - ActionFilters actionFilters, AsynchronousSearchService asynchronousSearchService, - Client client) { - super(transportService, clusterService, threadPool, client, DeleteAsynchronousSearchAction.NAME, actionFilters, - asynchronousSearchService, DeleteAsynchronousSearchRequest::new, AcknowledgedResponse::new); + public TransportDeleteAsynchronousSearchAction( + ThreadPool threadPool, + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + AsynchronousSearchService asynchronousSearchService, + Client client + ) { + super( + transportService, + clusterService, + threadPool, + client, + DeleteAsynchronousSearchAction.NAME, + actionFilters, + asynchronousSearchService, + DeleteAsynchronousSearchRequest::new, + AcknowledgedResponse::new + ); this.asynchronousSearchService = asynchronousSearchService; } @Override - public void handleRequest(AsynchronousSearchId asynchronousSearchId, DeleteAsynchronousSearchRequest request, - ActionListener listener, User user) { + public void handleRequest( + AsynchronousSearchId asynchronousSearchId, + DeleteAsynchronousSearchRequest request, + ActionListener listener, + User user + ) { try { - asynchronousSearchService.freeContext(request.getId(), asynchronousSearchId.getAsynchronousSearchContextId(), user, - ActionListener.wrap((complete) -> listener.onResponse(new AcknowledgedResponse(complete)), listener::onFailure)); + asynchronousSearchService.freeContext( + request.getId(), + asynchronousSearchId.getAsynchronousSearchContextId(), + user, + ActionListener.wrap((complete) -> listener.onResponse(new AcknowledgedResponse(complete)), listener::onFailure) + ); } catch (Exception e) { logger.error(() -> new ParameterizedMessage("Unable to delete asynchronous search [{}]", request.getId()), e); listener.onFailure(e); diff --git a/src/main/java/org/opensearch/search/asynchronous/transport/TransportGetAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/transport/TransportGetAsynchronousSearchAction.java index 3ec7f631..e17068a4 100644 --- a/src/main/java/org/opensearch/search/asynchronous/transport/TransportGetAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/transport/TransportGetAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.transport; import org.opensearch.commons.authuser.User; @@ -34,47 +37,79 @@ * whenever both the operations complete. If the search is however not RUNNING we simply need to update keep alive either in-memory * or disk and invoke the response with the search response */ -public class TransportGetAsynchronousSearchAction extends TransportAsynchronousSearchRoutingAction { +public class TransportGetAsynchronousSearchAction extends TransportAsynchronousSearchRoutingAction< + GetAsynchronousSearchRequest, + AsynchronousSearchResponse> { private static final Logger logger = LogManager.getLogger(TransportGetAsynchronousSearchAction.class); private final ThreadPool threadPool; private final AsynchronousSearchService asynchronousSearchService; @Inject - public TransportGetAsynchronousSearchAction(ThreadPool threadPool, TransportService transportService, ClusterService clusterService, - ActionFilters actionFilters, AsynchronousSearchService asynchronousSearchService, Client client) { - super(transportService, clusterService, threadPool, client, GetAsynchronousSearchAction.NAME, actionFilters, - asynchronousSearchService, GetAsynchronousSearchRequest::new, AsynchronousSearchResponse::new); + public TransportGetAsynchronousSearchAction( + ThreadPool threadPool, + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + AsynchronousSearchService asynchronousSearchService, + Client client + ) { + super( + transportService, + clusterService, + threadPool, + client, + GetAsynchronousSearchAction.NAME, + actionFilters, + asynchronousSearchService, + GetAsynchronousSearchRequest::new, + AsynchronousSearchResponse::new + ); this.threadPool = threadPool; this.asynchronousSearchService = asynchronousSearchService; } @Override - public void handleRequest(AsynchronousSearchId asynchronousSearchId, GetAsynchronousSearchRequest request, - ActionListener listener, User user) { + public void handleRequest( + AsynchronousSearchId asynchronousSearchId, + GetAsynchronousSearchRequest request, + ActionListener listener, + User user + ) { try { boolean updateNeeded = request.getKeepAlive() != null; if (updateNeeded) { - asynchronousSearchService.updateKeepAliveAndGetContext(request.getId(), request.getKeepAlive(), - asynchronousSearchId.getAsynchronousSearchContextId(), user, ActionListener.wrap( - // check if the context is active and is still RUNNING - (context) -> handleWaitForCompletion(context, request, listener), - (e) -> { - logger.debug(() -> new ParameterizedMessage("Unable to update and get asynchronous search request [{}]", - asynchronousSearchId), e); - listener.onFailure(e); - } - )); - } else { - // we don't need to update keep-alive, simply find one on the node if one exists or look up the index - asynchronousSearchService.findContext(request.getId(), asynchronousSearchId.getAsynchronousSearchContextId(), user, - ActionListener.wrap((context) -> handleWaitForCompletion(context, request, listener), + asynchronousSearchService.updateKeepAliveAndGetContext( + request.getId(), + request.getKeepAlive(), + asynchronousSearchId.getAsynchronousSearchContextId(), + user, + ActionListener.wrap( + // check if the context is active and is still RUNNING + (context) -> handleWaitForCompletion(context, request, listener), (e) -> { - logger.debug(() -> new ParameterizedMessage("Unable to get asynchronous search [{}]", - asynchronousSearchId), e); + logger.debug( + () -> new ParameterizedMessage( + "Unable to update and get asynchronous search request [{}]", + asynchronousSearchId + ), + e + ); listener.onFailure(e); - })); + } + ) + ); + } else { + // we don't need to update keep-alive, simply find one on the node if one exists or look up the index + asynchronousSearchService.findContext( + request.getId(), + asynchronousSearchId.getAsynchronousSearchContextId(), + user, + ActionListener.wrap((context) -> handleWaitForCompletion(context, request, listener), (e) -> { + logger.debug(() -> new ParameterizedMessage("Unable to get asynchronous search [{}]", asynchronousSearchId), e); + listener.onFailure(e); + }) + ); } } catch (Exception e) { logger.error(() -> new ParameterizedMessage("Unable to update and get asynchronous search [{}]", request.getId()), e); @@ -82,20 +117,26 @@ public void handleRequest(AsynchronousSearchId asynchronousSearchId, GetAsynchro } } - private void handleWaitForCompletion(AsynchronousSearchContext context, GetAsynchronousSearchRequest request, - ActionListener listener) { - //We wait for a response only if a wait for completion is non-null and the search execution is still in progress. + private void handleWaitForCompletion( + AsynchronousSearchContext context, + GetAsynchronousSearchRequest request, + ActionListener listener + ) { + // We wait for a response only if a wait for completion is non-null and the search execution is still in progress. if (context.isRunning() && request.getWaitForCompletionTimeout() != null) { logger.debug("Context is running for asynchronous search id [{}]", context.getAsynchronousSearchId()); AsynchronousSearchProgressListener progressActionListener = context.getAsynchronousSearchProgressListener(); assert progressActionListener != null : "progress listener cannot be null"; PrioritizedActionListener wrappedListener = AsynchronousSearchTimeoutWrapper.wrapScheduledTimeout( - threadPool, request.getWaitForCompletionTimeout(), OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - listener, - (actionListener) -> { - progressActionListener.searchProgressActionListener().removeListener(actionListener); - listener.onResponse(context.getAsynchronousSearchResponse()); - }); + threadPool, + request.getWaitForCompletionTimeout(), + OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + listener, + (actionListener) -> { + progressActionListener.searchProgressActionListener().removeListener(actionListener); + listener.onResponse(context.getAsynchronousSearchResponse()); + } + ); progressActionListener.searchProgressActionListener().addOrExecuteListener(wrappedListener); } else { // we don't need to wait any further on search completion diff --git a/src/main/java/org/opensearch/search/asynchronous/transport/TransportSubmitAsynchronousSearchAction.java b/src/main/java/org/opensearch/search/asynchronous/transport/TransportSubmitAsynchronousSearchAction.java index 780157cf..0cb9ee58 100644 --- a/src/main/java/org/opensearch/search/asynchronous/transport/TransportSubmitAsynchronousSearchAction.java +++ b/src/main/java/org/opensearch/search/asynchronous/transport/TransportSubmitAsynchronousSearchAction.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.transport; import org.opensearch.commons.ConfigConstants; @@ -42,8 +45,9 @@ * a {@link AsynchronousSearchProgressListener} set on the task. The listener is wrapped with a completion timeout wrapper via * {@link AsynchronousSearchTimeoutWrapper} which ensures that exactly one of action listener or the timeout listener gets executed */ -public class TransportSubmitAsynchronousSearchAction extends HandledTransportAction { +public class TransportSubmitAsynchronousSearchAction extends HandledTransportAction< + SubmitAsynchronousSearchRequest, + AsynchronousSearchResponse> { private static final Logger logger = LogManager.getLogger(TransportSubmitAsynchronousSearchAction.class); private final ThreadPool threadPool; @@ -53,9 +57,15 @@ public class TransportSubmitAsynchronousSearchAction extends HandledTransportAct private final SearchService searchService; @Inject - public TransportSubmitAsynchronousSearchAction(ThreadPool threadPool, TransportService transportService, ClusterService clusterService, - ActionFilters actionFilters, AsynchronousSearchService asynchronousSearchService, - TransportSearchAction transportSearchAction, SearchService searchService) { + public TransportSubmitAsynchronousSearchAction( + ThreadPool threadPool, + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + AsynchronousSearchService asynchronousSearchService, + TransportSearchAction transportSearchAction, + SearchService searchService + ) { super(SubmitAsynchronousSearchAction.NAME, transportService, actionFilters, SubmitAsynchronousSearchRequest::new); this.threadPool = threadPool; this.clusterService = clusterService; @@ -71,50 +81,78 @@ protected void doExecute(Task task, SubmitAsynchronousSearchRequest request, Act User user = User.parse(userStr); try { final long relativeStartTimeInMillis = threadPool.relativeTimeInMillis(); - asynchronousSearchContext = asynchronousSearchService.createAndStoreContext(request, relativeStartTimeInMillis, - () -> searchService.aggReduceContextBuilder(request.getSearchRequest().source()), user); + asynchronousSearchContext = asynchronousSearchService.createAndStoreContext( + request, + relativeStartTimeInMillis, + () -> searchService.aggReduceContextBuilder(request.getSearchRequest().source()), + user + ); assert asynchronousSearchContext.getAsynchronousSearchProgressListener() != null - : "missing progress listener for an active context"; + : "missing progress listener for an active context"; AsynchronousSearchProgressListener progressListener = asynchronousSearchContext.getAsynchronousSearchProgressListener(); - AsynchronousSearchContext context = asynchronousSearchContext; //making it effectively final for usage in anonymous class. + AsynchronousSearchContext context = asynchronousSearchContext; // making it effectively final for usage in anonymous class. SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) { @Override public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - AsynchronousSearchTask asynchronousSearchTask = new AsynchronousSearchTask(id, type, AsynchronousSearchTask.NAME, - parentTaskId, headers, - (AsynchronousSearchActiveContext) context, request, asynchronousSearchService::onCancelledFreeActiveContext); + AsynchronousSearchTask asynchronousSearchTask = new AsynchronousSearchTask( + id, + type, + AsynchronousSearchTask.NAME, + parentTaskId, + headers, + (AsynchronousSearchActiveContext) context, + request, + asynchronousSearchService::onCancelledFreeActiveContext + ); asynchronousSearchService.bootstrapSearch(asynchronousSearchTask, context.getContextId()); PrioritizedActionListener wrappedListener = AsynchronousSearchTimeoutWrapper - .wrapScheduledTimeout(threadPool, request.getWaitForCompletionTimeout(), - AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, listener, - (actionListener) -> { progressListener.searchProgressActionListener().removeListener(actionListener); - listener.onResponse(context.getAsynchronousSearchResponse()); - }); + .wrapScheduledTimeout( + threadPool, + request.getWaitForCompletionTimeout(), + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + listener, + (actionListener) -> { + progressListener.searchProgressActionListener().removeListener(actionListener); + listener.onResponse(context.getAsynchronousSearchResponse()); + } + ); progressListener.searchProgressActionListener().addOrExecuteListener(wrappedListener); return asynchronousSearchTask; } }; - //set the parent task as the submit task for cancellation on connection close + // set the parent task as the submit task for cancellation on connection close searchRequest.setParentTask(task.taskInfo(clusterService.localNode().getId(), false).getTaskId()); transportSearchAction.execute(searchRequest, progressListener); } catch (Exception e) { logger.error(() -> new ParameterizedMessage("Failed to submit asynchronous search request [{}]", request), e); if (asynchronousSearchContext != null) { - AsynchronousSearchActiveContext asynchronousSearchActiveContext = (AsynchronousSearchActiveContext) - asynchronousSearchContext; - asynchronousSearchService.freeContext(asynchronousSearchActiveContext.getAsynchronousSearchId(), - asynchronousSearchActiveContext.getContextId(), user, - ActionListener.wrap((r) -> { - logger.debug(() -> new ParameterizedMessage("Successfully cleaned up context on submit asynchronous" + - " search [{}] on failure", asynchronousSearchActiveContext.getAsynchronousSearchId()), e); - listener.onFailure(e); - }, (ex) -> { - logger.debug(() -> new ParameterizedMessage("Failed to cleaned up context on submit asynchronous search" + - " [{}] on failure", asynchronousSearchActiveContext.getAsynchronousSearchId()), ex); - listener.onFailure(e); - }) + AsynchronousSearchActiveContext asynchronousSearchActiveContext = + (AsynchronousSearchActiveContext) asynchronousSearchContext; + asynchronousSearchService.freeContext( + asynchronousSearchActiveContext.getAsynchronousSearchId(), + asynchronousSearchActiveContext.getContextId(), + user, + ActionListener.wrap((r) -> { + logger.debug( + () -> new ParameterizedMessage( + "Successfully cleaned up context on submit asynchronous" + " search [{}] on failure", + asynchronousSearchActiveContext.getAsynchronousSearchId() + ), + e + ); + listener.onFailure(e); + }, (ex) -> { + logger.debug( + () -> new ParameterizedMessage( + "Failed to cleaned up context on submit asynchronous search" + " [{}] on failure", + asynchronousSearchActiveContext.getAsynchronousSearchId() + ), + ex + ); + listener.onFailure(e); + }) ); } else { listener.onFailure(e); diff --git a/src/main/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchExceptionUtils.java b/src/main/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchExceptionUtils.java index 12b3783e..3390f4b6 100644 --- a/src/main/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchExceptionUtils.java +++ b/src/main/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchExceptionUtils.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.utils; import org.opensearch.ResourceNotFoundException; @@ -12,7 +15,8 @@ public class AsynchronousSearchExceptionUtils { public static ResourceNotFoundException buildResourceNotFoundException(String id) { - return new ResourceNotFoundException(String.format( - Locale.ROOT, "Either the resource [%s] does not exist or you do not have access", id)); + return new ResourceNotFoundException( + String.format(Locale.ROOT, "Either the resource [%s] does not exist or you do not have access", id) + ); } } diff --git a/src/main/java/org/opensearch/search/asynchronous/utils/UserAuthUtils.java b/src/main/java/org/opensearch/search/asynchronous/utils/UserAuthUtils.java index 76869ae5..724d48f0 100644 --- a/src/main/java/org/opensearch/search/asynchronous/utils/UserAuthUtils.java +++ b/src/main/java/org/opensearch/search/asynchronous/utils/UserAuthUtils.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.utils; import org.opensearch.commons.authuser.User; @@ -17,31 +20,27 @@ public class UserAuthUtils { @SuppressWarnings("unchecked") public static User parseUser(Map userDetails) throws IOException { - if(userDetails == null) { + if (userDetails == null) { return null; } String name = ""; List backendRoles = new ArrayList<>(); List roles = new ArrayList<>(); List customAttNames = new ArrayList<>(); - for (Map.Entry userDetail : userDetails.entrySet()) - { + for (Map.Entry userDetail : userDetails.entrySet()) { String fieldName = userDetail.getKey(); switch (fieldName) { case User.NAME_FIELD: name = (String) userDetail.getValue(); break; case User.BACKEND_ROLES_FIELD: - if(userDetail.getValue()!= null) - backendRoles = (List) userDetail.getValue(); + if (userDetail.getValue() != null) backendRoles = (List) userDetail.getValue(); break; case User.ROLES_FIELD: - if(userDetail.getValue()!= null) - roles = (List) userDetail.getValue(); + if (userDetail.getValue() != null) roles = (List) userDetail.getValue(); break; case User.CUSTOM_ATTRIBUTE_NAMES_FIELD: - if(userDetail.getValue()!= null) - customAttNames = (List) userDetail.getValue(); + if (userDetail.getValue() != null) customAttNames = (List) userDetail.getValue(); break; default: break; @@ -51,10 +50,10 @@ public static User parseUser(Map userDetails) throws IOException } public static boolean isUserValid(@Nullable User currentUser, @Nullable User originalUser) { - if(originalUser == null || currentUser == null) { + if (originalUser == null || currentUser == null) { return true; } - if(currentUser.getBackendRoles() == null) { + if (currentUser.getBackendRoles() == null) { return originalUser.getBackendRoles() == null; } return currentUser.getBackendRoles().containsAll(originalUser.getBackendRoles()); diff --git a/src/test/java/org/opensearch/search/asynchronous/SecurityEnabledRestTestCase.java b/src/test/java/org/opensearch/search/asynchronous/SecurityEnabledRestTestCase.java index 23b63f64..c22ddc75 100644 --- a/src/test/java/org/opensearch/search/asynchronous/SecurityEnabledRestTestCase.java +++ b/src/test/java/org/opensearch/search/asynchronous/SecurityEnabledRestTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous; import org.apache.http.Header; @@ -65,21 +68,28 @@ protected String getProtocol() { @Override protected Settings restAdminSettings() { - return Settings - .builder() - // disable the warning exception for admin client since it's only used for cleanup. - .put("strictDeprecationMode", false) - .put("http.port", 9200) - .put(OPENSEARCH_SECURITY_SSL_HTTP_ENABLED, isHttps()) - .put(OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, System.getProperty( - OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, "sample.pem")) - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, System.getProperty( - OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, "test-kirk.jks")) - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, System.getProperty( - OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, "changeit")) - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, System.getProperty( - OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, "changeit")) - .build(); + return Settings.builder() + // disable the warning exception for admin client since it's only used for cleanup. + .put("strictDeprecationMode", false) + .put("http.port", 9200) + .put(OPENSEARCH_SECURITY_SSL_HTTP_ENABLED, isHttps()) + .put( + OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, + System.getProperty(OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, "sample.pem") + ) + .put( + OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, + System.getProperty(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, "test-kirk.jks") + ) + .put( + OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, + System.getProperty(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, "changeit") + ) + .put( + OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, + System.getProperty(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, "changeit") + ) + .build(); } @Override @@ -115,13 +125,12 @@ protected void wipeAllOSIndices() throws IOException { Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType().getValue()); try ( - XContentParser parser = xContentType - .xContent() - .createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - response.getEntity().getContent() - ) + XContentParser parser = xContentType.xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent() + ) ) { XContentParser.Token token = parser.nextToken(); List> parserList = null; @@ -159,17 +168,19 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s builder.setHttpClientConfigCallback(httpClientBuilder -> { try { return httpClientBuilder - // disable the certificate since our testing cluster just uses the default security configuration - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()); + // disable the certificate since our testing cluster just uses the default security configuration + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()); } catch (Exception e) { throw new RuntimeException(e); } }); final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); - final TimeValue socketTimeout = TimeValue - .parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT); + final TimeValue socketTimeout = TimeValue.parseTimeValue( + socketTimeoutString == null ? "60s" : socketTimeoutString, + CLIENT_SOCKET_TIMEOUT + ); builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); if (settings.hasValue(CLIENT_PATH_PREFIX)) { builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); diff --git a/src/test/java/org/opensearch/search/asynchronous/bwc/AsyncSearchBackwardsCompatibilityIT.java b/src/test/java/org/opensearch/search/asynchronous/bwc/AsyncSearchBackwardsCompatibilityIT.java index 0891b731..e9cab5ac 100644 --- a/src/test/java/org/opensearch/search/asynchronous/bwc/AsyncSearchBackwardsCompatibilityIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/bwc/AsyncSearchBackwardsCompatibilityIT.java @@ -1,14 +1,11 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ - package org.opensearch.search.asynchronous.bwc; import org.junit.Assert; @@ -42,18 +39,15 @@ import static org.hamcrest.Matchers.containsString; public class AsyncSearchBackwardsCompatibilityIT extends AsynchronousSearchRestTestCase { - private static final ClusterType CLUSTER_TYPE = - ClusterType.parse(System.getProperty("tests.rest.bwcsuite_cluster")); + private static final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.bwcsuite_cluster")); private static final String CLUSTER_NAME = System.getProperty("tests.clustername"); public void testBackwardsCompatibility() throws Exception { String uri = getUri(); - Map> responseMap = - (Map>) getAsMap(uri).get("nodes"); + Map> responseMap = (Map>) getAsMap(uri).get("nodes"); for (Map response : responseMap.values()) { List> plugins = (List>) response.get("plugins"); - Set pluginNames = - plugins.stream().map(map -> map.get("name")).collect(Collectors.toSet()); + Set pluginNames = plugins.stream().map(map -> map.get("name")).collect(Collectors.toSet()); switch (CLUSTER_TYPE) { case OLD: case MIXED: @@ -100,117 +94,104 @@ private String getUri() { public void testSubmitWithRetainedResponse(boolean shouldUseLegacyApi) throws IOException { SearchRequest searchRequest = new SearchRequest("test"); searchRequest.source(new SearchSourceBuilder()); - SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = - new SubmitAsynchronousSearchRequest(searchRequest); + SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); - submitAsynchronousSearchRequest.waitForCompletionTimeout( - TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = - executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest, shouldUseLegacyApi); - List legalStates = - Arrays.asList( - AsynchronousSearchState.RUNNING, - AsynchronousSearchState.SUCCEEDED, - AsynchronousSearchState.PERSIST_SUCCEEDED, - AsynchronousSearchState.PERSISTING, - AsynchronousSearchState.CLOSED, - AsynchronousSearchState.STORE_RESIDENT); + submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest, shouldUseLegacyApi); + List legalStates = Arrays.asList( + AsynchronousSearchState.RUNNING, + AsynchronousSearchState.SUCCEEDED, + AsynchronousSearchState.PERSIST_SUCCEEDED, + AsynchronousSearchState.PERSISTING, + AsynchronousSearchState.CLOSED, + AsynchronousSearchState.STORE_RESIDENT + ); assertNotNull(submitResponse.getId()); assertTrue(submitResponse.getState().name(), legalStates.contains(submitResponse.getState())); - GetAsynchronousSearchRequest getAsynchronousSearchRequest = - new GetAsynchronousSearchRequest(submitResponse.getId()); + GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(submitResponse.getId()); AsynchronousSearchResponse getResponse; do { - getResponse = - getAssertedAsynchronousSearchResponse( - submitResponse, getAsynchronousSearchRequest, shouldUseLegacyApi); - if (getResponse.getState() == AsynchronousSearchState.RUNNING - && getResponse.getSearchResponse() != null) { + getResponse = getAssertedAsynchronousSearchResponse(submitResponse, getAsynchronousSearchRequest, shouldUseLegacyApi); + if (getResponse.getState() == AsynchronousSearchState.RUNNING && getResponse.getSearchResponse() != null) { assertEquals(getResponse.getSearchResponse().getHits().getHits().length, 0); } else { assertNotNull(getResponse.getSearchResponse()); assertNotEquals(getResponse.getSearchResponse().getTook(), -1L); } } while (AsynchronousSearchState.STORE_RESIDENT.equals(getResponse.getState()) == false); - getResponse = - getAssertedAsynchronousSearchResponse( - submitResponse, getAsynchronousSearchRequest, shouldUseLegacyApi); + getResponse = getAssertedAsynchronousSearchResponse(submitResponse, getAsynchronousSearchRequest, shouldUseLegacyApi); assertNotNull(getResponse.getSearchResponse()); assertEquals(AsynchronousSearchState.STORE_RESIDENT, getResponse.getState()); assertHitCount(getResponse.getSearchResponse(), 5); - executeDeleteAsynchronousSearch( - new DeleteAsynchronousSearchRequest(submitResponse.getId()), shouldUseLegacyApi); + executeDeleteAsynchronousSearch(new DeleteAsynchronousSearchRequest(submitResponse.getId()), shouldUseLegacyApi); } - Response executeDeleteAsynchronousSearch( - DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest, boolean shouldUseLegacyApi) - throws IOException { + Response executeDeleteAsynchronousSearch(DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest, boolean shouldUseLegacyApi) + throws IOException { Request request = RestTestUtils.buildHttpRequest(deleteAsynchronousSearchRequest, shouldUseLegacyApi); return client().performRequest(request); } public void testMaxKeepAliveSetting(boolean shouldUseLegacyApi) throws Exception { - SubmitAsynchronousSearchRequest validRequest = - new SubmitAsynchronousSearchRequest(new SearchRequest()); + SubmitAsynchronousSearchRequest validRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); validRequest.keepAlive(TimeValue.timeValueHours(7)); AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(validRequest, shouldUseLegacyApi); assertNotNull(asResponse.getSearchResponse()); updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING.getKey() - : AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), - TimeValue.timeValueHours(6)); - SubmitAsynchronousSearchRequest invalidRequest = - new SubmitAsynchronousSearchRequest(new SearchRequest()); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING.getKey() + : AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), + TimeValue.timeValueHours(6) + ); + SubmitAsynchronousSearchRequest invalidRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); invalidRequest.keepAlive(TimeValue.timeValueHours(7)); - ResponseException responseException = - expectThrows( - ResponseException.class, - () -> executeSubmitAsynchronousSearch(invalidRequest, shouldUseLegacyApi)); + ResponseException responseException = expectThrows( + ResponseException.class, + () -> executeSubmitAsynchronousSearch(invalidRequest, shouldUseLegacyApi) + ); assertThat( - responseException.getMessage(), - containsString( - "Keep alive for asynchronous search (" - + invalidRequest.getKeepAlive().getMillis() - + ") is too large")); + responseException.getMessage(), + containsString("Keep alive for asynchronous search (" + invalidRequest.getKeepAlive().getMillis() + ") is too large") + ); updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING.getKey() - : AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), - TimeValue.timeValueHours(24)); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING.getKey() + : AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), + TimeValue.timeValueHours(24) + ); } public void testSubmitInvalidWaitForCompletion(boolean shouldUseLegacyApi) throws Exception { - SubmitAsynchronousSearchRequest validRequest = - new SubmitAsynchronousSearchRequest(new SearchRequest()); + SubmitAsynchronousSearchRequest validRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); validRequest.waitForCompletionTimeout(TimeValue.timeValueSeconds(50)); AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(validRequest, shouldUseLegacyApi); assertNotNull(asResponse.getSearchResponse()); updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING - .getKey() - : AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), - TimeValue.timeValueSeconds(2)); - SubmitAsynchronousSearchRequest invalidRequest = - new SubmitAsynchronousSearchRequest(new SearchRequest()); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey() + : AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), + TimeValue.timeValueSeconds(2) + ); + SubmitAsynchronousSearchRequest invalidRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); invalidRequest.waitForCompletionTimeout(TimeValue.timeValueSeconds(50)); - ResponseException responseException = - expectThrows( - ResponseException.class, - () -> executeSubmitAsynchronousSearch(invalidRequest, shouldUseLegacyApi)); + ResponseException responseException = expectThrows( + ResponseException.class, + () -> executeSubmitAsynchronousSearch(invalidRequest, shouldUseLegacyApi) + ); assertThat( - responseException.getMessage(), - containsString( - "Wait for completion timeout for asynchronous search (" - + validRequest.getWaitForCompletionTimeout().getMillis() - + ") is too large")); + responseException.getMessage(), + containsString( + "Wait for completion timeout for asynchronous search (" + + validRequest.getWaitForCompletionTimeout().getMillis() + + ") is too large" + ) + ); updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING - .getKey() - : AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), - TimeValue.timeValueSeconds(60)); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey() + : AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), + TimeValue.timeValueSeconds(60) + ); } public void testMaxRunningAsynchronousSearchContexts(boolean shouldUseLegacyApi) throws Exception { @@ -218,26 +199,22 @@ public void testMaxRunningAsynchronousSearchContexts(boolean shouldUseLegacyApi) List threadsList = new LinkedList<>(); CyclicBarrier barrier = new CyclicBarrier(numThreads + 1); for (int i = 0; i < numThreads; i++) { - threadsList.add( - new Thread( - () -> { - try { - SubmitAsynchronousSearchRequest validRequest = - new SubmitAsynchronousSearchRequest(new SearchRequest()); - validRequest.keepAlive(TimeValue.timeValueHours(1)); - AsynchronousSearchResponse asResponse = - executeSubmitAsynchronousSearch(validRequest, shouldUseLegacyApi); - assertNotNull(asResponse.getSearchResponse()); - } catch (IOException e) { - fail("submit request failed"); - } finally { - try { - barrier.await(); - } catch (Exception e) { - fail(); - } - } - })); + threadsList.add(new Thread(() -> { + try { + SubmitAsynchronousSearchRequest validRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); + validRequest.keepAlive(TimeValue.timeValueHours(1)); + AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(validRequest, shouldUseLegacyApi); + assertNotNull(asResponse.getSearchResponse()); + } catch (IOException e) { + fail("submit request failed"); + } finally { + try { + barrier.await(); + } catch (Exception e) { + fail(); + } + } + })); } threadsList.forEach(Thread::start); barrier.await(); @@ -246,38 +223,32 @@ public void testMaxRunningAsynchronousSearchContexts(boolean shouldUseLegacyApi) } updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING - .getKey() - : AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), - 0); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey() + : AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), + 0 + ); threadsList.clear(); AtomicInteger numFailures = new AtomicInteger(); for (int i = 0; i < numThreads; i++) { - threadsList.add( - new Thread( - () -> { - try { - SubmitAsynchronousSearchRequest validRequest = - new SubmitAsynchronousSearchRequest(new SearchRequest()); - validRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(1)); - AsynchronousSearchResponse asResponse = - executeSubmitAsynchronousSearch(validRequest, shouldUseLegacyApi); - } catch (Exception e) { - assertTrue(e instanceof ResponseException); - assertThat( - e.getMessage(), - containsString("Trying to create too many concurrent searches")); - numFailures.getAndIncrement(); + threadsList.add(new Thread(() -> { + try { + SubmitAsynchronousSearchRequest validRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); + validRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(1)); + AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(validRequest, shouldUseLegacyApi); + } catch (Exception e) { + assertTrue(e instanceof ResponseException); + assertThat(e.getMessage(), containsString("Trying to create too many concurrent searches")); + numFailures.getAndIncrement(); - } finally { - try { - barrier.await(); - } catch (Exception e) { - fail(); - } - } - })); + } finally { + try { + barrier.await(); + } catch (Exception e) { + fail(); + } + } + })); } threadsList.forEach(Thread::start); barrier.await(); @@ -286,59 +257,52 @@ public void testMaxRunningAsynchronousSearchContexts(boolean shouldUseLegacyApi) } assertEquals(numFailures.get(), 50); updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING - .getKey() - : AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey() + : AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES + ); } public void testStoreAsyncSearchWithFailures(boolean shouldUseLegacyApi) throws Exception { - SubmitAsynchronousSearchRequest request = - new SubmitAsynchronousSearchRequest(new SearchRequest("non-existent-index")); + SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest("non-existent-index")); request.keepOnCompletion(true); request.waitForCompletionTimeout(TimeValue.timeValueMinutes(1)); AsynchronousSearchResponse response = executeSubmitAsynchronousSearch(request, shouldUseLegacyApi); - assertTrue( - Arrays.asList(AsynchronousSearchState.CLOSED, AsynchronousSearchState.FAILED) - .contains(AsynchronousSearchState.FAILED)); - waitUntil( - () -> { - try { - executeGetAsynchronousSearch( - new GetAsynchronousSearchRequest(response.getId()), shouldUseLegacyApi); - return false; - } catch (IOException e) { - return e.getMessage().contains("resource_not_found"); - } - }); + assertTrue(Arrays.asList(AsynchronousSearchState.CLOSED, AsynchronousSearchState.FAILED).contains(AsynchronousSearchState.FAILED)); + waitUntil(() -> { + try { + executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(response.getId()), shouldUseLegacyApi); + return false; + } catch (IOException e) { + return e.getMessage().contains("resource_not_found"); + } + }); updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING.getKey() - : AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), - true); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING.getKey() + : AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), + true + ); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(request, shouldUseLegacyApi); - waitUntil( - () -> { - try { - return executeGetAsynchronousSearch( - new GetAsynchronousSearchRequest(submitResponse.getId()), shouldUseLegacyApi) - .getState() - .equals(AsynchronousSearchState.STORE_RESIDENT); - } catch (IOException e) { - return false; - } - }); + waitUntil(() -> { + try { + return executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId()), shouldUseLegacyApi).getState() + .equals(AsynchronousSearchState.STORE_RESIDENT); + } catch (IOException e) { + return false; + } + }); assertEquals( - executeGetAsynchronousSearch( - new GetAsynchronousSearchRequest(submitResponse.getId()), shouldUseLegacyApi) - .getState(), - AsynchronousSearchState.STORE_RESIDENT); + executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId()), shouldUseLegacyApi).getState(), + AsynchronousSearchState.STORE_RESIDENT + ); updateClusterSettings( - shouldUseLegacyApi - ? LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING.getKey() - : AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), - false); + shouldUseLegacyApi + ? LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING.getKey() + : AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), + false + ); } private enum ClusterType { @@ -359,4 +323,4 @@ public static ClusterType parse(String value) { } } } -} \ No newline at end of file +} diff --git a/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchIntegTestCase.java b/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchIntegTestCase.java index d429f1ea..0f2f7934 100644 --- a/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchIntegTestCase.java +++ b/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchIntegTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.commons; import org.opensearch.search.asynchronous.action.GetAsynchronousSearchAction; @@ -51,10 +54,7 @@ public abstract class AsynchronousSearchIntegTestCase extends OpenSearchIntegTes @Override protected Collection> nodePlugins() { - return Arrays.asList( - ScriptedBlockPlugin.class, - AsynchronousSearchPlugin.class, - ReindexPlugin.class); + return Arrays.asList(ScriptedBlockPlugin.class, AsynchronousSearchPlugin.class, ReindexPlugin.class); } @Override @@ -113,8 +113,8 @@ protected SearchResponse ensureSearchWasCancelled(SearchResponse searchResponse, protected boolean verifyAsynchronousSearchDoesNotExists(String id) { GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(id); try { - AsynchronousSearchResponse response = client().execute(GetAsynchronousSearchAction.INSTANCE, - getAsynchronousSearchRequest).actionGet(); + AsynchronousSearchResponse response = client().execute(GetAsynchronousSearchAction.INSTANCE, getAsynchronousSearchRequest) + .actionGet(); return response == null; } catch (Exception e) { if (e instanceof ResourceNotFoundException) { @@ -128,9 +128,9 @@ protected boolean verifyAsynchronousSearchDoesNotExists(String id) { protected boolean verifyResponsePersisted(String id) { try { - boolean isExists = client().get(new GetRequest(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX) - .refresh(true).id(id)) - .actionGet().isExists(); + boolean isExists = client().get( + new GetRequest(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX).refresh(true).id(id) + ).actionGet().isExists(); return isExists; } catch (ResourceNotFoundException | NoShardAvailableActionException e) { return false; @@ -177,8 +177,7 @@ public void enableBlock() { public Map, Object>> pluginScripts() { return Collections.singletonMap(SCRIPT_NAME, params -> { LeafFieldsLookup fieldsLookup = (LeafFieldsLookup) params.get("_fields"); - LogManager.getLogger(AsynchronousSearchIntegTestCase.class).info("Blocking on the document {}", - fieldsLookup.get("_id")); + LogManager.getLogger(AsynchronousSearchIntegTestCase.class).info("Blocking on the document {}", fieldsLookup.get("_id")); hits.incrementAndGet(); try { assertBusy(() -> assertFalse(shouldBlock.get()), 60, TimeUnit.SECONDS); @@ -191,11 +190,11 @@ public Map, Object>> pluginScripts() { } public static AsynchronousSearchResponse executeSubmitAsynchronousSearch(Client client, SubmitAsynchronousSearchRequest request) - throws ExecutionException, InterruptedException { + throws ExecutionException, InterruptedException { return client.execute(SubmitAsynchronousSearchAction.INSTANCE, request).get(); } - //We need to apply blocks via ScriptedBlockPlugin, external clusters are immutable + // We need to apply blocks via ScriptedBlockPlugin, external clusters are immutable @Override protected boolean ignoreExternalCluster() { return true; diff --git a/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchSingleNodeTestCase.java b/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchSingleNodeTestCase.java index 80d1b1c6..5a424b5e 100644 --- a/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchSingleNodeTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.commons; import org.opensearch.search.asynchronous.action.DeleteAsynchronousSearchAction; @@ -69,12 +72,11 @@ protected boolean resetNodeAfterTest() { public void setUp() throws Exception { super.setUp(); Map allActiveContexts = getInstanceFromNode(AsynchronousSearchService.class) - .getAllActiveContexts(); + .getAllActiveContexts(); assertTrue(allActiveContexts.toString(), allActiveContexts.isEmpty()); createIndex(TEST_INDEX, Settings.builder().put("index.refresh_interval", -1).build()); for (int i = 0; i < 10; i++) - client().prepareIndex(TEST_INDEX).setId(String.valueOf(i)).setSource("field", "value" + i) - .setRefreshPolicy(IMMEDIATE).get(); + client().prepareIndex(TEST_INDEX).setId(String.valueOf(i)).setSource("field", "value" + i).setRefreshPolicy(IMMEDIATE).get(); } @Override @@ -87,33 +89,48 @@ protected Collection> getPlugins() { return plugins; } - public static ActionFuture executeSubmitAsynchronousSearch(Client client, - SubmitAsynchronousSearchRequest request) { + public static ActionFuture executeSubmitAsynchronousSearch( + Client client, + SubmitAsynchronousSearchRequest request + ) { return client.execute(SubmitAsynchronousSearchAction.INSTANCE, request); } - public static ActionFuture executeGetAsynchronousSearch(Client client, - GetAsynchronousSearchRequest request) { + public static ActionFuture executeGetAsynchronousSearch( + Client client, + GetAsynchronousSearchRequest request + ) { return client.execute(GetAsynchronousSearchAction.INSTANCE, request); } - public static ActionFuture executeDeleteAsynchronousSearch(Client client, - DeleteAsynchronousSearchRequest request) { + public static ActionFuture executeDeleteAsynchronousSearch( + Client client, + DeleteAsynchronousSearchRequest request + ) { return client.execute(DeleteAsynchronousSearchAction.INSTANCE, request); } - public static void executeDeleteAsynchronousSearch(Client client, DeleteAsynchronousSearchRequest request, - ActionListener listener) { + public static void executeDeleteAsynchronousSearch( + Client client, + DeleteAsynchronousSearchRequest request, + ActionListener listener + ) { client.execute(DeleteAsynchronousSearchAction.INSTANCE, request, listener); } - public static void executeSubmitAsynchronousSearch(Client client, SubmitAsynchronousSearchRequest request, - ActionListener listener) { + public static void executeSubmitAsynchronousSearch( + Client client, + SubmitAsynchronousSearchRequest request, + ActionListener listener + ) { client.execute(SubmitAsynchronousSearchAction.INSTANCE, request, listener); } - public static void executeGetAsynchronousSearch(Client client, GetAsynchronousSearchRequest request, - ActionListener listener) { + public static void executeGetAsynchronousSearch( + Client client, + GetAsynchronousSearchRequest request, + ActionListener listener + ) { client.execute(GetAsynchronousSearchAction.INSTANCE, request, listener); } @@ -151,13 +168,24 @@ public void enableBlocks(List plugins) { protected SearchResponse getMockSearchResponse() { int totalShards = randomInt(100); int successfulShards = totalShards - randomInt(100); - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), - "", totalShards, successfulShards, 0, randomNonNegativeLong(), - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + randomInt(5) + ), + "", + totalShards, + successfulShards, + 0, + randomNonNegativeLong(), + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } public static class SearchDelayPlugin extends MockScriptPlugin { @@ -195,9 +223,10 @@ public void tearDownData() throws InterruptedException { waitUntil(() -> getInstanceFromNode(AsynchronousSearchService.class).getAllActiveContexts().isEmpty()); logger.warn("delete asynchronous search response index"); CountDownLatch deleteLatch = new CountDownLatch(1); - client().admin().indices().prepareDelete(INDEX).execute(ActionListener.wrap(r -> deleteLatch.countDown(), e -> { - deleteLatch.countDown(); - })); + client().admin() + .indices() + .prepareDelete(INDEX) + .execute(ActionListener.wrap(r -> deleteLatch.countDown(), e -> { deleteLatch.countDown(); })); deleteLatch.await(); } diff --git a/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchTestCase.java b/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchTestCase.java index 2c2a2cf1..d93ca04f 100644 --- a/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchTestCase.java +++ b/src/test/java/org/opensearch/search/asynchronous/commons/AsynchronousSearchTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.commons; import org.opensearch.search.asynchronous.listener.AsynchronousSearchProgressListener; @@ -19,29 +22,46 @@ public abstract class AsynchronousSearchTestCase extends OpenSearchTestCase { public static AsynchronousSearchProgressListener mockAsynchronousSearchProgressListener(ThreadPool threadPool) { - return new AsynchronousSearchProgressListener(threadPool.absoluteTimeInMillis(), r -> null, e -> null, threadPool.generic(), - threadPool::relativeTimeInMillis, () -> getReduceContextBuilder()); + return new AsynchronousSearchProgressListener( + threadPool.absoluteTimeInMillis(), + r -> null, + e -> null, + threadPool.generic(), + threadPool::relativeTimeInMillis, + () -> getReduceContextBuilder() + ); } - public static AsynchronousSearchProgressListener mockAsynchronousSearchProgressListener(ThreadPool threadPool, - Function successFunction, - Function failureFunction) { - return new AsynchronousSearchProgressListener(threadPool.absoluteTimeInMillis(), successFunction, failureFunction, - threadPool.generic(), - threadPool::relativeTimeInMillis, () -> getReduceContextBuilder()); + public static AsynchronousSearchProgressListener mockAsynchronousSearchProgressListener( + ThreadPool threadPool, + Function successFunction, + Function failureFunction + ) { + return new AsynchronousSearchProgressListener( + threadPool.absoluteTimeInMillis(), + successFunction, + failureFunction, + threadPool.generic(), + threadPool::relativeTimeInMillis, + () -> getReduceContextBuilder() + ); } private static InternalAggregation.ReduceContextBuilder getReduceContextBuilder() { return new InternalAggregation.ReduceContextBuilder() { @Override public InternalAggregation.ReduceContext forPartialReduction() { - return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null, - null); + return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null, null); } public InternalAggregation.ReduceContext forFinalReduction() { return InternalAggregation.ReduceContext.forFinalReduction( - BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, PipelineAggregator.PipelineTree.EMPTY); - }}; + BigArrays.NON_RECYCLING_INSTANCE, + null, + b -> {}, + PipelineAggregator.PipelineTree.EMPTY + ); + } + }; } } diff --git a/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContextTests.java b/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContextTests.java index dbd71023..b65714e2 100644 --- a/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContextTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveContextTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.active; import org.opensearch.commons.authuser.User; @@ -49,25 +52,43 @@ public void testInitializeContext() { int writeThreadPoolSize = randomIntBetween(1, 2); int writeThreadPoolQueueSize = randomIntBetween(1, 2); Settings settings = Settings.builder() - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".size", writeThreadPoolSize) - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".queue_size", writeThreadPoolQueueSize) - .build(); + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".size", + writeThreadPoolSize + ) + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".queue_size", + writeThreadPoolQueueSize + ) + .build(); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); - ScalingExecutorBuilder scalingExecutorBuilder = - new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)); + ScalingExecutorBuilder scalingExecutorBuilder = new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ); threadPool = new TestThreadPool("Tests", settings, scalingExecutorBuilder); String node = UUID.randomUUID().toString(); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(threadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); boolean keepOnCompletion = randomBoolean(); User user = TestClientUtils.randomUser(); TimeValue keepAlive = TimeValue.timeValueDays(randomInt(100)); - AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext(asContextId, node, - keepAlive, keepOnCompletion, threadPool, threadPool::absoluteTimeInMillis, asProgressListener, user, () -> true); + AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext( + asContextId, + node, + keepAlive, + keepOnCompletion, + threadPool, + threadPool::absoluteTimeInMillis, + asProgressListener, + user, + () -> true + ); assertEquals(AsynchronousSearchState.INIT, context.getAsynchronousSearchState()); assertNull(context.getTask()); assertNull(context.getAsynchronousSearchId()); @@ -83,31 +104,56 @@ public void testTaskBootstrap() { int writeThreadPoolSize = randomIntBetween(1, 2); int writeThreadPoolQueueSize = randomIntBetween(1, 2); Settings settings = Settings.builder() - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".size", writeThreadPoolSize) - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".queue_size", writeThreadPoolQueueSize) - .build(); + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".size", + writeThreadPoolSize + ) + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".queue_size", + writeThreadPoolQueueSize + ) + .build(); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); - ScalingExecutorBuilder scalingExecutorBuilder = - new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)); + ScalingExecutorBuilder scalingExecutorBuilder = new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ); threadPool = new TestThreadPool("Tests", settings, scalingExecutorBuilder); String node = UUID.randomUUID().toString(); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(threadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); boolean keepOnCompletion = randomBoolean(); User user = TestClientUtils.randomUser(); TimeValue keepAlive = TimeValue.timeValueDays(randomInt(100)); - AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext(asContextId, node, - keepAlive, keepOnCompletion, threadPool, - threadPool::absoluteTimeInMillis, asProgressListener, user, () -> true); + AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext( + asContextId, + node, + keepAlive, + keepOnCompletion, + threadPool, + threadPool::absoluteTimeInMillis, + asProgressListener, + user, + () -> true + ); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest("test")); request.keepAlive(keepAlive); request.keepOnCompletion(keepOnCompletion); - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", - SearchAction.NAME, TaskId.EMPTY_TASK_ID, emptyMap(), context, request, (c) -> {}); + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + context, + request, + (c) -> {} + ); context.setTask(task); assertEquals(task, context.getTask()); assertEquals(task.getStartTime(), context.getStartTimeMillis()); @@ -137,31 +183,48 @@ public void testProcessSearchCompletion() throws InterruptedException { int writeThreadPoolSize = randomIntBetween(1, 2); int writeThreadPoolQueueSize = randomIntBetween(1, 2); Settings settings = Settings.builder() - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".size", writeThreadPoolSize) - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".queue_size", writeThreadPoolQueueSize) - .build(); + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".size", + writeThreadPoolSize + ) + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".queue_size", + writeThreadPoolQueueSize + ) + .build(); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); - ScalingExecutorBuilder scalingExecutorBuilder = - new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)); + ScalingExecutorBuilder scalingExecutorBuilder = new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ); threadPool = new TestThreadPool("Tests", settings, scalingExecutorBuilder); String node = UUID.randomUUID().toString(); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(threadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); boolean keepOnCompletion = true; TimeValue keepAlive = TimeValue.timeValueDays(randomInt(100)); - AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext(asContextId, node, - keepAlive, keepOnCompletion, threadPool, - threadPool::absoluteTimeInMillis, asProgressListener, null, () -> true); + AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext( + asContextId, + node, + keepAlive, + keepOnCompletion, + threadPool, + threadPool::absoluteTimeInMillis, + asProgressListener, + null, + () -> true + ); if (randomBoolean()) { SearchResponse mockSearchResponse = getMockSearchResponse(); try { context.processSearchResponse(mockSearchResponse); } catch (Exception ex) { - fail("Unexpected exception "+ ex); + fail("Unexpected exception " + ex); } if (mockSearchResponse.equals(context.getSearchResponse())) { assertNull(context.getSearchError()); @@ -171,7 +234,7 @@ public void testProcessSearchCompletion() throws InterruptedException { try { context.processSearchFailure(e); } catch (Exception ex) { - fail("Unexpected exception "+ ex); + fail("Unexpected exception " + ex); } if (e.equals(context.getSearchError())) { assertNull(context.getSearchResponse()); @@ -183,35 +246,58 @@ public void testProcessSearchCompletion() throws InterruptedException { } } - public void testClosedContext() throws InterruptedException { TestThreadPool threadPool = null; try { int writeThreadPoolSize = randomIntBetween(1, 2); int writeThreadPoolQueueSize = randomIntBetween(1, 2); Settings settings = Settings.builder() - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".size", writeThreadPoolSize) - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".queue_size", writeThreadPoolQueueSize) - .build(); + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".size", + writeThreadPoolSize + ) + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".queue_size", + writeThreadPoolQueueSize + ) + .build(); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); - ScalingExecutorBuilder scalingExecutorBuilder = - new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)); + ScalingExecutorBuilder scalingExecutorBuilder = new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ); threadPool = new TestThreadPool("Tests", settings, scalingExecutorBuilder); String node = UUID.randomUUID().toString(); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(threadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); boolean keepOnCompletion = randomBoolean(); TimeValue keepAlive = TimeValue.timeValueDays(randomInt(100)); - AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext(asContextId, node, - keepAlive, keepOnCompletion, threadPool, - threadPool::absoluteTimeInMillis, asProgressListener, null, () -> true); - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", - SearchAction.NAME, TaskId.EMPTY_TASK_ID, emptyMap(), context, null, (c) -> { - }); + AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext( + asContextId, + node, + keepAlive, + keepOnCompletion, + threadPool, + threadPool::absoluteTimeInMillis, + asProgressListener, + null, + () -> true + ); + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + context, + null, + (c) -> {} + ); context.setTask(task); assertEquals(task, context.getTask()); assertEquals(task.getStartTime(), context.getStartTimeMillis()); @@ -230,13 +316,23 @@ SearchAction.NAME, TaskId.EMPTY_TASK_ID, emptyMap(), context, null, (c) -> { } protected SearchResponse getMockSearchResponse() { - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), - "", 1, 1, 0, 0, - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 1 + ), + "", + 1, + 1, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } } - diff --git a/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStoreTests.java b/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStoreTests.java index ec029a79..8c8d5265 100644 --- a/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStoreTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/context/active/AsynchronousSearchActiveStoreTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.active; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -52,24 +55,36 @@ public class AsynchronousSearchActiveStoreTests extends OpenSearchTestCase { @Before public void createObjects() { Settings settings = Settings.builder() - .put("node.name", "test") - .put("cluster.name", "ClusterServiceTests") - .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), maxRunningContexts) - .build(); - final Set> settingsSet = - Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING)).collect(Collectors.toSet()); + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), maxRunningContexts) + .build(); + final Set> settingsSet = Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING) + ).collect(Collectors.toSet()); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); List> executorBuilders = new ArrayList<>(); - executorBuilders.add(new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30))); + executorBuilders.add( + new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ) + ); executorBuilder = executorBuilders.get(0); clusterSettings = new ClusterSettings(settings, settingsSet); } public void testPutContextRejection() throws InterruptedException, BrokenBarrierException, TimeoutException { - DiscoveryNode node = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode node = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(this.getClass().getName(), executorBuilder); @@ -88,16 +103,27 @@ public void testPutContextRejection() throws InterruptedException, BrokenBarrier runnables.add(() -> { try { AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener( - finalTestThreadPool1); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - runningContexts.incrementAndGet()); + finalTestThreadPool1 + ); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + runningContexts.incrementAndGet() + ); boolean keepOnCompletion = randomBoolean(); TimeValue keepAlive = TimeValue.timeValueDays(randomInt(100)); AsynchronousSearchContextEventListener asContextEventListener = new AsynchronousSearchContextEventListener() { }; - AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext(asContextId, node.getId(), - keepAlive, keepOnCompletion, finalTestThreadPool1, - finalTestThreadPool1::absoluteTimeInMillis, asProgressListener, null, () -> true); + AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext( + asContextId, + node.getId(), + keepAlive, + keepOnCompletion, + finalTestThreadPool1, + finalTestThreadPool1::absoluteTimeInMillis, + asProgressListener, + null, + () -> true + ); activeStore.putContext(asContextId, context, asContextEventListener::onContextRejected); numSuccesses.getAndIncrement(); Optional optional = activeStore.getContext(asContextId); @@ -126,13 +152,13 @@ public void testPutContextRejection() throws InterruptedException, BrokenBarrier } ThreadPool finalTestThreadPool = testThreadPool; runnables.forEach(r -> finalTestThreadPool.generic().execute(r)); - barrier.await(5, TimeUnit.SECONDS);//create contexts + barrier.await(5, TimeUnit.SECONDS);// create contexts assertEquals(activeStore.getAllContexts().size(), maxRunningContexts); assertEquals(numFailures.get(), 0); assertEquals(numRejected.get(), 1); assertEquals(numSuccesses.get(), maxRunningContexts); assertsLatch.countDown(); - barrier.await(5, TimeUnit.SECONDS); //free contexts + barrier.await(5, TimeUnit.SECONDS); // free contexts assertEquals(activeStore.getAllContexts().size(), 0); assertEquals(numFailures.get(), 0); @@ -143,15 +169,22 @@ public void testPutContextRejection() throws InterruptedException, BrokenBarrier } public void testGetNonExistentContext() { - DiscoveryNode node = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode node = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(this.getClass().getName(), executorBuilder); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, node, clusterSettings); AsynchronousSearchActiveStore activeStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); Optional optional = activeStore.getContext(asContextId); assertFalse(optional.isPresent()); assertEquals(activeStore.getAllContexts().size(), 0); @@ -161,15 +194,22 @@ public void testGetNonExistentContext() { } public void testFreeNonExistentContext() { - DiscoveryNode node = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode node = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(this.getClass().getName(), executorBuilder); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, node, clusterSettings); AsynchronousSearchActiveStore activeStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); assertFalse(activeStore.freeContext(asContextId)); assertEquals(activeStore.getAllContexts().size(), 0); } finally { @@ -178,8 +218,13 @@ public void testFreeNonExistentContext() { } public void testContextFoundWithContextIdMismatch() { - DiscoveryNode node = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode node = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(this.getClass().getName(), executorBuilder); diff --git a/src/test/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermitsTests.java b/src/test/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermitsTests.java index ea5431f3..fba4800b 100644 --- a/src/test/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermitsTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/context/permits/AsynchronousSearchContextPermitsTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.permits; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -58,15 +61,19 @@ public static void setupThreadPool() { int writeThreadPoolSize = randomIntBetween(1, 2); int writeThreadPoolQueueSize = randomIntBetween(1, 2); Settings settings = Settings.builder() - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".size", writeThreadPoolSize) - .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME - + ".queue_size", writeThreadPoolQueueSize) - .build(); + .put("thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".size", writeThreadPoolSize) + .put( + "thread_pool." + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME + ".queue_size", + writeThreadPoolQueueSize + ) + .build(); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); - ScalingExecutorBuilder scalingExecutorBuilder = - new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)); + ScalingExecutorBuilder scalingExecutorBuilder = new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ); threadPool = new TestThreadPool("PermitsTests", settings, scalingExecutorBuilder); } @@ -78,8 +85,10 @@ public static void shutdownThreadPool() { @Before public void createAsynchronousSearchContextPermit() { - permits = new AsynchronousSearchContextPermits(new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()), threadPool); + permits = new AsynchronousSearchContextPermits( + new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong()), + threadPool + ); } @After @@ -90,8 +99,7 @@ public void checkNoInflightOperations() { public void testAllOperationsInvoked() throws InterruptedException, TimeoutException { int numThreads = 10; - class DummyException extends RuntimeException { - } + class DummyException extends RuntimeException {} List> futures = new ArrayList<>(); List operationThreads = new ArrayList<>(); @@ -142,8 +150,10 @@ public void onResponse(Releasable releasable) { assertNotNull(future.get(1, TimeUnit.MINUTES)); } catch (ExecutionException e) { - assertThat(e.getCause(), either(instanceOf(DummyException.class)) - .or(instanceOf(OpenSearchRejectedExecutionException.class))); + assertThat( + e.getCause(), + either(instanceOf(DummyException.class)).or(instanceOf(OpenSearchRejectedExecutionException.class)) + ); } } @@ -154,14 +164,14 @@ public void onResponse(Releasable releasable) { blockFinished.await(); } - public void testAsyncBlockOperationsOperationBeforeBlocked() throws InterruptedException, BrokenBarrierException { final CyclicBarrier barrier = new CyclicBarrier(2); final CountDownLatch operationExecutingLatch = new CountDownLatch(1); final CountDownLatch firstOperationLatch = new CountDownLatch(1); final CountDownLatch firstOperationCompleteLatch = new CountDownLatch(1); - final Thread firstOperationThread = - new Thread(controlledAcquire(barrier, operationExecutingLatch, firstOperationLatch, firstOperationCompleteLatch)); + final Thread firstOperationThread = new Thread( + controlledAcquire(barrier, operationExecutingLatch, firstOperationLatch, firstOperationCompleteLatch) + ); firstOperationThread.start(); barrier.await(); @@ -181,22 +191,23 @@ public void testAsyncBlockOperationsOperationBeforeBlocked() throws InterruptedE final AtomicBoolean secondOperation = new AtomicBoolean(); final Thread secondOperationThread = new Thread(() -> { secondOperationExecuting.countDown(); - permits.asyncAcquirePermit( - new ActionListener() { - @Override - public void onResponse(Releasable releasable) { - secondOperation.set(true); - releasable.close(); - secondOperationComplete.countDown(); - } - - @Override - public void onFailure(Exception e) { - throw new RuntimeException(e); - } - }, - - TimeValue.timeValueMinutes(1), ""); + permits.asyncAcquirePermit(new ActionListener() { + @Override + public void onResponse(Releasable releasable) { + secondOperation.set(true); + releasable.close(); + secondOperationComplete.countDown(); + } + + @Override + public void onFailure(Exception e) { + throw new RuntimeException(e); + } + }, + + TimeValue.timeValueMinutes(1), + "" + ); }); secondOperationThread.start(); @@ -243,8 +254,10 @@ public void onFailure(Exception e) { } }; permits.asyncAcquirePermit( - new LatchedActionListener(onAcquired, operationLatch), - TimeValue.timeValueMinutes(1), ""); + new LatchedActionListener(onAcquired, operationLatch), + TimeValue.timeValueMinutes(1), + "" + ); }); thread.start(); threads.add(thread); @@ -269,8 +282,10 @@ public void onFailure(Exception e) { } }; permits.asyncAcquirePermit( - new LatchedActionListener(onAcquired, operationLatch), - TimeValue.timeValueMinutes(1), ""); + new LatchedActionListener(onAcquired, operationLatch), + TimeValue.timeValueMinutes(1), + "" + ); }); blockingThread.start(); @@ -303,15 +318,13 @@ public void onFailure(Exception e) { }); } - public void testTimeout() throws BrokenBarrierException, InterruptedException { final CyclicBarrier barrier = new CyclicBarrier(2); final CountDownLatch operationExecutingLatch = new CountDownLatch(1); final CountDownLatch operationLatch = new CountDownLatch(1); final CountDownLatch operationCompleteLatch = new CountDownLatch(1); - final Thread thread = new Thread(controlledAcquire(barrier, operationExecutingLatch, - operationLatch, operationCompleteLatch)); + final Thread thread = new Thread(controlledAcquire(barrier, operationExecutingLatch, operationLatch, operationCompleteLatch)); thread.start(); barrier.await(); @@ -320,13 +333,12 @@ public void testTimeout() throws BrokenBarrierException, InterruptedException { final CountDownLatch onFailureLatch = new CountDownLatch(2); permits.asyncAcquireAllPermits(new LatchedActionListener<>(ActionListener.wrap(releasable -> { - releasable.close(); - fail("Permit acquisition attempt should have timed out"); - }, e -> { - assertTrue(e instanceof TimeoutException); - assertThat(e, hasToString(containsString("timed out"))); - }), onFailureLatch), - TimeValue.timeValueMillis(1), ""); + releasable.close(); + fail("Permit acquisition attempt should have timed out"); + }, e -> { + assertTrue(e instanceof TimeoutException); + assertThat(e, hasToString(containsString("timed out"))); + }), onFailureLatch), TimeValue.timeValueMillis(1), ""); { final AtomicReference reference = new AtomicReference<>(); @@ -353,7 +365,6 @@ public void onFailure(final Exception e) { thread.join(); } - /** * Returns an operation that acquires a permit and synchronizes in the following manner: *
    @@ -370,36 +381,35 @@ public void onFailure(final Exception e) { * @return a controllable runnable that acquires a permit */ private Runnable controlledAcquire( - final CyclicBarrier barrier, - final CountDownLatch operationExecutingLatch, - final CountDownLatch operationLatch, - final CountDownLatch operationCompleteLatch) { + final CyclicBarrier barrier, + final CountDownLatch operationExecutingLatch, + final CountDownLatch operationLatch, + final CountDownLatch operationCompleteLatch + ) { return () -> { try { barrier.await(); } catch (final BrokenBarrierException | InterruptedException e) { throw new RuntimeException(e); } - permits.asyncAcquirePermit( - new ActionListener() { - @Override - public void onResponse(Releasable releasable) { - operationExecutingLatch.countDown(); - try { - operationLatch.await(); - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } - releasable.close(); - operationCompleteLatch.countDown(); - } - - @Override - public void onFailure(Exception e) { - throw new RuntimeException(e); - } - }, - TimeValue.timeValueMinutes(1), ""); + permits.asyncAcquirePermit(new ActionListener() { + @Override + public void onResponse(Releasable releasable) { + operationExecutingLatch.countDown(); + try { + operationLatch.await(); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + releasable.close(); + operationCompleteLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + throw new RuntimeException(e); + } + }, TimeValue.timeValueMinutes(1), ""); }; } @@ -421,6 +431,4 @@ public void onFailure(final Exception e) { }; } - } - diff --git a/src/test/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermitsTests.java b/src/test/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermitsTests.java index 69db6df8..52c3c315 100644 --- a/src/test/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermitsTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermitsTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.permits; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -20,33 +23,40 @@ public class NoopAsynchronousSearchContextPermitsTests extends OpenSearchTestCas public void testAcquireAllPermits() { NoopAsynchronousSearchContextPermits permits = new NoopAsynchronousSearchContextPermits( - new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong())); - expectThrows(IllegalStateException.class, - () -> permits.asyncAcquireAllPermits(ActionListener.wrap(Assert::fail), TimeValue.ZERO, "reason")); + new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong()) + ); + expectThrows( + IllegalStateException.class, + () -> permits.asyncAcquireAllPermits(ActionListener.wrap(Assert::fail), TimeValue.ZERO, "reason") + ); } public void testAcquireSinglePermit() throws InterruptedException { NoopAsynchronousSearchContextPermits permits = new NoopAsynchronousSearchContextPermits( - new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong())); + new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong()) + ); CountDownLatch countDownLatch = new CountDownLatch(1); - permits.asyncAcquirePermit(new LatchedActionListener<>(ActionListener.wrap(r -> {}, - e -> fail("expected permit acquisition to succeed")), countDownLatch), TimeValue.ZERO, "reason"); + permits.asyncAcquirePermit( + new LatchedActionListener<>(ActionListener.wrap(r -> {}, e -> fail("expected permit acquisition to succeed")), countDownLatch), + TimeValue.ZERO, + "reason" + ); countDownLatch.await(); } public void testAcquireSinglePermitAfterClosure() throws InterruptedException { AsynchronousSearchContextId contextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong()); - NoopAsynchronousSearchContextPermits permits = new NoopAsynchronousSearchContextPermits( - contextId); + NoopAsynchronousSearchContextPermits permits = new NoopAsynchronousSearchContextPermits(contextId); permits.close(); CountDownLatch countDownLatch = new CountDownLatch(1); - permits.asyncAcquirePermit(new LatchedActionListener<>(ActionListener.wrap( - r -> fail("expected permit acquisition to fail due to permit closure"), - e -> { - assertTrue("expected context closed exception, got " + e.getClass(), - e instanceof AsynchronousSearchContextClosedException); - assertTrue(((AsynchronousSearchContextClosedException) e).getAsynchronousSearchContextId().equals(contextId)); - }), countDownLatch), TimeValue.ZERO, "reason"); + permits.asyncAcquirePermit( + new LatchedActionListener<>(ActionListener.wrap(r -> fail("expected permit acquisition to fail due to permit closure"), e -> { + assertTrue("expected context closed exception, got " + e.getClass(), e instanceof AsynchronousSearchContextClosedException); + assertTrue(((AsynchronousSearchContextClosedException) e).getAsynchronousSearchContextId().equals(contextId)); + }), countDownLatch), + TimeValue.ZERO, + "reason" + ); countDownLatch.await(); } diff --git a/src/test/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContextTests.java b/src/test/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContextTests.java index 7c82773e..9733ed0e 100644 --- a/src/test/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContextTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContextTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.context.persistence; import org.opensearch.commons.authuser.User; @@ -40,24 +43,42 @@ public class AsynchronousSearchPersistenceContextTests extends OpenSearchTestCas */ public void testSerializationRoundTripWithSearchResponse() throws IOException { AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong()); - String id = AsynchronousSearchIdConverter.buildAsyncId(new AsynchronousSearchId(UUID.randomUUID().toString(), - randomNonNegativeLong(), asContextId)); + String id = AsynchronousSearchIdConverter.buildAsyncId( + new AsynchronousSearchId(UUID.randomUUID().toString(), randomNonNegativeLong(), asContextId) + ); long expirationTimeMillis = randomNonNegativeLong(); long startTimeMillis = randomNonNegativeLong(); SearchResponse searchResponse = getMockSearchResponse(); User user = TestClientUtils.randomUserOrNull(); SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - AsynchronousSearchPersistenceContext asPersistenceContext = - new AsynchronousSearchPersistenceContext(id, asContextId, new AsynchronousSearchPersistenceModel(startTimeMillis, - expirationTimeMillis, searchResponse, null, user), System::currentTimeMillis, - new NamedWriteableRegistry(searchModule.getNamedWriteables())); - assertEquals(asPersistenceContext, new AsynchronousSearchPersistenceContext(id, asContextId, + AsynchronousSearchPersistenceContext asPersistenceContext = new AsynchronousSearchPersistenceContext( + id, + asContextId, + new AsynchronousSearchPersistenceModel(startTimeMillis, expirationTimeMillis, searchResponse, null, user), + System::currentTimeMillis, + new NamedWriteableRegistry(searchModule.getNamedWriteables()) + ); + assertEquals( + asPersistenceContext, + new AsynchronousSearchPersistenceContext( + id, + asContextId, new AsynchronousSearchPersistenceModel(startTimeMillis, expirationTimeMillis, searchResponse, null, user), - System::currentTimeMillis, new NamedWriteableRegistry(Collections.emptyList()))); + System::currentTimeMillis, + new NamedWriteableRegistry(Collections.emptyList()) + ) + ); assertEquals( - asPersistenceContext.getAsynchronousSearchResponse(), - new AsynchronousSearchResponse(id, asPersistenceContext.getAsynchronousSearchState(), startTimeMillis, - expirationTimeMillis, searchResponse, null)); + asPersistenceContext.getAsynchronousSearchResponse(), + new AsynchronousSearchResponse( + id, + asPersistenceContext.getAsynchronousSearchState(), + startTimeMillis, + expirationTimeMillis, + searchResponse, + null + ) + ); } /** @@ -68,20 +89,27 @@ public void testSerializationRoundTripWithSearchResponse() throws IOException { */ public void testSerializationRoundTripWithError() throws IOException { AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong()); - String id = AsynchronousSearchIdConverter.buildAsyncId(new AsynchronousSearchId(UUID.randomUUID().toString(), - randomNonNegativeLong(), - asContextId)); + String id = AsynchronousSearchIdConverter.buildAsyncId( + new AsynchronousSearchId(UUID.randomUUID().toString(), randomNonNegativeLong(), asContextId) + ); long expirationTimeMillis = randomNonNegativeLong(); long startTimeMillis = randomNonNegativeLong(); ShardSearchFailure shardSearchFailure = new ShardSearchFailure(new RuntimeException("runtime-exception")); - SearchPhaseExecutionException exception = new SearchPhaseExecutionException("phase", "msg", new NullPointerException(), - new ShardSearchFailure[] {shardSearchFailure}); + SearchPhaseExecutionException exception = new SearchPhaseExecutionException( + "phase", + "msg", + new NullPointerException(), + new ShardSearchFailure[] { shardSearchFailure } + ); User user = TestClientUtils.randomUser(); SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - AsynchronousSearchPersistenceContext asPersistenceContext = new AsynchronousSearchPersistenceContext(id, asContextId, - new AsynchronousSearchPersistenceModel(startTimeMillis, expirationTimeMillis, null, exception, user), - System::currentTimeMillis, - new NamedWriteableRegistry(searchModule.getNamedWriteables())); + AsynchronousSearchPersistenceContext asPersistenceContext = new AsynchronousSearchPersistenceContext( + id, + asContextId, + new AsynchronousSearchPersistenceModel(startTimeMillis, expirationTimeMillis, null, exception, user), + System::currentTimeMillis, + new NamedWriteableRegistry(searchModule.getNamedWriteables()) + ); OpenSearchException deserializedException = asPersistenceContext.getAsynchronousSearchResponse().getError(); assertTrue(deserializedException instanceof SearchPhaseExecutionException); assertEquals("phase", ((SearchPhaseExecutionException) deserializedException).getPhaseName()); @@ -92,13 +120,23 @@ public void testSerializationRoundTripWithError() throws IOException { } protected SearchResponse getMockSearchResponse() { - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), - "", 1, 1, 0, 0, - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 1 + ), + "", + 1, + 1, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } } - diff --git a/src/test/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdTests.java b/src/test/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdTests.java index ea9342c5..10568cee 100644 --- a/src/test/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/id/AsynchronousSearchIdTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.id; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -16,20 +19,18 @@ public class AsynchronousSearchIdTests extends OpenSearchTestCase { public void testAsynchronousSearchIdParsing() { String node = "node" + randomIntBetween(1, 50); long taskId = randomNonNegativeLong(); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( - UUIDs.base64UUID(), randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUIDs.base64UUID(), randomNonNegativeLong()); AsynchronousSearchId original = new AsynchronousSearchId(node, taskId, asContextId); - //generate identifier to access the submitted asynchronous search + // generate identifier to access the submitted asynchronous search String id = AsynchronousSearchIdConverter.buildAsyncId(original); - //parse the AsynchronousSearchId object which will contain information regarding node running the search, the associated task and - //context id. + // parse the AsynchronousSearchId object which will contain information regarding node running the search, the associated task and + // context id. AsynchronousSearchId parsed = AsynchronousSearchIdConverter.parseAsyncId(id); assertEquals(original, parsed); assertEquals(parsed.toString(), "[" + node + "][" + taskId + "][" + asContextId + "]"); } - public void testAsynchronousSearchIdParsingFailure() { String id = UUID.randomUUID().toString(); expectThrows(IllegalArgumentException.class, () -> AsynchronousSearchIdConverter.parseAsyncId(id)); diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchQueryIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchQueryIT.java index 29a455ba..3c360add 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchQueryIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchQueryIT.java @@ -1,14 +1,11 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ - package org.opensearch.search.asynchronous.integTests; import org.apache.lucene.tests.analysis.MockTokenizer; @@ -102,9 +99,12 @@ protected int maximumNumberOfReplicas() { public void testEmptyQueryString() throws ExecutionException, InterruptedException, IOException { String index = UUID.randomUUID().toString(); createIndex(index); - indexRandom(true, client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), - client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), - client().prepareIndex(index).setId("3").setSource("field1", "quick")); + indexRandom( + true, + client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), + client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), + client().prepareIndex(index).setId("3").setSource("field1", "quick") + ); SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(new SearchSourceBuilder()); searchRequest.source().query(queryStringQuery("quick")); @@ -118,48 +118,56 @@ public void testEmptyQueryString() throws ExecutionException, InterruptedExcepti public void testAggregationQuery() throws InterruptedException, ExecutionException { // Tests that we can refer to nested elements under a sample in a path // statement - assertAcked(prepareCreate("test") - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, NUM_SHARDS).put(SETTING_NUMBER_OF_REPLICAS, 0)) - .setMapping( - "author", "type=keyword", "name", "type=text", "genre", "type=keyword", "price", "type=float")); + assertAcked( + prepareCreate("test").setSettings( + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, NUM_SHARDS).put(SETTING_NUMBER_OF_REPLICAS, 0) + ).setMapping("author", "type=keyword", "name", "type=text", "genre", "type=keyword", "price", "type=float") + ); createIndex("idx_unmapped"); // idx_unmapped_author is same as main index but missing author field - assertAcked(prepareCreate("idx_unmapped_author") - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, NUM_SHARDS).put(SETTING_NUMBER_OF_REPLICAS, 0)) - .setMapping("name", "type=text", "genre", "type=keyword", "price", "type=float")); + assertAcked( + prepareCreate("idx_unmapped_author").setSettings( + Settings.builder().put(SETTING_NUMBER_OF_SHARDS, NUM_SHARDS).put(SETTING_NUMBER_OF_REPLICAS, 0) + ).setMapping("name", "type=text", "genre", "type=keyword", "price", "type=float") + ); ensureGreen(); String data[] = { - // "id,cat,name,price,inStock,author_t,series_t,sequence_i,genre_s", - "0553573403,book,A Game of Thrones,7.99,true,George R.R. Martin,A Song of Ice and Fire,1,fantasy", - "0553579908,book,A Clash of Kings,7.99,true,George R.R. Martin,A Song of Ice and Fire,2,fantasy", - "055357342X,book,A Storm of Swords,7.99,true,George R.R. Martin,A Song of Ice and Fire,3,fantasy", - "0553293354,book,Foundation,17.99,true,Isaac Asimov,Foundation Novels,1,scifi", - "0812521390,book,The Black Company,6.99,false,Glen Cook,The Chronicles of The Black Company,1,fantasy", - "0812550706,book,Ender's Game,6.99,true,Orson Scott Card,Ender,1,scifi", - "0441385532,book,Jhereg,7.95,false,Steven Brust,Vlad Taltos,1,fantasy", - "0380014300,book,Nine Princes In Amber,6.99,true,Roger Zelazny,the Chronicles of Amber,1,fantasy", - "0805080481,book,The Book of Three,5.99,true,Lloyd Alexander,The Chronicles of Prydain,1,fantasy", - "080508049X,book,The Black Cauldron,5.99,true,Lloyd Alexander,The Chronicles of Prydain,2,fantasy" + // "id,cat,name,price,inStock,author_t,series_t,sequence_i,genre_s", + "0553573403,book,A Game of Thrones,7.99,true,George R.R. Martin,A Song of Ice and Fire,1,fantasy", + "0553579908,book,A Clash of Kings,7.99,true,George R.R. Martin,A Song of Ice and Fire,2,fantasy", + "055357342X,book,A Storm of Swords,7.99,true,George R.R. Martin,A Song of Ice and Fire,3,fantasy", + "0553293354,book,Foundation,17.99,true,Isaac Asimov,Foundation Novels,1,scifi", + "0812521390,book,The Black Company,6.99,false,Glen Cook,The Chronicles of The Black Company,1,fantasy", + "0812550706,book,Ender's Game,6.99,true,Orson Scott Card,Ender,1,scifi", + "0441385532,book,Jhereg,7.95,false,Steven Brust,Vlad Taltos,1,fantasy", + "0380014300,book,Nine Princes In Amber,6.99,true,Roger Zelazny,the Chronicles of Amber,1,fantasy", + "0805080481,book,The Book of Three,5.99,true,Lloyd Alexander,The Chronicles of Prydain,1,fantasy", + "080508049X,book,The Black Cauldron,5.99,true,Lloyd Alexander,The Chronicles of Prydain,2,fantasy" }; for (int i = 0; i < data.length; i++) { String[] parts = data[i].split(","); - client().prepareIndex("test").setId("" + i) - .setSource("author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])).get(); - client().prepareIndex("idx_unmapped_author").setId("" + i) - .setSource("name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])).get(); + client().prepareIndex("test") + .setId("" + i) + .setSource("author", parts[5], "name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])) + .get(); + client().prepareIndex("idx_unmapped_author") + .setId("" + i) + .setSource("name", parts[2], "genre", parts[8], "price", Float.parseFloat(parts[3])) + .get(); } client().admin().indices().refresh(new RefreshRequest("test")).get(); boolean asc = randomBoolean(); SearchRequest searchRequest = new SearchRequest("test"); searchRequest.source(new SearchSourceBuilder()); - searchRequest.source().aggregation(terms("genres") - .field("genre") - .order(BucketOrder.aggregation("sample>max_price.value", asc)) - .subAggregation(sampler("sample").shardSize(100) - .subAggregation(max("max_price").field("price")))); + searchRequest.source() + .aggregation( + terms("genres").field("genre") + .order(BucketOrder.aggregation("sample>max_price.value", asc)) + .subAggregation(sampler("sample").shardSize(100).subAggregation(max("max_price").field("price"))) + ); SearchResponse response = getPersistedAsynchronousSearchResponse(searchRequest).getSearchResponse(); assertSearchResponse(response); Terms genres = response.getAggregations().get("genres"); @@ -182,32 +190,31 @@ public void testAggregationQuery() throws InterruptedException, ExecutionExcepti } public void testIpRangeQuery() throws InterruptedException { - assertAcked(prepareCreate("idx") - .setMapping("ip", "type=ip", "ips", "type=ip")); + assertAcked(prepareCreate("idx").setMapping("ip", "type=ip", "ips", "type=ip")); waitForRelocation(ClusterHealthStatus.GREEN); - indexRandom(true, - client().prepareIndex("idx").setId("1").setSource( - "ip", "192.168.1.7", - "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), - client().prepareIndex("idx").setId("2").setSource( - "ip", "192.168.1.10", - "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), - client().prepareIndex("idx").setId("3").setSource( - "ip", "2001:db8::ff00:42:8329", - "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); + indexRandom( + true, + client().prepareIndex("idx").setId("1").setSource("ip", "192.168.1.7", "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), + client().prepareIndex("idx").setId("2").setSource("ip", "192.168.1.10", "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), + client().prepareIndex("idx") + .setId("3") + .setSource("ip", "2001:db8::ff00:42:8329", "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")) + ); assertAcked(prepareCreate("idx_unmapped")); waitForRelocation(ClusterHealthStatus.GREEN); refresh(); SearchRequest searchRequest = new SearchRequest("idx"); searchRequest.source(new SearchSourceBuilder()); - searchRequest.source().aggregation( + searchRequest.source() + .aggregation( AggregationBuilders.ipRange("my_range") - .field("ip") - .addUnboundedTo("192.168.1.0") - .addRange("192.168.1.0", "192.168.1.10") - .addUnboundedFrom("192.168.1.10")); + .field("ip") + .addUnboundedTo("192.168.1.0") + .addRange("192.168.1.0", "192.168.1.10") + .addUnboundedFrom("192.168.1.10") + ); SearchResponse rsp = getPersistedAsynchronousSearchResponse(searchRequest).getSearchResponse(); assertSearchResponse(rsp); Range range = rsp.getAggregations().get("my_range"); @@ -235,22 +242,21 @@ public void testIpRangeQuery() throws InterruptedException { public void testHighlighterQuery() throws IOException, InterruptedException { XContentBuilder mappings = jsonBuilder(); mappings.startObject() - .startObject("properties") - .startObject("text") - .field("type", "keyword") - .field("store", true) - .endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test1") - .setMapping(mappings)); - client().prepareIndex("test1").setId("1") - .setSource(jsonBuilder().startObject().field("text", "foo").endObject()) - .get(); + .startObject("properties") + .startObject("text") + .field("type", "keyword") + .field("store", true) + .endObject() + .endObject() + .endObject(); + assertAcked(prepareCreate("test1").setMapping(mappings)); + client().prepareIndex("test1").setId("1").setSource(jsonBuilder().startObject().field("text", "foo").endObject()).get(); refresh(); SearchRequest searchRequest = new SearchRequest("test1"); searchRequest.source(new SearchSourceBuilder()); - searchRequest.source().query(matchQuery("text", "foo")) - .highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("text"))); + searchRequest.source() + .query(matchQuery("text", "foo")) + .highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("text"))); SearchResponse searchResponse = getPersistedAsynchronousSearchResponse(searchRequest).getSearchResponse(); assertHighlight(searchResponse, 0, "text", 0, equalTo("foo")); } @@ -259,11 +265,9 @@ private AsynchronousSearchResponse getPersistedAsynchronousSearchResponse(Search SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); request.waitForCompletionTimeout(TimeValue.timeValueMillis(1)); request.keepOnCompletion(true); - AsynchronousSearchResponse asResponse = TestClientUtils.blockingSubmitAsynchronousSearch(client(), - request); + AsynchronousSearchResponse asResponse = TestClientUtils.blockingSubmitAsynchronousSearch(client(), request); TestClientUtils.assertResponsePersistence(client(), asResponse.getId()); - asResponse = TestClientUtils.blockingGetAsynchronousSearchResponse(client(), - new GetAsynchronousSearchRequest(asResponse.getId())); + asResponse = TestClientUtils.blockingGetAsynchronousSearchResponse(client(), new GetAsynchronousSearchRequest(asResponse.getId())); return asResponse; } @@ -291,8 +295,13 @@ public Reader create(Reader reader) { @Override public Map> getTokenizers() { - return singletonMap("keyword", (indexSettings, environment, name, settings) -> TokenizerFactory.newFactory(name, - () -> new MockTokenizer(MockTokenizer.KEYWORD, false))); + return singletonMap( + "keyword", + (indexSettings, environment, name, settings) -> TokenizerFactory.newFactory( + name, + () -> new MockTokenizer(MockTokenizer.KEYWORD, false) + ) + ); } } } diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchRejectionIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchRejectionIT.java index 58ec7451..99d1d029 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchRejectionIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchRejectionIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.integTests; import org.opensearch.search.asynchronous.action.DeleteAsynchronousSearchAction; @@ -58,15 +61,15 @@ public class AsynchronousSearchRejectionIT extends AsynchronousSearchIntegTestCa @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put("thread_pool.search.size", 1) - .put("thread_pool.search.queue_size", 1) - .put("thread_pool.write.size", 1) - .put("thread_pool.write.queue_size", 10) - .put("thread_pool.get.size", 1) - .put("thread_pool.get.queue_size", 10) - .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) - .build(); + .put(super.nodeSettings(nodeOrdinal)) + .put("thread_pool.search.size", 1) + .put("thread_pool.search.queue_size", 1) + .put("thread_pool.write.size", 1) + .put("thread_pool.write.queue_size", 10) + .put("thread_pool.get.size", 1) + .put("thread_pool.get.queue_size", 10) + .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) + .build(); } @TestLogging(value = "_root:DEBUG", reason = "flaky") @@ -83,71 +86,80 @@ public void testSimulatedSearchRejectionLoad() throws Throwable { final CopyOnWriteArrayList responses = new CopyOnWriteArrayList<>(); for (int i = 0; i < numberOfAsyncOps; i++) { SearchRequest request = client().prepareSearch("test") - .setSearchType(SearchType.QUERY_THEN_FETCH) - .setQuery(QueryBuilders.matchQuery("field", "1")) - .request(); + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setQuery(QueryBuilders.matchQuery("field", "1")) + .request(); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(request); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMinutes(1)); boolean keepOnCompletion = randomBoolean(); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); - client().execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, - new LatchedActionListener<>(new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asynchronousSearchResponse) { - if (asynchronousSearchResponse.getSearchResponse() != null) { - responses.add(asynchronousSearchResponse.getSearchResponse()); - } else if(asynchronousSearchResponse.getError() != null){ - responses.add(asynchronousSearchResponse.getError()); + client().execute( + SubmitAsynchronousSearchAction.INSTANCE, + submitAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asynchronousSearchResponse) { + if (asynchronousSearchResponse.getSearchResponse() != null) { + responses.add(asynchronousSearchResponse.getSearchResponse()); + } else if (asynchronousSearchResponse.getError() != null) { + responses.add(asynchronousSearchResponse.getError()); + } + if (asynchronousSearchResponse.getId() == null) { + // task cancelled by the time we process final response/error due to during partial merge failure. + // no delete required + latch.countDown(); + } else { + DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest( + asynchronousSearchResponse.getId() + ); + client().execute( + DeleteAsynchronousSearchAction.INSTANCE, + deleteAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + assertTrue(acknowledgedResponse.isAcknowledged()); } - if (asynchronousSearchResponse.getId() == null) { - // task cancelled by the time we process final response/error due to during partial merge failure. - // no delete required - latch.countDown(); - } else { - DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest - = new DeleteAsynchronousSearchRequest(asynchronousSearchResponse.getId()); - client().execute(DeleteAsynchronousSearchAction.INSTANCE, deleteAsynchronousSearchRequest, - new LatchedActionListener<>(new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - assertTrue(acknowledgedResponse.isAcknowledged()); - } - @Override - public void onFailure(Exception e) { - Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof OpenSearchRejectedExecutionException) { - numRejections.incrementAndGet(); - } else if (cause instanceof OpenSearchTimeoutException) { - numTimeouts.incrementAndGet(); - } else if(cause instanceof ResourceNotFoundException) { - // deletion is in race with task cancellation due to partial merge failure - numRnf.getAndIncrement(); - } else { - numFailures.incrementAndGet(); - } - } - }, latch)); + @Override + public void onFailure(Exception e) { + Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof OpenSearchRejectedExecutionException) { + numRejections.incrementAndGet(); + } else if (cause instanceof OpenSearchTimeoutException) { + numTimeouts.incrementAndGet(); + } else if (cause instanceof ResourceNotFoundException) { + // deletion is in race with task cancellation due to partial merge failure + numRnf.getAndIncrement(); + } else { + numFailures.incrementAndGet(); + } } + }, latch) + ); } - @Override - public void onFailure(Exception e) { - responses.add(e); - assertThat(e.getMessage(), startsWith("Trying to create too many concurrent searches")); - latch.countDown(); - } - }, latch)); + } + + @Override + public void onFailure(Exception e) { + responses.add(e); + assertThat(e.getMessage(), startsWith("Trying to create too many concurrent searches")); + latch.countDown(); + } + }, latch) + ); } latch.await(); - // validate all responses for (Object response : responses) { if (response instanceof SearchResponse) { SearchResponse searchResponse = (SearchResponse) response; for (ShardSearchFailure failure : searchResponse.getShardFailures()) { - assertTrue("got unexpected reason..." + failure.reason(), - failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected")); + assertTrue( + "got unexpected reason..." + failure.reason(), + failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected") + ); } } else { Exception t = (Exception) response; @@ -155,10 +167,12 @@ public void onFailure(Exception e) { if (unwrap instanceof SearchPhaseExecutionException) { SearchPhaseExecutionException e = (SearchPhaseExecutionException) unwrap; for (ShardSearchFailure failure : e.shardFailures()) { - assertTrue("got unexpected reason..." + failure.reason(), - // task cancellation can occur due to partial merge failures - failure.reason().toLowerCase(Locale.ENGLISH).contains("cancelled") || - failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected")); + assertTrue( + "got unexpected reason..." + failure.reason(), + // task cancellation can occur due to partial merge failures + failure.reason().toLowerCase(Locale.ENGLISH).contains("cancelled") + || failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected") + ); } // we have have null responses if submit completes before search starts } else if (unwrap instanceof OpenSearchRejectedExecutionException == false) { @@ -182,22 +196,25 @@ public void testSearchFailures() throws Exception { threadPool = new TestThreadPool(AsynchronousSearchProgressListenerIT.class.getName()); for (int i = 0; i < numberOfAsyncOps; i++) { SearchRequest request = client().prepareSearch("test") - .setSearchType(SearchType.QUERY_THEN_FETCH) - .setQuery(QueryBuilders.matchQuery("field", "1")) - .request(); + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setQuery(QueryBuilders.matchQuery("field", "1")) + .request(); AtomicReference responseRef = new AtomicReference<>(); AtomicInteger reduceContextInvocation = new AtomicInteger(); AsynchronousSearchProgressListener listener; SearchService service = internalCluster().getInstance(SearchService.class); InternalAggregation.ReduceContextBuilder reduceContextBuilder = service.aggReduceContextBuilder(request.source()); AtomicReference exceptionRef = new AtomicReference<>(); - Function responseFunction = - (r) -> null; - Function failureFunction = - (e) -> null; - listener = new AsynchronousSearchProgressListener(threadPool.relativeTimeInMillis(), responseFunction, - failureFunction, threadPool.generic(), threadPool::relativeTimeInMillis, - () -> reduceContextBuilder) { + Function responseFunction = (r) -> null; + Function failureFunction = (e) -> null; + listener = new AsynchronousSearchProgressListener( + threadPool.relativeTimeInMillis(), + responseFunction, + failureFunction, + threadPool.generic(), + threadPool::relativeTimeInMillis, + () -> reduceContextBuilder + ) { @Override public void onResponse(SearchResponse searchResponse) { assertTrue(responseRef.compareAndSet(null, searchResponse)); diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchStatsIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchStatsIT.java index 87b6dc10..426cdce5 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchStatsIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchStatsIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.integTests; import org.opensearch.search.asynchronous.action.AsynchronousSearchStatsAction; @@ -50,35 +53,39 @@ protected Settings nodeSettings(int nodeOrdinal) { boolean lowLevelCancellation = randomBoolean(); logger.info("Using lowLevelCancellation: {}", lowLevelCancellation); return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), asConcurrentLimit) - .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) - .build(); + .put(super.nodeSettings(nodeOrdinal)) + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), asConcurrentLimit) + .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) + .build(); } public void testNodewiseStats() throws InterruptedException { String index = "idx"; createIndex(index); - indexRandom(super.ignoreExternalCluster(), client().prepareIndex(index).setId("1") - .setSource("field1", "the quick brown fox jumps"), - client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), - client().prepareIndex(index).setId("3").setSource("field1", "quick")); + indexRandom( + super.ignoreExternalCluster(), + client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), + client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), + client().prepareIndex(index).setId("3").setSource("field1", "quick") + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest(index)); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueSeconds(2)); submitAsynchronousSearchRequest.keepOnCompletion(true); List dataNodes = new LinkedList<>(); - clusterService().state().nodes().getDataNodes().values().iterator().forEachRemaining(node -> { - dataNodes.add(node); - }); + clusterService().state().nodes().getDataNodes().values().iterator().forEachRemaining(node -> { dataNodes.add(node); }); assertFalse(dataNodes.isEmpty()); DiscoveryNode randomDataNode = dataNodes.get(randomInt(dataNodes.size() - 1)); try { - AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(client(randomDataNode.getName()), - submitAsynchronousSearchRequest); + AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch( + client(randomDataNode.getName()), + submitAsynchronousSearchRequest + ); assertNotNull(asResponse.getSearchResponse()); TestClientUtils.assertResponsePersistence(client(), asResponse.getId()); - AsynchronousSearchStatsResponse statsResponse = client().execute(AsynchronousSearchStatsAction.INSTANCE, - new AsynchronousSearchStatsRequest()).get(); + AsynchronousSearchStatsResponse statsResponse = client().execute( + AsynchronousSearchStatsAction.INSTANCE, + new AsynchronousSearchStatsRequest() + ).get(); String responseAsString = statsResponse.toString(); for (DiscoveryNode dataNode : dataNodes) { assertThat(responseAsString, containsString(dataNode.getId())); @@ -112,15 +119,15 @@ public void testStatsAcrossNodes() throws InterruptedException, ExecutionExcepti threadPool = new TestThreadPool(AsynchronousSearchStatsIT.class.getName()); String index = "idx"; createIndex(index); - indexRandom(super.ignoreExternalCluster(), client().prepareIndex(index).setId("1") - .setSource("field1", "the quick brown fox jumps"), - client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), - client().prepareIndex(index).setId("3").setSource("field1", "quick")); + indexRandom( + super.ignoreExternalCluster(), + client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), + client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), + client().prepareIndex(index).setId("3").setSource("field1", "quick") + ); List dataNodes = new LinkedList<>(); - clusterService().state().nodes().getDataNodes().values().iterator().forEachRemaining(node -> { - dataNodes.add(node); - }); + clusterService().state().nodes().getDataNodes().values().iterator().forEachRemaining(node -> { dataNodes.add(node); }); assertFalse(dataNodes.isEmpty()); int numThreads = 20; List threads = new ArrayList<>(); @@ -145,14 +152,14 @@ public void testStatsAcrossNodes() throws InterruptedException, ExecutionExcepti } else { expectedNumFailures.getAndIncrement(); - submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest( - "non_existent_index")); + submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest("non_existent_index")); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); } - AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(client( - dataNodes.get(randomInt(1)).getName()), - submitAsynchronousSearchRequest); + AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch( + client(dataNodes.get(randomInt(1)).getName()), + submitAsynchronousSearchRequest + ); if (keepOnCompletion) { TestClientUtils.assertResponsePersistence(client(), asResponse.getId()); } @@ -166,8 +173,10 @@ public void testStatsAcrossNodes() throws InterruptedException, ExecutionExcepti TestThreadPool finalThreadPool = threadPool; threads.forEach(t -> finalThreadPool.generic().execute(t)); latch.await(); - AsynchronousSearchStatsResponse statsResponse = client().execute(AsynchronousSearchStatsAction.INSTANCE, - new AsynchronousSearchStatsRequest()).get(); + AsynchronousSearchStatsResponse statsResponse = client().execute( + AsynchronousSearchStatsAction.INSTANCE, + new AsynchronousSearchStatsRequest() + ).get(); AtomicLong actualNumSuccesses = new AtomicLong(); AtomicLong actualNumFailures = new AtomicLong(); AtomicLong actualNumPersisted = new AtomicLong(); @@ -197,21 +206,24 @@ public void testStatsAcrossNodes() throws InterruptedException, ExecutionExcepti public void testRunningAsynchronousSearchCountStat() throws InterruptedException, ExecutionException { String index = "idx"; createIndex(index); - indexRandom(super.ignoreExternalCluster(), client().prepareIndex(index).setId("1") - .setSource("field1", "the quick brown fox jumps"), - client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), - client().prepareIndex(index).setId("3").setSource("field1", "quick")); + indexRandom( + super.ignoreExternalCluster(), + client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), + client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), + client().prepareIndex(index).setId("3").setSource("field1", "quick") + ); List plugins = initBlockFactory(); - SearchRequest searchRequest = client().prepareSearch(index).setQuery( - scriptQuery(new Script( - ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) - .request(); + SearchRequest searchRequest = client().prepareSearch(index) + .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + .request(); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest); - AsynchronousSearchStatsResponse statsResponse = client().execute(AsynchronousSearchStatsAction.INSTANCE, - new AsynchronousSearchStatsRequest()).get(); + AsynchronousSearchStatsResponse statsResponse = client().execute( + AsynchronousSearchStatsAction.INSTANCE, + new AsynchronousSearchStatsRequest() + ).get(); long runningSearchCount = 0; for (AsynchronousSearchStats node : statsResponse.getNodes()) { runningSearchCount += node.getAsynchronousSearchCountStats().getRunningCount(); @@ -237,24 +249,23 @@ public void testRunningAsynchronousSearchCountStat() throws InterruptedException public void testThrottledAsynchronousSearchCount() throws InterruptedException, ExecutionException { String index = "idx"; createIndex(index); - indexRandom(super.ignoreExternalCluster(), client().prepareIndex(index).setId("1") - .setSource("field1", "the quick brown fox jumps"), - client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), - client().prepareIndex(index).setId("3").setSource("field1", "quick")); + indexRandom( + super.ignoreExternalCluster(), + client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), + client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), + client().prepareIndex(index).setId("3").setSource("field1", "quick") + ); List dataNodes = new LinkedList<>(); - clusterService().state().nodes().getDataNodes().values().iterator().forEachRemaining(node -> { - dataNodes.add(node); - }); + clusterService().state().nodes().getDataNodes().values().iterator().forEachRemaining(node -> { dataNodes.add(node); }); assertFalse(dataNodes.isEmpty()); DiscoveryNode randomDataNode = dataNodes.get(randomInt(dataNodes.size() - 1)); int numThreads = 21; List threads = new ArrayList<>(); List plugins = initBlockFactory(); - SearchRequest searchRequest = client().prepareSearch(index).setQuery( - scriptQuery(new Script( - ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) - .request(); + SearchRequest searchRequest = client().prepareSearch(index) + .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + .request(); for (int i = 0; i < numThreads; i++) { Thread t = new Thread(() -> { try { @@ -279,8 +290,10 @@ public void testThrottledAsynchronousSearchCount() throws InterruptedException, private boolean verifyThrottlingFromStats() { try { - AsynchronousSearchStatsResponse statsResponse = client().execute(AsynchronousSearchStatsAction.INSTANCE, - new AsynchronousSearchStatsRequest()).get(); + AsynchronousSearchStatsResponse statsResponse = client().execute( + AsynchronousSearchStatsAction.INSTANCE, + new AsynchronousSearchStatsRequest() + ).get(); for (AsynchronousSearchStats nodeStats : statsResponse.getNodes()) { if (nodeStats.getAsynchronousSearchCountStats().getThrottledCount() == 1L) { return true; diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchTaskCancellationIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchTaskCancellationIT.java index 37f30576..27f568ae 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchTaskCancellationIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/AsynchronousSearchTaskCancellationIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.integTests; import org.opensearch.common.xcontent.XContentType; @@ -50,7 +53,7 @@ protected Collection> nodePlugins() { return Arrays.asList(ScriptedBlockPlugin.class, AsynchronousSearchPlugin.class); } - //We need to apply blocks via ScriptedBlockPlugin, external clusters are immutable + // We need to apply blocks via ScriptedBlockPlugin, external clusters are immutable @Override protected boolean ignoreExternalCluster() { return true; @@ -61,9 +64,9 @@ protected Settings nodeSettings(int nodeOrdinal) { boolean lowLevelCancellation = randomBoolean(); logger.info("Using lowLevelCancellation: {}", lowLevelCancellation); return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), lowLevelCancellation) - .build(); + .put(super.nodeSettings(nodeOrdinal)) + .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), lowLevelCancellation) + .build(); } private void indexTestData() { @@ -71,8 +74,7 @@ private void indexTestData() { // Make sure we have a few segments BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (int j = 0; j < 20; j++) { - bulkRequestBuilder.add(client().prepareIndex("test").setId(Integer.toString(i * 5 + j)) - .setSource("field", "value")); + bulkRequestBuilder.add(client().prepareIndex("test").setId(Integer.toString(i * 5 + j)).setSource("field", "value")); } assertNoFailures(bulkRequestBuilder.get()); } @@ -94,26 +96,23 @@ public void testCancellationDuringQueryPhase() throws Exception { List plugins = initBlockFactory(); indexTestData(); - SearchRequest searchRequest = client().prepareSearch("test").setQuery( - scriptQuery(new Script( - ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) - .request(); - //We need a NodeClient to make sure the listener gets injected in the search request execution. - //Randomized client randomly return NodeClient/TransportClient + SearchRequest searchRequest = client().prepareSearch("test") + .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + .request(); + // We need a NodeClient to make sure the listener gets injected in the search request execution. + // Randomized client randomly return NodeClient/TransportClient SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(10000)); testCase(internalCluster().smartClient(), submitAsynchronousSearchRequest, plugins); } - public void testCancellationDuringFetchPhase() throws Exception { List plugins = initBlockFactory(); indexTestData(); SearchRequest searchRequest = client().prepareSearch("test") - .addScriptField("test_field", - new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()) - ).request(); + .addScriptField("test_field", new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap())) + .request(); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(50000)); diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/DeleteAsynchronousSearchSingleNodeIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/DeleteAsynchronousSearchSingleNodeIT.java index a4ed93fd..fb235468 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/DeleteAsynchronousSearchSingleNodeIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/DeleteAsynchronousSearchSingleNodeIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.integTests; import org.opensearch.search.asynchronous.commons.AsynchronousSearchSingleNodeTestCase; @@ -43,15 +46,13 @@ public void testDeleteAsynchronousSearchForRetainedResponseRandomTime() throws I SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentDeletes(submitResponse.getId(), - (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { - assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); - assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); - }, concurrentRuns); + assertConcurrentDeletes(submitResponse.getId(), (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { + assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); + assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); + }, concurrentRuns); assertAsynchronousSearchResourceCleanUp(submitResponse.getId()); } @@ -62,15 +63,13 @@ public void testDeleteAsynchronousSearchNoRetainedResponseRandomTime() throws In SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentDeletes(submitResponse.getId(), - (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { - assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get()); - assertEquals(0, numDeleteUnAcknowledged.get()); - }, concurrentRuns); + assertConcurrentDeletes(submitResponse.getId(), (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { + assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get()); + assertEquals(0, numDeleteUnAcknowledged.get()); + }, concurrentRuns); assertAsynchronousSearchResourceCleanUp(submitResponse.getId()); } @@ -81,16 +80,14 @@ public void testDeleteAsynchronousSearchPostCompletionNoRetainedResponse() throw SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(5000)); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentDeletes(submitResponse.getId(), - (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { - assertEquals(0, numDeleteAcknowledged.get()); - assertEquals(0, numDeleteUnAcknowledged.get()); - assertEquals(concurrentRuns, numResourceNotFound.get()); - }, concurrentRuns); + assertConcurrentDeletes(submitResponse.getId(), (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { + assertEquals(0, numDeleteAcknowledged.get()); + assertEquals(0, numDeleteUnAcknowledged.get()); + assertEquals(concurrentRuns, numResourceNotFound.get()); + }, concurrentRuns); assertAsynchronousSearchResourceCleanUp(submitResponse.getId()); } @@ -101,16 +98,14 @@ public void testDeleteAsynchronousSearchPostCompletionForRetainedResponse() thro SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(5000)); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentDeletes(submitResponse.getId(), - (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { - assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); - assertEquals(0, numDeleteUnAcknowledged.get()); - assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); - }, concurrentRuns); + assertConcurrentDeletes(submitResponse.getId(), (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { + assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); + assertEquals(0, numDeleteUnAcknowledged.get()); + assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); + }, concurrentRuns); assertAsynchronousSearchResourceCleanUp(submitResponse.getId()); } @@ -122,15 +117,18 @@ public void testDeleteAsynchronousSearchInBlockedStateForRetainedResponse() thro SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(10, 20); - assertConcurrentDeletesForBlockedSearch(submitResponse.getId(), - (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { - assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); - assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); - }, concurrentRuns, plugins); + assertConcurrentDeletesForBlockedSearch( + submitResponse.getId(), + (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { + assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); + assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); + }, + concurrentRuns, + plugins + ); assertAsynchronousSearchResourceCleanUp(submitResponse.getId()); } @@ -142,20 +140,27 @@ public void testDeleteAsynchronousSearchInBlockedStateForNoRetainedResponse() th SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(10, 20); - assertConcurrentDeletesForBlockedSearch(submitResponse.getId(), - (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { - assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); - assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); - }, concurrentRuns, plugins); + assertConcurrentDeletesForBlockedSearch( + submitResponse.getId(), + (numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound) -> { + assertThat(numDeleteAcknowledged.get(), Matchers.greaterThan(0)); + assertEquals(concurrentRuns, numDeleteAcknowledged.get() + numResourceNotFound.get() + numDeleteUnAcknowledged.get()); + }, + concurrentRuns, + plugins + ); assertAsynchronousSearchResourceCleanUp(submitResponse.getId()); } - private void assertConcurrentDeletesForBlockedSearch(String id, TriConsumer assertionConsumer, int concurrentRuns, List plugins) throws Exception { + private void assertConcurrentDeletesForBlockedSearch( + String id, + TriConsumer assertionConsumer, + int concurrentRuns, + List plugins + ) throws Exception { AtomicInteger numDeleteAcknowledged = new AtomicInteger(); AtomicInteger numDeleteUnAcknowledged = new AtomicInteger(); AtomicInteger numResourceNotFound = new AtomicInteger(); @@ -169,24 +174,27 @@ private void assertConcurrentDeletesForBlockedSearch(String id, TriConsumer { logger.info("Triggering asynchronous search delete --->"); DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(id); - executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, new LatchedActionListener<> - (new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - if (acknowledgedResponse.isAcknowledged()) { - numDeleteAcknowledged.incrementAndGet(); - } else { - numDeleteUnAcknowledged.incrementAndGet(); - } + executeDeleteAsynchronousSearch( + client(), + deleteAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + if (acknowledgedResponse.isAcknowledged()) { + numDeleteAcknowledged.incrementAndGet(); + } else { + numDeleteUnAcknowledged.incrementAndGet(); } + } - @Override - public void onFailure(Exception e) { - if (e instanceof ResourceNotFoundException) { - numResourceNotFound.incrementAndGet(); - } + @Override + public void onFailure(Exception e) { + if (e instanceof ResourceNotFoundException) { + numResourceNotFound.incrementAndGet(); } - }, countDownLatch)); + } + }, countDownLatch) + ); }; operationThreads.add(thread); } @@ -200,8 +208,11 @@ public void onFailure(Exception e) { } } - private void assertConcurrentDeletes(String id, TriConsumer assertionConsumer, - int concurrentRuns) throws InterruptedException { + private void assertConcurrentDeletes( + String id, + TriConsumer assertionConsumer, + int concurrentRuns + ) throws InterruptedException { AtomicInteger numDeleteAcknowledged = new AtomicInteger(); AtomicInteger numDeleteUnAcknowledged = new AtomicInteger(); AtomicInteger numResourceNotFound = new AtomicInteger(); @@ -215,24 +226,27 @@ private void assertConcurrentDeletes(String id, TriConsumer { logger.info("Triggering asynchronous search delete --->"); DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(id); - executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, new LatchedActionListener<> - (new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - if (acknowledgedResponse.isAcknowledged()) { - numDeleteAcknowledged.incrementAndGet(); - } else { - numDeleteUnAcknowledged.incrementAndGet(); - } + executeDeleteAsynchronousSearch( + client(), + deleteAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + if (acknowledgedResponse.isAcknowledged()) { + numDeleteAcknowledged.incrementAndGet(); + } else { + numDeleteUnAcknowledged.incrementAndGet(); } + } - @Override - public void onFailure(Exception e) { - if (e instanceof ResourceNotFoundException) { - numResourceNotFound.incrementAndGet(); - } + @Override + public void onFailure(Exception e) { + if (e instanceof ResourceNotFoundException) { + numResourceNotFound.incrementAndGet(); } - }, countDownLatch)); + } + }, countDownLatch) + ); }; operationThreads.add(thread); } diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/GetAsynchronousSearchSingleNodeIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/GetAsynchronousSearchSingleNodeIT.java index bd64e20e..019a1c89 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/GetAsynchronousSearchSingleNodeIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/GetAsynchronousSearchSingleNodeIT.java @@ -1,11 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.integTests; - import org.opensearch.search.asynchronous.commons.AsynchronousSearchSingleNodeTestCase; import org.opensearch.search.asynchronous.request.DeleteAsynchronousSearchRequest; import org.opensearch.search.asynchronous.request.GetAsynchronousSearchRequest; @@ -48,16 +50,19 @@ public void testGetAsynchronousSearchForNoRetainedResponse() throws InterruptedE SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentGetOrUpdates(submitResponse, - (numGetSuccess, numGetFailures, numVersionConflictFailure, numResourceNotFoundFailures) -> { - assertEquals(concurrentRuns, numGetSuccess.get() + numResourceNotFoundFailures.get() - + numVersionConflictFailure.get()); - assertEquals(0, numGetFailures.get()); - }, false, concurrentRuns, false); + assertConcurrentGetOrUpdates( + submitResponse, + (numGetSuccess, numGetFailures, numVersionConflictFailure, numResourceNotFoundFailures) -> { + assertEquals(concurrentRuns, numGetSuccess.get() + numResourceNotFoundFailures.get() + numVersionConflictFailure.get()); + assertEquals(0, numGetFailures.get()); + }, + false, + concurrentRuns, + false + ); } public void testUpdateAsynchronousSearchForNoRetainedResponse() throws InterruptedException { @@ -67,16 +72,19 @@ public void testUpdateAsynchronousSearchForNoRetainedResponse() throws Interrupt SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentGetOrUpdates(submitResponse, - (numGetSuccess, numGetFailures, numVersionConflictFailures, numResourceNotFoundFailures) -> { - assertEquals(0, numGetFailures.get()); - assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get() - + numResourceNotFoundFailures.get()); - }, true, concurrentRuns, false); + assertConcurrentGetOrUpdates( + submitResponse, + (numGetSuccess, numGetFailures, numVersionConflictFailures, numResourceNotFoundFailures) -> { + assertEquals(0, numGetFailures.get()); + assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get() + numResourceNotFoundFailures.get()); + }, + true, + concurrentRuns, + false + ); } public void testUpdateAsynchronousSearchForRetainedResponse() throws InterruptedException { @@ -86,16 +94,13 @@ public void testUpdateAsynchronousSearchForRetainedResponse() throws Interrupted SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentGetOrUpdates(submitResponse, - (numGetSuccess, numGetFailures, numVersionConflictFailures, numTimeouts) -> { - assertEquals(0, numGetFailures.get()); - assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get() - + numTimeouts.get()); - }, true, concurrentRuns, true); + assertConcurrentGetOrUpdates(submitResponse, (numGetSuccess, numGetFailures, numVersionConflictFailures, numTimeouts) -> { + assertEquals(0, numGetFailures.get()); + assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get() + numTimeouts.get()); + }, true, concurrentRuns, true); } public void testGetAsynchronousSearchForRetainedResponse() throws InterruptedException { @@ -105,16 +110,19 @@ public void testGetAsynchronousSearchForRetainedResponse() throws InterruptedExc SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentGetOrUpdates(submitResponse, - (numGetSuccess, numGetFailures, numVersionConflictFailures, numResourceNotFoundFailures) -> { - assertEquals(0, numGetFailures.get()); - assertEquals(concurrentRuns, numGetSuccess.get() + numResourceNotFoundFailures.get() - + numVersionConflictFailures.get()); - }, false, concurrentRuns, true); + assertConcurrentGetOrUpdates( + submitResponse, + (numGetSuccess, numGetFailures, numVersionConflictFailures, numResourceNotFoundFailures) -> { + assertEquals(0, numGetFailures.get()); + assertEquals(concurrentRuns, numGetSuccess.get() + numResourceNotFoundFailures.get() + numVersionConflictFailures.get()); + }, + false, + concurrentRuns, + true + ); } public void testGetAsynchronousSearchInBlockedStateForNoRetainedResponse() throws Exception { @@ -125,16 +133,14 @@ public void testGetAsynchronousSearchInBlockedStateForNoRetainedResponse() throw SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(10, 20); - assertConcurrentGetForBlockedSearch(submitResponse, - (numGetSuccess, numGetFailures, numResourceNotFound) -> { - assertEquals(0, numGetFailures.get()); - assertEquals(concurrentRuns, numGetSuccess.get()); - assertEquals(0, numResourceNotFound.get()); - }, false, concurrentRuns, false, plugins); + assertConcurrentGetForBlockedSearch(submitResponse, (numGetSuccess, numGetFailures, numResourceNotFound) -> { + assertEquals(0, numGetFailures.get()); + assertEquals(concurrentRuns, numGetSuccess.get()); + assertEquals(0, numResourceNotFound.get()); + }, false, concurrentRuns, false, plugins); } public void testGetAsynchronousSearchInBlockedStateForRetainedResponse() throws Exception { @@ -145,16 +151,14 @@ public void testGetAsynchronousSearchInBlockedStateForRetainedResponse() throws SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(10, 20); - assertConcurrentGetForBlockedSearch(submitResponse, - (numGetSuccess, numGetFailures, numResourceNotFound) -> { - assertEquals(0, numGetFailures.get()); - assertEquals(concurrentRuns, numGetSuccess.get()); - assertEquals(0, numResourceNotFound.get()); - }, false, concurrentRuns, false, plugins); + assertConcurrentGetForBlockedSearch(submitResponse, (numGetSuccess, numGetFailures, numResourceNotFound) -> { + assertEquals(0, numGetFailures.get()); + assertEquals(concurrentRuns, numGetSuccess.get()); + assertEquals(0, numResourceNotFound.get()); + }, false, concurrentRuns, false, plugins); } public void testUpdateAsynchronousSearchInBlockedStateForRetainedResponse() throws Exception { @@ -165,15 +169,13 @@ public void testUpdateAsynchronousSearchInBlockedStateForRetainedResponse() thro SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(10, 20); - assertConcurrentGetForBlockedSearch(submitResponse, - (numGetSuccess, numGetFailures, numVersionConflictFailures) -> { - assertEquals(0, numGetFailures.get()); - assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get()); - }, true, concurrentRuns, false, plugins); + assertConcurrentGetForBlockedSearch(submitResponse, (numGetSuccess, numGetFailures, numVersionConflictFailures) -> { + assertEquals(0, numGetFailures.get()); + assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get()); + }, true, concurrentRuns, false, plugins); } public void testUpdateAsynchronousSearchInBlockedStateForNoRetainedResponse() throws Exception { @@ -184,21 +186,23 @@ public void testUpdateAsynchronousSearchInBlockedStateForNoRetainedResponse() th SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) - .actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest).actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(10, 20); - assertConcurrentGetForBlockedSearch(submitResponse, - (numGetSuccess, numGetFailures, numVersionConflictFailures) -> { - assertEquals(0, numGetFailures.get()); - assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get()); - }, true, concurrentRuns, false, plugins); + assertConcurrentGetForBlockedSearch(submitResponse, (numGetSuccess, numGetFailures, numVersionConflictFailures) -> { + assertEquals(0, numGetFailures.get()); + assertEquals(concurrentRuns, numGetSuccess.get() + numVersionConflictFailures.get()); + }, true, concurrentRuns, false, plugins); } - private void assertConcurrentGetForBlockedSearch(AsynchronousSearchResponse submitResponse, - TriConsumer assertionConsumer, - boolean update, int concurrentRuns, boolean retainResponse, - List plugins) throws Exception { + private void assertConcurrentGetForBlockedSearch( + AsynchronousSearchResponse submitResponse, + TriConsumer assertionConsumer, + boolean update, + int concurrentRuns, + boolean retainResponse, + List plugins + ) throws Exception { AtomicInteger numGetSuccess = new AtomicInteger(); AtomicInteger numGetFailures = new AtomicInteger(); @@ -207,7 +211,7 @@ private void assertConcurrentGetForBlockedSearch(AsynchronousSearchResponse subm try { testThreadPool = new TestThreadPool(GetAsynchronousSearchSingleNodeIT.class.getName()); int numThreads = concurrentRuns; - long lowerKeepAliveMillis = 5 * 1000 * 60 * 60 ; + long lowerKeepAliveMillis = 5 * 1000 * 60 * 60; long higherKeepAliveMillis = 10 * 1000 * 60 * 60; List operationThreads = new ArrayList<>(); CountDownLatch countDownLatch = new CountDownLatch(numThreads); @@ -216,15 +220,14 @@ private void assertConcurrentGetForBlockedSearch(AsynchronousSearchResponse subm GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(submitResponse.getId()); if (update) { logger.info("Triggering asynchronous search gets with keep alives --->"); - getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueMillis(randomLongBetween(lowerKeepAliveMillis, - higherKeepAliveMillis))); + getAsynchronousSearchRequest.setKeepAlive( + TimeValue.timeValueMillis(randomLongBetween(lowerKeepAliveMillis, higherKeepAliveMillis)) + ); } - //if waitForCompletionTimeout is null we return response immediately - TimeValue waitForCompletionTimeout = randomBoolean() ? null : - TimeValue.timeValueMillis(randomLongBetween(1, 5000)); + // if waitForCompletionTimeout is null we return response immediately + TimeValue waitForCompletionTimeout = randomBoolean() ? null : TimeValue.timeValueMillis(randomLongBetween(1, 5000)); getAsynchronousSearchRequest.setWaitForCompletionTimeout(waitForCompletionTimeout); - executeGetAsynchronousSearch(client(), getAsynchronousSearchRequest, - new ActionListener() { + executeGetAsynchronousSearch(client(), getAsynchronousSearchRequest, new ActionListener() { @Override public void onResponse(AsynchronousSearchResponse acknowledgedResponse) { numGetSuccess.incrementAndGet(); @@ -249,8 +252,7 @@ public void onFailure(Exception e) { countDownLatch.await(); DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(submitResponse.getId()); CountDownLatch deleteLatch = new CountDownLatch(1); - executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, - ActionListener.wrap(() -> deleteLatch.countDown())); + executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, ActionListener.wrap(() -> deleteLatch.countDown())); deleteLatch.await(); disableBlocks(plugins); assertionConsumer.apply(numGetSuccess, numGetFailures, numVersionConflictFailures); @@ -259,10 +261,13 @@ public void onFailure(Exception e) { } } - private void assertConcurrentGetOrUpdates(AsynchronousSearchResponse submitResponse, - QuadConsumer assertionConsumer, - boolean update, int concurrentRuns, boolean retainResponse) - throws InterruptedException { + private void assertConcurrentGetOrUpdates( + AsynchronousSearchResponse submitResponse, + QuadConsumer assertionConsumer, + boolean update, + int concurrentRuns, + boolean retainResponse + ) throws InterruptedException { AtomicInteger numGetSuccess = new AtomicInteger(); AtomicInteger numGetFailures = new AtomicInteger(); AtomicInteger numVersionConflictFailures = new AtomicInteger(); @@ -272,7 +277,7 @@ private void assertConcurrentGetOrUpdates(AsynchronousSearchResponse submitRespo try { testThreadPool = new TestThreadPool(GetAsynchronousSearchSingleNodeIT.class.getName()); int numThreads = concurrentRuns; - long lowerKeepAliveMillis = 5 * 1000 * 60 * 60 ; // 5 hours in millis + long lowerKeepAliveMillis = 5 * 1000 * 60 * 60; // 5 hours in millis long higherKeepAliveMillis = 10 * 1000 * 60 * 60; // 10 hours in millis List operationThreads = new ArrayList<>(); CountDownLatch countDownLatch = new CountDownLatch(numThreads); @@ -286,35 +291,43 @@ private void assertConcurrentGetOrUpdates(AsynchronousSearchResponse submitRespo getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueMillis(keepAlive)); } getAsynchronousSearchRequest.setWaitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); - executeGetAsynchronousSearch(client(), getAsynchronousSearchRequest, new LatchedActionListener<>( - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - if (update) { - // while updates we can run into version conflicts and hence the comparison is on the successful - // response. Since the final keep alive is calculated based on the current time of the server - // active contexts's expiration in memory are superseded by later writer so we are keeping a loose - // check - assertThat(asResponse.getExpirationTimeMillis(), greaterThanOrEqualTo( - System.currentTimeMillis() + lowerKeepAliveMillis)); - assertThat(asResponse.getExpirationTimeMillis(), lessThanOrEqualTo( - System.currentTimeMillis() + higherKeepAliveMillis)); + executeGetAsynchronousSearch( + client(), + getAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + if (update) { + // while updates we can run into version conflicts and hence the comparison is on the successful + // response. Since the final keep alive is calculated based on the current time of the server + // active contexts's expiration in memory are superseded by later writer so we are keeping a loose + // check + assertThat( + asResponse.getExpirationTimeMillis(), + greaterThanOrEqualTo(System.currentTimeMillis() + lowerKeepAliveMillis) + ); + assertThat( + asResponse.getExpirationTimeMillis(), + lessThanOrEqualTo(System.currentTimeMillis() + higherKeepAliveMillis) + ); + } + numGetSuccess.incrementAndGet(); } - numGetSuccess.incrementAndGet(); - } - @Override - public void onFailure(Exception e) { - if (e instanceof VersionConflictEngineException) { - numVersionConflictFailures.incrementAndGet(); - } else if (e instanceof ResourceNotFoundException) { - numResourceNotFoundFailures.incrementAndGet(); - } else if (e instanceof OpenSearchTimeoutException) { - numTimeouts.incrementAndGet(); - } else { - numGetFailures.incrementAndGet(); + + @Override + public void onFailure(Exception e) { + if (e instanceof VersionConflictEngineException) { + numVersionConflictFailures.incrementAndGet(); + } else if (e instanceof ResourceNotFoundException) { + numResourceNotFoundFailures.incrementAndGet(); + } else if (e instanceof OpenSearchTimeoutException) { + numTimeouts.incrementAndGet(); + } else { + numGetFailures.incrementAndGet(); + } } - } - }, countDownLatch)); + }, countDownLatch) + ); }; operationThreads.add(thread); } @@ -323,7 +336,8 @@ public void onFailure(Exception e) { countDownLatch.await(); if (retainResponse) { DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest( - submitResponse.getId()); + submitResponse.getId() + ); executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest).actionGet(); } if (retainResponse && update) { @@ -337,4 +351,3 @@ public void onFailure(Exception e) { } } } - diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/MixedOperationSingleNodeIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/MixedOperationSingleNodeIT.java index d319875f..6ef4394b 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/MixedOperationSingleNodeIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/MixedOperationSingleNodeIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.integTests; import org.opensearch.search.asynchronous.commons.AsynchronousSearchSingleNodeTestCase; @@ -43,29 +46,38 @@ public void testGetAndDeleteAsynchronousSearchForRetainedResponse() throws Inter SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); - AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), - submitAsynchronousSearchRequest).actionGet(); + AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest) + .actionGet(); assertNotNull(submitResponse); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentGetOrUpdatesWithDeletes(submitResponse, - (numSuccess, numGetFailures, numVersionConflictFailure, numResourceNotFoundFailures) -> { - assertEquals(concurrentRuns, numSuccess.get() + numResourceNotFoundFailures.get() - + numVersionConflictFailure.get()); - assertEquals(0, numGetFailures.get()); - }, false, concurrentRuns, true); + assertConcurrentGetOrUpdatesWithDeletes( + submitResponse, + (numSuccess, numGetFailures, numVersionConflictFailure, numResourceNotFoundFailures) -> { + assertEquals(concurrentRuns, numSuccess.get() + numResourceNotFoundFailures.get() + numVersionConflictFailure.get()); + assertEquals(0, numGetFailures.get()); + }, + false, + concurrentRuns, + true + ); assertAsynchronousSearchResourceCleanUp(submitResponse.getId()); } finally { CountDownLatch deleteLatch = new CountDownLatch(1); - client().admin().indices().prepareDelete(INDEX).execute(ActionListener.wrap(r -> deleteLatch.countDown(), e -> { - deleteLatch.countDown(); - })); + client().admin() + .indices() + .prepareDelete(INDEX) + .execute(ActionListener.wrap(r -> deleteLatch.countDown(), e -> { deleteLatch.countDown(); })); deleteLatch.await(); } } - private void assertConcurrentGetOrUpdatesWithDeletes(AsynchronousSearchResponse submitResponse, QuadConsumer assertionConsumer, boolean update, int concurrentRuns, boolean retainResponse) - throws InterruptedException { + private void assertConcurrentGetOrUpdatesWithDeletes( + AsynchronousSearchResponse submitResponse, + QuadConsumer assertionConsumer, + boolean update, + int concurrentRuns, + boolean retainResponse + ) throws InterruptedException { AtomicInteger numSuccess = new AtomicInteger(); AtomicInteger numGetFailures = new AtomicInteger(); AtomicInteger numVersionConflictFailures = new AtomicInteger(); @@ -86,59 +98,71 @@ private void assertConcurrentGetOrUpdatesWithDeletes(AsynchronousSearchResponse Runnable thread = () -> { if (currentThreadIteration == randomDeleteThread) { DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest( - submitResponse.getId()); - executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, - new LatchedActionListener<>(new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - assertTrue(acknowledgedResponse.isAcknowledged()); - numSuccess.incrementAndGet(); - } + submitResponse.getId() + ); + executeDeleteAsynchronousSearch( + client(), + deleteAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + assertTrue(acknowledgedResponse.isAcknowledged()); + numSuccess.incrementAndGet(); + } - @Override - public void onFailure(Exception e) { - if (e instanceof OpenSearchTimeoutException) { - numTimeouts.incrementAndGet(); - } else { - fail("Unexpected exception " + e.getMessage()); - } + @Override + public void onFailure(Exception e) { + if (e instanceof OpenSearchTimeoutException) { + numTimeouts.incrementAndGet(); + } else { + fail("Unexpected exception " + e.getMessage()); } - }, countDownLatch)); + } + }, countDownLatch) + ); } else { GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest( - submitResponse.getId()); + submitResponse.getId() + ); long requestedTime = System.currentTimeMillis() + keepAlive; if (update) { logger.info("Triggering asynchronous search gets with keep alive [{}] --->", requestedTime); getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueMillis(keepAlive)); } getAsynchronousSearchRequest.setWaitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); - executeGetAsynchronousSearch(client(), getAsynchronousSearchRequest, new LatchedActionListener<>( - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - if (update) { - assertThat(asResponse.getExpirationTimeMillis(), greaterThanOrEqualTo( - System.currentTimeMillis() + lowerKeepAliveMillis)); - assertThat(asResponse.getExpirationTimeMillis(), lessThanOrEqualTo( - System.currentTimeMillis() + higherKeepAliveMillis)); - } - numSuccess.incrementAndGet(); + executeGetAsynchronousSearch( + client(), + getAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + if (update) { + assertThat( + asResponse.getExpirationTimeMillis(), + greaterThanOrEqualTo(System.currentTimeMillis() + lowerKeepAliveMillis) + ); + assertThat( + asResponse.getExpirationTimeMillis(), + lessThanOrEqualTo(System.currentTimeMillis() + higherKeepAliveMillis) + ); } + numSuccess.incrementAndGet(); + } - @Override - public void onFailure(Exception e) { - if (e instanceof VersionConflictEngineException) { - numVersionConflictFailures.incrementAndGet(); - } else if (e instanceof ResourceNotFoundException) { - numResourceNotFoundFailures.incrementAndGet(); - } else if (e instanceof OpenSearchTimeoutException) { - numTimeouts.incrementAndGet(); - } else { - numGetFailures.incrementAndGet(); - } + @Override + public void onFailure(Exception e) { + if (e instanceof VersionConflictEngineException) { + numVersionConflictFailures.incrementAndGet(); + } else if (e instanceof ResourceNotFoundException) { + numResourceNotFoundFailures.incrementAndGet(); + } else if (e instanceof OpenSearchTimeoutException) { + numTimeouts.incrementAndGet(); + } else { + numGetFailures.incrementAndGet(); } - }, countDownLatch)); + } + }, countDownLatch) + ); } }; operationThreads.add(thread); diff --git a/src/test/java/org/opensearch/search/asynchronous/integTests/SubmitAsynchronousSearchSingleNodeIT.java b/src/test/java/org/opensearch/search/asynchronous/integTests/SubmitAsynchronousSearchSingleNodeIT.java index 1d820a9f..3ea5434d 100644 --- a/src/test/java/org/opensearch/search/asynchronous/integTests/SubmitAsynchronousSearchSingleNodeIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/integTests/SubmitAsynchronousSearchSingleNodeIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.integTests; import org.opensearch.search.asynchronous.commons.AsynchronousSearchSingleNodeTestCase; @@ -47,12 +50,12 @@ public class SubmitAsynchronousSearchSingleNodeIT extends AsynchronousSearchSing @Override protected Settings nodeSettings() { - return Settings.builder().put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), - asynchronousSearchConcurrentLimit).build(); + return Settings.builder() + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), asynchronousSearchConcurrentLimit) + .build(); } - public void - testSubmitAsynchronousSearchWithoutRetainedResponse() throws InterruptedException { + public void testSubmitAsynchronousSearchWithoutRetainedResponse() throws InterruptedException { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("index"); searchRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("field", "value0"))); @@ -61,13 +64,16 @@ protected Settings nodeSettings() { submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentSubmits(submitAsynchronousSearchRequest, searchResponse, (numStartedAsynchronousSearch, - numFailedAsynchronousSearch, - numErrorResponseAsynchronousSearch) -> { - assertEquals(concurrentRuns, numStartedAsynchronousSearch.get()); - assertEquals(0, numFailedAsynchronousSearch.get()); - assertEquals(0, numErrorResponseAsynchronousSearch.get()); - }, concurrentRuns); + assertConcurrentSubmits( + submitAsynchronousSearchRequest, + searchResponse, + (numStartedAsynchronousSearch, numFailedAsynchronousSearch, numErrorResponseAsynchronousSearch) -> { + assertEquals(concurrentRuns, numStartedAsynchronousSearch.get()); + assertEquals(0, numFailedAsynchronousSearch.get()); + assertEquals(0, numErrorResponseAsynchronousSearch.get()); + }, + concurrentRuns + ); AsynchronousSearchService asynchronousSearchService = getInstanceFromNode(AsynchronousSearchService.class); waitUntil(asynchronousSearchService.getAllActiveContexts()::isEmpty, 30, TimeUnit.SECONDS); } @@ -81,31 +87,38 @@ public void testSubmitAsynchronousSearchWithRetainedResponse() throws Interrupte submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000))); int concurrentRuns = randomIntBetween(20, 50); - assertConcurrentSubmits(submitAsynchronousSearchRequest, searchResponse, (numStartedAsynchronousSearch, - numFailedAsynchronousSearch, - numErrorResponseAsynchronousSearch) -> { - assertEquals(concurrentRuns, numStartedAsynchronousSearch.get()); - assertEquals(0, numFailedAsynchronousSearch.get()); - assertEquals(0, numErrorResponseAsynchronousSearch.get()); - }, concurrentRuns); + assertConcurrentSubmits( + submitAsynchronousSearchRequest, + searchResponse, + (numStartedAsynchronousSearch, numFailedAsynchronousSearch, numErrorResponseAsynchronousSearch) -> { + assertEquals(concurrentRuns, numStartedAsynchronousSearch.get()); + assertEquals(0, numFailedAsynchronousSearch.get()); + assertEquals(0, numErrorResponseAsynchronousSearch.get()); + }, + concurrentRuns + ); AsynchronousSearchService asynchronousSearchService = getInstanceFromNode(AsynchronousSearchService.class); waitUntil(asynchronousSearchService.getAllActiveContexts()::isEmpty, 30, TimeUnit.SECONDS); } public void testSubmitAsynchronousSearchWithNoRetainedResponseBlocking() throws Exception { int concurrentRuns = randomIntBetween(asynchronousSearchConcurrentLimit + 10, asynchronousSearchConcurrentLimit + 20); - assertConcurrentSubmitsForBlockedSearch((numStartedAsynchronousSearch, numFailedAsynchronousSearch, - numRejectedAsynchronousSearch) -> { - assertEquals(asynchronousSearchConcurrentLimit, numStartedAsynchronousSearch.get()); - assertEquals(concurrentRuns - asynchronousSearchConcurrentLimit, numFailedAsynchronousSearch.get()); - assertEquals(concurrentRuns - asynchronousSearchConcurrentLimit, numRejectedAsynchronousSearch.get()); - }, concurrentRuns); + assertConcurrentSubmitsForBlockedSearch( + (numStartedAsynchronousSearch, numFailedAsynchronousSearch, numRejectedAsynchronousSearch) -> { + assertEquals(asynchronousSearchConcurrentLimit, numStartedAsynchronousSearch.get()); + assertEquals(concurrentRuns - asynchronousSearchConcurrentLimit, numFailedAsynchronousSearch.get()); + assertEquals(concurrentRuns - asynchronousSearchConcurrentLimit, numRejectedAsynchronousSearch.get()); + }, + concurrentRuns + ); AsynchronousSearchService asynchronousSearchService = getInstanceFromNode(AsynchronousSearchService.class); waitUntil(asynchronousSearchService.getAllActiveContexts()::isEmpty, 30, TimeUnit.SECONDS); } - private void assertConcurrentSubmitsForBlockedSearch(TriConsumer assertionConsumer, - int concurrentRuns) throws Exception { + private void assertConcurrentSubmitsForBlockedSearch( + TriConsumer assertionConsumer, + int concurrentRuns + ) throws Exception { AtomicInteger numStartedAsynchronousSearch = new AtomicInteger(); AtomicInteger numFailedAsynchronousSearch = new AtomicInteger(); AtomicInteger numRejectedAsynchronousSearch = new AtomicInteger(); @@ -121,30 +134,35 @@ private void assertConcurrentSubmitsForBlockedSearch(TriConsumer"); SearchRequest searchRequest = new SearchRequest("index"); searchRequest.source(new SearchSourceBuilder()); - searchRequest.source().query(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SearchDelayPlugin.SCRIPT_NAME, - Collections.emptyMap()))); + searchRequest.source() + .query( + scriptQuery(new Script(ScriptType.INLINE, "mockscript", SearchDelayPlugin.SCRIPT_NAME, Collections.emptyMap())) + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(100)); - executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - if (asResponse.getId() != null) { - numStartedAsynchronousSearch.incrementAndGet(); - } - countDownLatch.countDown(); + executeSubmitAsynchronousSearch( + client(), + submitAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + if (asResponse.getId() != null) { + numStartedAsynchronousSearch.incrementAndGet(); } + countDownLatch.countDown(); + } - @Override - public void onFailure(Exception e) { - if (e instanceof OpenSearchRejectedExecutionException) { - numRejectedAsynchronousSearch.incrementAndGet(); - } - numFailedAsynchronousSearch.incrementAndGet(); - countDownLatch.countDown(); + @Override + public void onFailure(Exception e) { + if (e instanceof OpenSearchRejectedExecutionException) { + numRejectedAsynchronousSearch.incrementAndGet(); } - }); + numFailedAsynchronousSearch.incrementAndGet(); + countDownLatch.countDown(); + } + } + ); }; operationThreads.add(thread); } @@ -158,9 +176,12 @@ public void onFailure(Exception e) { } } - private void assertConcurrentSubmits(SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest, SearchResponse searchResponse, - TriConsumer assertionConsumer, int concurrentRuns) - throws InterruptedException { + private void assertConcurrentSubmits( + SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest, + SearchResponse searchResponse, + TriConsumer assertionConsumer, + int concurrentRuns + ) throws InterruptedException { AtomicInteger numStartedAsynchronousSearch = new AtomicInteger(); AtomicInteger numFailedAsynchronousSearch = new AtomicInteger(); AtomicInteger numErrorResponseAsynchronousSearch = new AtomicInteger(); @@ -173,7 +194,7 @@ private void assertConcurrentSubmits(SubmitAsynchronousSearchRequest submitAsync int numThreads = concurrentRuns; List operationThreads = new ArrayList<>(); if (submitAsynchronousSearchRequest.getKeepOnCompletion()) { - //we also need to delete asynchronous search response to ensure test completes gracefully with no background tasks + // we also need to delete asynchronous search response to ensure test completes gracefully with no background tasks // running countDownLatch = new CountDownLatch(2 * numThreads); } else { @@ -184,46 +205,53 @@ private void assertConcurrentSubmits(SubmitAsynchronousSearchRequest submitAsync CountDownLatch finalCountDownLatch = countDownLatch; Runnable thread = () -> { logger.info("Triggering asynchronous search submit --->"); - executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - if (asResponse.getId() != null) { - AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(asResponse.getId()); - assertEquals(state.nodes().getLocalNodeId(), asId.getNode()); - AsynchronousSearchAssertions.assertSearchResponses(searchResponse, asResponse.getSearchResponse()); - numStartedAsynchronousSearch.incrementAndGet(); - } - if (asResponse.getError() != null) { - numErrorResponseAsynchronousSearch.incrementAndGet(); - } - finalCountDownLatch.countDown(); - - if (submitAsynchronousSearchRequest.getKeepOnCompletion()) { - DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest - = new DeleteAsynchronousSearchRequest(asResponse.getId()); - executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, - new LatchedActionListener<>(new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - assertTrue(acknowledgedResponse.isAcknowledged()); - } - - @Override - public void onFailure(Exception e) { - fail("Search deletion failed for asynchronous search id " + e.getMessage()); - } - }, finalCountDownLatch)); - } - ; + executeSubmitAsynchronousSearch( + client(), + submitAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + if (asResponse.getId() != null) { + AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(asResponse.getId()); + assertEquals(state.nodes().getLocalNodeId(), asId.getNode()); + AsynchronousSearchAssertions.assertSearchResponses(searchResponse, asResponse.getSearchResponse()); + numStartedAsynchronousSearch.incrementAndGet(); + } + if (asResponse.getError() != null) { + numErrorResponseAsynchronousSearch.incrementAndGet(); } + finalCountDownLatch.countDown(); - @Override - public void onFailure(Exception e) { - numFailedAsynchronousSearch.incrementAndGet(); - finalCountDownLatch.countDown(); + if (submitAsynchronousSearchRequest.getKeepOnCompletion()) { + DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest( + asResponse.getId() + ); + executeDeleteAsynchronousSearch( + client(), + deleteAsynchronousSearchRequest, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + assertTrue(acknowledgedResponse.isAcknowledged()); + } + + @Override + public void onFailure(Exception e) { + fail("Search deletion failed for asynchronous search id " + e.getMessage()); + } + }, finalCountDownLatch) + ); } - }); + ; + } + + @Override + public void onFailure(Exception e) { + numFailedAsynchronousSearch.incrementAndGet(); + finalCountDownLatch.countDown(); + } + } + ); }; operationThreads.add(thread); } diff --git a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchCancellationIT.java b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchCancellationIT.java index 7a27a854..9da7bc3e 100644 --- a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchCancellationIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchCancellationIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.search.asynchronous.utils.AsynchronousSearchAssertions; @@ -60,9 +63,9 @@ protected Settings nodeSettings(int nodeOrdinal) { boolean lowLevelCancellation = randomBoolean(); logger.info("Using lowLevelCancellation: {}", lowLevelCancellation); return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), lowLevelCancellation) - .build(); + .put(super.nodeSettings(nodeOrdinal)) + .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), lowLevelCancellation) + .build(); } private void indexTestData() { @@ -70,8 +73,7 @@ private void indexTestData() { // Make sure we have a few segments BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (int j = 0; j < 20; j++) { - bulkRequestBuilder.add(client().prepareIndex("test").setId(Integer.toString(i * 5 + j)) - .setSource("field", "value")); + bulkRequestBuilder.add(client().prepareIndex("test").setId(Integer.toString(i * 5 + j)).setSource("field", "value")); } assertNoFailures(bulkRequestBuilder.get()); } @@ -93,23 +95,20 @@ public void testCancellationDuringQueryPhase() throws Exception { List plugins = initBlockFactory(); indexTestData(); - SearchRequest searchRequest = client().prepareSearch("test").setQuery( - scriptQuery(new Script( - ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) - .request(); - //We need a NodeClient to make sure the listener gets injected in the search request execution. - //Randomized client randomly return NodeClient/TransportClient + SearchRequest searchRequest = client().prepareSearch("test") + .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + .request(); + // We need a NodeClient to make sure the listener gets injected in the search request execution. + // Randomized client randomly return NodeClient/TransportClient testCase(client(), searchRequest, plugins); } - public void testCancellationDuringFetchPhase() throws Exception { List plugins = initBlockFactory(); indexTestData(); SearchRequest searchRequest = client().prepareSearch("test") - .addScriptField("test_field", - new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()) - ).request(); + .addScriptField("test_field", new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap())) + .request(); testCase(client(), searchRequest, plugins); } @@ -123,17 +122,19 @@ private void testCase(Client client, SearchRequest request, List exceptionRef = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); - Function responseFunction = - (r) -> null; - Function failureFunction = - (e) -> null; - AsynchronousSearchProgressListener listener = new AsynchronousSearchProgressListener(threadPool.relativeTimeInMillis(), - responseFunction, - failureFunction, threadPool.generic(), threadPool::relativeTimeInMillis, - () -> { - assertTrue(reduceContextInvocation.compareAndSet(false, true)); - return reduceContextBuilder; - }) { + Function responseFunction = (r) -> null; + Function failureFunction = (e) -> null; + AsynchronousSearchProgressListener listener = new AsynchronousSearchProgressListener( + threadPool.relativeTimeInMillis(), + responseFunction, + failureFunction, + threadPool.generic(), + threadPool::relativeTimeInMillis, + () -> { + assertTrue(reduceContextInvocation.compareAndSet(false, true)); + return reduceContextBuilder; + } + ) { @Override public void onResponse(SearchResponse searchResponse) { assertTrue(responseRef.compareAndSet(null, searchResponse)); diff --git a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchPartialResponseIT.java b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchPartialResponseIT.java index 804fa882..780fc81a 100644 --- a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchPartialResponseIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchPartialResponseIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.search.asynchronous.utils.AsynchronousSearchAssertions; @@ -62,14 +65,13 @@ protected int numberOfShards() { } protected void createIdx(String keyFieldMapping) { - assertAcked(prepareCreate("idx") - .setMapping("key", keyFieldMapping)); + assertAcked(prepareCreate("idx").setMapping("key", keyFieldMapping)); } protected void indexData() throws Exception { List docs = new ArrayList<>(); - for(int i = 0; i < shardCount; i++) { + for (int i = 0; i < shardCount; i++) { docs.addAll(indexDoc(routingKeyForShard("idx", i), "1", 3)); docs.addAll(indexDoc(routingKeyForShard("idx", i), "2", 1)); docs.addAll(indexDoc(routingKeyForShard("idx", i), "3", 5)); @@ -80,8 +82,7 @@ protected void indexData() throws Exception { indexRandom(true, docs); String shardRouting = routingKeyForShard("idx", randomIntBetween(0, shardCount - 1)); - SearchResponse resp = client().prepareSearch("idx").setRouting(shardRouting) - .setQuery(matchAllQuery()).get(); + SearchResponse resp = client().prepareSearch("idx").setRouting(shardRouting).setQuery(matchAllQuery()).get(); assertSearchResponse(resp); long totalHits = resp.getHits().getTotalHits().value; assertThat(totalHits, is(20L)); @@ -90,11 +91,9 @@ protected void indexData() throws Exception { protected List indexDoc(String shard, String key, int times) throws Exception { IndexRequestBuilder[] builders = new IndexRequestBuilder[times]; for (int i = 0; i < times; i++) { - builders[i] = client().prepareIndex("idx").setRouting(shard).setSource(jsonBuilder() - .startObject() - .field("key", key) - .field("value", 1) - .endObject()); + builders[i] = client().prepareIndex("idx") + .setRouting(shard) + .setSource(jsonBuilder().startObject().field("key", key).field("value", 1).endObject()); } return Arrays.asList(builders); } @@ -103,10 +102,14 @@ public void testPartialReduceBuckets() throws Exception { createIdx("type=keyword"); indexData(); SearchRequest request = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation(terms("keys").field("key").size(aggregationSize) - .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())).order(BucketOrder.count(false))) - .request(); + .setQuery(matchAllQuery()) + .addAggregation( + terms("keys").field("key") + .size(aggregationSize) + .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) + .order(BucketOrder.count(false)) + ) + .request(); request.setBatchedReduceSize(2); testCase(client(), request); } @@ -122,13 +125,16 @@ private void testCase(Client client, SearchRequest request) throws Exception { InternalAggregation.ReduceContextBuilder reduceContextBuilder = service.aggReduceContextBuilder(request.source()); AtomicReference exceptionRef = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); - Function responseFunction = - (r) -> null; - Function failureFunction = - (e) -> null; - listener = new AsynchronousSearchProgressListener(threadPool.relativeTimeInMillis(), responseFunction, - failureFunction, threadPool.generic(), threadPool::relativeTimeInMillis, - () -> reduceContextBuilder) { + Function responseFunction = (r) -> null; + Function failureFunction = (e) -> null; + listener = new AsynchronousSearchProgressListener( + threadPool.relativeTimeInMillis(), + responseFunction, + failureFunction, + threadPool.generic(), + threadPool::relativeTimeInMillis, + () -> reduceContextBuilder + ) { @Override public void onResponse(SearchResponse searchResponse) { assertTrue(responseRef.compareAndSet(null, searchResponse)); @@ -137,8 +143,7 @@ public void onResponse(SearchResponse searchResponse) { } @Override - protected void onPartialReduce(List shards, TotalHits totalHits, - InternalAggregations aggs, int reducePhase) { + protected void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { super.onPartialReduce(shards, totalHits, aggs, reducePhase); Terms terms = this.partialResponse().getAggregations().get("keys"); List buckets = terms.getBuckets(); diff --git a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListenerIT.java b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListenerIT.java index 74273d89..82e00423 100644 --- a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListenerIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchProgressListenerIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.search.asynchronous.utils.AsynchronousSearchAssertions; @@ -64,71 +67,46 @@ public void testEmptyQueryStringNoDocs() throws InterruptedException { public void testSearchProgressSimple() throws Exception { for (SearchType searchType : SearchType.values()) { - SearchRequest request = new SearchRequest("index-*") - .searchType(searchType) - .source(new SearchSourceBuilder().size(0)); + SearchRequest request = new SearchRequest("index-*").searchType(searchType).source(new SearchSourceBuilder().size(0)); testCase((NodeClient) client(), request); } } public void testSearchProgressWithHits() throws Exception { for (SearchType searchType : SearchType.values()) { - SearchRequest request = new SearchRequest("index-*") - .searchType(searchType) - .source( - new SearchSourceBuilder() - .size(10) - ); + SearchRequest request = new SearchRequest("index-*").searchType(searchType).source(new SearchSourceBuilder().size(10)); testCase((NodeClient) client(), request); } } public void testSearchProgressWithAggs() throws Exception { for (SearchType searchType : SearchType.values()) { - SearchRequest request = new SearchRequest("index-*") - .searchType(searchType) - .source( - new SearchSourceBuilder() - .size(0) - .aggregation(AggregationBuilders.max("max").field("number")) - ); + SearchRequest request = new SearchRequest("index-*").searchType(searchType) + .source(new SearchSourceBuilder().size(0).aggregation(AggregationBuilders.max("max").field("number"))); testCase((NodeClient) client(), request); } } public void testSearchProgressWithHitsAndAggs() throws Exception { for (SearchType searchType : SearchType.values()) { - SearchRequest request = new SearchRequest("index-*") - .searchType(searchType) - .source( - new SearchSourceBuilder() - .size(10) - .aggregation(AggregationBuilders.max("max").field("number")) - ); + SearchRequest request = new SearchRequest("index-*").searchType(searchType) + .source(new SearchSourceBuilder().size(10).aggregation(AggregationBuilders.max("max").field("number"))); testCase((NodeClient) client(), request); } } public void testSearchProgressWithQuery() throws Exception { for (SearchType searchType : SearchType.values()) { - SearchRequest request = new SearchRequest("index-*") - .searchType(searchType) - .source( - new SearchSourceBuilder() - .size(10) - .query(QueryBuilders.termQuery("foo", "bar")) - ); + SearchRequest request = new SearchRequest("index-*").searchType(searchType) + .source(new SearchSourceBuilder().size(10).query(QueryBuilders.termQuery("foo", "bar"))); testCase((NodeClient) client(), request); } } public void testSearchProgressWithShardSort() throws Exception { - SearchRequest request = new SearchRequest("index-*") - .source( - new SearchSourceBuilder() - .size(0) - .sort(new FieldSortBuilder("number").order(SortOrder.DESC)) - ); + SearchRequest request = new SearchRequest("index-*").source( + new SearchSourceBuilder().size(0).sort(new FieldSortBuilder("number").order(SortOrder.DESC)) + ); request.setPreFilterShardSize(1); testCase((NodeClient) client(), request); } @@ -142,13 +120,16 @@ private void testCase(NodeClient client, SearchRequest request) throws Interrupt AtomicReference responseRef = new AtomicReference<>(); AtomicReference exceptionRef = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); - Function responseFunction = - (r) -> null; - Function failureFunction = - (e) -> null; - AsynchronousSearchProgressListener listener = new AsynchronousSearchProgressListener(threadPool.relativeTimeInMillis(), - responseFunction, - failureFunction, threadPool.generic(), threadPool::relativeTimeInMillis, () -> reduceContextBuilder){ + Function responseFunction = (r) -> null; + Function failureFunction = (e) -> null; + AsynchronousSearchProgressListener listener = new AsynchronousSearchProgressListener( + threadPool.relativeTimeInMillis(), + responseFunction, + failureFunction, + threadPool.generic(), + threadPool::relativeTimeInMillis, + () -> reduceContextBuilder + ) { @Override public void onResponse(SearchResponse searchResponse) { assertTrue(responseRef.compareAndSet(null, searchResponse)); @@ -180,15 +161,12 @@ public SearchTask createTask(long id, String type, String action, TaskId parentT private static List createRandomIndices(Client client) { int numIndices = randomIntBetween(3, 20); for (int i = 0; i < numIndices; i++) { - String indexName = String.format(Locale.ROOT, "index-%03d" , i); + String indexName = String.format(Locale.ROOT, "index-%03d", i); assertAcked(client.admin().indices().prepareCreate(indexName).get()); client.prepareIndex(indexName).setId(Integer.toString(i)).setSource("number", i, "foo", "bar").get(); } client.admin().indices().prepareRefresh("index-*").get(); ClusterSearchShardsResponse resp = client.admin().cluster().prepareSearchShards("index-*").get(); - return Arrays.stream(resp.getGroups()) - .map(e -> new SearchShard(null, e.getShardId())) - .sorted() - .collect(Collectors.toList()); + return Arrays.stream(resp.getGroups()).map(e -> new SearchShard(null, e.getShardId())).sorted().collect(Collectors.toList()); } } diff --git a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapperTests.java b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapperTests.java index 5be5a85c..834582db 100644 --- a/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapperTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapperTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.search.asynchronous.plugin.AsynchronousSearchPlugin; @@ -37,20 +40,24 @@ public void testTimeoutConsumerInvokedOnTimeout() { AtomicReference exception = new AtomicReference<>(); PrioritizedActionListener listener = mockListener(onResponseInvoked, exception); PrioritizedActionListener prioritizedActionListener = AsynchronousSearchTimeoutWrapper.wrapScheduledTimeout( - deterministicTaskQueue.getThreadPool(), timeout, AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - listener, (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true))); - //simulate timeout by advancing time + deterministicTaskQueue.getThreadPool(), + timeout, + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + listener, + (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true)) + ); + // simulate timeout by advancing time assertTrue(deterministicTaskQueue.hasDeferredTasks()); deterministicTaskQueue.advanceTime(); deterministicTaskQueue.runAllRunnableTasks(); - //Fire onResponse/onFailure from search action listener + // Fire onResponse/onFailure from search action listener if (randomBoolean()) { prioritizedActionListener.onResponse(null); } else { prioritizedActionListener.onFailure(new RuntimeException("random exception")); } - //assert only the timeout consumer gets executed + // assert only the timeout consumer gets executed assertTrue(onTimeoutInvoked.get()); assertFalse(onResponseInvoked.get()); assertNull(exception.get()); @@ -63,13 +70,17 @@ public void testResponseBeforeTimeout() { AtomicReference exception = new AtomicReference<>(); ActionListener listener = mockListener(onResponseInvoked, exception); PrioritizedActionListener prioritizedActionListener = AsynchronousSearchTimeoutWrapper.wrapScheduledTimeout( - deterministicTaskQueue.getThreadPool(), timeout, AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - listener, (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true))); - - //Fire on Response of the action listener + deterministicTaskQueue.getThreadPool(), + timeout, + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + listener, + (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true)) + ); + + // Fire on Response of the action listener prioritizedActionListener.onResponse(null); - //simulate timeout by advancing time + // simulate timeout by advancing time assertTrue(deterministicTaskQueue.hasDeferredTasks()); deterministicTaskQueue.advanceTime(); deterministicTaskQueue.runAllRunnableTasks(); @@ -86,12 +97,16 @@ public void testExceptionBeforeTimeout() { AtomicReference exception = new AtomicReference<>(); ActionListener listener = mockListener(onResponseInvoked, exception); PrioritizedActionListener prioritizedActionListener = AsynchronousSearchTimeoutWrapper.wrapScheduledTimeout( - deterministicTaskQueue.getThreadPool(), timeout, AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - listener, (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true))); + deterministicTaskQueue.getThreadPool(), + timeout, + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + listener, + (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true)) + ); prioritizedActionListener.onFailure(new RuntimeException("random exception")); - //simulate timeout by advancing time + // simulate timeout by advancing time assertTrue(deterministicTaskQueue.hasDeferredTasks()); deterministicTaskQueue.advanceTime(); deterministicTaskQueue.runAllRunnableTasks(); @@ -108,17 +123,21 @@ public void testExecuteImmediately() { AtomicReference exception = new AtomicReference<>(); ActionListener listener = mockListener(onResponseInvoked, exception); PrioritizedActionListener prioritizedActionListener = AsynchronousSearchTimeoutWrapper.wrapScheduledTimeout( - deterministicTaskQueue.getThreadPool(), timeout, AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - listener, (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true))); - - //execute the listener immediately + deterministicTaskQueue.getThreadPool(), + timeout, + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + listener, + (r) -> assertTrue(onTimeoutInvoked.compareAndSet(false, true)) + ); + + // execute the listener immediately prioritizedActionListener.executeImmediately(); assertTrue(onTimeoutInvoked.get()); assertFalse(onResponseInvoked.get()); assertNull(exception.get()); - //simulate timeout by advancing time + // simulate timeout by advancing time assertTrue(deterministicTaskQueue.hasDeferredTasks()); deterministicTaskQueue.advanceTime(); deterministicTaskQueue.runAllRunnableTasks(); diff --git a/src/test/java/org/opensearch/search/asynchronous/listener/SearchProgressActionListenerTests.java b/src/test/java/org/opensearch/search/asynchronous/listener/SearchProgressActionListenerTests.java index ec59d860..d53c0fa3 100644 --- a/src/test/java/org/opensearch/search/asynchronous/listener/SearchProgressActionListenerTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/listener/SearchProgressActionListenerTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.listener; import org.opensearch.search.asynchronous.commons.AsynchronousSearchTestCase; @@ -32,7 +35,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; - public class SearchProgressActionListenerTests extends AsynchronousSearchTestCase { private final AtomicReference responseRef = new AtomicReference<>(); @@ -47,17 +49,27 @@ public class SearchProgressActionListenerTests extends AsynchronousSearchTestCas public void setUpMocks() { mockSearchException = new RuntimeException("random-search-exception"); mockPostProcessingException = new RuntimeException("random-post-processing-exception"); - mockSearchResponse = new SearchResponse(new InternalSearchResponse( + mockSearchResponse = new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), - "", 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 1 + ), + "", + 1, + 1, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); mockAsynchronousSearchResp = AsynchronousSearchResponse.empty("random-id", mockSearchResponse, null); - mockAsynchronousSearchFailResp = AsynchronousSearchResponse.empty("random-id", null, - new OpenSearchException(mockSearchException)); + mockAsynchronousSearchFailResp = AsynchronousSearchResponse.empty("random-id", null, new OpenSearchException(mockSearchException)); } public void testListenerOnResponseForSuccessfulSearch() throws InterruptedException { @@ -65,31 +77,35 @@ public void testListenerOnResponseForSuccessfulSearch() throws InterruptedExcept try { final int numListeners = randomIntBetween(1, 20); threadPool = new TestThreadPool(getClass().getName()); - Function responseFunction = - (r) -> { - assertTrue(responseRef.compareAndSet(null, r)); - return mockAsynchronousSearchResp; - }; - Function failureFunction = - (e) -> { - assertTrue(exceptionRef.compareAndSet(null, e)); - return mockAsynchronousSearchFailResp; - }; - AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener(threadPool, responseFunction, - failureFunction); - Tuple>, List>> respTuple = - processListeners(progressActionListener, () -> progressActionListener.onResponse(mockSearchResponse), numListeners); + Function responseFunction = (r) -> { + assertTrue(responseRef.compareAndSet(null, r)); + return mockAsynchronousSearchResp; + }; + Function failureFunction = (e) -> { + assertTrue(exceptionRef.compareAndSet(null, e)); + return mockAsynchronousSearchFailResp; + }; + AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener( + threadPool, + responseFunction, + failureFunction + ); + Tuple>, List>> respTuple = processListeners( + progressActionListener, + () -> progressActionListener.onResponse(mockSearchResponse), + numListeners + ); List> responseList = respTuple.v1(); List> exceptionList = respTuple.v2(); - //assert all response listeners that were added were invoked + // assert all response listeners that were added were invoked assertEquals(numListeners, responseList.size()); assertEquals(0, exceptionList.size()); assertNull(exceptionRef.get()); assertEquals(mockSearchResponse, responseRef.get()); for (int i = 0; i < numListeners; i++) { - //assert all response listeners that were added were invoked with the search response + // assert all response listeners that were added were invoked with the search response assertEquals(mockAsynchronousSearchResp, responseList.get(i).get()); } } finally { @@ -102,31 +118,35 @@ public void testListenerOnResponseForFailedSearch() throws InterruptedException try { final int numListeners = randomIntBetween(1, 20); threadPool = new TestThreadPool(getClass().getName()); - Function responseFunction = - (r) -> { - assertTrue(responseRef.compareAndSet(null, r)); - return mockAsynchronousSearchResp; - }; - Function failureFunction = - (e) -> { - assertTrue(exceptionRef.compareAndSet(null, e)); - return mockAsynchronousSearchFailResp; - }; - AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener(threadPool, responseFunction, - failureFunction); - Tuple>, List>> respTuple = - processListeners(progressActionListener, () -> progressActionListener.onFailure(mockSearchException), numListeners); + Function responseFunction = (r) -> { + assertTrue(responseRef.compareAndSet(null, r)); + return mockAsynchronousSearchResp; + }; + Function failureFunction = (e) -> { + assertTrue(exceptionRef.compareAndSet(null, e)); + return mockAsynchronousSearchFailResp; + }; + AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener( + threadPool, + responseFunction, + failureFunction + ); + Tuple>, List>> respTuple = processListeners( + progressActionListener, + () -> progressActionListener.onFailure(mockSearchException), + numListeners + ); List> responseList = respTuple.v1(); List> exceptionList = respTuple.v2(); - //assert all response listeners that were added were invoked + // assert all response listeners that were added were invoked assertEquals(numListeners, responseList.size()); assertEquals(0, exceptionList.size()); assertEquals(mockSearchException, exceptionRef.get()); assertNull(responseRef.get()); for (int i = 0; i < numListeners; i++) { - //assert all response listeners that were added were invoked with the search response + // assert all response listeners that were added were invoked with the search response assertEquals(mockAsynchronousSearchFailResp, responseList.get(i).get()); } } finally { @@ -139,31 +159,35 @@ public void testListenerOnFailureForFailedSearch() throws InterruptedException { try { final int numListeners = randomIntBetween(1, 20); threadPool = new TestThreadPool(getClass().getName()); - Function responseFunction = - (r) -> { - assertTrue(responseRef.compareAndSet(null, r)); - throw mockPostProcessingException; - }; - Function failureFunction = - (e) -> { - assertTrue(exceptionRef.compareAndSet(null, e)); - throw mockPostProcessingException; - }; - AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener(threadPool, responseFunction, - failureFunction); - Tuple>, List>> respTuple = - processListeners(progressActionListener, () -> progressActionListener.onFailure(mockSearchException), numListeners); + Function responseFunction = (r) -> { + assertTrue(responseRef.compareAndSet(null, r)); + throw mockPostProcessingException; + }; + Function failureFunction = (e) -> { + assertTrue(exceptionRef.compareAndSet(null, e)); + throw mockPostProcessingException; + }; + AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener( + threadPool, + responseFunction, + failureFunction + ); + Tuple>, List>> respTuple = processListeners( + progressActionListener, + () -> progressActionListener.onFailure(mockSearchException), + numListeners + ); List> responseList = respTuple.v1(); List> exceptionList = respTuple.v2(); - //assert all response listeners that were added were invoked + // assert all response listeners that were added were invoked assertEquals(0, responseList.size()); assertEquals(numListeners, exceptionList.size()); assertEquals(mockSearchException, exceptionRef.get()); assertEquals(null, responseRef.get()); for (int i = 0; i < numListeners; i++) { - //assert all response listeners that were added were invoked with the search response + // assert all response listeners that were added were invoked with the search response assertEquals(mockPostProcessingException, exceptionList.get(i).get()); } } finally { @@ -176,31 +200,35 @@ public void testListenerOnFailureForSuccessfulSearch() throws InterruptedExcepti try { final int numListeners = randomIntBetween(1, 20); threadPool = new TestThreadPool(getClass().getName()); - Function responseFunction = - (r) -> { - assertTrue(responseRef.compareAndSet(null, r)); - throw mockPostProcessingException; - }; - Function failureFunction = - (e) -> { - assertTrue(exceptionRef.compareAndSet(null, e)); - throw mockPostProcessingException; - }; - AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener(threadPool, responseFunction, - failureFunction); - Tuple>, List>> respTuple = - processListeners(progressActionListener, () -> progressActionListener.onResponse(mockSearchResponse), numListeners); + Function responseFunction = (r) -> { + assertTrue(responseRef.compareAndSet(null, r)); + throw mockPostProcessingException; + }; + Function failureFunction = (e) -> { + assertTrue(exceptionRef.compareAndSet(null, e)); + throw mockPostProcessingException; + }; + AsynchronousSearchProgressListener progressActionListener = mockAsynchronousSearchProgressListener( + threadPool, + responseFunction, + failureFunction + ); + Tuple>, List>> respTuple = processListeners( + progressActionListener, + () -> progressActionListener.onResponse(mockSearchResponse), + numListeners + ); List> responseList = respTuple.v1(); List> exceptionList = respTuple.v2(); - //assert all response listeners that were added were invoked + // assert all response listeners that were added were invoked assertEquals(0, responseList.size()); assertEquals(numListeners, exceptionList.size()); assertNull(exceptionRef.get()); assertEquals(mockSearchResponse, responseRef.get()); for (int i = 0; i < numListeners; i++) { - //assert all response listeners that were added were invoked with the search response + // assert all response listeners that were added were invoked with the search response assertEquals(mockPostProcessingException, exceptionList.get(i).get()); } } finally { @@ -209,29 +237,31 @@ public void testListenerOnFailureForSuccessfulSearch() throws InterruptedExcepti } public Tuple>, List>> processListeners( - AsynchronousSearchProgressListener progressActionListener, Runnable listenerAction, - int numListeners) throws InterruptedException { + AsynchronousSearchProgressListener progressActionListener, + Runnable listenerAction, + int numListeners + ) throws InterruptedException { final CountDownLatch latch = new CountDownLatch(numListeners); final List> responseList = new ArrayList<>(); final List> exceptionList = new ArrayList<>(); final AtomicInteger immediateExecution = new AtomicInteger(); for (int i = 0; i < numListeners; i++) { - progressActionListener.searchProgressActionListener().addOrExecuteListener(createMockListener(responseList, exceptionList, - immediateExecution, latch)); + progressActionListener.searchProgressActionListener() + .addOrExecuteListener(createMockListener(responseList, exceptionList, immediateExecution, latch)); } listenerAction.run(); - //wait for all listeners to be executed since on response is forked to a separate thread pool + // wait for all listeners to be executed since on response is forked to a separate thread pool latch.await(); return new Tuple<>(responseList, exceptionList); } - private PrioritizedActionListener createMockListener( - List> responseList, - List> exceptionList, - AtomicInteger immediateExecution, - CountDownLatch latch) { + List> responseList, + List> exceptionList, + AtomicInteger immediateExecution, + CountDownLatch latch + ) { final AtomicBoolean completed = new AtomicBoolean(); final AtomicReference asResponseRef = new AtomicReference<>(); diff --git a/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceIT.java b/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceIT.java index e1cc44d2..2e799cd5 100644 --- a/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.management; import org.opensearch.common.xcontent.XContentType; @@ -54,13 +57,10 @@ public class AsynchronousSearchManagementServiceIT extends AsynchronousSearchInt @Override protected Collection> nodePlugins() { - return Arrays.asList( - ScriptedBlockPlugin.class, - AsynchronousSearchPlugin.class, - ReindexPlugin.class); + return Arrays.asList(ScriptedBlockPlugin.class, AsynchronousSearchPlugin.class, ReindexPlugin.class); } - //We need to apply blocks via ScriptedBlockPlugin, external clusters are immutable + // We need to apply blocks via ScriptedBlockPlugin, external clusters are immutable @Override protected boolean ignoreExternalCluster() { return true; @@ -71,13 +71,12 @@ protected Settings nodeSettings(int nodeOrdinal) { boolean lowLevelCancellation = randomBoolean(); logger.info("Using lowLevelCancellation: {}", lowLevelCancellation); return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put("node.attr.asynchronous_search_enabled", true) - .put(AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(5)) - .put(AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.getKey(), - TimeValue.timeValueSeconds(5)) - .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), lowLevelCancellation) - .build(); + .put(super.nodeSettings(nodeOrdinal)) + .put("node.attr.asynchronous_search_enabled", true) + .put(AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(5)) + .put(AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(5)) + .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), lowLevelCancellation) + .build(); } private void indexTestData() { @@ -85,8 +84,7 @@ private void indexTestData() { // Make sure we have a few segments BulkRequestBuilder bulkRequestBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (int j = 0; j < 20; j++) { - bulkRequestBuilder.add(client().prepareIndex("test").setId(Integer.toString(i * 5 + j)) - .setSource("field", "value")); + bulkRequestBuilder.add(client().prepareIndex("test").setId(Integer.toString(i * 5 + j)).setSource("field", "value")); } assertNoFailures(bulkRequestBuilder.get()); } @@ -96,26 +94,23 @@ public void testCleansUpExpiredAsynchronousSearchDuringQueryPhase() throws Excep List plugins = initBlockFactory(); indexTestData(); - SearchRequest searchRequest = client().prepareSearch("test").setQuery( - scriptQuery(new Script( - ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) - .request(); - //We need a NodeClient to make sure the listener gets injected in the search request execution. - //Randomized client randomly return NodeClient/TransportClient + SearchRequest searchRequest = client().prepareSearch("test") + .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + .request(); + // We need a NodeClient to make sure the listener gets injected in the search request execution. + // Randomized client randomly return NodeClient/TransportClient SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(1)); testCase(internalCluster().smartClient(), submitAsynchronousSearchRequest, plugins); } - public void testCleansUpExpiredAsynchronousSearchDuringFetchPhase() throws Exception { List plugins = initBlockFactory(); indexTestData(); SearchRequest searchRequest = client().prepareSearch("test") - .addScriptField("test_field", - new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()) - ).request(); + .addScriptField("test_field", new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap())) + .request(); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(1)); @@ -124,19 +119,16 @@ public void testCleansUpExpiredAsynchronousSearchDuringFetchPhase() throws Excep public void testDeletesExpiredAsynchronousSearchResponseFromPersistedStore() throws Exception { String idx = "idx"; - assertAcked(prepareCreate(idx) - .setMapping("ip", "type=ip", "ips", "type=ip")); + assertAcked(prepareCreate(idx).setMapping("ip", "type=ip", "ips", "type=ip")); waitForRelocation(ClusterHealthStatus.GREEN); - indexRandom(true, - client().prepareIndex(idx).setId("1").setSource( - "ip", "192.168.1.7", - "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), - client().prepareIndex(idx).setId("2").setSource( - "ip", "192.168.1.10", - "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), - client().prepareIndex(idx).setId("3").setSource( - "ip", "2001:db8::ff00:42:8329", - "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); + indexRandom( + true, + client().prepareIndex(idx).setId("1").setSource("ip", "192.168.1.7", "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), + client().prepareIndex(idx).setId("2").setSource("ip", "192.168.1.10", "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), + client().prepareIndex(idx) + .setId("3") + .setSource("ip", "2001:db8::ff00:42:8329", "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")) + ); assertAcked(prepareCreate("idx_unmapped")); waitForRelocation(ClusterHealthStatus.GREEN); @@ -150,36 +142,42 @@ public void testDeletesExpiredAsynchronousSearchResponseFromPersistedStore() thr submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(5000)); CountDownLatch latch = new CountDownLatch(2); - client().execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - asResponseRef.set(asResponse); - exceptionRef.set(asResponse.getError()); - latch.countDown(); - } + client().execute( + SubmitAsynchronousSearchAction.INSTANCE, + submitAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + asResponseRef.set(asResponse); + exceptionRef.set(asResponse.getError()); + latch.countDown(); + } - @Override - public void onFailure(Exception e) { - exceptionRef.set(e); - latch.countDown(); + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } } - }); + ); - //submit another request to verify that the second request is not cancelled - client().execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - nonExpiredAsynchronousSearchResponseRef.set(asResponse); - latch.countDown(); - } + // submit another request to verify that the second request is not cancelled + client().execute( + SubmitAsynchronousSearchAction.INSTANCE, + submitAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + nonExpiredAsynchronousSearchResponseRef.set(asResponse); + latch.countDown(); + } - @Override - public void onFailure(Exception e) { - latch.countDown(); + @Override + public void onFailure(Exception e) { + latch.countDown(); + } } - }); + ); latch.await(); waitUntil(() -> verifyResponsePersisted(asResponseRef.get().getId())); @@ -187,42 +185,49 @@ public void onFailure(Exception e) { CountDownLatch updateLatch = new CountDownLatch(1); GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(asResponseRef.get().getId()); getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueMillis(1)); - client().execute(GetAsynchronousSearchAction.INSTANCE, getAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - asResponseRef.set(asResponse); - exceptionRef.set(asResponse.getError()); - updateLatch.countDown(); - } + client().execute( + GetAsynchronousSearchAction.INSTANCE, + getAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + asResponseRef.set(asResponse); + exceptionRef.set(asResponse.getError()); + updateLatch.countDown(); + } - @Override - public void onFailure(Exception e) { - exceptionRef.set(e); - updateLatch.countDown(); + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + updateLatch.countDown(); + } } - }); + ); updateLatch.await(); waitUntil(() -> verifyResponseRemoved(asResponseRef.get().getId())); assertBusy(() -> assertTrue(verifyResponsePersisted(nonExpiredAsynchronousSearchResponseRef.get().getId()))); // delete the non expired response explicitly CountDownLatch deleteLatch = new CountDownLatch(1); DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest( - nonExpiredAsynchronousSearchResponseRef.get().getId()); - client().execute(DeleteAsynchronousSearchAction.INSTANCE, deleteAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse response) { - assertTrue(response.isAcknowledged()); - deleteLatch.countDown(); - } + nonExpiredAsynchronousSearchResponseRef.get().getId() + ); + client().execute( + DeleteAsynchronousSearchAction.INSTANCE, + deleteAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + assertTrue(response.isAcknowledged()); + deleteLatch.countDown(); + } - @Override - public void onFailure(Exception e) { - deleteLatch.countDown(); - fail("Cleanup failed"); + @Override + public void onFailure(Exception e) { + deleteLatch.countDown(); + fail("Cleanup failed"); + } } - }); + ); deleteLatch.await(); } @@ -246,7 +251,7 @@ public void onFailure(Exception e) { } }); - //submit another request to verify that the second request is not cancelled + // submit another request to verify that the second request is not cancelled client.execute(SubmitAsynchronousSearchAction.INSTANCE, request, new ActionListener() { @Override public void onResponse(AsynchronousSearchResponse asResponse) { @@ -265,51 +270,57 @@ public void onFailure(Exception e) { CountDownLatch updateLatch = new CountDownLatch(1); GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(asResponseRef.get().getId()); getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueMillis(1)); - client.execute(GetAsynchronousSearchAction.INSTANCE, getAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse asResponse) { - asResponseRef.set(asResponse); - exceptionRef.set(asResponse.getError()); - updateLatch.countDown(); - } + client.execute( + GetAsynchronousSearchAction.INSTANCE, + getAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse asResponse) { + asResponseRef.set(asResponse); + exceptionRef.set(asResponse.getError()); + updateLatch.countDown(); + } - @Override - public void onFailure(Exception e) { - exceptionRef.set(e); - updateLatch.countDown(); + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + updateLatch.countDown(); + } } - }); + ); updateLatch.await(); - assertThat(asResponseRef.get().getExpirationTimeMillis(), - lessThan((System.currentTimeMillis()) + randomLongBetween(100, 200))); + assertThat(asResponseRef.get().getExpirationTimeMillis(), lessThan((System.currentTimeMillis()) + randomLongBetween(100, 200))); boolean cleanedUp = waitUntil(() -> verifyAsynchronousSearchDoesNotExists(asResponseRef.get().getId())); assertTrue(cleanedUp); disableBlocks(plugins); AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(asResponseRef.get().getId()); TaskId taskId = new TaskId(asId.getNode(), asId.getTaskId()); waitUntil(() -> verifyTaskCancelled(AsynchronousSearchTask.NAME, taskId)); - //ensure the second asynchronous search is not cleaned up + // ensure the second asynchronous search is not cleaned up assertBusy(() -> assertFalse(verifyAsynchronousSearchDoesNotExists(nonExpiredAsynchronousSearchResponseRef.get().getId()))); logger.info("Segments {}", Strings.toString(XContentType.JSON, client().admin().indices().prepareSegments("test").get())); CountDownLatch deleteLatch = new CountDownLatch(1); - //explicitly clean up the second request + // explicitly clean up the second request DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest( - nonExpiredAsynchronousSearchResponseRef.get().getId()); - client.execute(DeleteAsynchronousSearchAction.INSTANCE, deleteAsynchronousSearchRequest, - new ActionListener() { - @Override - public void onResponse(AcknowledgedResponse response) { - assertTrue(response.isAcknowledged()); - deleteLatch.countDown(); - } + nonExpiredAsynchronousSearchResponseRef.get().getId() + ); + client.execute( + DeleteAsynchronousSearchAction.INSTANCE, + deleteAsynchronousSearchRequest, + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + assertTrue(response.isAcknowledged()); + deleteLatch.countDown(); + } - @Override - public void onFailure(Exception e) { - deleteLatch.countDown(); - fail("Cleanup failed"); + @Override + public void onFailure(Exception e) { + deleteLatch.countDown(); + fail("Cleanup failed"); + } } - }); + ); deleteLatch.await(); } } diff --git a/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceTests.java b/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceTests.java index 683f8e7d..288ef4c3 100644 --- a/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/management/AsynchronousSearchManagementServiceTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.management; import org.opensearch.search.asynchronous.service.AsynchronousSearchPersistenceService; @@ -52,26 +55,34 @@ public class AsynchronousSearchManagementServiceTests extends OpenSearchTestCase private static final String NODE_ID_PREFIX = "node_"; private static final String INITIAL_CLUSTER_ID = UUIDs.randomBase64UUID(); // the initial indices which every cluster state test starts out with - private static final List initialIndices = Arrays.asList(new Index("idx1", UUIDs.randomBase64UUID()), - new Index("idx2", UUIDs.randomBase64UUID()), - new Index("idx3", UUIDs.randomBase64UUID())); + private static final List initialIndices = Arrays.asList( + new Index("idx1", UUIDs.randomBase64UUID()), + new Index("idx2", UUIDs.randomBase64UUID()), + new Index("idx3", UUIDs.randomBase64UUID()) + ); @Before public void createObjects() { - Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node") - .put("node.attr.asynchronous_search_enabled", true).build(); + Settings settings = Settings.builder() + .put(NODE_NAME_SETTING.getKey(), "node") + .put("node.attr.asynchronous_search_enabled", true) + .build(); deterministicTaskQueue = new DeterministicTaskQueue(settings, random()); } public void testSchedulesContextReaperAtRefreshIntervals() { long refreshInterval = randomLongBetween(100000, 200000); final Settings settings = Settings.builder() - .put(AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.getKey(), refreshInterval + "ms") - .build(); - AsynchronousSearchManagementService managementService = new AsynchronousSearchManagementService(settings, - Mockito.mock(ClusterService.class), - deterministicTaskQueue.getThreadPool(), Mockito.mock(AsynchronousSearchService.class), Mockito.mock(TransportService.class), - Mockito.mock(AsynchronousSearchPersistenceService.class)); + .put(AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.getKey(), refreshInterval + "ms") + .build(); + AsynchronousSearchManagementService managementService = new AsynchronousSearchManagementService( + settings, + Mockito.mock(ClusterService.class), + deterministicTaskQueue.getThreadPool(), + Mockito.mock(AsynchronousSearchService.class), + Mockito.mock(TransportService.class), + Mockito.mock(AsynchronousSearchPersistenceService.class) + ); final long startTimeMillis = deterministicTaskQueue.getCurrentTimeMillis(); managementService.doStart(); assertFalse(deterministicTaskQueue.hasRunnableTasks()); @@ -97,11 +108,15 @@ public void testSchedulesContextReaperAtRefreshIntervals() { public void testSchedulesResponseCleanupAtRefreshIntervals() { long refreshInterval = randomLongBetween(60000, 120000); final Settings settings = Settings.builder() - .put(AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.getKey(), refreshInterval + "ms") - .build(); - DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), - Collections.singletonMap("asynchronous_search_enabled", "true"), Sets.newHashSet(DiscoveryNodeRole.DATA_ROLE), - Version.CURRENT); + .put(AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.getKey(), refreshInterval + "ms") + .build(); + DiscoveryNode localNode = new DiscoveryNode( + "local-node", + buildNewFakeTransportAddress(), + Collections.singletonMap("asynchronous_search_enabled", "true"), + Sets.newHashSet(DiscoveryNodeRole.DATA_ROLE), + Version.CURRENT + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(deterministicTaskQueue.getThreadPool(), localNode); final MockTransport mockTransport = new MockTransport() { @Override @@ -114,15 +129,25 @@ protected void onSendRequest(long requestId, String action, TransportRequest req } } }; - final TransportService transportService = mockTransport.createTransportService(settings, - deterministicTaskQueue.getThreadPool(), NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> - new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT), null, emptySet(), - NoopTracer.INSTANCE); + final TransportService transportService = mockTransport.createTransportService( + settings, + deterministicTaskQueue.getThreadPool(), + NOOP_TRANSPORT_INTERCEPTOR, + boundTransportAddress -> new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT), + null, + emptySet(), + NoopTracer.INSTANCE + ); transportService.start(); transportService.acceptIncomingRequests(); - AsynchronousSearchManagementService managementService = new AsynchronousSearchManagementService(settings, mockClusterService, - deterministicTaskQueue.getThreadPool(), Mockito.mock(AsynchronousSearchService.class), transportService, - Mockito.mock(AsynchronousSearchPersistenceService.class)); + AsynchronousSearchManagementService managementService = new AsynchronousSearchManagementService( + settings, + mockClusterService, + deterministicTaskQueue.getThreadPool(), + Mockito.mock(AsynchronousSearchService.class), + transportService, + Mockito.mock(AsynchronousSearchPersistenceService.class) + ); final long startTimeMillis = deterministicTaskQueue.getCurrentTimeMillis(); final int numNodesInCluster = 3; ClusterState previousState = createSimpleClusterState(); @@ -155,15 +180,24 @@ private static ClusterState createSimpleClusterState() { private static ClusterState createState(final int numNodes, final boolean isLocalClusterManager, final List indices) { final Metadata metadata = createMetadata(indices); return ClusterState.builder(TEST_CLUSTER_NAME) - .nodes(createDiscoveryNodes(numNodes, isLocalClusterManager)) - .metadata(metadata) - .routingTable(createRoutingTable(1, metadata)) - .build(); + .nodes(createDiscoveryNodes(numNodes, isLocalClusterManager)) + .metadata(metadata) + .routingTable(createRoutingTable(1, metadata)) + .build(); } private static DiscoveryNode newNode(final String nodeId, Set roles) { - return new DiscoveryNode(nodeId, nodeId, nodeId, "host", "host_address", buildNewFakeTransportAddress(), - Collections.singletonMap("asynchronous_search_enabled", "true"), roles, Version.CURRENT); + return new DiscoveryNode( + nodeId, + nodeId, + nodeId, + "host", + "host_address", + buildNewFakeTransportAddress(), + Collections.singletonMap("asynchronous_search_enabled", "true"), + roles, + Version.CURRENT + ); } // Create the metadata for a cluster state. @@ -183,16 +217,17 @@ private static IndexMetadata createIndexMetadata(final Index index) { // Create the index metadata for a given index, with the specified version. private static IndexMetadata createIndexMetadata(final Index index, final long version) { - final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) - .build(); + final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); return IndexMetadata.builder(index.getName()) - .settings(settings) - .numberOfShards(1) - .numberOfReplicas(0) - .creationDate(System.currentTimeMillis()) - .version(version) - .build(); + .settings(settings) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .version(version) + .build(); } // Create the routing table for a cluster state. @@ -204,12 +239,12 @@ private static RoutingTable createRoutingTable(final long version, final Metadat return builder.build(); } - // Create the discovery nodes for a cluster state. For our testing purposes, we want + // Create the discovery nodes for a cluster state. For our testing purposes, we want // the first to be cluster_manager, the second to be cluster_manager eligible, the third to be a data node, // and the remainder can be any kinds of nodes (cluster_manager eligible, data, or both). private static DiscoveryNodes createDiscoveryNodes(final int numNodes, final boolean isLocalClusterManager) { - assert (numNodes >= 3) : "the initial cluster state for event change tests should have a minimum of 3 nodes " + - "so there are a minimum of 2 cluster_manager nodes for testing cluster_manager change events."; + assert (numNodes >= 3) : "the initial cluster state for event change tests should have a minimum of 3 nodes " + + "so there are a minimum of 2 cluster_manager nodes for testing cluster_manager change events."; final DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); // randomly assign the local node if not cluster_manager @@ -218,7 +253,7 @@ private static DiscoveryNodes createDiscoveryNodes(final int numNodes, final boo final String nodeId = NODE_ID_PREFIX + i; Set roles = new HashSet<>(); if (i == 0) { - //local node id + // local node id builder.localNodeId(nodeId); roles.add(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE); } else if (i == 1) { diff --git a/src/test/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRequestRoutingIT.java b/src/test/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRequestRoutingIT.java index 7991cc75..a8deb284 100644 --- a/src/test/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRequestRoutingIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/request/AsynchronousSearchRequestRoutingIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.request; import org.opensearch.search.asynchronous.action.DeleteAsynchronousSearchAction; @@ -47,19 +50,16 @@ protected int maximumNumberOfReplicas() { public void testRequestForwardingToCoordinatorNodeForPersistedAsynchronousSearch() throws Exception { String idx = "idx"; - assertAcked(prepareCreate(idx) - .setMapping("ip", "type=ip", "ips", "type=ip")); + assertAcked(prepareCreate(idx).setMapping("ip", "type=ip", "ips", "type=ip")); waitForRelocation(ClusterHealthStatus.GREEN); - indexRandom(true, - client().prepareIndex(idx).setId("1").setSource( - "ip", "192.168.1.7", - "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), - client().prepareIndex(idx).setId("2").setSource( - "ip", "192.168.1.10", - "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), - client().prepareIndex(idx).setId("3").setSource( - "ip", "2001:db8::ff00:42:8329", - "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); + indexRandom( + true, + client().prepareIndex(idx).setId("1").setSource("ip", "192.168.1.7", "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), + client().prepareIndex(idx).setId("2").setSource("ip", "192.168.1.10", "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), + client().prepareIndex(idx) + .setId("3") + .setSource("ip", "2001:db8::ff00:42:8329", "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")) + ); assertAcked(prepareCreate("idx_unmapped")); waitForRelocation(ClusterHealthStatus.GREEN); @@ -70,24 +70,35 @@ public void testRequestForwardingToCoordinatorNodeForPersistedAsynchronousSearch AsynchronousSearchResponse submitResponse = client().execute(SubmitAsynchronousSearchAction.INSTANCE, request).get(); AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(submitResponse.getId()); assertNotNull(submitResponse.getId()); - waitUntil(() -> TestClientUtils.blockingGetAsynchronousSearchResponse(client(), new GetAsynchronousSearchRequest( - submitResponse.getId())) - .getState().equals(AsynchronousSearchState.STORE_RESIDENT)); + waitUntil( + () -> TestClientUtils.blockingGetAsynchronousSearchResponse(client(), new GetAsynchronousSearchRequest(submitResponse.getId())) + .getState() + .equals(AsynchronousSearchState.STORE_RESIDENT) + ); assertNotNull(submitResponse.getSearchResponse()); ClusterService clusterService = internalCluster().getInstance(ClusterService.class); assertEquals(clusterService.state().nodes().getDataNodes().size(), 5); List nonCoordinatorNodeNames = new LinkedList<>(); clusterService.state().nodes().iterator().forEachRemaining(node -> { - if (asId.getNode().equals(node.getId()) == false) - nonCoordinatorNodeNames.add(node.getName()); + if (asId.getNode().equals(node.getId()) == false) nonCoordinatorNodeNames.add(node.getName()); }); nonCoordinatorNodeNames.forEach(n -> { try { - AsynchronousSearchResponse getResponse = client(n).execute(GetAsynchronousSearchAction.INSTANCE, - new GetAsynchronousSearchRequest(submitResponse.getId())).get(); - assertEquals(getResponse, new AsynchronousSearchResponse(submitResponse.getId(), AsynchronousSearchState.STORE_RESIDENT, - submitResponse.getStartTimeMillis(), submitResponse.getExpirationTimeMillis(), submitResponse.getSearchResponse(), - submitResponse.getError())); + AsynchronousSearchResponse getResponse = client(n).execute( + GetAsynchronousSearchAction.INSTANCE, + new GetAsynchronousSearchRequest(submitResponse.getId()) + ).get(); + assertEquals( + getResponse, + new AsynchronousSearchResponse( + submitResponse.getId(), + AsynchronousSearchState.STORE_RESIDENT, + submitResponse.getStartTimeMillis(), + submitResponse.getExpirationTimeMillis(), + submitResponse.getSearchResponse(), + submitResponse.getError() + ) + ); } catch (InterruptedException | ExecutionException e) { fail("Get asynchronous search request should not have failed"); } @@ -97,196 +108,200 @@ public void testRequestForwardingToCoordinatorNodeForPersistedAsynchronousSearch public void testRequestForwardingToCoordinatorNodeForRunningAsynchronousSearch() throws Exception { List plugins = initBlockFactory(); String index = "idx"; - assertAcked(prepareCreate(index) - .setMapping("ip", "type=ip", "ips", "type=ip")); + assertAcked(prepareCreate(index).setMapping("ip", "type=ip", "ips", "type=ip")); waitForRelocation(ClusterHealthStatus.GREEN); - indexRandom(true, - client().prepareIndex(index).setId("1").setSource( - "ip", "192.168.1.7", - "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), - client().prepareIndex(index).setId("2").setSource( - "ip", "192.168.1.10", - "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), - client().prepareIndex(index).setId("3").setSource( - "ip", "2001:db8::ff00:42:8329", - "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); + indexRandom( + true, + client().prepareIndex(index).setId("1").setSource("ip", "192.168.1.7", "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), + client().prepareIndex(index).setId("2").setSource("ip", "192.168.1.10", "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), + client().prepareIndex(index) + .setId("3") + .setSource("ip", "2001:db8::ff00:42:8329", "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")) + ); assertAcked(prepareCreate("idx_unmapped")); waitForRelocation(ClusterHealthStatus.GREEN); refresh(); - SearchRequest searchRequest = client().prepareSearch(index).setQuery( - scriptQuery(new Script( - ScriptType.INLINE, "mockscript", SCRIPT_NAME, - Collections.emptyMap()))) - .request(); + SearchRequest searchRequest = client().prepareSearch(index) + .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))) + .request(); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); request.keepOnCompletion(false); request.waitForCompletionTimeout(TimeValue.timeValueMillis(0)); CountDownLatch latch = new CountDownLatch(1); - client().execute(SubmitAsynchronousSearchAction.INSTANCE, request, new LatchedActionListener<>( - new ActionListener() { - @Override - public void onResponse(AsynchronousSearchResponse submitResponse) { - String id = submitResponse.getId(); - assertNotNull(id); - assertEquals(AsynchronousSearchState.RUNNING, submitResponse.getState()); - AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(id); - ClusterService clusterService = internalCluster().getInstance(ClusterService.class); - assertEquals(clusterService.state().nodes().getDataNodes().size(), 5); - List nonCoordinatorNodeNames = new LinkedList<>(); - clusterService.state().nodes().iterator().forEachRemaining(node -> { - if (asId.getNode().equals(node.getId()) == false) - nonCoordinatorNodeNames.add(node.getName()); - }); - nonCoordinatorNodeNames.forEach(n -> { + client().execute( + SubmitAsynchronousSearchAction.INSTANCE, + request, + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(AsynchronousSearchResponse submitResponse) { + String id = submitResponse.getId(); + assertNotNull(id); + assertEquals(AsynchronousSearchState.RUNNING, submitResponse.getState()); + AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(id); + ClusterService clusterService = internalCluster().getInstance(ClusterService.class); + assertEquals(clusterService.state().nodes().getDataNodes().size(), 5); + List nonCoordinatorNodeNames = new LinkedList<>(); + clusterService.state().nodes().iterator().forEachRemaining(node -> { + if (asId.getNode().equals(node.getId()) == false) nonCoordinatorNodeNames.add(node.getName()); + }); + nonCoordinatorNodeNames.forEach(n -> { + try { + AsynchronousSearchResponse getResponse = client(n).execute( + GetAsynchronousSearchAction.INSTANCE, + new GetAsynchronousSearchRequest(id) + ).get(); + assertEquals(getResponse.getState(), AsynchronousSearchState.RUNNING); + } catch (InterruptedException | ExecutionException e) { + fail("Get asynchronous search request should not have failed"); + } + }); + String randomNonCoordinatorNode = nonCoordinatorNodeNames.get(randomInt(nonCoordinatorNodeNames.size() - 1)); try { - AsynchronousSearchResponse getResponse = client(n).execute(GetAsynchronousSearchAction.INSTANCE, - new GetAsynchronousSearchRequest(id)).get(); - assertEquals(getResponse.getState(), AsynchronousSearchState.RUNNING); + AcknowledgedResponse acknowledgedResponse = client(randomNonCoordinatorNode).execute( + DeleteAsynchronousSearchAction.INSTANCE, + new DeleteAsynchronousSearchRequest(id) + ).get(); + assertTrue(acknowledgedResponse.isAcknowledged()); + ExecutionException executionException = expectThrows( + ExecutionException.class, + () -> client().execute(GetAsynchronousSearchAction.INSTANCE, new GetAsynchronousSearchRequest(id)).get() + ); + assertThat(executionException.getMessage(), containsString("ResourceNotFoundException")); } catch (InterruptedException | ExecutionException e) { - fail("Get asynchronous search request should not have failed"); + fail("Delete asynchronous search request from random non-coordinator node should have succeeded."); } - }); - String randomNonCoordinatorNode = nonCoordinatorNodeNames.get(randomInt(nonCoordinatorNodeNames.size() - 1)); - try { - AcknowledgedResponse acknowledgedResponse = - client(randomNonCoordinatorNode).execute(DeleteAsynchronousSearchAction.INSTANCE, - new DeleteAsynchronousSearchRequest(id)).get(); - assertTrue(acknowledgedResponse.isAcknowledged()); - ExecutionException executionException = expectThrows(ExecutionException.class, - () -> client().execute(GetAsynchronousSearchAction.INSTANCE, new GetAsynchronousSearchRequest(id)).get()); - assertThat(executionException.getMessage(), containsString("ResourceNotFoundException")); - } catch (InterruptedException | ExecutionException e) { - fail("Delete asynchronous search request from random non-coordinator node should have succeeded."); - } - } + } - @Override - public void onFailure(Exception e) { - fail(e.getMessage()); - } - }, latch)); + @Override + public void onFailure(Exception e) { + fail(e.getMessage()); + } + }, latch) + ); latch.await(); disableBlocks(plugins); } -//TODO -// public void testCoordinatorNodeDropOnPersistedSearch() throws Exception { -// String idx = "idx"; -// assertAcked(prepareCreate(idx) -// .addMapping("type", "ip", "type=ip", "ips", "type=ip")); -// waitForRelocation(ClusterHealthStatus.GREEN); -// indexRandom(true, -// client().prepareIndex(idx).setId("1").setSource( -// "ip", "192.168.1.7", -// "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), -// client().prepareIndex(idx).setId("2").setSource( -// "ip", "192.168.1.10", -// "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), -// client().prepareIndex(idx).setId("3").setSource( -// "ip", "2001:db8::ff00:42:8329", -// "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); -// -// assertAcked(prepareCreate("idx_unmapped")); -// waitForRelocation(ClusterHealthStatus.GREEN); -// refresh(); -// List dataNodes = new ArrayList<>(); -// clusterService().state().nodes().getDataNodes().iterator().forEachRemaining(entry -> dataNodes.add(entry.value)); -// assertFalse(dataNodes.isEmpty()); -// DiscoveryNode coordinatorNode = dataNodes.get(0); -// SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest(idx)); -// request.keepOnCompletion(true); -// AsynchronousSearchResponse submitResponse = client(coordinatorNode.getName()).execute(SubmitAsynchronousSearchAction.INSTANCE, -// request).get(); -// TestClientUtils.assertResponsePersistence(client(), submitResponse.getId()); -// CountDownLatch latch = new CountDownLatch(1); -// internalCluster().restartNode(coordinatorNode.getName(), new InternalTestCluster.RestartCallback() { -// @Override -// public Settings onNodeStopped(String nodeName) throws Exception { -// try { -// AsynchronousSearchResponse getResponse = client(dataNodes.get(1).getName()) -// .execute(GetAsynchronousSearchAction.INSTANCE, -// new GetAsynchronousSearchRequest(submitResponse.getId())).get(); -// assertEquals(getResponse.getState(), AsynchronousSearchState.PERSISTED); -// assertEquals(getResponse.getId(), submitResponse.getId()); -// return super.onNodeStopped(nodeName); -// } finally { -// latch.countDown(); -// } -// } -// }); -// latch.await(); -// -// } -//TODO after test completes MockSearchService.assertNoInFlightContext() FAILS because we test killing a node with a running blocked search + // TODO + // public void testCoordinatorNodeDropOnPersistedSearch() throws Exception { + // String idx = "idx"; + // assertAcked(prepareCreate(idx) + // .addMapping("type", "ip", "type=ip", "ips", "type=ip")); + // waitForRelocation(ClusterHealthStatus.GREEN); + // indexRandom(true, + // client().prepareIndex(idx).setId("1").setSource( + // "ip", "192.168.1.7", + // "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), + // client().prepareIndex(idx).setId("2").setSource( + // "ip", "192.168.1.10", + // "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), + // client().prepareIndex(idx).setId("3").setSource( + // "ip", "2001:db8::ff00:42:8329", + // "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); + // + // assertAcked(prepareCreate("idx_unmapped")); + // waitForRelocation(ClusterHealthStatus.GREEN); + // refresh(); + // List dataNodes = new ArrayList<>(); + // clusterService().state().nodes().getDataNodes().iterator().forEachRemaining(entry -> dataNodes.add(entry.value)); + // assertFalse(dataNodes.isEmpty()); + // DiscoveryNode coordinatorNode = dataNodes.get(0); + // SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest(idx)); + // request.keepOnCompletion(true); + // AsynchronousSearchResponse submitResponse = client(coordinatorNode.getName()).execute(SubmitAsynchronousSearchAction.INSTANCE, + // request).get(); + // TestClientUtils.assertResponsePersistence(client(), submitResponse.getId()); + // CountDownLatch latch = new CountDownLatch(1); + // internalCluster().restartNode(coordinatorNode.getName(), new InternalTestCluster.RestartCallback() { + // @Override + // public Settings onNodeStopped(String nodeName) throws Exception { + // try { + // AsynchronousSearchResponse getResponse = client(dataNodes.get(1).getName()) + // .execute(GetAsynchronousSearchAction.INSTANCE, + // new GetAsynchronousSearchRequest(submitResponse.getId())).get(); + // assertEquals(getResponse.getState(), AsynchronousSearchState.PERSISTED); + // assertEquals(getResponse.getId(), submitResponse.getId()); + // return super.onNodeStopped(nodeName); + // } finally { + // latch.countDown(); + // } + // } + // }); + // latch.await(); + // + // } + // TODO after test completes MockSearchService.assertNoInFlightContext() FAILS because we test killing a node with a running blocked + // search -// public void testCoordinatorNodeDropOnRunningSearch() throws Exception { -// List plugins = initBlockFactory(); -// String idx = "idx"; -// assertAcked(prepareCreate(idx) -// .addMapping("type", "ip", "type=ip", "ips", "type=ip")); -// waitForRelocation(ClusterHealthStatus.GREEN); -// indexRandom(true, -// client().prepareIndex(idx).setId("1").setSource( -// "ip", "192.168.1.7", -// "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), -// client().prepareIndex(idx).setId("2").setSource( -// "ip", "192.168.1.10", -// "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), -// client().prepareIndex(idx).setId("3").setSource( -// "ip", "2001:db8::ff00:42:8329", -// "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); -// -// assertAcked(prepareCreate("idx_unmapped")); -// waitForRelocation(ClusterHealthStatus.GREEN); -// refresh(); -// -// List dataNodes = new ArrayList<>(); -// clusterService().state().nodes().getDataNodes().iterator().forEachRemaining(entry -> dataNodes.add(entry.value)); -// assertFalse(dataNodes.isEmpty()); -// DiscoveryNode coordinatorNode = dataNodes.get(0); -// SearchRequest searchRequest = client().prepareSearch(idx).setQuery( -// scriptQuery(new Script( -// ScriptType.INLINE, "mockscript", SCRIPT_NAME, -// Collections.emptyMap()))) -// .request(); -// SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); -// request.keepOnCompletion(true); -// AsynchronousSearchResponse submitResponse = client(coordinatorNode.getName()).execute(SubmitAsynchronousSearchAction.INSTANCE, -// request).get(); -// CountDownLatch latch = new CountDownLatch(1); -// internalCluster().restartNode(coordinatorNode.getName(), new InternalTestCluster.RestartCallback() { -// @Override -// public boolean clearData(String nodeName) { -// return super.clearData(nodeName); -// } -// -// @Override -// public Settings onNodeStopped(String nodeName) throws Exception { -// try { -// ExecutionException executionException = expectThrows(ExecutionException.class, -// () -> client().execute(GetAsynchronousSearchAction.INSTANCE, new GetAsynchronousSearchRequest( -// submitResponse.getId())) -// .get()); -// assertThat(executionException.getMessage(), containsString("ResourceNotFoundException")); -// disableBlocks(plugins); -// return super.onNodeStopped(nodeName); -// } finally { -// latch.countDown(); -// } -// } -// }); -// latch.await(); -// SearchService instance = internalCluster().getInstance(SearchService.class); -// assertTrue(instance instanceof MockSearchService); -// } + // public void testCoordinatorNodeDropOnRunningSearch() throws Exception { + // List plugins = initBlockFactory(); + // String idx = "idx"; + // assertAcked(prepareCreate(idx) + // .addMapping("type", "ip", "type=ip", "ips", "type=ip")); + // waitForRelocation(ClusterHealthStatus.GREEN); + // indexRandom(true, + // client().prepareIndex(idx).setId("1").setSource( + // "ip", "192.168.1.7", + // "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), + // client().prepareIndex(idx).setId("2").setSource( + // "ip", "192.168.1.10", + // "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), + // client().prepareIndex(idx).setId("3").setSource( + // "ip", "2001:db8::ff00:42:8329", + // "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380"))); + // + // assertAcked(prepareCreate("idx_unmapped")); + // waitForRelocation(ClusterHealthStatus.GREEN); + // refresh(); + // + // List dataNodes = new ArrayList<>(); + // clusterService().state().nodes().getDataNodes().iterator().forEachRemaining(entry -> dataNodes.add(entry.value)); + // assertFalse(dataNodes.isEmpty()); + // DiscoveryNode coordinatorNode = dataNodes.get(0); + // SearchRequest searchRequest = client().prepareSearch(idx).setQuery( + // scriptQuery(new Script( + // ScriptType.INLINE, "mockscript", SCRIPT_NAME, + // Collections.emptyMap()))) + // .request(); + // SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); + // request.keepOnCompletion(true); + // AsynchronousSearchResponse submitResponse = client(coordinatorNode.getName()).execute(SubmitAsynchronousSearchAction.INSTANCE, + // request).get(); + // CountDownLatch latch = new CountDownLatch(1); + // internalCluster().restartNode(coordinatorNode.getName(), new InternalTestCluster.RestartCallback() { + // @Override + // public boolean clearData(String nodeName) { + // return super.clearData(nodeName); + // } + // + // @Override + // public Settings onNodeStopped(String nodeName) throws Exception { + // try { + // ExecutionException executionException = expectThrows(ExecutionException.class, + // () -> client().execute(GetAsynchronousSearchAction.INSTANCE, new GetAsynchronousSearchRequest( + // submitResponse.getId())) + // .get()); + // assertThat(executionException.getMessage(), containsString("ResourceNotFoundException")); + // disableBlocks(plugins); + // return super.onNodeStopped(nodeName); + // } finally { + // latch.countDown(); + // } + // } + // }); + // latch.await(); + // SearchService instance = internalCluster().getInstance(SearchService.class); + // assertTrue(instance instanceof MockSearchService); + // } public void testInvalidIdRequestHandling() { - ExecutionException executionException = expectThrows(ExecutionException.class, () -> client().execute( - GetAsynchronousSearchAction.INSTANCE, - new GetAsynchronousSearchRequest(randomAlphaOfLength(16))).get()); + ExecutionException executionException = expectThrows( + ExecutionException.class, + () -> client().execute(GetAsynchronousSearchAction.INSTANCE, new GetAsynchronousSearchRequest(randomAlphaOfLength(16))).get() + ); assertThat(executionException.getMessage(), containsString("ResourceNotFoundException")); } } diff --git a/src/test/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequestTests.java b/src/test/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequestTests.java index 21cd12b3..f7955ccd 100644 --- a/src/test/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequestTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/request/SubmitAsynchronousSearchRequestTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.request; import org.opensearch.search.asynchronous.action.SubmitAsynchronousSearchAction; @@ -22,20 +25,25 @@ public class SubmitAsynchronousSearchRequestTests extends OpenSearchTestCase { public void testValidRequest() { SearchSourceBuilder source = new SearchSourceBuilder(); - SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); + SearchRequest searchRequest = new SearchRequest(new String[] { "test" }, source); searchRequest.setCcsMinimizeRoundtrips(false); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); ValidationException validationException = request.validate(); assertNull(validationException); assertEquals(request, new SubmitAsynchronousSearchRequest(searchRequest)); - String description = request.createTask(randomNonNegativeLong(), "taskType", SubmitAsynchronousSearchAction.NAME, - new TaskId("test", -1), new HashMap<>()).getDescription(); + String description = request.createTask( + randomNonNegativeLong(), + "taskType", + SubmitAsynchronousSearchAction.NAME, + new TaskId("test", -1), + new HashMap<>() + ).getDescription(); assertThat(description, containsString("indices[test]")); } public void testSuggestOnlyQueryFailsValidation() { SearchSourceBuilder source = new SearchSourceBuilder().suggest(new SuggestBuilder()); - SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); + SearchRequest searchRequest = new SearchRequest(new String[] { "test" }, source); searchRequest.setCcsMinimizeRoundtrips(false); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); ValidationException validationException = request.validate(); @@ -46,7 +54,7 @@ public void testSuggestOnlyQueryFailsValidation() { public void testSearchScrollFailsValidation() { SearchSourceBuilder source = new SearchSourceBuilder(); - SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); + SearchRequest searchRequest = new SearchRequest(new String[] { "test" }, source); searchRequest.setCcsMinimizeRoundtrips(false); searchRequest.scroll(randomTimeValue()); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); @@ -58,9 +66,8 @@ public void testSearchScrollFailsValidation() { public void testCcsMinimizeRoundtripsFlagFailsValidation() { SearchSourceBuilder source = new SearchSourceBuilder(); - SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); - SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest( - searchRequest); + SearchRequest searchRequest = new SearchRequest(new String[] { "test" }, source); + SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); searchRequest.setCcsMinimizeRoundtrips(true); ValidationException validationException = request.validate(); assertNotNull(validationException); @@ -70,7 +77,7 @@ public void testCcsMinimizeRoundtripsFlagFailsValidation() { public void testInvalidKeepAliveFailsValidation() { SearchSourceBuilder source = new SearchSourceBuilder(); - SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); + SearchRequest searchRequest = new SearchRequest(new String[] { "test" }, source); searchRequest.setCcsMinimizeRoundtrips(false); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); request.keepAlive(TimeValue.timeValueSeconds(59)); @@ -80,7 +87,7 @@ public void testInvalidKeepAliveFailsValidation() { public void testInvalidWaitForCompletionFailsValidation() { SearchSourceBuilder source = new SearchSourceBuilder(); - SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); + SearchRequest searchRequest = new SearchRequest(new String[] { "test" }, source); searchRequest.setCcsMinimizeRoundtrips(false); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); request.waitForCompletionTimeout(TimeValue.timeValueMillis(-1)); diff --git a/src/test/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponseTests.java b/src/test/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponseTests.java index 385b5844..866aa6d9 100644 --- a/src/test/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponseTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/response/AsynchronousSearchResponseTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.response; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -44,21 +47,27 @@ protected Writeable.Reader instanceReader() { @Override protected AsynchronousSearchResponse createTestInstance() { - return new AsynchronousSearchResponse(UUID.randomUUID().toString(), - getRandomAsynchronousSearchState(), - randomNonNegativeLong(), - randomNonNegativeLong(), getMockSearchResponse(), null); + return new AsynchronousSearchResponse( + UUID.randomUUID().toString(), + getRandomAsynchronousSearchState(), + randomNonNegativeLong(), + randomNonNegativeLong(), + getMockSearchResponse(), + null + ); } @Override protected AsynchronousSearchResponse mutateInstance(AsynchronousSearchResponse instance) { - return new AsynchronousSearchResponse(randomBoolean() ? instance.getId() : UUID.randomUUID().toString(), - getRandomAsynchronousSearchState(), - randomBoolean() ? instance.getStartTimeMillis() : randomNonNegativeLong(), - randomBoolean() ? instance.getExpirationTimeMillis() : randomNonNegativeLong(), - getMockSearchResponse(), - instance.getError()); + return new AsynchronousSearchResponse( + randomBoolean() ? instance.getId() : UUID.randomUUID().toString(), + getRandomAsynchronousSearchState(), + randomBoolean() ? instance.getStartTimeMillis() : randomNonNegativeLong(), + randomBoolean() ? instance.getExpirationTimeMillis() : randomNonNegativeLong(), + getMockSearchResponse(), + instance.getError() + ); } private AsynchronousSearchState getRandomAsynchronousSearchState() { @@ -68,13 +77,24 @@ private AsynchronousSearchState getRandomAsynchronousSearchState() { private SearchResponse getMockSearchResponse() { int totalShards = randomInt(100); int successfulShards = totalShards - randomInt(100); - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), - "", totalShards, successfulShards, 0, randomNonNegativeLong(), - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + randomInt(5) + ), + "", + totalShards, + successfulShards, + 0, + randomNonNegativeLong(), + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } /* @@ -83,21 +103,30 @@ private SearchResponse getMockSearchResponse() { * message appears somewhere in the rendered xContent. */ public void testXContentRoundTripForAsynchronousSearchResponseContainingError() throws IOException { - AsynchronousSearchResponse asResponse = new AsynchronousSearchResponse(UUID.randomUUID().toString(), - getRandomAsynchronousSearchState(), - randomNonNegativeLong(), randomNonNegativeLong(), null, new RuntimeException("test")); + AsynchronousSearchResponse asResponse = new AsynchronousSearchResponse( + UUID.randomUUID().toString(), + getRandomAsynchronousSearchState(), + randomNonNegativeLong(), + randomNonNegativeLong(), + null, + new RuntimeException("test") + ); BytesReference serializedResponse; MediaType xContentType = Requests.INDEX_CONTENT_TYPE; serializedResponse = org.opensearch.core.xcontent.XContentHelper.toXContent(asResponse, xContentType, true); - try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, serializedResponse, xContentType)) { + try ( + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + serializedResponse, + xContentType + ) + ) { AsynchronousSearchResponse asResponse1 = AsynchronousSearchResponse.fromXContent(parser); String originalMsg = asResponse1.getError().getCause().getMessage(); - assertEquals(originalMsg, - "OpenSearch exception [type=runtime_exception, reason=test]"); + assertEquals(originalMsg, "OpenSearch exception [type=runtime_exception, reason=test]"); } } } - diff --git a/src/test/java/org/opensearch/search/asynchronous/restIT/ApiParamsValidationIT.java b/src/test/java/org/opensearch/search/asynchronous/restIT/ApiParamsValidationIT.java index 28aa90ad..40651656 100644 --- a/src/test/java/org/opensearch/search/asynchronous/restIT/ApiParamsValidationIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/restIT/ApiParamsValidationIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.restIT; import org.opensearch.action.search.SearchRequest; @@ -28,34 +31,43 @@ public void testSubmitInvalidKeepAlive() throws IOException { SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest()); request.keepAlive(TimeValue.timeValueDays(100)); ResponseException responseException = expectThrows(ResponseException.class, () -> executeSubmitAsynchronousSearch(request)); - assertThat(responseException.getMessage(), containsString("Keep alive for asynchronous search (" + - request.getKeepAlive().getMillis() + ") is too large")); + assertThat( + responseException.getMessage(), + containsString("Keep alive for asynchronous search (" + request.getKeepAlive().getMillis() + ") is too large") + ); } public void testSubmitInvalidWaitForCompletion() throws IOException { SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest()); request.waitForCompletionTimeout(TimeValue.timeValueMinutes(2)); ResponseException responseException = expectThrows(ResponseException.class, () -> executeSubmitAsynchronousSearch(request)); - assertThat(responseException.getMessage(), containsString("Wait for completion timeout for asynchronous search (" + - request.getWaitForCompletionTimeout().getMillis() + ") is too large")); + assertThat( + responseException.getMessage(), + containsString( + "Wait for completion timeout for asynchronous search (" + + request.getWaitForCompletionTimeout().getMillis() + + ") is too large" + ) + ); } public void testSubmitDefaultKeepAlive() throws IOException { - SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest( - new SearchRequest("test")); + SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest("test")); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest); List legalStates = Arrays.asList(AsynchronousSearchState.SUCCEEDED, AsynchronousSearchState.CLOSED); assertTrue(legalStates.contains(submitResponse.getState())); - assertTrue((submitResponse.getExpirationTimeMillis() - > System.currentTimeMillis() + TimeValue.timeValueHours(23).getMillis() + TimeValue.timeValueMinutes(59).millis()) && - (submitResponse.getExpirationTimeMillis() < System.currentTimeMillis() + TimeValue.timeValueHours(24).getMillis() + - TimeValue.timeValueMillis(1).getMillis())); + assertTrue( + (submitResponse.getExpirationTimeMillis() > System.currentTimeMillis() + TimeValue.timeValueHours(23).getMillis() + TimeValue + .timeValueMinutes(59) + .millis()) + && (submitResponse.getExpirationTimeMillis() < System.currentTimeMillis() + TimeValue.timeValueHours(24).getMillis() + + TimeValue.timeValueMillis(1).getMillis()) + ); assertHitCount(submitResponse.getSearchResponse(), 5); } public void testSubmitDefaultWaitForCompletion() throws IOException { - SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest( - new SearchRequest("test")); + SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest("test")); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest); List legalStates = Arrays.asList(AsynchronousSearchState.SUCCEEDED, AsynchronousSearchState.CLOSED); assertTrue(legalStates.contains(submitResponse.getState())); @@ -64,7 +76,8 @@ public void testSubmitDefaultWaitForCompletion() throws IOException { public void testSubmitSearchOnInvalidIndex() throws IOException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest( - new SearchRequest("invalid-index")); + new SearchRequest("invalid-index") + ); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest); List legalStates = Arrays.asList(AsynchronousSearchState.FAILED, AsynchronousSearchState.CLOSED); assertNull(submitResponse.getSearchResponse()); @@ -81,16 +94,22 @@ public void testGetWithInvalidKeepAliveUpdate() throws IOException { AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest); GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(submitResponse.getId()); getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueDays(100)); - ResponseException responseException = expectThrows(ResponseException.class, () -> executeGetAsynchronousSearch( - getAsynchronousSearchRequest)); - assertThat(responseException.getMessage(), containsString("Keep alive for asynchronous search (" + - getAsynchronousSearchRequest.getKeepAlive().getMillis() + ") is too large")); + ResponseException responseException = expectThrows( + ResponseException.class, + () -> executeGetAsynchronousSearch(getAsynchronousSearchRequest) + ); + assertThat( + responseException.getMessage(), + containsString( + "Keep alive for asynchronous search (" + getAsynchronousSearchRequest.getKeepAlive().getMillis() + ") is too large" + ) + ); executeDeleteAsynchronousSearch(new DeleteAsynchronousSearchRequest(submitResponse.getId())); } public void testSuggestOnlySearchRequest() { SearchSourceBuilder source = new SearchSourceBuilder().suggest(new SuggestBuilder()); - SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest(new String[]{"test"}, source)); + SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(new SearchRequest(new String[] { "test" }, source)); try { AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(request); } catch (Exception e) { diff --git a/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestIT.java b/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestIT.java index c3cb674b..03c8a5e7 100644 --- a/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.restIT; import org.opensearch.search.asynchronous.context.state.AsynchronousSearchState; @@ -38,7 +41,10 @@ public void testSubmitWithoutRetainedResponse() throws IOException { submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest); List legalStates = Arrays.asList( - AsynchronousSearchState.RUNNING, AsynchronousSearchState.SUCCEEDED, AsynchronousSearchState.CLOSED); + AsynchronousSearchState.RUNNING, + AsynchronousSearchState.SUCCEEDED, + AsynchronousSearchState.CLOSED + ); assertTrue(legalStates.contains(submitResponse.getState())); GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(submitResponse.getId()); AsynchronousSearchResponse getResponse; @@ -47,7 +53,7 @@ public void testSubmitWithoutRetainedResponse() throws IOException { try { getResponse = getAssertedAsynchronousSearchResponse(submitResponse, getAsynchronousSearchRequest); if (AsynchronousSearchState.SUCCEEDED.equals(getResponse.getState()) - || AsynchronousSearchState.CLOSED.equals(getResponse.getState())) { + || AsynchronousSearchState.CLOSED.equals(getResponse.getState())) { assertNotNull(getResponse.getSearchResponse()); assertHitCount(getResponse.getSearchResponse(), 5L); } @@ -65,9 +71,13 @@ public void testSubmitWithRetainedResponse() throws IOException { submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 500))); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest); List legalStates = Arrays.asList( - AsynchronousSearchState.RUNNING, AsynchronousSearchState.SUCCEEDED, AsynchronousSearchState.PERSIST_SUCCEEDED, - AsynchronousSearchState.PERSISTING, - AsynchronousSearchState.CLOSED, AsynchronousSearchState.STORE_RESIDENT); + AsynchronousSearchState.RUNNING, + AsynchronousSearchState.SUCCEEDED, + AsynchronousSearchState.PERSIST_SUCCEEDED, + AsynchronousSearchState.PERSISTING, + AsynchronousSearchState.CLOSED, + AsynchronousSearchState.STORE_RESIDENT + ); assertNotNull(submitResponse.getId()); assertTrue(submitResponse.getState().name(), legalStates.contains(submitResponse.getState())); GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(submitResponse.getId()); @@ -96,9 +106,13 @@ public void testSubmitSearchCompletesBeforeWaitForCompletionTimeout() throws IOE submitAsynchronousSearchRequest.keepAlive(TimeValue.timeValueHours(5)); submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMinutes(1)); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest); - List legalStates = Arrays.asList(AsynchronousSearchState.SUCCEEDED, - AsynchronousSearchState.PERSIST_SUCCEEDED, - AsynchronousSearchState.PERSISTING, AsynchronousSearchState.CLOSED, AsynchronousSearchState.STORE_RESIDENT); + List legalStates = Arrays.asList( + AsynchronousSearchState.SUCCEEDED, + AsynchronousSearchState.PERSIST_SUCCEEDED, + AsynchronousSearchState.PERSISTING, + AsynchronousSearchState.CLOSED, + AsynchronousSearchState.STORE_RESIDENT + ); assertTrue(submitResponse.getState().name(), legalStates.contains(submitResponse.getState())); assertHitCount(submitResponse.getSearchResponse(), 5L); GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(submitResponse.getId()); @@ -116,8 +130,10 @@ public void testGetWithoutKeepAliveUpdate() throws IOException { AsynchronousSearchResponse getResponse = executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId())); assertEquals(getResponse.getExpirationTimeMillis(), submitResponse.getExpirationTimeMillis()); executeDeleteAsynchronousSearch(new DeleteAsynchronousSearchRequest(submitResponse.getId())); - ResponseException responseException = expectThrows(ResponseException.class, () -> executeGetAsynchronousSearch( - new GetAsynchronousSearchRequest(submitResponse.getId()))); + ResponseException responseException = expectThrows( + ResponseException.class, + () -> executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId())) + ); assertRnf(responseException); } @@ -135,29 +151,37 @@ public void testGetWithKeepAliveUpdate() throws IOException { AsynchronousSearchResponse getResponse = executeGetAsynchronousSearch(getAsynchronousSearchRequest); assertThat(getResponse.getExpirationTimeMillis(), greaterThan(submitResponse.getExpirationTimeMillis())); executeDeleteAsynchronousSearch(new DeleteAsynchronousSearchRequest(submitResponse.getId())); - ResponseException responseException = expectThrows(ResponseException.class, () -> executeGetAsynchronousSearch( - new GetAsynchronousSearchRequest(submitResponse.getId()))); + ResponseException responseException = expectThrows( + ResponseException.class, + () -> executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId())) + ); assertRnf(responseException); } public void testBackwardCompatibilityWithOpenDistro() throws IOException { - //submit async search - Request request = new Request(HttpPost.METHOD_NAME, AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + - "?keep_on_completion=true"); + // submit async search + Request request = new Request( + HttpPost.METHOD_NAME, + AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "?keep_on_completion=true" + ); Response resp = client().performRequest(request); assertEquals(resp.getStatusLine().getStatusCode(), 200); AsynchronousSearchResponse asynchronousSearchResponse = parseEntity(resp.getEntity(), AsynchronousSearchResponse::fromXContent); - //get async search - request = new Request(HttpGet.METHOD_NAME, - AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/" + asynchronousSearchResponse.getId()); + // get async search + request = new Request( + HttpGet.METHOD_NAME, + AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/" + asynchronousSearchResponse.getId() + ); resp = client().performRequest(request); assertEquals(resp.getStatusLine().getStatusCode(), 200); - //delete async search - request = new Request(HttpDelete.METHOD_NAME, - AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/" + asynchronousSearchResponse.getId()); + // delete async search + request = new Request( + HttpDelete.METHOD_NAME, + AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/" + asynchronousSearchResponse.getId() + ); resp = client().performRequest(request); assertEquals(resp.getStatusLine().getStatusCode(), 200); - //async search stats + // async search stats request = new Request(HttpGet.METHOD_NAME, AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/stats"); resp = client().performRequest(request); assertEquals(resp.getStatusLine().getStatusCode(), 200); diff --git a/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestTestCase.java b/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestTestCase.java index 2537c99b..df901425 100644 --- a/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestTestCase.java +++ b/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchRestTestCase.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.restIT; import org.opensearch.common.xcontent.XContentFactory; @@ -47,14 +50,14 @@ import static org.hamcrest.Matchers.containsString; - /** * Verifies asynchronous search APIs - submit, get, delete end to end using rest client */ public abstract class AsynchronousSearchRestTestCase extends SecurityEnabledRestTestCase { private final NamedXContentRegistry registry = new NamedXContentRegistry( - new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()); + new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents() + ); @Before public void indexDocuments() throws IOException { @@ -86,28 +89,30 @@ public void indexDocuments() throws IOException { client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); } - protected AsynchronousSearchResponse executeGetAsynchronousSearch( - GetAsynchronousSearchRequest getAsynchronousSearchRequest) throws IOException { - return executeGetAsynchronousSearch(getAsynchronousSearchRequest, false); - } - - protected AsynchronousSearchResponse executeGetAsynchronousSearch( - GetAsynchronousSearchRequest getAsynchronousSearchRequest, boolean shouldUseLegacyApi) - throws IOException { - Request getRequest = RestTestUtils.buildHttpRequest(getAsynchronousSearchRequest, shouldUseLegacyApi); - Response resp = client().performRequest(getRequest); - return parseEntity(resp.getEntity(), AsynchronousSearchResponse::fromXContent); - } - - protected AsynchronousSearchResponse executeSubmitAsynchronousSearch( - @Nullable SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest) - throws IOException { - return executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest, false); - } - - protected AsynchronousSearchResponse executeSubmitAsynchronousSearch( - @Nullable SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest, boolean shouldUseLegacyApi) - throws IOException { + protected AsynchronousSearchResponse executeGetAsynchronousSearch(GetAsynchronousSearchRequest getAsynchronousSearchRequest) + throws IOException { + return executeGetAsynchronousSearch(getAsynchronousSearchRequest, false); + } + + protected AsynchronousSearchResponse executeGetAsynchronousSearch( + GetAsynchronousSearchRequest getAsynchronousSearchRequest, + boolean shouldUseLegacyApi + ) throws IOException { + Request getRequest = RestTestUtils.buildHttpRequest(getAsynchronousSearchRequest, shouldUseLegacyApi); + Response resp = client().performRequest(getRequest); + return parseEntity(resp.getEntity(), AsynchronousSearchResponse::fromXContent); + } + + protected AsynchronousSearchResponse executeSubmitAsynchronousSearch( + @Nullable SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest + ) throws IOException { + return executeSubmitAsynchronousSearch(submitAsynchronousSearchRequest, false); + } + + protected AsynchronousSearchResponse executeSubmitAsynchronousSearch( + @Nullable SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest, + boolean shouldUseLegacyApi + ) throws IOException { Request request = RestTestUtils.buildHttpRequest(submitAsynchronousSearchRequest, shouldUseLegacyApi); Response resp = client().performRequest(request); return parseEntity(resp.getEntity(), AsynchronousSearchResponse::fromXContent); @@ -134,7 +139,6 @@ public interface IProxy { void reset(); } - @AfterClass public static void dumpCoverage() throws IOException, MalformedObjectNameException { // jacoco.dir is set in esplugin-coverage.gradle, if it doesn't exist we don't @@ -147,8 +151,11 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti String serverUrl = "service:jmx:rmi:///jndi/rmi://127.0.0.1:7777/jmxrmi"; try (JMXConnector connector = JMXConnectorFactory.connect(new JMXServiceURL(serverUrl))) { IProxy proxy = MBeanServerInvocationHandler.newProxyInstance( - connector.getMBeanServerConnection(), new ObjectName("org.jacoco:type=Runtime"), IProxy.class, - false); + connector.getMBeanServerConnection(), + new ObjectName("org.jacoco:type=Runtime"), + IProxy.class, + false + ); Path path = org.opensearch.common.io.PathUtils.get(jacocoBuildPath + "/integTestRunner.exec"); Files.write(path, proxy.getExecutionData(false)); @@ -157,9 +164,8 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti } } - protected final Resp parseEntity(final HttpEntity entity, - final CheckedFunction entityParser) - throws IOException { + protected final Resp parseEntity(final HttpEntity entity, final CheckedFunction entityParser) + throws IOException { if (entity == null) { throw new IllegalStateException("Response body expected but not returned"); } @@ -170,23 +176,26 @@ protected final Resp parseEntity(final HttpEntity entity, if (xContentType == null) { throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue()); } - try (XContentParser parser = xContentType.xContent().createParser( - registry, DeprecationHandler.IGNORE_DEPRECATIONS, entity.getContent())) { + try ( + XContentParser parser = xContentType.xContent() + .createParser(registry, DeprecationHandler.IGNORE_DEPRECATIONS, entity.getContent()) + ) { return entityParser.apply(parser); } } - protected AsynchronousSearchResponse getAssertedAsynchronousSearchResponse(AsynchronousSearchResponse submitResponse, - GetAsynchronousSearchRequest getAsynchronousSearchRequest) - throws IOException { + protected AsynchronousSearchResponse getAssertedAsynchronousSearchResponse( + AsynchronousSearchResponse submitResponse, + GetAsynchronousSearchRequest getAsynchronousSearchRequest + ) throws IOException { return getAssertedAsynchronousSearchResponse(submitResponse, getAsynchronousSearchRequest, false); } - protected AsynchronousSearchResponse getAssertedAsynchronousSearchResponse( - AsynchronousSearchResponse submitResponse, - GetAsynchronousSearchRequest getAsynchronousSearchRequest, - boolean shouldUseLegacyApi) - throws IOException { + protected AsynchronousSearchResponse getAssertedAsynchronousSearchResponse( + AsynchronousSearchResponse submitResponse, + GetAsynchronousSearchRequest getAsynchronousSearchRequest, + boolean shouldUseLegacyApi + ) throws IOException { AsynchronousSearchResponse getResponse; getResponse = executeGetAsynchronousSearch(getAsynchronousSearchRequest, shouldUseLegacyApi); assertEquals(submitResponse.getId(), getResponse.getId()); @@ -194,7 +203,6 @@ protected AsynchronousSearchResponse getAssertedAsynchronousSearchResponse( return getResponse; } - protected void assertRnf(Exception e) { assertTrue(e instanceof ResponseException); assertThat(e.getMessage(), containsString("resource_not_found_exception")); @@ -204,8 +212,7 @@ protected void assertRnf(Exception e) { protected static void assertHitCount(SearchResponse countResponse, long expectedHitCount) { final TotalHits totalHits = countResponse.getHits().getTotalHits(); if (totalHits.relation != TotalHits.Relation.EQUAL_TO || totalHits.value != expectedHitCount) { - fail("Count is " + totalHits + " but " + expectedHitCount - + " was expected. " + countResponse.toString()); + fail("Count is " + totalHits + " but " + expectedHitCount + " was expected. " + countResponse.toString()); } } @@ -216,11 +223,11 @@ protected boolean preserveClusterUponCompletion() { protected void updateClusterSettings(String settingKey, Object value) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("persistent") - .field(settingKey, value) - .endObject() - .endObject(); + .startObject() + .startObject("persistent") + .field(settingKey, value) + .endObject() + .endObject(); Request request = new Request("PUT", "_cluster/settings"); request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); diff --git a/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchSettingsIT.java b/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchSettingsIT.java index d2a6cf17..07e59e87 100644 --- a/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchSettingsIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/restIT/AsynchronousSearchSettingsIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.restIT; import org.opensearch.search.asynchronous.context.active.AsynchronousSearchActiveStore; @@ -34,10 +37,11 @@ public void testMaxKeepAliveSetting() throws Exception { updateClusterSettings(AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), TimeValue.timeValueHours(6)); SubmitAsynchronousSearchRequest invalidRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); invalidRequest.keepAlive(TimeValue.timeValueHours(7)); - ResponseException responseException = expectThrows(ResponseException.class, () -> executeSubmitAsynchronousSearch( - invalidRequest)); - assertThat(responseException.getMessage(), containsString("Keep alive for asynchronous search (" + - invalidRequest.getKeepAlive().getMillis() + ") is too large")); + ResponseException responseException = expectThrows(ResponseException.class, () -> executeSubmitAsynchronousSearch(invalidRequest)); + assertThat( + responseException.getMessage(), + containsString("Keep alive for asynchronous search (" + invalidRequest.getKeepAlive().getMillis() + ") is too large") + ); updateClusterSettings(AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), TimeValue.timeValueHours(24)); } @@ -46,16 +50,19 @@ public void testSubmitInvalidWaitForCompletion() throws Exception { validRequest.waitForCompletionTimeout(TimeValue.timeValueSeconds(50)); AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(validRequest); assertNotNull(asResponse.getSearchResponse()); - updateClusterSettings(AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), - TimeValue.timeValueSeconds(2)); + updateClusterSettings(AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(2)); SubmitAsynchronousSearchRequest invalidRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); invalidRequest.waitForCompletionTimeout(TimeValue.timeValueSeconds(50)); - ResponseException responseException = expectThrows(ResponseException.class, () -> executeSubmitAsynchronousSearch( - invalidRequest)); - assertThat(responseException.getMessage(), containsString("Wait for completion timeout for asynchronous search (" + - validRequest.getWaitForCompletionTimeout().getMillis() + ") is too large")); - updateClusterSettings(AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), - TimeValue.timeValueSeconds(60)); + ResponseException responseException = expectThrows(ResponseException.class, () -> executeSubmitAsynchronousSearch(invalidRequest)); + assertThat( + responseException.getMessage(), + containsString( + "Wait for completion timeout for asynchronous search (" + + validRequest.getWaitForCompletionTimeout().getMillis() + + ") is too large" + ) + ); + updateClusterSettings(AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(60)); } public void testMaxRunningAsynchronousSearchContexts() throws Exception { @@ -78,8 +85,7 @@ public void testMaxRunningAsynchronousSearchContexts() throws Exception { fail(); } } - } - )); + })); } threadsList.forEach(Thread::start); barrier.await(); @@ -108,8 +114,7 @@ public void testMaxRunningAsynchronousSearchContexts() throws Exception { fail(); } } - } - )); + })); } threadsList.forEach(Thread::start); barrier.await(); @@ -117,8 +122,10 @@ public void testMaxRunningAsynchronousSearchContexts() throws Exception { thread.join(); } assertEquals(numFailures.get(), 50); - updateClusterSettings(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES); + updateClusterSettings( + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES + ); } public void testStoreAsyncSearchWithFailures() throws Exception { @@ -135,20 +142,20 @@ public void testStoreAsyncSearchWithFailures() throws Exception { return e.getMessage().contains("resource_not_found"); } }); - updateClusterSettings(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), - true); + updateClusterSettings(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true); AsynchronousSearchResponse submitResponse = executeSubmitAsynchronousSearch(request); waitUntil(() -> { try { return executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId())).getState() - .equals(AsynchronousSearchState.STORE_RESIDENT); + .equals(AsynchronousSearchState.STORE_RESIDENT); } catch (IOException e) { return false; } }); - assertEquals(executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId())).getState(), - AsynchronousSearchState.STORE_RESIDENT); - updateClusterSettings(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), - false); + assertEquals( + executeGetAsynchronousSearch(new GetAsynchronousSearchRequest(submitResponse.getId())).getState(), + AsynchronousSearchState.STORE_RESIDENT + ); + updateClusterSettings(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), false); } } diff --git a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPostProcessorTests.java b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPostProcessorTests.java index 7aeedb16..39bd1999 100644 --- a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPostProcessorTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchPostProcessorTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service; import org.opensearch.search.asynchronous.context.active.AsynchronousSearchActiveContext; @@ -75,28 +78,42 @@ public class AsynchronousSearchPostProcessorTests extends OpenSearchTestCase { @Before public void createObjects() { Settings settings = Settings.builder() - .put("node.name", "test") - .put("cluster.name", "ClusterServiceTests") - .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) - .build(); - final Set> settingsSet = - Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING)).collect(Collectors.toSet()); + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) + .build(); + final Set> settingsSet = Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of( + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING + ) + ).collect(Collectors.toSet()); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); List> executorBuilders = new ArrayList<>(); - executorBuilders.add(new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30))); + executorBuilders.add( + new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ) + ); executorBuilder = executorBuilders.get(0); clusterSettings = new ClusterSettings(settings, settingsSet); } public void testProcessSearchFailureOnDeletedContext() throws AsynchronousSearchStateMachineClosedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); AtomicBoolean activeContextCleanUpConsumerInvocation = new AtomicBoolean(); ThreadPool testThreadPool = null; try { @@ -105,27 +122,49 @@ public void testProcessSearchFailureOnDeletedContext() throws AsynchronousSearch ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); AsynchronousSearchStateMachine asStateMachine = asService.getStateMachine(); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); - AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor(persistenceService, - asActiveStore, asStateMachine, - (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), testThreadPool, clusterService); + AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor( + persistenceService, + asActiveStore, + asStateMachine, + (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), + testThreadPool, + clusterService + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.keepAlive(TimeValue.timeValueHours(1)); AsynchronousSearchActiveContext context = (AsynchronousSearchActiveContext) asService.createAndStoreContext( - submitAsynchronousSearchRequest, - System.currentTimeMillis(), () -> InternalAggregationTestCase.emptyReduceContextBuilder(), null); + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> InternalAggregationTestCase.emptyReduceContextBuilder(), + null + ); ShardSearchFailure shardSearchFailure = new ShardSearchFailure(new RuntimeException("runtime-exception")); - SearchPhaseExecutionException exception = new SearchPhaseExecutionException("phase", "msg", new NullPointerException(), - new ShardSearchFailure[]{shardSearchFailure}); - asStateMachine.trigger(new SearchStartedEvent(context, - new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()))); + SearchPhaseExecutionException exception = new SearchPhaseExecutionException( + "phase", + "msg", + new NullPointerException(), + new ShardSearchFailure[] { shardSearchFailure } + ); + asStateMachine.trigger( + new SearchStartedEvent(context, new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap())) + ); asStateMachine.trigger(new SearchDeletedEvent(context)); AsynchronousSearchResponse asResponse = postProcessor.processSearchFailure(exception, context.getContextId()); assertNull(asResponse.getId()); @@ -142,34 +181,59 @@ public void testProcessSearchFailureOnDeletedContext() throws AsynchronousSearch } public void testProcessSearchResponseBeginPersistence() throws AsynchronousSearchStateMachineClosedException, InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); AtomicBoolean activeContextCleanUpConsumerInvocation = new AtomicBoolean(); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); AsynchronousSearchStateMachine asStateMachine = asService.getStateMachine(); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); - AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor(persistenceService, - asActiveStore, asStateMachine, - (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), testThreadPool, clusterService); + AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor( + persistenceService, + asActiveStore, + asStateMachine, + (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), + testThreadPool, + clusterService + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.keepAlive(TimeValue.timeValueHours(1)); AsynchronousSearchActiveContext context = (AsynchronousSearchActiveContext) asService.createAndStoreContext( - submitAsynchronousSearchRequest, - System.currentTimeMillis(), () -> InternalAggregationTestCase.emptyReduceContextBuilder(), null); - asStateMachine.trigger(new SearchStartedEvent(context, - new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()))); + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> InternalAggregationTestCase.emptyReduceContextBuilder(), + null + ); + asStateMachine.trigger( + new SearchStartedEvent(context, new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap())) + ); SearchResponse mockSearchResponse = getMockSearchResponse(); AsynchronousSearchResponse asResponse = postProcessor.processSearchResponse(mockSearchResponse, context.getContextId()); assertNotNull(asResponse.getId()); @@ -185,34 +249,59 @@ public void testProcessSearchResponseBeginPersistence() throws AsynchronousSearc } public void testProcessSearchResponsePersisted() throws AsynchronousSearchStateMachineClosedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); AtomicBoolean activeContextCleanUpConsumerInvocation = new AtomicBoolean(); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); AsynchronousSearchStateMachine asStateMachine = asService.getStateMachine(); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); - AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor(persistenceService, - asActiveStore, asStateMachine, - (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), testThreadPool, clusterService); + AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor( + persistenceService, + asActiveStore, + asStateMachine, + (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), + testThreadPool, + clusterService + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.keepAlive(TimeValue.timeValueHours(1)); AsynchronousSearchActiveContext context = (AsynchronousSearchActiveContext) asService.createAndStoreContext( - submitAsynchronousSearchRequest, - System.currentTimeMillis(), () -> InternalAggregationTestCase.emptyReduceContextBuilder(), null); - asStateMachine.trigger(new SearchStartedEvent(context, - new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()))); + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> InternalAggregationTestCase.emptyReduceContextBuilder(), + null + ); + asStateMachine.trigger( + new SearchStartedEvent(context, new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap())) + ); SearchResponse mockSearchResponse = getMockSearchResponse(); AsynchronousSearchResponse asResponse = postProcessor.processSearchResponse(mockSearchResponse, context.getContextId()); assertNotNull(asResponse.getId()); @@ -231,34 +320,59 @@ public void testProcessSearchResponsePersisted() throws AsynchronousSearchStateM } public void testProcessSearchResponseForExpiredContext() throws AsynchronousSearchStateMachineClosedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); AtomicBoolean activeContextCleanUpConsumerInvocation = new AtomicBoolean(); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); AsynchronousSearchStateMachine asStateMachine = asService.getStateMachine(); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); - AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor(persistenceService, - asActiveStore, asStateMachine, - (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), testThreadPool, clusterService); + AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor( + persistenceService, + asActiveStore, + asStateMachine, + (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), + testThreadPool, + clusterService + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.keepAlive(TimeValue.timeValueMillis(1)); AsynchronousSearchActiveContext context = (AsynchronousSearchActiveContext) asService.createAndStoreContext( - submitAsynchronousSearchRequest, - System.currentTimeMillis(), () -> InternalAggregationTestCase.emptyReduceContextBuilder(), null); - asStateMachine.trigger(new SearchStartedEvent(context, - new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()))); + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> InternalAggregationTestCase.emptyReduceContextBuilder(), + null + ); + asStateMachine.trigger( + new SearchStartedEvent(context, new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap())) + ); SearchResponse mockSearchResponse = getMockSearchResponse(); waitUntil(() -> context.isExpired()); AsynchronousSearchResponse asResponse = postProcessor.processSearchResponse(mockSearchResponse, context.getContextId()); @@ -276,8 +390,13 @@ public void testProcessSearchResponseForExpiredContext() throws AsynchronousSear } public void testProcessSearchResponseOnClosedContext() throws AsynchronousSearchStateMachineClosedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); AtomicBoolean activeContextCleanUpConsumerInvocation = new AtomicBoolean(); ThreadPool testThreadPool = null; try { @@ -285,24 +404,42 @@ public void testProcessSearchResponseOnClosedContext() throws AsynchronousSearch ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); AsynchronousSearchStateMachine asStateMachine = asService.getStateMachine(); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); - AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor(persistenceService, - asActiveStore, asStateMachine, - (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), testThreadPool, clusterService); + AsynchronousSearchPostProcessor postProcessor = new AsynchronousSearchPostProcessor( + persistenceService, + asActiveStore, + asStateMachine, + (context) -> activeContextCleanUpConsumerInvocation.compareAndSet(false, true), + testThreadPool, + clusterService + ); SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest()); submitAsynchronousSearchRequest.keepOnCompletion(true); submitAsynchronousSearchRequest.keepAlive(TimeValue.timeValueHours(1)); AsynchronousSearchActiveContext context = (AsynchronousSearchActiveContext) asService.createAndStoreContext( - submitAsynchronousSearchRequest, - System.currentTimeMillis(), () -> InternalAggregationTestCase.emptyReduceContextBuilder(), null); - asStateMachine.trigger(new SearchStartedEvent(context, - new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()))); + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> InternalAggregationTestCase.emptyReduceContextBuilder(), + null + ); + asStateMachine.trigger( + new SearchStartedEvent(context, new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap())) + ); asStateMachine.trigger(new SearchDeletedEvent(context)); SearchResponse mockSearchResponse = getMockSearchResponse(); AsynchronousSearchResponse asResponse = postProcessor.processSearchResponse(mockSearchResponse, context.getContextId()); @@ -322,13 +459,24 @@ public void testProcessSearchResponseOnClosedContext() throws AsynchronousSearch protected SearchResponse getMockSearchResponse() { int totalShards = randomInt(100); int successfulShards = totalShards - randomInt(100); - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(emptyList()), new Suggest(emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), - "", totalShards, successfulShards, 0, randomNonNegativeLong(), - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + randomInt(5) + ), + "", + totalShards, + successfulShards, + 0, + randomNonNegativeLong(), + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } private static class FakeClient extends NoOpClient { @@ -340,9 +488,11 @@ private static class FakeClient extends NoOpClient { } @Override - protected void doExecute(ActionType action, - Request request, - ActionListener listener) { + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { if (action instanceof IndexAction) { persistenceCount++; } diff --git a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceFreeContextTests.java b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceFreeContextTests.java index f9ed251b..142cb7ae 100644 --- a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceFreeContextTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceFreeContextTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service; import org.opensearch.commons.authuser.User; @@ -100,25 +103,33 @@ public class AsynchronousSearchServiceFreeContextTests extends OpenSearchTestCas private static boolean simulateTimedOut = false; private static boolean simulateUncheckedException = false; - @Before public void createObjects() { Settings settings = Settings.builder() - .put("node.name", "test") - .put("cluster.name", "ClusterServiceTests") - .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) - .build(); - final Set> settingsSet = - Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING)).collect(Collectors.toSet()); + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) + .build(); + final Set> settingsSet = Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of( + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING + ) + ).collect(Collectors.toSet()); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); List> executorBuilders = new ArrayList<>(); - executorBuilders.add(new ScalingExecutorBuilder(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30))); + executorBuilders.add( + new ScalingExecutorBuilder( + OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ) + ); executorBuilder = executorBuilders.get(0); clusterSettings = new ClusterSettings(settings, settingsSet); persisted = false; @@ -128,28 +139,52 @@ public void createObjects() { } public void testFreePersistedContextUserMatches() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - AsynchronousSearchId asId = new AsynchronousSearchId(discoveryNode.getId(), randomNonNegativeLong(), - asContextId); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + AsynchronousSearchId asId = new AsynchronousSearchId(discoveryNode.getId(), randomNonNegativeLong(), asContextId); persisted = true; CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(AsynchronousSearchIdConverter.buildAsyncId(asId), asContextId, null, - new LatchedActionListener<>(ActionListener.wrap(Assert::assertTrue, - e -> fail("expected successful delete because persistence is true. but got " + e.getMessage())), latch)); + asService.freeContext( + AsynchronousSearchIdConverter.buildAsyncId(asId), + asContextId, + null, + new LatchedActionListener<>( + ActionListener.wrap( + Assert::assertTrue, + e -> fail("expected successful delete because persistence is true. but got " + e.getMessage()) + ), + latch + ) + ); latch.await(); mockClusterService.stop(); } finally { @@ -158,34 +193,59 @@ public void testFreePersistedContextUserMatches() throws InterruptedException { } public void testFreePersistedContextUserNotMatches() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - AsynchronousSearchId asId = new AsynchronousSearchId(discoveryNode.getId(), randomNonNegativeLong(), - asContextId); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + AsynchronousSearchId asId = new AsynchronousSearchId(discoveryNode.getId(), randomNonNegativeLong(), asContextId); persisted = true; userMatches = false; CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(AsynchronousSearchIdConverter.buildAsyncId(asId), asContextId, randomUser(), - new LatchedActionListener<>(ActionListener.wrap( - r -> { - fail("Expected resource_not_found_exception due to user mismatch security exception. received delete " + - "acknowledgement : " + r); - - }, e -> assertTrue("expected resource_not_found_exception got " + e.getClass().getName(), - e instanceof ResourceNotFoundException)), latch)); + asService.freeContext( + AsynchronousSearchIdConverter.buildAsyncId(asId), + asContextId, + randomUser(), + new LatchedActionListener<>(ActionListener.wrap(r -> { + fail( + "Expected resource_not_found_exception due to user mismatch security exception. received delete " + + "acknowledgement : " + + r + ); + + }, + e -> assertTrue( + "expected resource_not_found_exception got " + e.getClass().getName(), + e instanceof ResourceNotFoundException + ) + ), latch) + ); latch.await(); mockClusterService.stop(); } finally { @@ -194,18 +254,33 @@ public void testFreePersistedContextUserNotMatches() throws InterruptedException } public void testFreeContextRunningUserMatches() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueDays(9); boolean keepOnCompletion = true; @@ -215,14 +290,32 @@ public void testFreeContextRunningUserMatches() throws InterruptedException { submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - AsynchronousSearchActiveContext asActiveContext = new AsynchronousSearchActiveContext(asContextId, discoveryNode.getId(), - keepAlive, true, testThreadPool, testThreadPool::absoluteTimeInMillis, asProgressListener, user1, () -> true); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), asActiveContext, null, (c) -> {}) { + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + AsynchronousSearchActiveContext asActiveContext = new AsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1, + () -> true + ); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ) { @Override public boolean isCancelled() { return true; @@ -232,38 +325,53 @@ public boolean isCancelled() { asActiveContext.setState(AsynchronousSearchState.RUNNING); when(mockStore.getContext(any())).thenReturn(Optional.of(asActiveContext)); persisted = false; - User user = randomBoolean() ? null : user1; //user - //task cancellation fails + User user = randomBoolean() ? null : user1; // user + // task cancellation fails CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), - user, new LatchedActionListener<>(ActionListener.wrap( - Assert::assertTrue, - e -> { - fail("active context should have been deleted"); - } - ), latch)); + asService.freeContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + user, + new LatchedActionListener<>(ActionListener.wrap(Assert::assertTrue, e -> { + fail("active context should have been deleted"); + }), latch) + ); latch.await(); mockClusterService.stop(); - } finally { ThreadPool.terminate(testThreadPool, 30, TimeUnit.SECONDS); } } public void testFreeActiveContextWithCancelledTask() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueDays(9); boolean keepOnCompletion = true; @@ -273,15 +381,32 @@ public void testFreeActiveContextWithCancelledTask() throws InterruptedException submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - AsynchronousSearchActiveContext asActiveContext = new AsynchronousSearchActiveContext(asContextId, discoveryNode.getId(), - keepAlive, true, testThreadPool, testThreadPool::absoluteTimeInMillis, asProgressListener, user1, () -> true); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), asActiveContext, null, (c) -> { - }) { + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + AsynchronousSearchActiveContext asActiveContext = new AsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1, + () -> true + ); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ) { @Override public boolean isCancelled() { return true; @@ -291,39 +416,53 @@ public boolean isCancelled() { asActiveContext.setState(AsynchronousSearchState.RUNNING); when(mockStore.getContext(any())).thenReturn(Optional.of(asActiveContext)); persisted = false; - //task cancellation fails + // task cancellation fails CountDownLatch latch = new CountDownLatch(1); - User user = randomBoolean() ? null : user1; //user - asService.freeContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), - user, new LatchedActionListener<>(ActionListener.wrap( - Assert::assertTrue, - e -> { - fail("active context should have been deleted"); - } - ), latch)); + User user = randomBoolean() ? null : user1; // user + asService.freeContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + user, + new LatchedActionListener<>(ActionListener.wrap(Assert::assertTrue, e -> { + fail("active context should have been deleted"); + }), latch) + ); latch.await(); mockClusterService.stop(); - } finally { ThreadPool.terminate(testThreadPool, 30, TimeUnit.SECONDS); } } public void testFreeContextUserNotMatches() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, - new AsynchronousSearchActiveStore(mockClusterService), mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + new AsynchronousSearchActiveStore(mockClusterService), + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -332,50 +471,75 @@ public void testFreeContextUserNotMatches() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchActiveContext context = (AsynchronousSearchActiveContext) - asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), () -> null, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), - "transport", SearchAction.NAME, TaskId.EMPTY_TASK_ID, emptyMap(), context, null, (c) -> {}); + AsynchronousSearchActiveContext context = (AsynchronousSearchActiveContext) asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + context, + null, + (c) -> {} + ); asService.bootstrapSearch(task, context.getContextId()); persisted = false; CountDownLatch latch = new CountDownLatch(1); User user2 = randomUser(); - asService.freeContext(context.getAsynchronousSearchId(), context.getContextId(), - user2, new LatchedActionListener<>(ActionListener.wrap( - r -> { - fail("expected security exception but got delete ack " + r + " search creator " + user1 + " " + - "accessing user " + user2); - }, - e -> { - assertTrue(e.getClass().getName(), e instanceof ResourceNotFoundException); - } - ), latch)); + asService.freeContext( + context.getAsynchronousSearchId(), + context.getContextId(), + user2, + new LatchedActionListener<>(ActionListener.wrap(r -> { + fail( + "expected security exception but got delete ack " + r + " search creator " + user1 + " " + "accessing user " + user2 + ); + }, e -> { assertTrue(e.getClass().getName(), e instanceof ResourceNotFoundException); }), latch) + ); latch.await(); mockClusterService.stop(); - } finally { ThreadPool.terminate(testThreadPool, 30, TimeUnit.SECONDS); } } public void testFreeContextTimedOut() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueDays(9); boolean keepOnCompletion = true; @@ -385,25 +549,46 @@ public void testFreeContextTimedOut() throws InterruptedException { submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext(asContextId, - discoveryNode.getId(), keepAlive, - true, testThreadPool, testThreadPool::absoluteTimeInMillis, asProgressListener, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), asActiveContext, null, (c) -> {}); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ); asActiveContext.setTask(task); simulateTimedOut = true; persisted = true; when(mockStore.getContext(asContextId)).thenReturn(Optional.of(asActiveContext)); CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), - null, new LatchedActionListener<>( - wrap(r -> fail("expected timedout exception"), - e -> assertTrue(e instanceof OpenSearchTimeoutException)), latch)); + asService.freeContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + null, + new LatchedActionListener<>( + wrap(r -> fail("expected timedout exception"), e -> assertTrue(e instanceof OpenSearchTimeoutException)), + latch + ) + ); latch.await(); assertEquals(0, (int) mockClient.deleteCount); mockClusterService.stop(); @@ -413,19 +598,33 @@ public void testFreeContextTimedOut() throws InterruptedException { } public void testFreeContextPermitAcquisitionFailure() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueDays(9); boolean keepOnCompletion = true; @@ -435,27 +634,45 @@ public void testFreeContextPermitAcquisitionFailure() throws InterruptedExceptio submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext(asContextId, - discoveryNode.getId(), keepAlive, - true, testThreadPool, testThreadPool::absoluteTimeInMillis, asProgressListener, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), asActiveContext, null, (c) -> {}); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ); asActiveContext.setTask(task); simulateUncheckedException = true; persisted = false; when(mockStore.getContext(asContextId)).thenReturn(Optional.of(asActiveContext)); CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), - null, new LatchedActionListener<>( - wrap(r -> fail("Expected resource_not_found_exception. Got acknowledgement " + r), - e -> { - assertTrue(e.getClass().getName(), e instanceof ResourceNotFoundException); - }), latch)); + asService.freeContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + null, + new LatchedActionListener<>(wrap(r -> fail("Expected resource_not_found_exception. Got acknowledgement " + r), e -> { + assertTrue(e.getClass().getName(), e instanceof ResourceNotFoundException); + }), latch) + ); latch.await(); assertEquals(1, (int) mockClient.deleteCount); mockClusterService.stop(); @@ -465,19 +682,33 @@ public void testFreeContextPermitAcquisitionFailure() throws InterruptedExceptio } public void testFreeContextPermitAcquisitionFailureKeepOnCompletionFalse() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueDays(9); boolean keepOnCompletion = true; @@ -487,28 +718,45 @@ public void testFreeContextPermitAcquisitionFailureKeepOnCompletionFalse() throw submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext(asContextId, - discoveryNode.getId(), keepAlive, - false, testThreadPool, testThreadPool::absoluteTimeInMillis, asProgressListener, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), asActiveContext, null, (c) -> { - }); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + false, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ); asActiveContext.setTask(task); simulateUncheckedException = true; persisted = false; when(mockStore.getContext(asContextId)).thenReturn(Optional.of(asActiveContext)); CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), - null, new LatchedActionListener<>( - wrap(Assert::assertFalse, - e -> { - fail("Expected acknowledgement false. Got error " + e.getMessage()); - }), latch)); + asService.freeContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + null, + new LatchedActionListener<>(wrap(Assert::assertFalse, e -> { + fail("Expected acknowledgement false. Got error " + e.getMessage()); + }), latch) + ); latch.await(); assertEquals(0, (int) mockClient.deleteCount); mockClusterService.stop(); @@ -518,18 +766,33 @@ public void testFreeContextPermitAcquisitionFailureKeepOnCompletionFalse() throw } public void testFreeActiveContextKeepOnCompletionFalse() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueDays(9); boolean keepOnCompletion = true; @@ -539,14 +802,32 @@ public void testFreeActiveContextKeepOnCompletionFalse() throws InterruptedExcep submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - AsynchronousSearchActiveContext asActiveContext = new AsynchronousSearchActiveContext(asContextId, discoveryNode.getId(), - keepAlive, false, testThreadPool, testThreadPool::absoluteTimeInMillis, asProgressListener, user1, () -> true); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), asActiveContext, null, (c) -> {}) { + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + AsynchronousSearchActiveContext asActiveContext = new AsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + false, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1, + () -> true + ); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ) { @Override public boolean isCancelled() { return true; @@ -556,40 +837,38 @@ public boolean isCancelled() { asActiveContext.setState(AsynchronousSearchState.RUNNING); when(mockStore.getContext(any())).thenReturn(Optional.of(asActiveContext)); persisted = false; - //task cancellation fails - User user = randomBoolean() ? null : user1; //user + // task cancellation fails + User user = randomBoolean() ? null : user1; // user CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), - user, new LatchedActionListener<>(ActionListener.wrap( - Assert::assertTrue, - e -> { - fail("active context should have been deleted"); - } - ), latch)); + asService.freeContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + user, + new LatchedActionListener<>(ActionListener.wrap(Assert::assertTrue, e -> { + fail("active context should have been deleted"); + }), latch) + ); latch.await(); CountDownLatch latch1 = new CountDownLatch(1); - asService.freeContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), - user, new LatchedActionListener<>(ActionListener.wrap( - r -> { - fail("Expected resource_not_found_exception"); - }, - e -> { - assertTrue(e.getClass().getName(), e instanceof ResourceNotFoundException); - } - ), latch1)); + asService.freeContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + user, + new LatchedActionListener<>(ActionListener.wrap(r -> { + fail("Expected resource_not_found_exception"); + }, e -> { assertTrue(e.getClass().getName(), e instanceof ResourceNotFoundException); }), latch1) + ); latch1.await(); mockClusterService.stop(); - } finally { ThreadPool.terminate(testThreadPool, 30, TimeUnit.SECONDS); } } - private static class MockClient extends NoOpClient { Integer persistenceCount; @@ -603,30 +882,42 @@ private static class MockClient extends NoOpClient { @Override @SuppressWarnings("unchecked") - protected void doExecute(ActionType action, - Request request, - ActionListener listener) { + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { if (action instanceof IndexAction) { persistenceCount++; listener.onResponse(null); - } else if (action instanceof UpdateAction) { //even delete is being done by UpdateAction + } else if (action instanceof UpdateAction) { // even delete is being done by UpdateAction deleteCount++; - ShardId shardId = new ShardId(new Index(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, - UUID.randomUUID().toString()), 1); - UpdateResponse updateResponse = new UpdateResponse(shardId, "testId", 1L, 1L, 1L, - persisted ? (userMatches ? DocWriteResponse.Result.DELETED : DocWriteResponse.Result.NOOP) - : DocWriteResponse.Result.NOT_FOUND); + ShardId shardId = new ShardId( + new Index(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, UUID.randomUUID().toString()), + 1 + ); + UpdateResponse updateResponse = new UpdateResponse( + shardId, + "testId", + 1L, + 1L, + 1L, + persisted + ? (userMatches ? DocWriteResponse.Result.DELETED : DocWriteResponse.Result.NOOP) + : DocWriteResponse.Result.NOT_FOUND + ); listener.onResponse((Response) updateResponse); } else if (action instanceof DeleteAction) { deleteCount++; - ShardId shardId = new ShardId(new Index(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, - UUID.randomUUID().toString()), 1); - DeleteResponse deleteResponse = new DeleteResponse(shardId, "testId", - 1L, 1L, 1L, persisted); + ShardId shardId = new ShardId( + new Index(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, UUID.randomUUID().toString()), + 1 + ); + DeleteResponse deleteResponse = new DeleteResponse(shardId, "testId", 1L, 1L, 1L, persisted); listener.onResponse((Response) deleteResponse); } else if (action instanceof CancelTasksAction) { - listener.onFailure(new RuntimeException("message")); + listener.onFailure(new RuntimeException("message")); } else { listener.onResponse(null); @@ -634,36 +925,66 @@ protected void } } - public static SearchResponse getMockSearchResponse() { int totalShards = randomInt(100); int successfulShards = totalShards - randomInt(100); - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), - "", totalShards, successfulShards, 0, randomNonNegativeLong(), - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + randomInt(5) + ), + "", + totalShards, + successfulShards, + 0, + randomNonNegativeLong(), + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } private ClusterService getClusterService(DiscoveryNode discoveryNode, ThreadPool testThreadPool) { ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); - ClusterServiceUtils.setState(clusterService, - ClusterStateCreationUtils.stateWithActivePrimary(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, - true, randomInt(5))); + ClusterServiceUtils.setState( + clusterService, + ClusterStateCreationUtils.stateWithActivePrimary( + AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX, + true, + randomInt(5) + ) + ); return clusterService; } static class MockAsynchronousSearchActiveContext extends AsynchronousSearchActiveContext { - MockAsynchronousSearchActiveContext(AsynchronousSearchContextId asContextId, String nodeId, TimeValue keepAlive, - boolean keepOnCompletion, ThreadPool threadPool, LongSupplier currentTimeSupplier, - AsynchronousSearchProgressListener searchProgressActionListener, User user) { - super(asContextId, nodeId, keepAlive, keepOnCompletion, threadPool, currentTimeSupplier, searchProgressActionListener, - user, () -> true); + MockAsynchronousSearchActiveContext( + AsynchronousSearchContextId asContextId, + String nodeId, + TimeValue keepAlive, + boolean keepOnCompletion, + ThreadPool threadPool, + LongSupplier currentTimeSupplier, + AsynchronousSearchProgressListener searchProgressActionListener, + User user + ) { + super( + asContextId, + nodeId, + keepAlive, + keepOnCompletion, + threadPool, + currentTimeSupplier, + searchProgressActionListener, + user, + () -> true + ); } - @Override public void acquireContextPermitIfRequired(ActionListener onPermitAcquired, TimeValue timeout, String reason) { if (simulateTimedOut) { diff --git a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceTests.java b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceTests.java index c37f4eeb..c13b3e99 100644 --- a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceTests.java @@ -1,11 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service; - import org.opensearch.commons.authuser.User; import org.opensearch.search.asynchronous.context.AsynchronousSearchContext; import org.opensearch.search.asynchronous.context.AsynchronousSearchContextId; @@ -80,45 +82,71 @@ public class AsynchronousSearchServiceTests extends OpenSearchTestCase { private Settings settings; private ExecutorBuilder executorBuilder; static boolean blockPersistence; + @Before public void createObjects() { settings = Settings.builder() - .put("node.name", "test") - .put("cluster.name", "ClusterServiceTests") - .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) - .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) - .build(); - final Set> settingsSet = - Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING)).collect(Collectors.toSet()); + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) + .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) + .build(); + final Set> settingsSet = Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of( + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING + ) + ).collect(Collectors.toSet()); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); List> executorBuilders = new ArrayList<>(); - executorBuilders.add(new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30))); + executorBuilders.add( + new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ) + ); executorBuilder = executorBuilders.get(0); clusterSettings = new ClusterSettings(settings, settingsSet); blockPersistence = false; } public void testFindContext() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = TestUtils.createClusterService(settings, testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = randomBoolean(); @@ -128,59 +156,89 @@ public void testFindContext() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, user1); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext asActiveContext = (AsynchronousSearchActiveContext) context; assertNull(asActiveContext.getTask()); assertNull(asActiveContext.getAsynchronousSearchId()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.INIT); assertEquals(asActiveContext.getUser(), user1); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> {}); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ); asService.bootstrapSearch(task, context.getContextId()); assertEquals(asActiveContext.getTask(), task); assertEquals(asActiveContext.getStartTimeMillis(), task.getStartTime()); assertEquals(asActiveContext.getExpirationTimeMillis(), task.getStartTime() + keepAlive.millis()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.RUNNING); CountDownLatch findContextLatch = new CountDownLatch(3); - ActionListener expectedSuccessfulActive = new LatchedActionListener<>(wrap( - r -> { - assertTrue(r instanceof AsynchronousSearchActiveContext); - assertEquals(r, context); - }, e -> fail("Find context shouldn't have failed. " + e.getMessage())), findContextLatch); - ActionListener expectedSecurityException = new LatchedActionListener<>(wrap( - r -> fail("Expecting security exception"), e -> assertTrue(e instanceof ResourceNotFoundException) - ), findContextLatch); - asService.findContext(asActiveContext.getAsynchronousSearchId(), - asActiveContext.getContextId(), user1, expectedSuccessfulActive); - asService.findContext(asActiveContext.getAsynchronousSearchId(), - asActiveContext.getContextId(), user2, expectedSecurityException); - asService.findContext(asActiveContext.getAsynchronousSearchId(), - asActiveContext.getContextId(), null, expectedSuccessfulActive); + ActionListener expectedSuccessfulActive = new LatchedActionListener<>(wrap(r -> { + assertTrue(r instanceof AsynchronousSearchActiveContext); + assertEquals(r, context); + }, e -> fail("Find context shouldn't have failed. " + e.getMessage())), findContextLatch); + ActionListener expectedSecurityException = new LatchedActionListener<>( + wrap(r -> fail("Expecting security exception"), e -> assertTrue(e instanceof ResourceNotFoundException)), + findContextLatch + ); + asService.findContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + user1, + expectedSuccessfulActive + ); + asService.findContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + user2, + expectedSecurityException + ); + asService.findContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + null, + expectedSuccessfulActive + ); findContextLatch.await(); AsynchronousSearchProgressListener asProgressListener = asActiveContext.getAsynchronousSearchProgressListener(); boolean success = randomBoolean(); - if (success) { //successful search response + if (success) { // successful search response asProgressListener.onResponse(getMockSearchResponse()); } else { // exception occurred in search asProgressListener.onFailure(new RuntimeException("test")); } waitUntil(() -> asService.getAllActiveContexts().isEmpty()); - if (keepOnCompletion) { //persist to disk + if (keepOnCompletion) { // persist to disk assertEquals(1, fakeClient.persistenceCount.intValue()); } else { assertEquals(fakeClient.persistenceCount, Integer.valueOf(0)); CountDownLatch freeContextLatch = new CountDownLatch(1); - asService.findContext(context.getAsynchronousSearchId(), context.getContextId(), null, - new LatchedActionListener<>(wrap( - r -> fail("No context should have been found but found " + - asService.getAllActiveContexts().size()), - e -> assertTrue(e instanceof ResourceNotFoundException)), freeContextLatch)); + asService.findContext( + context.getAsynchronousSearchId(), + context.getContextId(), + null, + new LatchedActionListener<>( + wrap( + r -> fail("No context should have been found but found " + asService.getAllActiveContexts().size()), + e -> assertTrue(e instanceof ResourceNotFoundException) + ), + freeContextLatch + ) + ); freeContextLatch.await(); } } finally { @@ -189,21 +247,37 @@ public void testFindContext() throws InterruptedException { } public void testUpdateExpirationTimesOutBlockedOnPersistence() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); - ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, - clusterSettings); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); + ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, testThreadPool); - - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -211,17 +285,28 @@ public void testUpdateExpirationTimesOutBlockedOnPersistence() throws Interrupte SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, null); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + null + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext asActiveContext = (AsynchronousSearchActiveContext) context; assertNull(asActiveContext.getTask()); assertNull(asActiveContext.getAsynchronousSearchId()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.INIT); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> {}); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ); asService.bootstrapSearch(task, context.getContextId()); assertEquals(asActiveContext.getTask(), task); @@ -234,10 +319,19 @@ public void testUpdateExpirationTimesOutBlockedOnPersistence() throws Interrupte CountDownLatch updateLatch = new CountDownLatch(1); TimeValue newKeepAlive = timeValueHours(10); fakeClient.awaitBlock(); - asService.updateKeepAliveAndGetContext(asActiveContext.getAsynchronousSearchId(), newKeepAlive, - asActiveContext.getContextId(), null, new LatchedActionListener<>(wrap(r -> fail("expected update req to timeout"), - e -> assertTrue("expected timeout got " + e.getClass(), e instanceof OpenSearchTimeoutException)), - updateLatch)); + asService.updateKeepAliveAndGetContext( + asActiveContext.getAsynchronousSearchId(), + newKeepAlive, + asActiveContext.getContextId(), + null, + new LatchedActionListener<>( + wrap( + r -> fail("expected update req to timeout"), + e -> assertTrue("expected timeout got " + e.getClass(), e instanceof OpenSearchTimeoutException) + ), + updateLatch + ) + ); updateLatch.await(); fakeClient.releaseBlock(); } finally { @@ -246,20 +340,36 @@ public void testUpdateExpirationTimesOutBlockedOnPersistence() throws Interrupte } public void testUpdateExpirationOnRunningSearch() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); - ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, - clusterSettings); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); + ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = false; @@ -267,17 +377,28 @@ public void testUpdateExpirationOnRunningSearch() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, null); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + null + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext asActiveContext = (AsynchronousSearchActiveContext) context; assertNull(asActiveContext.getTask()); assertNull(asActiveContext.getAsynchronousSearchId()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.INIT); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, - emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> {}); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ); asService.bootstrapSearch(task, context.getContextId()); assertEquals(asActiveContext.getTask(), task); @@ -286,21 +407,28 @@ public void testUpdateExpirationOnRunningSearch() throws InterruptedException { assertEquals(originalExpirationTimeMillis, task.getStartTime() + keepAlive.millis()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.RUNNING); CountDownLatch findContextLatch = new CountDownLatch(1); - asService.findContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), null, - new LatchedActionListener<>(wrap( - r -> { - assertTrue(r instanceof AsynchronousSearchActiveContext); - assertEquals(r, context); - }, e -> fail("Find context shouldn't have failed") - ), findContextLatch)); + asService.findContext( + asActiveContext.getAsynchronousSearchId(), + asActiveContext.getContextId(), + null, + new LatchedActionListener<>(wrap(r -> { + assertTrue(r instanceof AsynchronousSearchActiveContext); + assertEquals(r, context); + }, e -> fail("Find context shouldn't have failed")), findContextLatch) + ); findContextLatch.await(); CountDownLatch updateLatch = new CountDownLatch(1); TimeValue newKeepAlive = timeValueHours(10); - asService.updateKeepAliveAndGetContext(asActiveContext.getAsynchronousSearchId(), newKeepAlive, - asActiveContext.getContextId(), null, new LatchedActionListener<>(wrap(r -> { - assertTrue(r instanceof AsynchronousSearchActiveContext); - assertThat(r.getExpirationTimeMillis(), greaterThan(originalExpirationTimeMillis)); - }, e -> fail()), updateLatch)); + asService.updateKeepAliveAndGetContext( + asActiveContext.getAsynchronousSearchId(), + newKeepAlive, + asActiveContext.getContextId(), + null, + new LatchedActionListener<>(wrap(r -> { + assertTrue(r instanceof AsynchronousSearchActiveContext); + assertThat(r.getExpirationTimeMillis(), greaterThan(originalExpirationTimeMillis)); + }, e -> fail()), updateLatch) + ); updateLatch.await(); } finally { ThreadPool.terminate(testThreadPool, 200, TimeUnit.MILLISECONDS); @@ -308,19 +436,36 @@ public void testUpdateExpirationOnRunningSearch() throws InterruptedException { } public void testFindContextOnNonExistentSearch() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); User user1 = TestClientUtils.randomUser(); @@ -329,12 +474,22 @@ public void testFindContextOnNonExistentSearch() throws InterruptedException { submitAsynchronousSearchRequest.keepOnCompletion(false); submitAsynchronousSearchRequest.keepAlive(keepAlive); CountDownLatch findContextLatch = new CountDownLatch(2); - ActionListener failureExpectingListener = new LatchedActionListener<>(wrap(r -> fail(), - e -> assertTrue(e instanceof ResourceNotFoundException)), findContextLatch); - asService.findContext("nonExistentId", new AsynchronousSearchContextId(randomAlphaOfLength(10), - randomNonNegativeLong()), null, failureExpectingListener); - asService.findContext("nonExistentId", new AsynchronousSearchContextId(randomAlphaOfLength(10), - randomNonNegativeLong()), user1, failureExpectingListener); + ActionListener failureExpectingListener = new LatchedActionListener<>( + wrap(r -> fail(), e -> assertTrue(e instanceof ResourceNotFoundException)), + findContextLatch + ); + asService.findContext( + "nonExistentId", + new AsynchronousSearchContextId(randomAlphaOfLength(10), randomNonNegativeLong()), + null, + failureExpectingListener + ); + asService.findContext( + "nonExistentId", + new AsynchronousSearchContextId(randomAlphaOfLength(10), randomNonNegativeLong()), + user1, + failureExpectingListener + ); findContextLatch.await(); } finally { ThreadPool.terminate(testThreadPool, 200, TimeUnit.MILLISECONDS); @@ -342,19 +497,36 @@ public void testFindContextOnNonExistentSearch() throws InterruptedException { } public void testBootStrapOnClosedSearch() { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = false; @@ -362,41 +534,69 @@ public void testBootStrapOnClosedSearch() { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, null); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + null + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext asActiveContext = (AsynchronousSearchActiveContext) context; assertNull(asActiveContext.getTask()); assertNull(asActiveContext.getAsynchronousSearchId()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.INIT); - //close context + // close context asActiveContext.close(); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", - SearchAction.NAME, TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> {}); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ); asActiveContext.setState(AsynchronousSearchState.CLOSED); - expectThrows(IllegalStateException.class, () -> asService.bootstrapSearch(task, - context.getContextId())); + expectThrows(IllegalStateException.class, () -> asService.bootstrapSearch(task, context.getContextId())); } finally { ThreadPool.terminate(testThreadPool, 200, TimeUnit.MILLISECONDS); } } public void testFreeActiveContext() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = randomBoolean(); @@ -405,17 +605,29 @@ public void testFreeActiveContext() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, user1); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext asActiveContext = (AsynchronousSearchActiveContext) context; assertNull(asActiveContext.getTask()); assertNull(asActiveContext.getAsynchronousSearchId()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.INIT); assertEquals(asActiveContext.getUser(), user1); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> {}) { + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ) { @Override public boolean isCancelled() { return true; @@ -429,8 +641,12 @@ public boolean isCancelled() { assertTrue(asService.freeActiveContext(asActiveContext)); assertTrue(asService.getAllActiveContexts().isEmpty()); CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(context.getAsynchronousSearchId(), context.getContextId(), user1, - new LatchedActionListener<>(wrap(r -> fail(), e -> assertTrue(e instanceof ResourceNotFoundException)), latch)); + asService.freeContext( + context.getAsynchronousSearchId(), + context.getContextId(), + user1, + new LatchedActionListener<>(wrap(r -> fail(), e -> assertTrue(e instanceof ResourceNotFoundException)), latch) + ); latch.await(); } finally { ThreadPool.terminate(testThreadPool, 200, TimeUnit.MILLISECONDS); @@ -438,20 +654,36 @@ public boolean isCancelled() { } public void testFreeContext() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder); + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ); ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = randomBoolean(); @@ -460,17 +692,29 @@ public void testFreeContext() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, user1); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext asActiveContext = (AsynchronousSearchActiveContext) context; assertNull(asActiveContext.getTask()); assertNull(asActiveContext.getAsynchronousSearchId()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.INIT); assertEquals(asActiveContext.getUser(), user1); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> {}) { + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ) { @Override public boolean isCancelled() { return true; @@ -482,8 +726,12 @@ public boolean isCancelled() { assertEquals(asActiveContext.getExpirationTimeMillis(), task.getStartTime() + keepAlive.millis()); assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.RUNNING); CountDownLatch latch = new CountDownLatch(1); - asService.freeContext(context.getAsynchronousSearchId(), context.getContextId(), user1, - new LatchedActionListener<>(wrap(Assert::assertTrue, e -> fail()), latch)); + asService.freeContext( + context.getAsynchronousSearchId(), + context.getContextId(), + user1, + new LatchedActionListener<>(wrap(Assert::assertTrue, e -> fail()), latch) + ); latch.await(); } finally { ThreadPool.terminate(testThreadPool, 200, TimeUnit.MILLISECONDS); @@ -491,12 +739,19 @@ public boolean isCancelled() { } public void testFindContextsToReap() { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { - testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, - executorBuilder) { + testThreadPool = new TestThreadPool( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + executorBuilder + ) { @Override public long absoluteTimeInMillis() { // simulate search has over run) return System.currentTimeMillis() - 24 * 3600 * 1000; @@ -506,10 +761,20 @@ public long absoluteTimeInMillis() { // simulate search has over run) ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = randomBoolean(); @@ -518,8 +783,12 @@ public long absoluteTimeInMillis() { // simulate search has over run) SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, user1); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); assertTrue(asService.getContextsToReap().contains(context)); } finally { ThreadPool.terminate(testThreadPool, 30, TimeUnit.SECONDS); @@ -538,9 +807,11 @@ private static class FakeClient extends NoOpClient { } @Override - protected void doExecute(ActionType action, - Request request, - ActionListener listener) { + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { if (action instanceof IndexAction) { persistenceCount++; if (blockPersistence) { @@ -554,6 +825,7 @@ protected void } listener.onResponse(null); } + public void awaitBlock() throws InterruptedException { waitUntil(() -> block); } @@ -566,12 +838,23 @@ public void releaseBlock() { protected SearchResponse getMockSearchResponse() { int totalShards = randomInt(100); int successfulShards = totalShards - randomInt(100); - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), - "", totalShards, successfulShards, 0, randomNonNegativeLong(), - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + randomInt(5) + ), + "", + totalShards, + successfulShards, + 0, + randomNonNegativeLong(), + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } } diff --git a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceUpdateContextTests.java b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceUpdateContextTests.java index 5c9e44ff..6ae5c2bc 100644 --- a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceUpdateContextTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchServiceUpdateContextTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service; import org.opensearch.common.xcontent.XContentType; @@ -111,21 +114,30 @@ public class AsynchronousSearchServiceUpdateContextTests extends OpenSearchTestC @Before public void createObjects() { Settings settings = Settings.builder() - .put("node.name", "test") - .put("cluster.name", "ClusterServiceTests") - .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) - .build(); - final Set> settingsSet = - Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING)).collect(Collectors.toSet()); + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) + .build(); + final Set> settingsSet = Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of( + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING + ) + ).collect(Collectors.toSet()); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); List> executorBuilders = new ArrayList<>(); - executorBuilders.add(new ScalingExecutorBuilder(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30))); + executorBuilders.add( + new ScalingExecutorBuilder( + OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ) + ); executorBuilder = executorBuilders.get(0); clusterSettings = new ClusterSettings(settings, settingsSet); simulateUncheckedException = false; @@ -134,18 +146,33 @@ public void createObjects() { } public void testUpdateContextWhenContextCloseAndKeepOnCompletionTrue() throws InterruptedException, IOException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -155,35 +182,49 @@ public void testUpdateContextWhenContextCloseAndKeepOnCompletionTrue() throws In submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext(asContextId, - discoveryNode.getId(), keepAlive, - true, testThreadPool, testThreadPool::absoluteTimeInMillis, asProgressListener, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), asActiveContext, null, (c) -> { - }); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ); asActiveContext.setTask(task); long oldExpirationTimeMillis = asActiveContext.getExpirationTimeMillis(); simulateIsAlive = false; simulateTimedOut = false; CountDownLatch updateLatch = new CountDownLatch(1); when(mockStore.getContext(asContextId)).thenReturn(Optional.of(asActiveContext)); - asService.updateKeepAliveAndGetContext(asActiveContext.getAsynchronousSearchId(), keepAlive, - asActiveContext.getContextId(), - user1, - new LatchedActionListener<>(wrap( - r -> { - assertTrue(r instanceof AsynchronousSearchPersistenceContext); - //assert active context expiration time is not updated - assertEquals(asActiveContext.getExpirationTimeMillis(), oldExpirationTimeMillis); - }, - e -> { - fail("expected successful update got " + e.getMessage()); - } - ), updateLatch)); + asService.updateKeepAliveAndGetContext( + asActiveContext.getAsynchronousSearchId(), + keepAlive, + asActiveContext.getContextId(), + user1, + new LatchedActionListener<>(wrap(r -> { + assertTrue(r instanceof AsynchronousSearchPersistenceContext); + // assert active context expiration time is not updated + assertEquals(asActiveContext.getExpirationTimeMillis(), oldExpirationTimeMillis); + }, e -> { fail("expected successful update got " + e.getMessage()); }), updateLatch) + ); updateLatch.await(); mockClusterService.stop(); } finally { @@ -193,18 +234,33 @@ TaskId.EMPTY_TASK_ID, emptyMap(), asActiveContext, null, (c) -> { } public void testUpdateContextTimedOut() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -214,24 +270,47 @@ public void testUpdateContextTimedOut() throws InterruptedException { submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext(asContextId, - discoveryNode.getId(), keepAlive, true, testThreadPool, testThreadPool::absoluteTimeInMillis, - asProgressListener, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), asActiveContext, null, (c) -> {}); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ); asActiveContext.setTask(task); simulateIsAlive = true; simulateTimedOut = true; when(mockStore.getContext(asContextId)).thenReturn(Optional.of(asActiveContext)); CountDownLatch latch = new CountDownLatch(1); - asService.updateKeepAliveAndGetContext(asActiveContext.getAsynchronousSearchId(), timeValueHours(9), - asActiveContext.getContextId(), randomUser(), new LatchedActionListener<>( - wrap(r -> fail("expected timedout exception"), - e -> assertTrue(e instanceof OpenSearchTimeoutException)), latch)); + asService.updateKeepAliveAndGetContext( + asActiveContext.getAsynchronousSearchId(), + timeValueHours(9), + asActiveContext.getContextId(), + randomUser(), + new LatchedActionListener<>( + wrap(r -> fail("expected timedout exception"), e -> assertTrue(e instanceof OpenSearchTimeoutException)), + latch + ) + ); latch.await(); mockClusterService.stop(); } finally { @@ -240,18 +319,33 @@ public void testUpdateContextTimedOut() throws InterruptedException { } public void testUpdateContextPermitAcquisitionFailure() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -261,27 +355,55 @@ public void testUpdateContextPermitAcquisitionFailure() throws InterruptedExcept submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext(asContextId, - discoveryNode.getId(), keepAlive, true, testThreadPool, testThreadPool::absoluteTimeInMillis, - asProgressListener, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), asActiveContext, null, (c) -> { - }); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ); asActiveContext.setTask(task); simulateIsAlive = true; simulateUncheckedException = true; long oldExpiration = asActiveContext.getExpirationTimeMillis(); when(mockStore.getContext(asContextId)).thenReturn(Optional.of(asActiveContext)); CountDownLatch latch = new CountDownLatch(1); - asService.updateKeepAliveAndGetContext(asActiveContext.getAsynchronousSearchId(), timeValueHours(9), - asActiveContext.getContextId(), randomUser(), new LatchedActionListener<>( - wrap(r -> assertEquals("active context should not have been updated on permit acquisition failure", - asActiveContext.getExpirationTimeMillis(), oldExpiration), - e -> fail("expected update to succeed but got " + e.getMessage())), latch)); + asService.updateKeepAliveAndGetContext( + asActiveContext.getAsynchronousSearchId(), + timeValueHours(9), + asActiveContext.getContextId(), + randomUser(), + new LatchedActionListener<>( + wrap( + r -> assertEquals( + "active context should not have been updated on permit acquisition failure", + asActiveContext.getExpirationTimeMillis(), + oldExpiration + ), + e -> fail("expected update to succeed but got " + e.getMessage()) + ), + latch + ) + ); latch.await(); assertEquals("update should have been attempted on index", mockClient.updateCount.intValue(), 1); mockClusterService.stop(); @@ -291,18 +413,33 @@ TaskId.EMPTY_TASK_ID, emptyMap(), asActiveContext, null, (c) -> { } public void testUpdateContextPermitAcquisitionFailureKeepOnCompletionFalse() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore mockStore = mock(AsynchronousSearchActiveStore.class); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, mockStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + mockStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = false; @@ -312,26 +449,48 @@ public void testUpdateContextPermitAcquisitionFailureKeepOnCompletionFalse() thr submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); - MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext(asContextId, - discoveryNode.getId(), keepAlive, false, testThreadPool, testThreadPool::absoluteTimeInMillis, - asProgressListener, user1); - - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), asActiveContext, null, (c) -> { - }); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); + MockAsynchronousSearchActiveContext asActiveContext = new MockAsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + false, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user1 + ); + + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + asActiveContext, + null, + (c) -> {} + ); asActiveContext.setTask(task); simulateIsAlive = true; simulateUncheckedException = true; long oldExpiration = asActiveContext.getExpirationTimeMillis(); when(mockStore.getContext(asContextId)).thenReturn(Optional.of(asActiveContext)); CountDownLatch latch = new CountDownLatch(1); - asService.updateKeepAliveAndGetContext(asActiveContext.getAsynchronousSearchId(), timeValueHours(9), - asActiveContext.getContextId(), randomUser(), new LatchedActionListener<>( - wrap(r -> fail("expected update to fail but"), - e -> assertTrue(e instanceof ResourceNotFoundException)), latch)); + asService.updateKeepAliveAndGetContext( + asActiveContext.getAsynchronousSearchId(), + timeValueHours(9), + asActiveContext.getContextId(), + randomUser(), + new LatchedActionListener<>( + wrap(r -> fail("expected update to fail but"), e -> assertTrue(e instanceof ResourceNotFoundException)), + latch + ) + ); latch.await(); assertEquals("update should not have been attempted on index", mockClient.updateCount.intValue(), 0); mockClusterService.stop(); @@ -341,18 +500,33 @@ TaskId.EMPTY_TASK_ID, emptyMap(), asActiveContext, null, (c) -> { } public void testUpdateRunningContextValidUser() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -361,18 +535,29 @@ public void testUpdateRunningContextValidUser() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, user1); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext activeContext = (AsynchronousSearchActiveContext) context; assertNull(activeContext.getTask()); assertNull(activeContext.getAsynchronousSearchId()); assertEquals(activeContext.getAsynchronousSearchState(), INIT); assertEquals(activeContext.getUser(), user1); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> { - }); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ); asService.bootstrapSearch(task, context.getContextId()); assertEquals(activeContext.getTask(), task); assertEquals(activeContext.getStartTimeMillis(), task.getStartTime()); @@ -380,11 +565,15 @@ TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, nul assertEquals(activeContext.getAsynchronousSearchState(), RUNNING); CountDownLatch latch = new CountDownLatch(1); Long oldExpiration = context.getExpirationTimeMillis(); - asService.updateKeepAliveAndGetContext(context.getAsynchronousSearchId(), timeValueHours(10), context.getContextId(), - user1, new LatchedActionListener<>(wrap(r -> assertThat(context.getExpirationTimeMillis(), greaterThan(oldExpiration)), - e -> { - fail("Expected successful update but got failure " + e.getMessage()); - }), latch)); + asService.updateKeepAliveAndGetContext( + context.getAsynchronousSearchId(), + timeValueHours(10), + context.getContextId(), + user1, + new LatchedActionListener<>(wrap(r -> assertThat(context.getExpirationTimeMillis(), greaterThan(oldExpiration)), e -> { + fail("Expected successful update but got failure " + e.getMessage()); + }), latch) + ); latch.await(); mockClusterService.stop(); } finally { @@ -393,18 +582,33 @@ TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, nul } public void testUpdateContextNoActiveContextFound() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -415,18 +619,40 @@ public void testUpdateContextNoActiveContextFound() throws InterruptedException submitAsynchronousSearchRequest.keepAlive(keepAlive); String node = UUID.randomUUID().toString(); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(testThreadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); User user = randomUser(); - AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext(asContextId, node, - keepAlive, keepOnCompletion, testThreadPool, - testThreadPool::absoluteTimeInMillis, asProgressListener, user, () -> true); + AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext( + asContextId, + node, + keepAlive, + keepOnCompletion, + testThreadPool, + testThreadPool::absoluteTimeInMillis, + asProgressListener, + user, + () -> true + ); CountDownLatch latch = new CountDownLatch(1); docNotFound = true; - asService.updateKeepAliveAndGetContext(context.getAsynchronousSearchId(), keepAlive, context.getContextId(), - user1, new LatchedActionListener<>(wrap(r -> fail("Expected resource_not_found_exception"), - e -> assertTrue("Expected resource_not_found_exception but got " + e.getMessage(), - e instanceof ResourceNotFoundException)), latch)); + asService.updateKeepAliveAndGetContext( + context.getAsynchronousSearchId(), + keepAlive, + context.getContextId(), + user1, + new LatchedActionListener<>( + wrap( + r -> fail("Expected resource_not_found_exception"), + e -> assertTrue( + "Expected resource_not_found_exception but got " + e.getMessage(), + e instanceof ResourceNotFoundException + ) + ), + latch + ) + ); latch.await(); assertEquals(mockClient.updateCount.intValue(), 1); mockClusterService.stop(); @@ -436,18 +662,33 @@ public void testUpdateContextNoActiveContextFound() throws InterruptedException } public void testUpdateActiveContextInvalidUser() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = true; @@ -456,18 +697,29 @@ public void testUpdateActiveContextInvalidUser() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, user1); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext activeContext = (AsynchronousSearchActiveContext) context; assertNull(activeContext.getTask()); assertNull(activeContext.getAsynchronousSearchId()); assertEquals(activeContext.getAsynchronousSearchState(), INIT); assertEquals(activeContext.getUser(), user1); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> { - }); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ); asService.bootstrapSearch(task, context.getContextId()); assertEquals(activeContext.getTask(), task); assertEquals(activeContext.getStartTimeMillis(), task.getStartTime()); @@ -475,12 +727,27 @@ TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, nul assertEquals(activeContext.getAsynchronousSearchState(), RUNNING); CountDownLatch latch = new CountDownLatch(1); User differenteUser = randomUser(); - asService.updateKeepAliveAndGetContext(context.getAsynchronousSearchId(), timeValueHours(9), context.getContextId(), - differenteUser, new LatchedActionListener<>(wrap(r -> fail("expected security exception Users must be different, " + - "actual user " + user1 + " random user " + differenteUser), - e -> { - assertTrue(e instanceof ResourceNotFoundException); - }), latch)); + asService.updateKeepAliveAndGetContext( + context.getAsynchronousSearchId(), + timeValueHours(9), + context.getContextId(), + differenteUser, + new LatchedActionListener<>( + wrap( + r -> fail( + "expected security exception Users must be different, " + + "actual user " + + user1 + + " random user " + + differenteUser + ), + e -> { + assertTrue(e instanceof ResourceNotFoundException); + } + ), + latch + ) + ); latch.await(); mockClusterService.stop(); } finally { @@ -489,18 +756,33 @@ TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, nul } public void testUpdateClosedContext() throws InterruptedException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); ThreadPool testThreadPool = null; try { testThreadPool = new TestThreadPool(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder); ClusterService mockClusterService = getClusterService(discoveryNode, testThreadPool); MockClient mockClient = new MockClient(testThreadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(mockClient, - mockClusterService, testThreadPool); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, mockClient, - mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList())); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + mockClient, + mockClusterService, + testThreadPool + ); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + mockClient, + mockClusterService, + testThreadPool, + new InternalAsynchronousSearchStats(), + new NamedWriteableRegistry(emptyList()) + ); TimeValue keepAlive = timeValueHours(9); boolean keepOnCompletion = randomBoolean(); @@ -509,18 +791,29 @@ public void testUpdateClosedContext() throws InterruptedException { SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion); submitAsynchronousSearchRequest.keepAlive(keepAlive); - AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), - () -> null, user1); + AsynchronousSearchContext context = asService.createAndStoreContext( + submitAsynchronousSearchRequest, + System.currentTimeMillis(), + () -> null, + user1 + ); assertTrue(context instanceof AsynchronousSearchActiveContext); AsynchronousSearchActiveContext activeContext = (AsynchronousSearchActiveContext) context; assertNull(activeContext.getTask()); assertNull(activeContext.getAsynchronousSearchId()); assertEquals(activeContext.getAsynchronousSearchState(), INIT); assertEquals(activeContext.getUser(), user1); - //bootstrap search - AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, - TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> { - }); + // bootstrap search + AsynchronousSearchTask task = new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + (AsynchronousSearchActiveContext) context, + null, + (c) -> {} + ); asService.bootstrapSearch(task, context.getContextId()); assertEquals(activeContext.getTask(), task); assertEquals(activeContext.getStartTimeMillis(), task.getStartTime()); @@ -530,21 +823,25 @@ TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, nul ((AsynchronousSearchActiveContext) context).close(); CountDownLatch latch = new CountDownLatch(1); - asService.updateKeepAliveAndGetContext(context.getAsynchronousSearchId(), timeValueHours(9), context.getContextId(), - user1, new LatchedActionListener<>(wrap(r -> { - if (keepOnCompletion) { - assertTrue(r instanceof AsynchronousSearchPersistenceContext); - } else { - fail("expected resource not found exception, got result."); - } - }, - e -> { - if (keepOnCompletion) { - fail("expected resource not found exception, got result"); - } else { - assertTrue(e instanceof ResourceNotFoundException); - } - }), latch)); + asService.updateKeepAliveAndGetContext( + context.getAsynchronousSearchId(), + timeValueHours(9), + context.getContextId(), + user1, + new LatchedActionListener<>(wrap(r -> { + if (keepOnCompletion) { + assertTrue(r instanceof AsynchronousSearchPersistenceContext); + } else { + fail("expected resource not found exception, got result."); + } + }, e -> { + if (keepOnCompletion) { + fail("expected resource not found exception, got result"); + } else { + assertTrue(e instanceof ResourceNotFoundException); + } + }), latch) + ); latch.await(); mockClusterService.stop(); } finally { @@ -565,23 +862,22 @@ private static class MockClient extends NoOpClient { @Override @SuppressWarnings("unchecked") - protected void doExecute(ActionType action, - Request request, - ActionListener listener) { + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { if (action instanceof IndexAction) { persistenceCount++; listener.onResponse(null); } else if (action instanceof UpdateAction) { updateCount++; - ShardId shardId = new ShardId(new Index(ASYNC_SEARCH_RESPONSE_INDEX, - UUID.randomUUID().toString()), 1); + ShardId shardId = new ShardId(new Index(ASYNC_SEARCH_RESPONSE_INDEX, UUID.randomUUID().toString()), 1); if (docNotFound) { - UpdateResponse updateResponse = new UpdateResponse(shardId, "testId", 1L, 1L, 1L, - DocWriteResponse.Result.NOT_FOUND); + UpdateResponse updateResponse = new UpdateResponse(shardId, "testId", 1L, 1L, 1L, DocWriteResponse.Result.NOT_FOUND); listener.onResponse((Response) updateResponse); } else { - UpdateResponse updateResponse = new UpdateResponse(shardId, "testId", 1L, 1L, 1L, - DocWriteResponse.Result.UPDATED); + UpdateResponse updateResponse = new UpdateResponse(shardId, "testId", 1L, 1L, 1L, DocWriteResponse.Result.UPDATED); try { Map sourceMap = new HashMap<>(); sourceMap.put(EXPIRATION_TIME_MILLIS, randomNonNegativeLong()); @@ -589,9 +885,9 @@ protected void XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(sourceMap); BytesReference source = BytesReference.bytes(builder); - updateResponse.setGetResult(new GetResult(ASYNC_SEARCH_RESPONSE_INDEX, - "testId", 1L, 1L, 1L, - true, source, emptyMap(), null)); + updateResponse.setGetResult( + new GetResult(ASYNC_SEARCH_RESPONSE_INDEX, "testId", 1L, 1L, 1L, true, source, emptyMap(), null) + ); listener.onResponse((Response) updateResponse); } catch (IOException e) { fail("Fake client failed to build mock update response"); @@ -605,11 +901,27 @@ protected void } static class MockAsynchronousSearchActiveContext extends AsynchronousSearchActiveContext { - MockAsynchronousSearchActiveContext(AsynchronousSearchContextId asContextId, String nodeId, TimeValue keepAlive, - boolean keepOnCompletion, ThreadPool threadPool, LongSupplier currentTimeSupplier, - AsynchronousSearchProgressListener searchProgressActionListener, User user) { - super(asContextId, nodeId, keepAlive, keepOnCompletion, threadPool, currentTimeSupplier, searchProgressActionListener, - user, () -> true); + MockAsynchronousSearchActiveContext( + AsynchronousSearchContextId asContextId, + String nodeId, + TimeValue keepAlive, + boolean keepOnCompletion, + ThreadPool threadPool, + LongSupplier currentTimeSupplier, + AsynchronousSearchProgressListener searchProgressActionListener, + User user + ) { + super( + asContextId, + nodeId, + keepAlive, + keepOnCompletion, + threadPool, + currentTimeSupplier, + searchProgressActionListener, + user, + () -> true + ); } @Override @@ -635,20 +947,32 @@ public void acquireContextPermitIfRequired(ActionListener onPermitAc public static SearchResponse getMockSearchResponse() { int totalShards = randomInt(100); int successfulShards = totalShards - randomInt(100); - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), - "", totalShards, successfulShards, 0, randomNonNegativeLong(), - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + randomInt(5) + ), + "", + totalShards, + successfulShards, + 0, + randomNonNegativeLong(), + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } private ClusterService getClusterService(DiscoveryNode discoveryNode, ThreadPool testThreadPool) { ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings); - ClusterServiceUtils.setState(clusterService, - ClusterStateCreationUtils.stateWithActivePrimary(ASYNC_SEARCH_RESPONSE_INDEX, - true, randomInt(5))); + ClusterServiceUtils.setState( + clusterService, + ClusterStateCreationUtils.stateWithActivePrimary(ASYNC_SEARCH_RESPONSE_INDEX, true, randomInt(5)) + ); return clusterService; } diff --git a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchStateMachineTests.java b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchStateMachineTests.java index 776b7f9c..6fe462fa 100644 --- a/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchStateMachineTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/service/AsynchronousSearchStateMachineTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service; import org.opensearch.search.asynchronous.commons.AsynchronousSearchTestCase; @@ -89,79 +92,152 @@ public class AsynchronousSearchStateMachineTests extends AsynchronousSearchTestC @Before public void createObjects() { settings = Settings.builder() - .put("node.name", "test") - .put("cluster.name", "ClusterServiceTests") - .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) - .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) - .build(); - final Set> settingsSet = - Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING)).collect(Collectors.toSet()); + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10) + .put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true) + .build(); + final Set> settingsSet = Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Stream.of( + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING + ) + ).collect(Collectors.toSet()); final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings); List> executorBuilders = new ArrayList<>(); - executorBuilders.add(new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, - Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30))); + executorBuilders.add( + new ScalingExecutorBuilder( + AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, + 1, + Math.min(2 * availableProcessors, Math.max(128, 512)), + TimeValue.timeValueMinutes(30) + ) + ); executorBuilder = executorBuilders.get(0); clusterSettings = new ClusterSettings(settings, settingsSet); } public void testStateMachine() throws InterruptedException, BrokenBarrierException { - DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); TestThreadPool threadPool = null; try { threadPool = new TestThreadPool("test", executorBuilder); ClusterService mockClusterService = createClusterService(settings, threadPool, discoveryNode, clusterSettings); FakeClient fakeClient = new FakeClient(threadPool); AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService); - AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, - mockClusterService, - threadPool); + AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService( + fakeClient, + mockClusterService, + threadPool + ); CustomContextListener customContextListener = new CustomContextListener(); - AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, - mockClusterService, threadPool, customContextListener, new NamedWriteableRegistry(emptyList())); + AsynchronousSearchService asService = new AsynchronousSearchService( + persistenceService, + asActiveStore, + fakeClient, + mockClusterService, + threadPool, + customContextListener, + new NamedWriteableRegistry(emptyList()) + ); AsynchronousSearchProgressListener asProgressListener = mockAsynchronousSearchProgressListener(threadPool); - AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), - randomNonNegativeLong()); + AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( + UUID.randomUUID().toString(), + randomNonNegativeLong() + ); TimeValue keepAlive = TimeValue.timeValueDays(randomInt(100)); - AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext(asContextId, discoveryNode.getId(), - keepAlive, true, threadPool, - threadPool::absoluteTimeInMillis, asProgressListener, null, () -> true); + AsynchronousSearchActiveContext context = new AsynchronousSearchActiveContext( + asContextId, + discoveryNode.getId(), + keepAlive, + true, + threadPool, + threadPool::absoluteTimeInMillis, + asProgressListener, + null, + () -> true + ); assertNull(context.getTask()); assertEquals(context.getAsynchronousSearchState(), INIT); AsynchronousSearchStateMachine stateMachine = asService.getStateMachine(); AtomicInteger numCompleted = new AtomicInteger(); AtomicInteger numFailure = new AtomicInteger(); - doConcurrentStateMachineTrigger(stateMachine, new SearchStartedEvent(context, new AsynchronousSearchTask( - randomNonNegativeLong(), "transport", SearchAction.NAME, TaskId.EMPTY_TASK_ID, emptyMap(), context, null, - (a) -> {})), - RUNNING, IllegalStateException.class, Optional.empty()); + doConcurrentStateMachineTrigger( + stateMachine, + new SearchStartedEvent( + context, + new AsynchronousSearchTask( + randomNonNegativeLong(), + "transport", + SearchAction.NAME, + TaskId.EMPTY_TASK_ID, + emptyMap(), + context, + null, + (a) -> {} + ) + ), + RUNNING, + IllegalStateException.class, + Optional.empty() + ); boolean success = randomBoolean(); assertNotNull(context.getTask()); - if (randomBoolean()) { //delete running context - doConcurrentStateMachineTrigger(stateMachine, new SearchDeletedEvent(context), CLOSED, - AsynchronousSearchStateMachineClosedException.class, Optional.empty()); + if (randomBoolean()) { // delete running context + doConcurrentStateMachineTrigger( + stateMachine, + new SearchDeletedEvent(context), + CLOSED, + AsynchronousSearchStateMachineClosedException.class, + Optional.empty() + ); } else { if (success) { - doConcurrentStateMachineTrigger(stateMachine, new SearchFailureEvent(context, new RuntimeException("test")), FAILED, - IllegalStateException.class, Optional.empty()); + doConcurrentStateMachineTrigger( + stateMachine, + new SearchFailureEvent(context, new RuntimeException("test")), + FAILED, + IllegalStateException.class, + Optional.empty() + ); numFailure.getAndIncrement(); - } else {//success or failure - doConcurrentStateMachineTrigger(stateMachine, new SearchSuccessfulEvent(context, getMockSearchResponse()), SUCCEEDED, - IllegalStateException.class, Optional.empty()); + } else {// success or failure + doConcurrentStateMachineTrigger( + stateMachine, + new SearchSuccessfulEvent(context, getMockSearchResponse()), + SUCCEEDED, + IllegalStateException.class, + Optional.empty() + ); numCompleted.getAndIncrement(); } - doConcurrentStateMachineTrigger(stateMachine, new BeginPersistEvent(context), PERSISTING, - IllegalStateException.class, Optional.of(AsynchronousSearchStateMachineClosedException.class)); + doConcurrentStateMachineTrigger( + stateMachine, + new BeginPersistEvent(context), + PERSISTING, + IllegalStateException.class, + Optional.of(AsynchronousSearchStateMachineClosedException.class) + ); waitUntil(() -> context.getAsynchronousSearchState().equals(CLOSED), 1, TimeUnit.MINUTES); - assertTrue(context.getAsynchronousSearchState().toString() + " numFailure : " + numFailure.get() + " numSuccess : " - + numCompleted.get(), - context.getAsynchronousSearchState().equals(CLOSED)); + assertTrue( + context.getAsynchronousSearchState().toString() + + " numFailure : " + + numFailure.get() + + " numSuccess : " + + numCompleted.get(), + context.getAsynchronousSearchState().equals(CLOSED) + ); assertEquals(1, customContextListener.getPersistedCount() + customContextListener.getPersistFailedCount()); } assertEquals(numCompleted.get(), customContextListener.getCompletedCount()); @@ -176,19 +252,33 @@ public void testStateMachine() throws InterruptedException, BrokenBarrierExcepti private SearchResponse getMockSearchResponse() { int totalShards = randomInt(100); int successfulShards = totalShards - randomInt(100); - return new SearchResponse(new InternalSearchResponse( + return new SearchResponse( + new InternalSearchResponse( new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), InternalAggregations.from(Collections.emptyList()), new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), - "", totalShards, successfulShards, 0, randomNonNegativeLong(), - ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + randomInt(5) + ), + "", + totalShards, + successfulShards, + 0, + randomNonNegativeLong(), + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); } - private void doConcurrentStateMachineTrigger( - AsynchronousSearchStateMachine asStateMachine, AsynchronousSearchContextEvent event, AsynchronousSearchState finalState, - Class throwable, Optional> terminalStateException) throws InterruptedException, BrokenBarrierException { + AsynchronousSearchStateMachine asStateMachine, + AsynchronousSearchContextEvent event, + AsynchronousSearchState finalState, + Class throwable, + Optional> terminalStateException + ) throws InterruptedException, BrokenBarrierException { int numThreads = 10; List operationThreads = new ArrayList<>(); AtomicInteger numTriggerSuccess = new AtomicInteger(); @@ -230,9 +320,11 @@ private static class FakeClient extends NoOpClient { } @Override - protected void doExecute(ActionType action, - Request request, - ActionListener listener) { + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { if (action instanceof CreateIndexAction) { listener.onResponse(null); return; diff --git a/src/test/java/org/opensearch/search/asynchronous/service/persistence/AsynchronousSearchPersistenceServiceIT.java b/src/test/java/org/opensearch/search/asynchronous/service/persistence/AsynchronousSearchPersistenceServiceIT.java index bef1651e..f94bfef2 100644 --- a/src/test/java/org/opensearch/search/asynchronous/service/persistence/AsynchronousSearchPersistenceServiceIT.java +++ b/src/test/java/org/opensearch/search/asynchronous/service/persistence/AsynchronousSearchPersistenceServiceIT.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.service.persistence; import org.opensearch.commons.ConfigConstants; @@ -64,56 +67,82 @@ public void setUp() throws Exception { super.setUp(); threadPool = new TestThreadPool("persistenceServiceTests"); } + public void setThreadPool(ThreadPool threadPool) { this.threadPool = threadPool; } + public void testCreateAndGetAndDelete() throws IOException, InterruptedException { AsynchronousSearchPersistenceService persistenceService = getInstanceFromNode(AsynchronousSearchPersistenceService.class); TransportService transportService = getInstanceFromNode(TransportService.class); AsynchronousSearchResponse asResponse = submitAndGetPersistedAsynchronousSearchResponse(); AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUIDs.base64UUID(), randomInt(100)); - AsynchronousSearchId newAsynchronousSearchId = new AsynchronousSearchId(transportService.getLocalNode().getId(), - 1, asContextId); + AsynchronousSearchId newAsynchronousSearchId = new AsynchronousSearchId(transportService.getLocalNode().getId(), 1, asContextId); String id = AsynchronousSearchIdConverter.buildAsyncId(newAsynchronousSearchId); User user1 = TestClientUtils.randomUser(); User user2 = TestClientUtils.randomUser(); for (User user : Arrays.asList(user1, null)) { - AsynchronousSearchResponse newAsynchronousSearchResponse = new AsynchronousSearchResponse(id, - AsynchronousSearchState.STORE_RESIDENT, - asResponse.getStartTimeMillis(), - asResponse.getExpirationTimeMillis(), - asResponse.getSearchResponse(), - asResponse.getError()); + AsynchronousSearchResponse newAsynchronousSearchResponse = new AsynchronousSearchResponse( + id, + AsynchronousSearchState.STORE_RESIDENT, + asResponse.getStartTimeMillis(), + asResponse.getExpirationTimeMillis(), + asResponse.getSearchResponse(), + asResponse.getError() + ); createDoc(persistenceService, newAsynchronousSearchResponse, user); if (user != null) { CountDownLatch getLatch1 = new CountDownLatch(1); ActionListener getListener = ActionListener.wrap( - r -> fail("Expected exception. Got " + r), e -> assertTrue(e instanceof OpenSearchSecurityException)); - persistenceService.getResponse(newAsynchronousSearchResponse.getId(), user2, new LatchedActionListener<>(getListener, - getLatch1)); + r -> fail("Expected exception. Got " + r), + e -> assertTrue(e instanceof OpenSearchSecurityException) + ); + persistenceService.getResponse( + newAsynchronousSearchResponse.getId(), + user2, + new LatchedActionListener<>(getListener, getLatch1) + ); getLatch1.await(); } CountDownLatch getLatch2 = new CountDownLatch(1); - persistenceService.getResponse(newAsynchronousSearchResponse.getId(), user, new LatchedActionListener<>( - ActionListener.wrap(r -> assertEquals( - new AsynchronousSearchPersistenceModel(asResponse.getStartTimeMillis(), - asResponse.getExpirationTimeMillis(), asResponse.getSearchResponse(), - null, user), r), - e -> { - logger.error("Expected get result got ", e); - fail(e.getMessage()); - }), getLatch2)); + persistenceService.getResponse( + newAsynchronousSearchResponse.getId(), + user, + new LatchedActionListener<>( + ActionListener.wrap( + r -> assertEquals( + new AsynchronousSearchPersistenceModel( + asResponse.getStartTimeMillis(), + asResponse.getExpirationTimeMillis(), + asResponse.getSearchResponse(), + null, + user + ), + r + ), + e -> { + logger.error("Expected get result got ", e); + fail(e.getMessage()); + } + ), + getLatch2 + ) + ); getLatch2.await(); if (user != null) { CountDownLatch deleteLatch1 = new CountDownLatch(1); User diffUser = TestClientUtils.randomUser(); ActionListener deleteListener = ActionListener.wrap( - r -> fail("Expected exception on delete. Got acknowledgment" + r), - e -> assertTrue(e instanceof OpenSearchSecurityException)); - persistenceService.deleteResponse(newAsynchronousSearchResponse.getId(), diffUser, - new LatchedActionListener<>(deleteListener, deleteLatch1)); + r -> fail("Expected exception on delete. Got acknowledgment" + r), + e -> assertTrue(e instanceof OpenSearchSecurityException) + ); + persistenceService.deleteResponse( + newAsynchronousSearchResponse.getId(), + diffUser, + new LatchedActionListener<>(deleteListener, deleteLatch1) + ); deleteLatch1.await(); } CountDownLatch deleteLatch2 = new CountDownLatch(1); @@ -121,18 +150,29 @@ public void testCreateAndGetAndDelete() throws IOException, InterruptedException logger.debug(() -> new ParameterizedMessage("Delete failed unexpectedly "), e); fail("delete failed.expected success"); }); - persistenceService.deleteResponse(newAsynchronousSearchResponse.getId(), user, - new LatchedActionListener<>(deleteListener, deleteLatch2)); + persistenceService.deleteResponse( + newAsynchronousSearchResponse.getId(), + user, + new LatchedActionListener<>(deleteListener, deleteLatch2) + ); deleteLatch2.await(); - //assert failure + // assert failure CountDownLatch getLatch3 = new CountDownLatch(2); - ActionListener getListener = ActionListener.wrap((r) -> fail("Expected RNF, Got " + r), - exception -> assertTrue(exception instanceof ResourceNotFoundException)); - persistenceService.getResponse(newAsynchronousSearchResponse.getId(), - null, new LatchedActionListener<>(getListener, getLatch3)); - persistenceService.getResponse(newAsynchronousSearchResponse.getId(), user2, - new LatchedActionListener<>(getListener, getLatch3)); + ActionListener getListener = ActionListener.wrap( + (r) -> fail("Expected RNF, Got " + r), + exception -> assertTrue(exception instanceof ResourceNotFoundException) + ); + persistenceService.getResponse( + newAsynchronousSearchResponse.getId(), + null, + new LatchedActionListener<>(getListener, getLatch3) + ); + persistenceService.getResponse( + newAsynchronousSearchResponse.getId(), + user2, + new LatchedActionListener<>(getListener, getLatch3) + ); getLatch3.await(); } } @@ -145,31 +185,49 @@ public void testGetAndDeleteNonExistentId() throws InterruptedException, IOExcep User user2 = TestClientUtils.randomUser(); for (User originalUser : Arrays.asList(user1, null)) { AsynchronousSearchId asId = generateNewAsynchronousSearchId(transportService); - AsynchronousSearchPersistenceModel model1 = new AsynchronousSearchPersistenceModel(System.currentTimeMillis(), - System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), - searchResponse, null, originalUser); + AsynchronousSearchPersistenceModel model1 = new AsynchronousSearchPersistenceModel( + System.currentTimeMillis(), + System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), + searchResponse, + null, + originalUser + ); CountDownLatch createLatch = new CountDownLatch(1); String id = AsynchronousSearchIdConverter.buildAsyncId(asId); - persistenceService.storeResponse(id, model1, new LatchedActionListener<>(ActionListener.wrap( - r -> assertSuccessfulResponseCreation(id, r), e -> { - logger.debug("expect successful create, got", e); - fail("Expected successful create, got " + e.getMessage()); - }), createLatch)); + persistenceService.storeResponse( + id, + model1, + new LatchedActionListener<>(ActionListener.wrap(r -> assertSuccessfulResponseCreation(id, r), e -> { + logger.debug("expect successful create, got", e); + fail("Expected successful create, got " + e.getMessage()); + }), createLatch) + ); createLatch.await(); for (User currentuser : Arrays.asList(originalUser, user2)) { CountDownLatch latch = new CountDownLatch(2); - //assert failure - persistenceService.getResponse("id", currentuser, new LatchedActionListener<>( + // assert failure + persistenceService.getResponse( + "id", + currentuser, + new LatchedActionListener<>( ActionListener.wrap( - (AsynchronousSearchPersistenceModel r) -> fail("Excepted resource_not_found_exception, got " + r), - exception -> assertTrue("Expected resource_not_found expection, got " + - exception.getClass().toString(), - exception instanceof ResourceNotFoundException)), latch)); - //assert failure + (AsynchronousSearchPersistenceModel r) -> fail("Excepted resource_not_found_exception, got " + r), + exception -> assertTrue( + "Expected resource_not_found expection, got " + exception.getClass().toString(), + exception instanceof ResourceNotFoundException + ) + ), + latch + ) + ); + // assert failure ActionListener wrap = ActionListener.wrap( - r -> fail("Expected resource_not_found expection on delete, got acknowledgement " + r), - ex -> assertTrue("Expected resource_not_found expection, got " + ex.getClass().toString(), - ex instanceof ResourceNotFoundException)); + r -> fail("Expected resource_not_found expection on delete, got acknowledgement " + r), + ex -> assertTrue( + "Expected resource_not_found expection, got " + ex.getClass().toString(), + ex instanceof ResourceNotFoundException + ) + ); persistenceService.deleteResponse("id", currentuser, new LatchedActionListener<>(wrap, latch)); latch.await(); } @@ -183,46 +241,68 @@ public void testCreateConcurrentDocsWhenIndexNotExists() throws InterruptedExcep SearchResponse searchResponse = client().search(new SearchRequest(TEST_INDEX)).get(); AsynchronousSearchId asId1 = generateNewAsynchronousSearchId(transportService); AsynchronousSearchId asId2 = generateNewAsynchronousSearchId(transportService); - AsynchronousSearchPersistenceModel model1 = new AsynchronousSearchPersistenceModel(System.currentTimeMillis(), - System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), searchResponse, - null, null); + AsynchronousSearchPersistenceModel model1 = new AsynchronousSearchPersistenceModel( + System.currentTimeMillis(), + System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), + searchResponse, + null, + null + ); String id1 = AsynchronousSearchIdConverter.buildAsyncId(asId1); - AsynchronousSearchPersistenceModel model2 = new AsynchronousSearchPersistenceModel(System.currentTimeMillis(), - System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), searchResponse, - null, null); + AsynchronousSearchPersistenceModel model2 = new AsynchronousSearchPersistenceModel( + System.currentTimeMillis(), + System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), + searchResponse, + null, + null + ); String id2 = AsynchronousSearchIdConverter.buildAsyncId(asId2); CountDownLatch createLatch = new CountDownLatch(2); threadPool.generic() - .execute(() -> persistenceService.storeResponse(id1, model1, new LatchedActionListener<>(ActionListener.wrap( - r -> assertSuccessfulResponseCreation(id1, r), e -> { - logger.debug("expect successful create, got", e); - fail("Expected successful create, got " + e.getMessage()); - }), createLatch))); + .execute( + () -> persistenceService.storeResponse( + id1, + model1, + new LatchedActionListener<>(ActionListener.wrap(r -> assertSuccessfulResponseCreation(id1, r), e -> { + logger.debug("expect successful create, got", e); + fail("Expected successful create, got " + e.getMessage()); + }), createLatch) + ) + ); threadPool.generic() - .execute(() -> persistenceService.storeResponse(id2, model2, new LatchedActionListener<>(ActionListener.wrap( - r -> assertSuccessfulResponseCreation(id2, r), e -> { - logger.debug("expect successful create, got", e); - fail("Expected successful create, got " + e.getMessage()); - }), createLatch))); + .execute( + () -> persistenceService.storeResponse( + id2, + model2, + new LatchedActionListener<>(ActionListener.wrap(r -> assertSuccessfulResponseCreation(id2, r), e -> { + logger.debug("expect successful create, got", e); + fail("Expected successful create, got " + e.getMessage()); + }), createLatch) + ) + ); createLatch.await(); CountDownLatch getLatch1 = new CountDownLatch(1); - persistenceService.getResponse(id1, null, new LatchedActionListener<>(ActionListener.wrap( - (AsynchronousSearchPersistenceModel r) -> - assertEquals(model1, r), e -> { - logger.debug("expect successful get result, got", e); - fail("Expected successful get result, got " + e.getMessage()); - }), getLatch1)); + persistenceService.getResponse( + id1, + null, + new LatchedActionListener<>(ActionListener.wrap((AsynchronousSearchPersistenceModel r) -> assertEquals(model1, r), e -> { + logger.debug("expect successful get result, got", e); + fail("Expected successful get result, got " + e.getMessage()); + }), getLatch1) + ); getLatch1.await(); CountDownLatch getLatch2 = new CountDownLatch(1); - persistenceService.getResponse(id2, null, new LatchedActionListener<>(ActionListener.wrap( - (AsynchronousSearchPersistenceModel r) -> - assertEquals(model2, r), e -> { - logger.debug("expect successful create, got", e); - fail("Expected successful create, got " + e.getMessage()); - }), getLatch2)); + persistenceService.getResponse( + id2, + null, + new LatchedActionListener<>(ActionListener.wrap((AsynchronousSearchPersistenceModel r) -> assertEquals(model2, r), e -> { + logger.debug("expect successful create, got", e); + fail("Expected successful create, got " + e.getMessage()); + }), getLatch2) + ); getLatch2.await(); } @@ -233,39 +313,56 @@ public void testUpdateExpiration() throws InterruptedException, IOException { User user2 = TestClientUtils.randomUser(); for (User originalUser : Arrays.asList(user1, null)) { try (ThreadContext.StoredContext ctx = threadPool1.getThreadContext().stashContext()) { - threadPool1.getThreadContext().putTransient( - ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT, getUserRolesString(originalUser)); + threadPool1.getThreadContext() + .putTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT, getUserRolesString(originalUser)); AsynchronousSearchResponse asResponse = submitAndGetPersistedAsynchronousSearchResponse(); long newExpirationTime = System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(); final AsynchronousSearchPersistenceModel newPersistenceModel = new AsynchronousSearchPersistenceModel( - asResponse.getStartTimeMillis(), - newExpirationTime, asResponse.getSearchResponse(), null, originalUser); + asResponse.getStartTimeMillis(), + newExpirationTime, + asResponse.getSearchResponse(), + null, + originalUser + ); for (User currentUser : Arrays.asList(user2, user1, null)) { CountDownLatch updateLatch = new CountDownLatch(1); if (originalUser != null && currentUser != null && currentUser.equals(originalUser) == false) { ActionListener updateListener = ActionListener.wrap( - r -> fail("Expected security exception. Unauthorized update. Got " + r), - e -> assertTrue(e instanceof OpenSearchSecurityException)); - persistenceService.updateExpirationTime(asResponse.getId(), newExpirationTime, currentUser, - new LatchedActionListener<>(updateListener, updateLatch)); + r -> fail("Expected security exception. Unauthorized update. Got " + r), + e -> assertTrue(e instanceof OpenSearchSecurityException) + ); + persistenceService.updateExpirationTime( + asResponse.getId(), + newExpirationTime, + currentUser, + new LatchedActionListener<>(updateListener, updateLatch) + ); } else { - persistenceService.updateExpirationTime(asResponse.getId(), - newExpirationTime, currentUser, new LatchedActionListener<>( - ActionListener.wrap(persistenceModel -> assertEquals(newPersistenceModel, persistenceModel), - e -> { - logger.debug("expect successful create, got", e); - fail("Expected successful create, got " + e.getMessage()); - }), updateLatch)); + persistenceService.updateExpirationTime( + asResponse.getId(), + newExpirationTime, + currentUser, + new LatchedActionListener<>( + ActionListener.wrap(persistenceModel -> assertEquals(newPersistenceModel, persistenceModel), e -> { + logger.debug("expect successful create, got", e); + fail("Expected successful create, got " + e.getMessage()); + }), + updateLatch + ) + ); } updateLatch.await(); } CountDownLatch getLatch = new CountDownLatch(1); - persistenceService.getResponse(asResponse.getId(), originalUser, new LatchedActionListener<>(ActionListener.wrap( - r -> assertEquals(newPersistenceModel, r), e -> { - logger.debug("expect successful get result, got", e); - fail("Expected successful get result, got " + e.getMessage()); - }), getLatch)); + persistenceService.getResponse( + asResponse.getId(), + originalUser, + new LatchedActionListener<>(ActionListener.wrap(r -> assertEquals(newPersistenceModel, r), e -> { + logger.debug("expect successful get result, got", e); + fail("Expected successful get result, got " + e.getMessage()); + }), getLatch) + ); getLatch.await(); } } @@ -280,29 +377,47 @@ public void testPersistenceServiceRetryTotalTime() { assertEquals(600000L, total); } - public void testCreateResponseFailureOnClusterBlock() throws Exception { AsynchronousSearchPersistenceService persistenceService = getInstanceFromNode(AsynchronousSearchPersistenceService.class); AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUIDs.base64UUID(), randomInt(100)); - AsynchronousSearchId newAsynchronousSearchId = new AsynchronousSearchId(getInstanceFromNode(TransportService.class) - .getLocalNode().getId(), 1, asContextId); + AsynchronousSearchId newAsynchronousSearchId = new AsynchronousSearchId( + getInstanceFromNode(TransportService.class).getLocalNode().getId(), + 1, + asContextId + ); String id = AsynchronousSearchIdConverter.buildAsyncId(newAsynchronousSearchId); - AsynchronousSearchResponse mockResponse = new AsynchronousSearchResponse(id, - AsynchronousSearchState.STORE_RESIDENT, randomNonNegativeLong(), randomNonNegativeLong(), getMockSearchResponse(), null); + AsynchronousSearchResponse mockResponse = new AsynchronousSearchResponse( + id, + AsynchronousSearchState.STORE_RESIDENT, + randomNonNegativeLong(), + randomNonNegativeLong(), + getMockSearchResponse(), + null + ); createDoc(getInstanceFromNode(AsynchronousSearchPersistenceService.class), mockResponse, null); - client().admin().indices().prepareUpdateSettings(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX) - .setSettings(Settings.builder().put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, true).build()) - .execute().actionGet(); + client().admin() + .indices() + .prepareUpdateSettings(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, true).build()) + .execute() + .actionGet(); SearchRequest searchRequest = new SearchRequest().indices("index").source(new SearchSourceBuilder()); SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); request.keepOnCompletion(true); request.waitForCompletionTimeout(TimeValue.timeValueMillis(5000)); AsynchronousSearchResponse asResponse = TestClientUtils.blockingSubmitAsynchronousSearch(client(), request); - waitUntil(() -> assertRnf(() -> TestClientUtils.blockingGetAsynchronousSearchResponse(client(), - new GetAsynchronousSearchRequest(asResponse.getId())))); + waitUntil( + () -> assertRnf( + () -> TestClientUtils.blockingGetAsynchronousSearchResponse(client(), new GetAsynchronousSearchRequest(asResponse.getId())) + ) + ); assertRnf(() -> TestClientUtils.blockingGetAsynchronousSearchResponse(client(), new GetAsynchronousSearchRequest(id))); - client().admin().indices().prepareUpdateSettings(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX) - .setSettings(Settings.builder().putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE).build()).execute().actionGet(); + client().admin() + .indices() + .prepareUpdateSettings(AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX) + .setSettings(Settings.builder().putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE).build()) + .execute() + .actionGet(); } public void testDeleteExpiredResponse() throws InterruptedException, IOException { @@ -312,23 +427,32 @@ public void testDeleteExpiredResponse() throws InterruptedException, IOException CountDownLatch updateLatch = new CountDownLatch(1); long newExpirationTime = System.currentTimeMillis() + new TimeValue(100, TimeUnit.MILLISECONDS).getMillis(); final AsynchronousSearchPersistenceModel newPersistenceModel = new AsynchronousSearchPersistenceModel( - asResponse.getStartTimeMillis(), - newExpirationTime, asResponse.getSearchResponse(), null, null); - persistenceService.updateExpirationTime(asResponse.getId(), - newExpirationTime, null, new LatchedActionListener<>( - ActionListener.wrap(persistenceModel -> assertEquals(newPersistenceModel, persistenceModel), - e -> { - logger.debug("expect successful create, got", e); - fail("Expected successful update result, got " + e.getMessage()); - }), updateLatch)); + asResponse.getStartTimeMillis(), + newExpirationTime, + asResponse.getSearchResponse(), + null, + null + ); + persistenceService.updateExpirationTime( + asResponse.getId(), + newExpirationTime, + null, + new LatchedActionListener<>(ActionListener.wrap(persistenceModel -> assertEquals(newPersistenceModel, persistenceModel), e -> { + logger.debug("expect successful create, got", e); + fail("Expected successful update result, got " + e.getMessage()); + }), updateLatch) + ); updateLatch.await(); CountDownLatch getLatch = new CountDownLatch(1); - persistenceService.getResponse(asResponse.getId(), null, new LatchedActionListener<>( - ActionListener.wrap(r -> assertEquals(newPersistenceModel, r), e -> { - logger.debug("expect successful create, got", e); - fail("Expected successful create, got " + e.getMessage()); - }), getLatch)); + persistenceService.getResponse( + asResponse.getId(), + null, + new LatchedActionListener<>(ActionListener.wrap(r -> assertEquals(newPersistenceModel, r), e -> { + logger.debug("expect successful create, got", e); + fail("Expected successful create, got " + e.getMessage()); + }), getLatch) + ); getLatch.await(); CountDownLatch deleteLatch = new CountDownLatch(1); @@ -358,20 +482,19 @@ public void testConcurrentDeletes() throws InterruptedException { AtomicInteger numFailure = new AtomicInteger(); for (int i = 0; i < numThreads; i++) { Thread t = new Thread(() -> { - persistenceService.deleteResponse(asResponse.getId(), null, new LatchedActionListener<>(ActionListener.wrap( - r -> { - if (r) { - numSuccess.getAndIncrement(); - } else { - numFailure.getAndIncrement(); - } - }, e -> { - if (e instanceof ResourceNotFoundException) { - numRnf.getAndIncrement(); - } else { - numFailure.getAndIncrement(); - } - }), latch)); + persistenceService.deleteResponse(asResponse.getId(), null, new LatchedActionListener<>(ActionListener.wrap(r -> { + if (r) { + numSuccess.getAndIncrement(); + } else { + numFailure.getAndIncrement(); + } + }, e -> { + if (e instanceof ResourceNotFoundException) { + numRnf.getAndIncrement(); + } else { + numFailure.getAndIncrement(); + } + }), latch)); }); threads.add(t); } @@ -398,24 +521,26 @@ public void testConcurrentUpdates() throws InterruptedException { for (int i = 0; i < numThreads; i++) { Thread t = new Thread(() -> { long expirationTimeMillis = System.currentTimeMillis() + timeValueDays(10).millis(); - persistenceService.updateExpirationTime(asResponse.getId(), - expirationTimeMillis, null, - new LatchedActionListener<>(ActionListener.wrap( - r -> { - if (r.getExpirationTimeMillis() == expirationTimeMillis) { - numSuccess.getAndIncrement(); - } else if (r.getExpirationTimeMillis() == asResponse.getExpirationTimeMillis()) { - numNoOp.getAndIncrement(); - } else { - numFailure.getAndIncrement(); - } - }, e -> { - if (e instanceof VersionConflictEngineException) { - numVersionConflictException.getAndIncrement(); - } else { - numFailure.getAndIncrement(); - } - }), latch)); + persistenceService.updateExpirationTime( + asResponse.getId(), + expirationTimeMillis, + null, + new LatchedActionListener<>(ActionListener.wrap(r -> { + if (r.getExpirationTimeMillis() == expirationTimeMillis) { + numSuccess.getAndIncrement(); + } else if (r.getExpirationTimeMillis() == asResponse.getExpirationTimeMillis()) { + numNoOp.getAndIncrement(); + } else { + numFailure.getAndIncrement(); + } + }, e -> { + if (e instanceof VersionConflictEngineException) { + numVersionConflictException.getAndIncrement(); + } else { + numFailure.getAndIncrement(); + } + }), latch) + ); }); threads.add(t); } @@ -445,39 +570,40 @@ public void testConcurrentUpdatesAndDeletesRace() throws InterruptedException { if (iter % 2 == 0 || iter < 20) /*letting few updates to queue up before starting to fire deletes*/ { long expirationTimeMillis = System.currentTimeMillis() + timeValueDays(10).millis(); - persistenceService.updateExpirationTime(asResponse.getId(), - expirationTimeMillis, null, - new LatchedActionListener<>(ActionListener.wrap( - r -> { - if (r.getExpirationTimeMillis() != expirationTimeMillis - && r.getExpirationTimeMillis() != asResponse.getExpirationTimeMillis()) { - numFailure.getAndIncrement(); - } - }, e -> { - // only version conflict from a concurrent update or RNF due to a concurrent delete is acceptable. - // rest all failures are unexpected - if (!(e instanceof VersionConflictEngineException) && !(e instanceof ResourceNotFoundException)) { - numFailure.getAndIncrement(); - } - }), latch)); + persistenceService.updateExpirationTime( + asResponse.getId(), + expirationTimeMillis, + null, + new LatchedActionListener<>(ActionListener.wrap(r -> { + if (r.getExpirationTimeMillis() != expirationTimeMillis + && r.getExpirationTimeMillis() != asResponse.getExpirationTimeMillis()) { + numFailure.getAndIncrement(); + } + }, e -> { + // only version conflict from a concurrent update or RNF due to a concurrent delete is acceptable. + // rest all failures are unexpected + if (!(e instanceof VersionConflictEngineException) && !(e instanceof ResourceNotFoundException)) { + numFailure.getAndIncrement(); + } + }), latch) + ); } else { numDeleteAttempts.getAndIncrement(); - persistenceService.deleteResponse(asResponse.getId(), null, new LatchedActionListener<>(ActionListener.wrap( - r -> { - if (r) { - numDelete.getAndIncrement(); - } else { - numFailure.getAndIncrement(); - } - }, e -> { - //only a failure due to concurrent delete causing RNF or concurrent update causing IllegalState is - // acceptable. rest all failures are unexpected - if (e instanceof ResourceNotFoundException || e instanceof IllegalStateException) { - numDeleteFailedAttempts.getAndIncrement(); - } else { - numFailure.getAndIncrement(); - } - }), latch)); + persistenceService.deleteResponse(asResponse.getId(), null, new LatchedActionListener<>(ActionListener.wrap(r -> { + if (r) { + numDelete.getAndIncrement(); + } else { + numFailure.getAndIncrement(); + } + }, e -> { + // only a failure due to concurrent delete causing RNF or concurrent update causing IllegalState is + // acceptable. rest all failures are unexpected + if (e instanceof ResourceNotFoundException || e instanceof IllegalStateException) { + numDeleteFailedAttempts.getAndIncrement(); + } else { + numFailure.getAndIncrement(); + } + }), latch)); } }); threads.add(t); @@ -490,21 +616,28 @@ public void testConcurrentUpdatesAndDeletesRace() throws InterruptedException { for (Thread t : threads) { t.join(); } - expectThrows(ResourceNotFoundException.class, () -> executeDeleteAsynchronousSearch(client(), - new DeleteAsynchronousSearchRequest(asResponse.getId())).actionGet()); + expectThrows( + ResourceNotFoundException.class, + () -> executeDeleteAsynchronousSearch(client(), new DeleteAsynchronousSearchRequest(asResponse.getId())).actionGet() + ); } private void createDoc(AsynchronousSearchPersistenceService persistenceService, AsynchronousSearchResponse asResponse, User user) - throws IOException, InterruptedException { + throws IOException, InterruptedException { CountDownLatch latch = new CountDownLatch(1); - persistenceService.storeResponse(asResponse.getId(), - new AsynchronousSearchPersistenceModel(asResponse.getStartTimeMillis(), - asResponse.getExpirationTimeMillis(), - asResponse.getSearchResponse(), null, user), - new LatchedActionListener<>( - ActionListener.wrap(r -> assertSuccessfulResponseCreation(asResponse.getId(), r), e -> { - logger.debug(() -> new ParameterizedMessage("Unexpected failure in create due to "), e); - }), latch)); + persistenceService.storeResponse( + asResponse.getId(), + new AsynchronousSearchPersistenceModel( + asResponse.getStartTimeMillis(), + asResponse.getExpirationTimeMillis(), + asResponse.getSearchResponse(), + null, + user + ), + new LatchedActionListener<>(ActionListener.wrap(r -> assertSuccessfulResponseCreation(asResponse.getId(), r), e -> { + logger.debug(() -> new ParameterizedMessage("Unexpected failure in create due to "), e); + }), latch) + ); latch.await(); } @@ -528,6 +661,7 @@ private boolean assertRnf(Runnable runnable) { return false; } } + private AsynchronousSearchId generateNewAsynchronousSearchId(TransportService transportService) { AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUIDs.base64UUID(), randomInt(100)); return new AsynchronousSearchId(transportService.getLocalNode().getId(), randomInt(100), asContextId); @@ -542,9 +676,10 @@ public void tearDown() throws Exception { @After public void deleteAsynchronousSearchIndex() throws InterruptedException { CountDownLatch deleteLatch = new CountDownLatch(1); - client().admin().indices().prepareDelete(INDEX).execute(ActionListener.wrap(r -> deleteLatch.countDown(), e -> { - deleteLatch.countDown(); - })); + client().admin() + .indices() + .prepareDelete(INDEX) + .execute(ActionListener.wrap(r -> deleteLatch.countDown(), e -> { deleteLatch.countDown(); })); deleteLatch.await(); } diff --git a/src/test/java/org/opensearch/search/asynchronous/settings/AsynchronousSearchSettingsTests.java b/src/test/java/org/opensearch/search/asynchronous/settings/AsynchronousSearchSettingsTests.java index e1322cfa..b21b5f1c 100644 --- a/src/test/java/org/opensearch/search/asynchronous/settings/AsynchronousSearchSettingsTests.java +++ b/src/test/java/org/opensearch/search/asynchronous/settings/AsynchronousSearchSettingsTests.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.settings; import org.junit.Before; @@ -18,7 +21,7 @@ import java.util.Arrays; import java.util.List; -@SuppressWarnings({"rawtypes"}) +@SuppressWarnings({ "rawtypes" }) public class AsynchronousSearchSettingsTests extends OpenSearchTestCase { AsynchronousSearchPlugin plugin; @@ -30,89 +33,96 @@ public void setup() { public void testAllLegacyOpenDistroSettingsReturned() { List> settings = plugin.getSettings(); assertTrue( - "legacy setting must be returned from settings", - settings.containsAll(Arrays.asList( - LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING, - LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING, - LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, - LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, - LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, - LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING - ))); + "legacy setting must be returned from settings", + settings.containsAll( + Arrays.asList( + LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING, + LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING, + LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, + LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, + LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, + LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING + ) + ) + ); } public void testAllOpenSearchSettingsReturned() { List> settings = plugin.getSettings(); assertTrue( - "legacy setting must be returned from settings", - settings.containsAll(Arrays.asList( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, - AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, - AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING - ))); + "legacy setting must be returned from settings", + settings.containsAll( + Arrays.asList( + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING, + AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING, + AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING, + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING + ) + ) + ); } public void testAllLegacyOpenDistroSettingsFallback() { assertEquals( - AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.get(Settings.EMPTY), - LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.get(Settings.EMPTY) + AsynchronousSearchManagementService.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.get(Settings.EMPTY), + LegacyOpendistroAsynchronousSearchSettings.ACTIVE_CONTEXT_REAPER_INTERVAL_SETTING.get(Settings.EMPTY) ); assertEquals( - AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.get(Settings.EMPTY), - LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.get(Settings.EMPTY) + AsynchronousSearchManagementService.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.get(Settings.EMPTY), + LegacyOpendistroAsynchronousSearchSettings.PERSISTED_RESPONSE_CLEAN_UP_INTERVAL_SETTING.get(Settings.EMPTY) ); assertEquals( - AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(Settings.EMPTY), - LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(Settings.EMPTY) + AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(Settings.EMPTY), + LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(Settings.EMPTY) ); assertEquals( - AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.get(Settings.EMPTY), - LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING.get(Settings.EMPTY) + AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.get(Settings.EMPTY), + LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING.get(Settings.EMPTY) ); assertEquals( - AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING.get(Settings.EMPTY), - LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING.get(Settings.EMPTY) + AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING.get(Settings.EMPTY), + LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING.get(Settings.EMPTY) ); assertEquals( - AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(Settings.EMPTY), - LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(Settings.EMPTY) + AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(Settings.EMPTY), + LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(Settings.EMPTY) ); assertEquals( - AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.get(Settings.EMPTY), - LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING.get(Settings.EMPTY) + AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.get(Settings.EMPTY), + LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING.get(Settings.EMPTY) ); } public void testSettingsGetValue() { - Settings settings = - Settings.builder().put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), "200").build(); + Settings settings = Settings.builder() + .put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), "200") + .build(); assertEquals(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(settings).intValue(), 200); assertEquals(LegacyOpendistroAsynchronousSearchSettings.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(settings).intValue(), 20); - settings = - Settings.builder().put(AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), "10d").build(); + settings = Settings.builder().put(AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.getKey(), "10d").build(); assertEquals(AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING.get(settings), TimeValue.timeValueDays(10)); assertEquals(LegacyOpendistroAsynchronousSearchSettings.MAX_KEEP_ALIVE_SETTING.get(settings), TimeValue.timeValueDays(5)); - settings = - Settings.builder().put(AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING.getKey(), "2d").build(); + settings = Settings.builder().put(AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING.getKey(), "2d").build(); assertEquals(AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING.get(settings), TimeValue.timeValueDays(2)); - assertEquals(LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING.get(settings), - TimeValue.timeValueHours(12)); + assertEquals( + LegacyOpendistroAsynchronousSearchSettings.MAX_SEARCH_RUNNING_TIME_SETTING.get(settings), + TimeValue.timeValueHours(12) + ); - settings = - Settings.builder().put(AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), "2m").build(); + settings = Settings.builder().put(AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.getKey(), "2m").build(); assertEquals(AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(settings), TimeValue.timeValueMinutes(2)); - assertEquals(LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(settings), - TimeValue.timeValueMinutes(1)); + assertEquals( + LegacyOpendistroAsynchronousSearchSettings.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING.get(settings), + TimeValue.timeValueMinutes(1) + ); - settings = - Settings.builder().put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), "true").build(); + settings = Settings.builder().put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), "true").build(); assertEquals(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.get(settings), true); assertEquals(LegacyOpendistroAsynchronousSearchSettings.PERSIST_SEARCH_FAILURES_SETTING.get(settings), false); } diff --git a/src/test/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchAssertions.java b/src/test/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchAssertions.java index d4021eff..4eff4dec 100644 --- a/src/test/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchAssertions.java +++ b/src/test/java/org/opensearch/search/asynchronous/utils/AsynchronousSearchAssertions.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.utils; import org.opensearch.action.search.SearchResponse; diff --git a/src/test/java/org/opensearch/search/asynchronous/utils/QuadConsumer.java b/src/test/java/org/opensearch/search/asynchronous/utils/QuadConsumer.java index 9ed5e098..6de8f586 100644 --- a/src/test/java/org/opensearch/search/asynchronous/utils/QuadConsumer.java +++ b/src/test/java/org/opensearch/search/asynchronous/utils/QuadConsumer.java @@ -1,11 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.utils; - /** * Represents an operation that accepts four arguments and returns no result. * diff --git a/src/test/java/org/opensearch/search/asynchronous/utils/RestTestUtils.java b/src/test/java/org/opensearch/search/asynchronous/utils/RestTestUtils.java index e5ee2576..19d3a83b 100644 --- a/src/test/java/org/opensearch/search/asynchronous/utils/RestTestUtils.java +++ b/src/test/java/org/opensearch/search/asynchronous/utils/RestTestUtils.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.utils; import org.opensearch.common.xcontent.XContentType; @@ -53,19 +56,20 @@ public static Request buildHttpRequest(SubmitAsynchronousSearchRequest submitAsy return buildHttpRequest(submitAsynchronousSearchRequest, false); } - public static Request buildHttpRequest(SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest - , boolean shouldUseLegacyApi) throws IOException { + public static Request buildHttpRequest(SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest, boolean shouldUseLegacyApi) + throws IOException { SearchRequest searchRequest = submitAsynchronousSearchRequest.getSearchRequest(); - Request request = - new Request( + Request request = new Request( HttpPost.METHOD_NAME, /*trim first backslash*/ endpoint( searchRequest.indices(), - shouldUseLegacyApi + shouldUseLegacyApi ? AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI.substring(1) - : AsynchronousSearchPlugin.BASE_URI.substring(1))); + : AsynchronousSearchPlugin.BASE_URI.substring(1) + ) + ); Params params = new Params(); addSearchRequestParams(params, searchRequest); @@ -83,41 +87,40 @@ public static Request buildHttpRequest(GetAsynchronousSearchRequest getAsynchron } public static Request buildHttpRequest(GetAsynchronousSearchRequest getAsynchronousSearchRequest, boolean shouldUseLegacyApi) { - Request request = - new Request( + Request request = new Request( HttpGet.METHOD_NAME, - shouldUseLegacyApi - ? AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI - + "/" - + getAsynchronousSearchRequest.getId() - : AsynchronousSearchPlugin.BASE_URI + "/" + getAsynchronousSearchRequest.getId()); + shouldUseLegacyApi + ? AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/" + getAsynchronousSearchRequest.getId() + : AsynchronousSearchPlugin.BASE_URI + "/" + getAsynchronousSearchRequest.getId() + ); Params params = new Params(); addGetAsynchronousSearchRequestParams(params, getAsynchronousSearchRequest); request.addParameters(params.asMap()); return request; } + public static Request buildHttpRequest(DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest) { return buildHttpRequest(deleteAsynchronousSearchRequest, false); } - public static Request buildHttpRequest( - DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest, boolean shouldUseLegacyApi) { - return new Request( - HttpDelete.METHOD_NAME, - shouldUseLegacyApi - ? AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI - + "/" - + deleteAsynchronousSearchRequest.getId() - : AsynchronousSearchPlugin.BASE_URI + "/" + deleteAsynchronousSearchRequest.getId()); - } + public static Request buildHttpRequest(DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest, boolean shouldUseLegacyApi) { + return new Request( + HttpDelete.METHOD_NAME, + shouldUseLegacyApi + ? AsynchronousSearchPlugin.LEGACY_OPENDISTRO_BASE_URI + "/" + deleteAsynchronousSearchRequest.getId() + : AsynchronousSearchPlugin.BASE_URI + "/" + deleteAsynchronousSearchRequest.getId() + ); + } private static void addGetAsynchronousSearchRequestParams(Params params, GetAsynchronousSearchRequest getAsynchronousSearchRequest) { params.withKeepAlive(getAsynchronousSearchRequest.getKeepAlive()); params.withWaitForCompletionTimeout(getAsynchronousSearchRequest.getWaitForCompletionTimeout()); } - private static void addSubmitAsynchronousSearchRequestParams(Params params, SubmitAsynchronousSearchRequest - submitAsynchronousSearchRequest) { + private static void addSubmitAsynchronousSearchRequestParams( + Params params, + SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest + ) { params.withKeepAlive(submitAsynchronousSearchRequest.getKeepAlive()); params.withWaitForCompletionTimeout(submitAsynchronousSearchRequest.getWaitForCompletionTimeout()); params.withKeepOnCompletion(submitAsynchronousSearchRequest.getKeepOnCompletion()); @@ -147,8 +150,7 @@ static void addSearchRequestParams(Params params, SearchRequest searchRequest) { } public static String endpoint(String[] indices, String endpoint) { - return new EndpointBuilder().addCommaSeparatedPathParts(indices) - .addPathPartAsIs(endpoint).build(); + return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).build(); } static class EndpointBuilder { @@ -189,13 +191,13 @@ String build() { private static String encodePart(String pathPart) { try { - //encode each part (e.g. index, type and id) separately before merging them into the path - //we prepend "/" to the path part to make this path absolute, otherwise there can be issues with - //paths that start with `-` or contain `:` - //the authority must be an empty string and not null, else paths that being with slashes could have them - //misinterpreted as part of the authority. + // encode each part (e.g. index, type and id) separately before merging them into the path + // we prepend "/" to the path part to make this path absolute, otherwise there can be issues with + // paths that start with `-` or contain `:` + // the authority must be an empty string and not null, else paths that being with slashes could have them + // misinterpreted as part of the authority. URI uri = new URI(null, "", "/" + pathPart, null, null); - //manually encode any slash that each part may contain + // manually encode any slash that each part may contain return uri.getRawPath().substring(1).replaceAll("/", "%2F"); } catch (URISyntaxException e) { throw new IllegalArgumentException("Path part [" + pathPart + "] couldn't be encoded", e); @@ -206,8 +208,7 @@ private static String encodePart(String pathPart) { static class Params { private final Map parameters = new HashMap<>(); - Params() { - } + Params() {} Params putParam(String name, String value) { if (Strings.hasLength(value)) { @@ -504,9 +505,9 @@ static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) } static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType, ToXContent.Params toXContentParams) - throws IOException { - BytesRef source = org.opensearch.core.xcontent.XContentHelper.toXContent( - toXContent, xContentType, toXContentParams, false).toBytesRef(); + throws IOException { + BytesRef source = org.opensearch.core.xcontent.XContentHelper.toXContent(toXContent, xContentType, toXContentParams, false) + .toBytesRef(); return new NByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType)); } diff --git a/src/test/java/org/opensearch/search/asynchronous/utils/TestClientUtils.java b/src/test/java/org/opensearch/search/asynchronous/utils/TestClientUtils.java index 49fcb548..47d0039f 100644 --- a/src/test/java/org/opensearch/search/asynchronous/utils/TestClientUtils.java +++ b/src/test/java/org/opensearch/search/asynchronous/utils/TestClientUtils.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.utils; import org.opensearch.commons.authuser.User; @@ -35,8 +38,7 @@ public class TestClientUtils { static final String INDEX = AsynchronousSearchPersistenceService.ASYNC_SEARCH_RESPONSE_INDEX; - static final BackoffPolicy STORE_BACKOFF_POLICY = - BackoffPolicy.exponentialBackoff(timeValueMillis(100), 20); + static final BackoffPolicy STORE_BACKOFF_POLICY = BackoffPolicy.exponentialBackoff(timeValueMillis(100), 20); public static AsynchronousSearchResponse blockingSubmitAsynchronousSearch(Client client, SubmitAsynchronousSearchRequest request) { ActionFuture execute = submitAsynchronousSearch(client, request); @@ -68,16 +70,22 @@ static ActionFuture deleteAsynchronousSearch(Client client /** * Match with submit asynchronous search response. */ - static AsynchronousSearchResponse blockingGetAsynchronousSearchResponse(Client client, AsynchronousSearchResponse submitResponse, - GetAsynchronousSearchRequest getAsynchronousSearchRequest) { + static AsynchronousSearchResponse blockingGetAsynchronousSearchResponse( + Client client, + AsynchronousSearchResponse submitResponse, + GetAsynchronousSearchRequest getAsynchronousSearchRequest + ) { AsynchronousSearchResponse getResponse = blockingGetAsynchronousSearchResponse(client, getAsynchronousSearchRequest); assert getResponse.getId().equals(submitResponse.getId()); assert getResponse.getStartTimeMillis() == submitResponse.getStartTimeMillis(); return getResponse; } - public static AsynchronousSearchResponse getFinalAsynchronousSearchResponse(Client client, AsynchronousSearchResponse submitResponse, - GetAsynchronousSearchRequest getAsynchronousSearchRequest) { + public static AsynchronousSearchResponse getFinalAsynchronousSearchResponse( + Client client, + AsynchronousSearchResponse submitResponse, + GetAsynchronousSearchRequest getAsynchronousSearchRequest + ) { AsynchronousSearchResponse getResponse; do { getResponse = blockingGetAsynchronousSearchResponse(client, submitResponse, getAsynchronousSearchRequest); @@ -123,10 +131,12 @@ public void onFailure(Exception e) { } public static User randomUser() { - return new User(OpenSearchRestTestCase.randomAlphaOfLength(10), Arrays.asList( - OpenSearchRestTestCase.randomAlphaOfLength(10), - OpenSearchRestTestCase.randomAlphaOfLength(10)), - Arrays.asList(OpenSearchRestTestCase.randomAlphaOfLength(10), "all_access"), Arrays.asList()); + return new User( + OpenSearchRestTestCase.randomAlphaOfLength(10), + Arrays.asList(OpenSearchRestTestCase.randomAlphaOfLength(10), OpenSearchRestTestCase.randomAlphaOfLength(10)), + Arrays.asList(OpenSearchRestTestCase.randomAlphaOfLength(10), "all_access"), + Arrays.asList() + ); } public static User randomUserOrNull() { diff --git a/src/test/java/org/opensearch/search/asynchronous/utils/TestUtils.java b/src/test/java/org/opensearch/search/asynchronous/utils/TestUtils.java index 13bd4996..55e0b392 100644 --- a/src/test/java/org/opensearch/search/asynchronous/utils/TestUtils.java +++ b/src/test/java/org/opensearch/search/asynchronous/utils/TestUtils.java @@ -1,8 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ - package org.opensearch.search.asynchronous.utils; import org.opensearch.action.search.SearchResponse; @@ -28,19 +31,21 @@ public class TestUtils { - public static ClusterService createClusterService(Settings settings, ThreadPool threadPool, DiscoveryNode localNode, - ClusterSettings clusterSettings) { + public static ClusterService createClusterService( + Settings settings, + ThreadPool threadPool, + DiscoveryNode localNode, + ClusterSettings clusterSettings + ) { ClusterService clusterService = new ClusterService(settings, clusterSettings, threadPool); clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService()); ClusterState initialClusterState = ClusterState.builder(new ClusterName(TestUtils.class.getSimpleName())) - .nodes(DiscoveryNodes.builder() - .add(localNode) - .localNodeId(localNode.getId()) - .clusterManagerNodeId(localNode.getId())) - .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build(); + .nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).clusterManagerNodeId(localNode.getId())) + .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK) + .build(); clusterService.getClusterApplierService().setInitialState(initialClusterState); - clusterService.getClusterManagerService().setClusterStatePublisher( - createClusterStatePublisher(clusterService.getClusterApplierService())); + clusterService.getClusterManagerService() + .setClusterStatePublisher(createClusterStatePublisher(clusterService.getClusterApplierService())); clusterService.getClusterManagerService().setClusterStateSupplier(clusterService.getClusterApplierService()::state); clusterService.start(); return clusterService; @@ -50,7 +55,10 @@ public static Map getResponseAsMap(SearchResponse searchResponse if (searchResponse != null) { BytesReference response = org.opensearch.core.xcontent.XContentHelper.toXContent( - searchResponse, Requests.INDEX_CONTENT_TYPE, true); + searchResponse, + Requests.INDEX_CONTENT_TYPE, + true + ); if (response == null) { return emptyMap(); }