Merge branch 'main' into ipfield_iodvq

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

CHANGELOG.md

@@ -103,7 +103,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add search query categorizer ([#10255](https://github.com/opensearch-project/OpenSearch/pull/10255))
 - Per request phase latency ([#10351](https://github.com/opensearch-project/OpenSearch/issues/10351))
 - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670))
-- Remove ingest processor supports excluding fields ([#10967](https://github.com/opensearch-project/OpenSearch/pull/10967))
+- Remove ingest processor supports excluding fields ([#10967](https://github.com/opensearch-project/OpenSearch/pull/10967), [#11983](https://github.com/opensearch-project/OpenSearch/pull/11983))
 - [Tiered caching] Enabling serialization for IndicesRequestCache key object ([#10275](https://github.com/opensearch-project/OpenSearch/pull/10275))
 - [Tiered caching] Defining interfaces, listeners and extending IndicesRequestCache with Tiered cache support ([#10753](https://github.com/opensearch-project/OpenSearch/pull/10753))
 - [Remote cluster state] Restore cluster state version during remote state auto restore ([#10853](https://github.com/opensearch-project/OpenSearch/pull/10853))
@@ -211,6 +211,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890))
 - Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877))
 - Updates IpField to be searchable when only `doc_values` are enabled ([#11508](https://github.com/opensearch-project/OpenSearch/pull/11508))
+- Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877)), ([#12000](https://github.com/opensearch-project/OpenSearch/pull/12000))
 - Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2 ([#11968](https://github.com/opensearch-project/OpenSearch/pull/11968))
 
 ### Deprecated

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "17.0.9+9";
+    private static final String SYSTEM_JDK_VERSION = "21.0.2+13";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.9+9";
+    private static final String GRADLE_JDK_VERSION = "21.0.2+13";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -2,7 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.9.1
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 21.0.1+12
+bundled_jdk = 21.0.2+13
 
 # optional dependencies
 spatial4j         = 0.7

modules/ingest-common/src/main/java/org/opensearch/ingest/common/RemoveProcessor.java

@@ -72,7 +72,7 @@ public final class RemoveProcessor extends AbstractProcessor {
     ) {
         super(tag, description);
         if (fields == null && excludeFields == null || fields != null && excludeFields != null) {
-            throw new IllegalArgumentException("ether fields and excludeFields must be set");
+            throw new IllegalArgumentException("either fields or excludeFields must be set");
         }
         if (fields != null) {
             this.fields = new ArrayList<>(fields);
@@ -188,7 +188,7 @@ public RemoveProcessor create(
             final Object excludeField = ConfigurationUtils.readOptionalObject(config, "exclude_field");
 
             if (field == null && excludeField == null || field != null && excludeField != null) {
-                throw newConfigurationException(TYPE, processorTag, "field", "ether field or exclude_field must be set");
+                throw newConfigurationException(TYPE, processorTag, "field", "either field or exclude_field must be set");
             }
 
             boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);

modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorFactoryTests.java

@@ -97,13 +97,13 @@ public void testCreateWithExcludeField() throws Exception {
             OpenSearchParseException.class,
             () -> factory.create(null, processorTag, null, config)
         );
-        assertThat(exception.getMessage(), equalTo("[field] ether field or exclude_field must be set"));
+        assertThat(exception.getMessage(), equalTo("[field] either field or exclude_field must be set"));
 
         Map<String, Object> config2 = new HashMap<>();
         config2.put("field", "field1");
         config2.put("exclude_field", "field2");
         exception = expectThrows(OpenSearchParseException.class, () -> factory.create(null, processorTag, null, config2));
-        assertThat(exception.getMessage(), equalTo("[field] ether field or exclude_field must be set"));
+        assertThat(exception.getMessage(), equalTo("[field] either field or exclude_field must be set"));
 
         Map<String, Object> config6 = new HashMap<>();
         config6.put("exclude_field", "exclude_field");

modules/ingest-common/src/test/java/org/opensearch/ingest/common/RemoveProcessorTests.java

@@ -203,7 +203,7 @@ public void testRemoveMetadataField() throws Exception {
 
     public void testCreateRemoveProcessorWithBothFieldsAndExcludeFields() throws Exception {
         assertThrows(
-            "ether fields and excludeFields must be set",
+            "either fields or excludeFields must be set",
             IllegalArgumentException.class,
             () -> new RemoveProcessor(randomAlphaOfLength(10), null, null, null, false)
         );
@@ -223,7 +223,7 @@ public void testCreateRemoveProcessorWithBothFieldsAndExcludeFields() throws Exc
         }
 
         assertThrows(
-            "ether fields and excludeFields must be set",
+            "either fields or excludeFields must be set",
             IllegalArgumentException.class,
             () -> new RemoveProcessor(randomAlphaOfLength(10), null, fields, excludeFields, false)
         );

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/TermsFixedDocCountErrorIT.java

@@ -23,6 +23,7 @@
 import java.util.Collection;
 
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.opensearch.index.IndexSettings.MINIMUM_REFRESH_INTERVAL;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.opensearch.search.aggregations.AggregationBuilders.terms;
 import static org.opensearch.test.OpenSearchIntegTestCase.Scope.TEST;
@@ -71,7 +72,10 @@ public void testSimpleAggErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_1").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -89,7 +93,10 @@ public void testSimpleAggErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_2").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -127,7 +134,10 @@ public void testSimpleAggErrorSingleShard() throws Exception {
         assertAcked(
             prepareCreate("idx_shard_error").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_shard_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -170,7 +180,10 @@ public void testSliceLevelDocCountErrorSingleShard() throws Exception {
         assertAcked(
             prepareCreate("idx_slice_error").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_slice_error").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -248,7 +261,10 @@ public void testSliceLevelDocCountErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_1").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_1").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();
@@ -288,7 +304,10 @@ public void testSliceLevelDocCountErrorMultiShard() throws Exception {
         assertAcked(
             prepareCreate("idx_mshard_2").setMapping(STRING_FIELD_NAME, "type=keyword")
                 .setSettings(
-                    Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", MINIMUM_REFRESH_INTERVAL)
                 )
         );
         client().prepareIndex("idx_mshard_2").setSource(jsonBuilder().startObject().field(STRING_FIELD_NAME, "A").endObject()).get();

server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java

@@ -60,6 +60,7 @@
 import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS;
 import static org.opensearch.core.common.util.CollectionUtils.iterableAsArrayList;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -789,6 +790,47 @@ public void testDefaultShardPreference() throws Exception {
         }
     }
 
+    public void testRestoreSearchableSnapshotWithIndexStoreTypeThrowsException() throws Exception {
+        final String snapshotName = "test-snap";
+        final String repoName = "test-repo";
+        final String indexName1 = "test-idx-1";
+        final int numReplicasIndex1 = randomIntBetween(1, 4);
+        final Client client = client();
+
+        internalCluster().ensureAtLeastNumDataNodes(numReplicasIndex1 + 1);
+        createIndexWithDocsAndEnsureGreen(numReplicasIndex1, 100, indexName1);
+
+        createRepositoryWithSettings(null, repoName);
+        takeSnapshot(client, snapshotName, repoName, indexName1);
+        deleteIndicesAndEnsureGreen(client, indexName1);
+
+        internalCluster().ensureAtLeastNumSearchNodes(numReplicasIndex1 + 1);
+
+        // set "index.store.type" to "remote_snapshot" in index settings of restore API and assert appropriate exception with error message
+        // is thrown.
+        final SnapshotRestoreException error = expectThrows(
+            SnapshotRestoreException.class,
+            () -> client.admin()
+                .cluster()
+                .prepareRestoreSnapshot(repoName, snapshotName)
+                .setRenamePattern("(.+)")
+                .setRenameReplacement("$1-copy")
+                .setIndexSettings(
+                    Settings.builder()
+                        .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT)
+                )
+                .setWaitForCompletion(true)
+                .execute()
+                .actionGet()
+        );
+        assertThat(
+            error.getMessage(),
+            containsString(
+                "cannot restore remote snapshot with index settings \"index.store.type\" set to \"remote_snapshot\". Instead use \"storage_type\": \"remote_snapshot\" as argument to restore."
+            )
+        );
+    }
+
     /**
      * Asserts the cache folder count to match the number of shards and the number of indices within the cache folder
      * as provided.

server/src/main/java/org/opensearch/snapshots/RestoreService.java

@@ -121,6 +121,7 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
 import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
 import static org.opensearch.common.util.set.Sets.newHashSet;
+import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING;
 import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING;
@@ -226,6 +227,16 @@ public RestoreService(
      */
     public void restoreSnapshot(final RestoreSnapshotRequest request, final ActionListener<RestoreCompletionResponse> listener) {
         try {
+            // Setting INDEX_STORE_TYPE_SETTING as REMOTE_SNAPSHOT is intended to be a system-managed index setting that is configured when
+            // restoring a snapshot and should not be manually set by user.
+            String storeTypeSetting = request.indexSettings().get(INDEX_STORE_TYPE_SETTING.getKey());
+            if (storeTypeSetting != null && storeTypeSetting.equals(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT.toString())) {
+                throw new SnapshotRestoreException(
+                    request.repository(),
+                    request.snapshot(),
+                    "cannot restore remote snapshot with index settings \"index.store.type\" set to \"remote_snapshot\". Instead use \"storage_type\": \"remote_snapshot\" as argument to restore."
+                );
+            }
             // Read snapshot info and metadata from the repository
             final String repositoryName = request.repository();
             Repository repository = repositoriesService.repository(repositoryName);

test/framework/src/main/java/org/opensearch/test/OpenSearchTestClusterRule.java

@@ -56,6 +56,8 @@
  * on the way cluster settings are being managed.
  */
 class OpenSearchTestClusterRule implements MethodRule {
+    // Maps each TestCluster instance to the exact test suite instance that triggered its creation
+    private final Map<TestCluster, OpenSearchIntegTestCase> suites = new IdentityHashMap<>();
     private final Map<Class<?>, TestCluster> clusters = new IdentityHashMap<>();
     private final Logger logger = LogManager.getLogger(getClass());
 
@@ -86,7 +88,13 @@ void afterClass() throws Exception {
                 printTestMessage("cleaning up after");
                 afterInternal(true, null);
                 OpenSearchTestCase.checkStaticState(true);
-                clusters.remove(getTestClass());
+                synchronized (clusters) {
+                    final TestCluster cluster = clusters.remove(getTestClass());
+                    IOUtils.closeWhileHandlingException(cluster);
+                    if (cluster != null) {
+                        suites.remove(cluster);
+                    }
+                }
             }
             StrictCheckSpanProcessor.validateTracingStateOnShutdown();
         } finally {
@@ -226,8 +234,11 @@ private static boolean isSuiteScopedTest(Class<?> clazz) {
         return clazz.getAnnotation(SuiteScopeTestCase.class) != null;
     }
 
-    private boolean hasParametersChanged(final ParameterizedOpenSearchIntegTestCase target) {
-        return !((ParameterizedOpenSearchIntegTestCase) suiteInstance).hasSameParametersAs(target);
+    private static boolean hasParametersChanged(
+        final ParameterizedOpenSearchIntegTestCase instance,
+        final ParameterizedOpenSearchIntegTestCase target
+    ) {
+        return !instance.hasSameParametersAs(target);
     }
 
     private boolean runTestScopeLifecycle() {
@@ -242,8 +253,24 @@ private TestCluster buildAndPutCluster(Scope currentClusterScope, long seed, Ope
             clearClusters(); // all leftovers are gone by now... this is really just a double safety if we miss something somewhere
             switch (currentClusterScope) {
                 case SUITE:
+                    if (testCluster != null && target instanceof ParameterizedOpenSearchIntegTestCase) {
+                        final OpenSearchIntegTestCase instance = suites.get(testCluster);
+                        if (instance != null) {
+                            assert instance instanceof ParameterizedOpenSearchIntegTestCase;
+                            if (hasParametersChanged(
+                                (ParameterizedOpenSearchIntegTestCase) instance,
+                                (ParameterizedOpenSearchIntegTestCase) target
+                            )) {
+                                IOUtils.closeWhileHandlingException(testCluster);
+                                printTestMessage("new instance of parameterized test class, recreating test cluster for suite");
+                                testCluster = null;
+                            }
+                        }
+                    }
+
                     if (testCluster == null) { // only build if it's not there yet
                         testCluster = buildWithPrivateContext(currentClusterScope, seed, target);
+                        suites.put(testCluster, target);
                     }
                     break;
                 case TEST:
@@ -310,6 +337,7 @@ private void clearClusters() throws Exception {
         synchronized (clusters) {
             if (!clusters.isEmpty()) {
                 IOUtils.close(clusters.values());
+                suites.clear();
                 clusters.clear();
             }
         }
@@ -363,7 +391,10 @@ private void initializeSuiteScope(OpenSearchIntegTestCase target, FrameworkMetho
             // Catching the case when parameterized test cases are run: the test class stays the same but the test instances changes.
             if (target instanceof ParameterizedOpenSearchIntegTestCase) {
                 assert suiteInstance instanceof ParameterizedOpenSearchIntegTestCase;
-                if (hasParametersChanged((ParameterizedOpenSearchIntegTestCase) target)) {
+                if (hasParametersChanged(
+                    (ParameterizedOpenSearchIntegTestCase) suiteInstance,
+                    (ParameterizedOpenSearchIntegTestCase) target
+                )) {
                     printTestMessage("new instance of parameterized test class, recreating cluster scope", method);
                     afterClass();
                     beforeClass();