Merge remote-tracking branch 'upstream/main' into fix-issue-15989

Signed-off-by: Ashish Singh <[email protected]>
opensearch-project · Sep 25, 2024 · 94f998e · 94f998e
2 parents d32dea3 + dc4dbce
commit 94f998e
Show file tree

Hide file tree

Showing 26 changed files with 1,120 additions and 253 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,8 +23,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946))
 - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011))
 - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039))
+- Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041))
 
 ### Changed
+- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
 
 
 ### Deprecated
@@ -35,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737))
 - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882))
 - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501))
+- Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988))
 - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985))
 - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931))
 

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java
@@ -106,6 +106,7 @@ public DockerAvailability getDockerAvailability() {
             Version version = null;
             boolean isVersionHighEnough = false;
             boolean isComposeAvailable = false;
+            boolean isComposeV2Available = false;
 
             // Check if the Docker binary exists
             final Optional<String> dockerBinary = getDockerPath();
@@ -129,6 +130,8 @@ public DockerAvailability getDockerAvailability() {
                         if (lastResult.isSuccess() && composePath.isPresent()) {
                             isComposeAvailable = runCommand(composePath.get(), "version").isSuccess();
                         }
+
+                        isComposeV2Available = runCommand(dockerPath, "compose", "version").isSuccess();
                     }
                 }
             }
@@ -138,6 +141,7 @@ public DockerAvailability getDockerAvailability() {
             this.dockerAvailability = new DockerAvailability(
                 isAvailable,
                 isComposeAvailable,
+                isComposeV2Available,
                 isVersionHighEnough,
                 dockerPath,
                 version,
@@ -356,6 +360,11 @@ public static class DockerAvailability {
          */
         public final boolean isComposeAvailable;
 
+        /**
+         * True if docker compose is available.
+         */
+        public final boolean isComposeV2Available;
+
         /**
          * True if the installed Docker version is &gt;= 17.05
          */
@@ -379,13 +388,15 @@ public static class DockerAvailability {
         DockerAvailability(
             boolean isAvailable,
             boolean isComposeAvailable,
+            boolean isComposeV2Available,
             boolean isVersionHighEnough,
             String path,
             Version version,
             Result lastCommand
         ) {
             this.isAvailable = isAvailable;
             this.isComposeAvailable = isComposeAvailable;
+            this.isComposeV2Available = isComposeV2Available;
             this.isVersionHighEnough = isVersionHighEnough;
             this.path = path;
             this.version = version;

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java
@@ -171,7 +171,11 @@ public void execute(Task task) {
                 .findFirst();
 
             composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker");
-            composeExtension.getUseDockerComposeV2().set(false);
+            if (dockerSupport.get().getDockerAvailability().isComposeV2Available) {
+                composeExtension.getUseDockerComposeV2().set(true);
+            } else if (dockerSupport.get().getDockerAvailability().isComposeAvailable) {
+                composeExtension.getUseDockerComposeV2().set(false);
+            }
 
             tasks.named("composeUp").configure(t -> {
                 // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
@@ -228,7 +232,8 @@ private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskPro
 
     private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, Task task) {
         task.onlyIf(spec -> {
-            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeAvailable;
+            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeV2Available
+                || dockerSupport.get().getDockerAvailability().isComposeAvailable;
             if (isComposeAvailable == false) {
                 LOGGER.info("Task {} requires docker-compose but it is unavailable. Task will be skipped.", task.getPath());
             }

diff --git a/...s/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/...s/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java
@@ -264,7 +264,15 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) {
             }
             ft.setPrefixField(prefixFieldType);
             ft.setShingleFields(shingleFieldTypes);
-            return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this);
+            return new SearchAsYouTypeFieldMapper(
+                name,
+                ft,
+                multiFieldsBuilder.build(this, context),
+                copyTo.build(),
+                prefixFieldMapper,
+                shingleFieldMappers,
+                this
+            );
         }
     }
 
@@ -623,12 +631,13 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew
     public SearchAsYouTypeFieldMapper(
         String simpleName,
         SearchAsYouTypeFieldType mappedFieldType,
+        MultiFields multiFields,
         CopyTo copyTo,
         PrefixFieldMapper prefixField,
         ShingleFieldMapper[] shingleFields,
         Builder builder
     ) {
-        super(simpleName, mappedFieldType, MultiFields.empty(), copyTo);
+        super(simpleName, mappedFieldType, multiFields, copyTo);
         this.prefixField = prefixField;
         this.shingleFields = shingleFields;
         this.maxShingleSize = builder.maxShingleSize.getValue();

diff --git a/...per-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/...per-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
@@ -298,6 +298,20 @@ private void assertMultiField(int shingleSize) throws IOException {
         }
     }
 
+    public void testSubField() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(
+                b -> b.field("type", "search_as_you_type")
+                    .startObject("fields")
+                    .startObject("subField")
+                    .field("type", "keyword")
+                    .endObject()
+                    .endObject()
+            )
+        );
+        assertThat(mapperService.fieldType("field.subField"), instanceOf(KeywordFieldMapper.KeywordFieldType.class));
+    }
+
     public void testIndexOptions() throws IOException {
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(b -> b.field("type", "search_as_you_type").field("index_options", "offsets"))

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml
@@ -76,7 +76,7 @@ teardown:
 ---
 "Invalid docs":
   - skip:
-      version: "- 2.99.99"
+      version: "- 2.17.99"
       reason: "parsing of these objects would infinite loop prior to 2.18"
   # The following documents are invalid.
   - do:

diff --git a/...spec/test/search/370_bitmap_filtering.yml → ...spec/test/search/380_bitmap_filtering.yml b/...spec/test/search/370_bitmap_filtering.yml → ...spec/test/search/380_bitmap_filtering.yml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml
@@ -0,0 +1,75 @@
+setup:
+  - do:
+      indices.create:
+        index:  test_1
+        body:
+          mappings:
+            properties:
+              text:
+                type: search_as_you_type
+                fields:
+                  subField:
+                    type: keyword
+  - do:
+      index:
+        index:  test_1
+        id:     1
+        body:   { text: test search as you type }
+
+  - do:
+      indices.refresh:
+        index: [test_1]
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: test_1
+
+# related issue: https://github.com/opensearch-project/OpenSearch/issues/5035
+---
+"Test search_as_you_type data type supports multi-fields":
+  - skip:
+      version: " - 2.99.99"
+      reason: "the bug was fixed since 3.0.0"
+
+  - do:
+      indices.get_mapping: {
+        index: test_1
+      }
+
+  - match: {test_1.mappings.properties.text.type:     search_as_you_type}
+  - match: {test_1.mappings.properties.text.fields.subField.type:     keyword}
+
+  - do:
+      search:
+        index: test_1
+        body:
+          query:
+            multi_match:
+              query: "test search"
+              type: "bool_prefix"
+
+  - match: {hits.total.value: 1}
+
+  - do:
+      search:
+        index: test_1
+        body:
+          query:
+            multi_match:
+              query: "test search"
+              type: "bool_prefix"
+              fields: ["text.subField"]
+
+  - match: {hits.total.value: 1}
+
+  - do:
+      search:
+        index: test_1
+        body:
+          query:
+            term:
+              text.subField: "test search as you type"
+
+  - match: {hits.total.value: 1}
diff --git a/...ClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/...ClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
@@ -12,6 +12,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
@@ -68,15 +69,17 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode()
         assertRemoteStoreBackedIndex(indexName2);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15793")
     public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception {
         logger.info("Initialize cluster: gives non remote cluster manager");
         initializeCluster(false);
 
         logger.info("Add remote and non-remote nodes");
         setClusterMode(MIXED.mode);
         addRemote = false;
-        String nonRemoteNodeName = internalCluster().startNode();
+        Settings settings = Settings.builder()
+            .put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : "")
+            .build();
+        String nonRemoteNodeName = internalCluster().startNode(settings);
         addRemote = true;
         String remoteNodeName = internalCluster().startNode();
         internalCluster().validateClusterFormed();

diff --git a/...internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/...internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java
@@ -42,6 +42,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Before;
 
+import java.io.IOError;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -339,10 +340,11 @@ public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathTh
             for (UploadedIndexMetadata md : manifest.getIndices()) {
                 Files.move(segmentRepoPath.resolve(md.getUploadedFilename()), segmentRepoPath.resolve("cluster-state/"));
             }
+            internalCluster().stopAllNodes();
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-        assertThrows(IllegalStateException.class, () -> addNewNodes(dataNodeCount, clusterManagerNodeCount));
+        assertThrows(IOError.class, () -> internalCluster().client());
         // Test is complete
 
         // Starting a node without remote state to ensure test cleanup