Merge branch 'main' into lib-toml

opensearch-project · Oct 16, 2024 · b9e799d · b9e799d
2 parents 8e5eb4b + 23d1c7a
commit b9e799d
Show file tree

Hide file tree

Showing 59 changed files with 3,572 additions and 790 deletions.
diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json
@@ -221,5 +221,23 @@
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+  },
+  "id_14": {
+    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0. Enables range query approximation.",
+    "supported_major_versions": ["3"],
+    "cluster-benchmark-configs": {
+      "SINGLE_NODE_CLUSTER": "true",
+      "MIN_DISTRIBUTION": "true",
+      "TEST_WORKLOAD": "big5",
+      "ADDITIONAL_CONFIG": "opensearch.experimental.feature.approximate_point_range_query.enabled:true",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
+      "CAPTURE_NODE_STAT": "true",
+      "TEST_PROCEDURE": "restore-from-snapshot"
+    },
+    "cluster_configuration": {
+      "size": "Single-Node",
+      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   }
 }
diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v1.10.0
+        uses: lycheeverse/lychee-action@v2.0.2
         with:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,11 +19,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718))
 - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976))
 - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611))
+- Add support to dynamically resize threadpools size. ([#16236](https://github.com/opensearch-project/OpenSearch/pull/16236))
 - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978))
 - Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986))
 - New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915))
 - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641))
 - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993))
+- Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383))
 - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284))
 
 ### Dependencies
@@ -33,7 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863))
 - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862))
 - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945))
-- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946))
+- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.10 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307))
 - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011))
 - Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040))
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038))
@@ -42,20 +44,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042))
 - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137))
 - Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333))
-- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
+- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.5 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133), [#16311](https://github.com/opensearch-project/OpenSearch/pull/16311))
 - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
 - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182))
 - Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216))
 - Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217))
 - Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213))
 - Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212))
+- Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309))
+- Bump `com.google.oauth-client:google-oauth-client` from 1.35.0 to 1.36.0 ([#16306](https://github.com/opensearch-project/OpenSearch/pull/16306))
+- Bump `lycheeverse/lychee-action` from 1.10.0 to 2.0.2 ([#16310](https://github.com/opensearch-project/OpenSearch/pull/16310))
+- Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#16308](https://github.com/opensearch-project/OpenSearch/pull/16308))
 
 ### Changed
 - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
 - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430))
 - Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024))
 - Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154))
-
+- Enable coordinator search.request_stats_enabled by default ([#16290](https://github.com/opensearch-project/OpenSearch/pull/16290))
+- Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273))
+- Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296))
+- Update last seen cluster state in the commit phase ([#16215](https://github.com/opensearch-project/OpenSearch/pull/16215))
 
 ### Deprecated
 
@@ -68,10 +77,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988))
 - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985))
 - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931))
+- Fix update settings with null replica not honoring cluster setting bug ([#14948](https://github.com/opensearch-project/OpenSearch/pull/14948))
 - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521))
 - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
 - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))
 - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254))
+- Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265))
 
 ### Security
 

diff --git a/...ng-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java b/...ng-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.script.mustache;
 
+import org.opensearch.ExceptionsHelper;
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.search.MultiSearchResponse;
 import org.opensearch.common.Nullable;
@@ -167,6 +168,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
             if (item.isFailure()) {
                 builder.startObject();
                 OpenSearchException.generateFailureXContent(builder, params, item.getFailure(), true);
+                builder.field(Fields.STATUS, ExceptionsHelper.status(item.getFailure()).getStatus());
                 builder.endObject();
             } else {
                 item.getResponse().toXContent(builder, params);
@@ -179,6 +181,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
 
     static final class Fields {
         static final String RESPONSES = "responses";
+        static final String STATUS = "status";
     }
 
     public static MultiSearchTemplateResponse fromXContext(XContentParser parser) {

diff --git a/...es/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java b/...es/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java
@@ -120,7 +120,10 @@ public static SearchTemplateResponse fromXContent(XContentParser parser) throws
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         if (hasResponse()) {
-            response.toXContent(builder, params);
+            builder.startObject();
+            response.innerToXContent(builder, params);
+            builder.field(MultiSearchTemplateResponse.Fields.STATUS, response.status().getStatus());
+            builder.endObject();
         } else {
             builder.startObject();
             // we can assume the template is always json as we convert it before compiling it

diff --git a/...stache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateResponseTests.java b/...stache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateResponseTests.java
@@ -34,8 +34,12 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.ShardSearchFailure;
+import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.Strings;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.search.internal.InternalSearchResponse;
 import org.opensearch.test.AbstractXContentTestCase;
@@ -44,6 +48,7 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
@@ -177,4 +182,78 @@ public void testFromXContentWithFailures() throws IOException {
             ToXContent.EMPTY_PARAMS
         );
     }
+
+    public void testToXContentWithFailures() throws Exception {
+        long overallTookInMillis = randomNonNegativeLong();
+        MultiSearchTemplateResponse.Item[] items = new MultiSearchTemplateResponse.Item[2];
+
+        long tookInMillis = 1L;
+        int totalShards = 2;
+        int successfulShards = 2;
+        int skippedShards = 0;
+        InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
+        SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse();
+        SearchResponse searchResponse = new SearchResponse(
+            internalSearchResponse,
+            null,
+            totalShards,
+            successfulShards,
+            skippedShards,
+            tookInMillis,
+            ShardSearchFailure.EMPTY_ARRAY,
+            SearchResponse.Clusters.EMPTY
+        );
+        searchTemplateResponse.setResponse(searchResponse);
+        items[0] = new MultiSearchTemplateResponse.Item(searchTemplateResponse, null);
+
+        items[1] = new MultiSearchTemplateResponse.Item(null, new IllegalArgumentException("Invalid argument"));
+
+        MultiSearchTemplateResponse response = new MultiSearchTemplateResponse(items, overallTookInMillis);
+
+        XContentType contentType = randomFrom(XContentType.values());
+        XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType)
+            .startObject()
+            .field("took", overallTookInMillis)
+            .startArray("responses")
+            .startObject()
+            .field("took", 1)
+            .field("timed_out", false)
+            .startObject("_shards")
+            .field("total", 2)
+            .field("successful", 2)
+            .field("skipped", 0)
+            .field("failed", 0)
+            .endObject()
+            .startObject("hits")
+            .startObject("total")
+            .field("value", 0)
+            .field("relation", "eq")
+            .endObject()
+            .field("max_score", 0.0F)
+            .startArray("hits")
+            .endArray()
+            .endObject()
+            .field("status", 200)
+            .endObject()
+            .startObject()
+            .startObject("error")
+            .field("type", "illegal_argument_exception")
+            .field("reason", "Invalid argument")
+            .startArray("root_cause")
+            .startObject()
+            .field("type", "illegal_argument_exception")
+            .field("reason", "Invalid argument")
+            .endObject()
+            .endArray()
+            .endObject()
+            .field("status", 400)
+            .endObject()
+            .endArray()
+            .endObject();
+
+        XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType);
+        response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS);
+
+        assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType);
+    }
 }
diff --git a/...ng-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java b/...ng-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java
@@ -234,6 +234,7 @@ public void testSearchResponseToXContent() throws IOException {
             .endObject()
             .endArray()
             .endObject()
+            .field("status", 200)
             .endObject();
 
         XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType);

diff --git a/.../src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/.../src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml
@@ -205,3 +205,51 @@ setup:
   - match:  { responses.1.hits.total.relation:  eq }
   - match:  { responses.2.hits.total.value:     1  }
   - match:  { responses.1.hits.total.relation:  eq }
+
+---
+"Test multi-search template returns status code":
+  - skip:
+      version: " - 2.17.99"
+      reason: Fixed in 2.18.0.
+  - do:
+      msearch_template:
+        rest_total_hits_as_int: true
+        body:
+          # Search 0 is OK
+          - index: index_*
+          - source: '{"query": {"match": {"foo": "{{value}}"} } }'
+            params:
+              value: "foo"
+          # Search 1 has an unclosed JSON template
+          - index: index_*
+          - source: '{"query": {"match": {'
+            params:
+              field: "foo"
+              value: "bar"
+          # Search 2 is OK
+          - index: _all
+          - source:
+              query:
+                match: {foo: "{{text}}"}
+            params:
+              text: "baz"
+          # Search 3 has an unknown query type
+          - index: index_*
+          - source: '{"query": {"{{query_type}}": {} }' # Unknown query type
+            params:
+              query_type: "unknown"
+          # Search 4 has an unsupported track_total_hits
+          - index: index_*
+          - source: '{"query": {"match_all": {} }, "track_total_hits": "{{trackTotalHits}}" }'
+            params:
+              trackTotalHits: 1
+          # Search 5 has an unknown index
+          - index: unknown_index
+          - source: '{"query": {"match_all": {} } }'
+
+  - match:  { responses.0.status:     200  }
+  - match:  { responses.1.status:     400  }
+  - match:  { responses.2.status:     200  }
+  - match:  { responses.3.status:     400  }
+  - match:  { responses.4.status:     400  }
+  - match:  { responses.5.status:     404  }
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
@@ -20,7 +20,7 @@ opensearchplugin {
 dependencies {
   api "com.google.apis:google-api-services-compute:v1-rev20240407-2.0.0"
   api "com.google.api-client:google-api-client:1.35.2"
-  api "com.google.oauth-client:google-oauth-client:1.35.0"
+  api "com.google.oauth-client:google-oauth-client:1.36.0"
   api "com.google.http-client:google-http-client:${versions.google_http_client}"
   api "com.google.http-client:google-http-client-gson:${versions.google_http_client}"
   api "com.google.http-client:google-http-client-jackson2:${versions.google_http_client}"

diff --git a/plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1 b/plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1
diff --git a/plugins/discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1 b/plugins/discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1
@@ -0,0 +1 @@
+dc3f07bc8f49dd52fe8fcc15958f3cfeb003e20f
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
@@ -48,7 +48,7 @@ dependencies {
   api 'com.azure:azure-json:1.3.0'
   api 'com.azure:azure-xml:1.1.0'
   api 'com.azure:azure-storage-common:12.25.1'
-  api 'com.azure:azure-core-http-netty:1.15.4'
+  api 'com.azure:azure-core-http-netty:1.15.5'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"

diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1
diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1
@@ -0,0 +1 @@
+44d99705d3759e2ad7ee8110f811d4ed304a6a7c
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
@@ -70,7 +70,7 @@ dependencies {
   api 'com.google.cloud:google-cloud-core-http:2.23.0'
   api 'com.google.cloud:google-cloud-storage:1.113.1'
 
-  api 'com.google.code.gson:gson:2.10.1'
+  api 'com.google.code.gson:gson:2.11.0'
 
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api 'com.google.guava:failureaccess:1.0.1'

diff --git a/plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1 b/plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1
diff --git a/plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1 b/plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1
@@ -0,0 +1 @@
+527175ca6d81050b53bdd4c457a6d6e017626b0e
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml
@@ -671,38 +671,6 @@ teardown:
   - match: { error.root_cause.0.reason: "Mapping definition for [data] has unsupported parameters:  [analyzer : standard]"}
   - match: { status: 400 }
 
-  # Wildcard Query with dot path.
-  - do:
-      catch: bad_request
-      search:
-        body: {
-          _source: true,
-          query: {
-            "wildcard": {
-              "catalog.title": "Mock*"
-            }
-          }
-        }
-  - match: { error.root_cause.0.type: "query_shard_exception" }
-  - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog.title] which is of type [flat_object]"}
-  - match: { status: 400 }
-
-  # Wildcard Query without dot path.
-  - do:
-      catch: bad_request
-      search:
-        body: {
-          _source: true,
-          query: {
-            "wildcard": {
-              "catalog": "Mock*"
-            }
-          }
-        }
-  - match: { error.root_cause.0.type: "query_shard_exception" }
-  - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog] which is of type [flat_object]" }
-  - match: { status: 400 }
-
   # Aggregation and Match Query with dot path.
   - do:
       catch: bad_request