Merge branch 'main' into add_esql_hash

idegtiarenko · Dec 11, 2024 · c2811b7 · c2811b7
2 parents ed4f01f + d745315
commit c2811b7
Show file tree

Hide file tree

Showing 403 changed files with 10,863 additions and 5,982 deletions.
diff --git a/...ain/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/...ain/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java
@@ -63,7 +63,7 @@ public void setup() throws IOException {
         );
         includesSet = Set.of(fetchContext.includes());
         excludesSet = Set.of(fetchContext.excludes());
-        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false);
+        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, includesSet, excludesSet, false);
     }
 
     private BytesReference read300BytesExample() throws IOException {

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java
@@ -170,7 +170,7 @@ private XContentParserConfiguration buildParseConfig(boolean matchDotsInFieldNam
             includes = null;
             excludes = filters;
         }
-        return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames);
+        return XContentParserConfiguration.EMPTY.withFiltering(null, includes, excludes, matchDotsInFieldNames);
     }
 
     private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException {

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -13,8 +13,8 @@
  * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch.
  */
 public enum DockerBase {
-    // "latest" here is intentional, since the image name specifies "8"
-    DEFAULT("docker.elastic.co/ubi8/ubi-minimal:latest", "", "microdnf"),
+    // "latest" here is intentional, since the image name specifies "9"
+    DEFAULT("docker.elastic.co/ubi9/ubi-minimal:latest", "", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),

diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.gradle.test.GradleTestPolicySetupPlugin;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.gradle.api.Action;
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
@@ -112,7 +113,6 @@ public void execute(Task t) {
             test.jvmArgs(
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
-                "-Djava.security.manager=allow",
                 "-Dtests.testfeatures.enabled=true",
                 "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
@@ -127,6 +127,13 @@ public void execute(Task t) {
             );
 
             test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir));
+            test.getJvmArgumentProviders().add(() -> {
+                if (test.getJavaVersion().compareTo(JavaVersion.VERSION_23) <= 0) {
+                    return List.of("-Djava.security.manager=allow");
+                } else {
+                    return List.of();
+                }
+            });
 
             String argline = System.getProperty("tests.jvm.argline");
             if (argline != null) {

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java
@@ -9,11 +9,14 @@
 
 package org.elasticsearch.gradle.test;
 
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.invocation.Gradle;
 import org.gradle.api.tasks.testing.Test;
 
+import java.util.List;
+
 public class GradleTestPolicySetupPlugin implements Plugin<Project> {
 
     @Override
@@ -23,8 +26,13 @@ public void apply(Project project) {
             test.systemProperty("tests.gradle", true);
             test.systemProperty("tests.task", test.getPath());
 
-            // Flag is required for later Java versions since our tests use a custom security manager
-            test.jvmArgs("-Djava.security.manager=allow");
+            test.getJvmArgumentProviders().add(() -> {
+                if (test.getJavaVersion().compareTo(JavaVersion.VERSION_23) <= 0) {
+                    return List.of("-Djava.security.manager=allow");
+                } else {
+                    return List.of();
+                }
+            });
 
             SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider();
             // don't track these as inputs since they contain absolute paths and break cache relocatability

diff --git a/distribution/build.gradle b/distribution/build.gradle
@@ -70,10 +70,10 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) {
   // Explicitly add the dependency on the RHEL UBI Docker base image
   String[] rhelUbiFields = [
     'Red Hat Universal Base Image minimal',
-    '8',
-    'https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8',
+    '9',
+    'https://catalog.redhat.com/software/containers/ubi9-minimal/61832888c0d15aff4912fe0d',
     'Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf',
-    'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/8/ubi-minimal-8-source.tar.gz'
+    'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/9/ubi-minimal-9-source.tar.gz'
   ]
   additionalLines << rhelUbiFields.join(',')
 }

diff --git a/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -11,6 +11,8 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.jdk.RuntimeVersionFeature;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -137,9 +139,13 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         return Stream.of();
     }
 
+    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
     private static Stream<String> maybeAllowSecurityManager() {
-        // Will become conditional on useEntitlements once entitlements can run without SM
-        return Stream.of("-Djava.security.manager=allow");
+        if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
+            // Will become conditional on useEntitlements once entitlements can run without SM
+            return Stream.of("-Djava.security.manager=allow");
+        }
+        return Stream.of();
     }
 
     private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {

diff --git a/docs/changelog/113827.yaml b/docs/changelog/113827.yaml
@@ -0,0 +1,5 @@
+pr: 113827
+summary: Add Optional Source Filtering to Source Loaders
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/114618.yaml b/docs/changelog/114618.yaml
@@ -0,0 +1,5 @@
+pr: 114618
+summary: Add a new index setting to skip recovery source when synthetic source is enabled
+area: Logs
+type: enhancement
+issues: []
diff --git a/docs/changelog/116663.yaml b/docs/changelog/116663.yaml
@@ -0,0 +1,5 @@
+pr: 116663
+summary: KNN vector rescoring for quantized vectors
+area: Vector Search
+type: feature
+issues: []
diff --git a/docs/changelog/117469.yaml b/docs/changelog/117469.yaml
@@ -0,0 +1,6 @@
+pr: 117469
+summary: Handle exceptions in query phase can match
+area: Search
+type: bug
+issues:
+ - 104994
diff --git a/docs/changelog/117939.yaml b/docs/changelog/117939.yaml
@@ -0,0 +1,5 @@
+pr: 117939
+summary: Adding default endpoint for Elastic Rerank
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/118025.yaml b/docs/changelog/118025.yaml
@@ -0,0 +1,5 @@
+pr: 118025
+summary: Update sparse text embeddings API route for Inference Service
+area: Inference
+type: enhancement
+issues: []
diff --git a/docs/changelog/118104.yaml b/docs/changelog/118104.yaml
@@ -0,0 +1,12 @@
+pr: 118104
+summary: Remove old `_knn_search` tech preview API in v9
+area: Vector Search
+type: breaking
+issues: []
+breaking:
+  title: Remove old `_knn_search` tech preview API in v9
+  area: REST API
+  details: The original, tech-preview api for vector search, `_knn_search`, has been removed in v9. For all vector search
+    operations, you should utilize the `_search` endpoint.
+  impact: The `_knn_search` API is now inaccessible without providing a compatible-with flag for v8.
+  notable: false
diff --git a/docs/changelog/118177.yaml b/docs/changelog/118177.yaml
@@ -0,0 +1,6 @@
+pr: 118177
+summary: Fixing bedrock event executor terminated cache issue
+area: Machine Learning
+type: bug
+issues:
+ - 117916
diff --git a/docs/changelog/118267.yaml b/docs/changelog/118267.yaml
@@ -0,0 +1,5 @@
+pr: 118267
+summary: Adding get migration reindex status
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/118354.yaml b/docs/changelog/118354.yaml
@@ -0,0 +1,5 @@
+pr: 118354
+summary: Fix log message format bugs
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/118370.yaml b/docs/changelog/118370.yaml
@@ -0,0 +1,6 @@
+pr: 118370
+summary: Fix concurrency issue with `ReinitializingSourceProvider`
+area: Mapping
+type: bug
+issues:
+ - 118238
diff --git a/docs/changelog/118378.yaml b/docs/changelog/118378.yaml
@@ -0,0 +1,5 @@
+pr: 118378
+summary: Opt into extra data stream resolution
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/118380.yaml b/docs/changelog/118380.yaml
@@ -0,0 +1,5 @@
+pr: 118380
+summary: Restore original "is within leaf" value in `SparseVectorFieldMapper`
+area: Mapping
+type: bug
+issues: []
diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc
@@ -475,7 +475,7 @@ The input is untokenized text and the result is the single term attribute emitte
 - 영영칠 -> 7
 - 일영영영 -> 1000
 - 삼천2백2십삼 -> 3223
-- 조육백만오천일 -> 1000006005001
+- 일조육백만오천일 -> 1000006005001
 - ３.２천 ->  3200
 - １.２만３４５.６７ -> 12345.67
 - 4,647.100 -> 4647.1

diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
@@ -257,6 +257,10 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh]
 (Optional, Boolean) If `true`, the request's actions must target an index alias.
 Defaults to `false`.
 
+`require_data_stream`::
+(Optional, Boolean) If `true`, the request's actions must target a data stream (existing or to-be-created).
+Defaults to `false`.
+
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source]

diff --git a/docs/reference/inference/service-openai.asciidoc b/docs/reference/inference/service-openai.asciidoc
@@ -76,6 +76,12 @@ https://platform.openai.com/api-keys[API keys section].
 include::inference-shared.asciidoc[tag=api-key-admonition]
 --
 
+`dimensions`:::
+(Optional, integer)
+The number of dimensions the resulting output embeddings should have.
+Only supported in `text-embedding-3` and later models.
+If not set the OpenAI defined default for the model is used.
+
 `model_id`:::
 (Required, string)
 The name of the model to use for the {infer} task.
@@ -134,8 +140,8 @@ Specifies the user issuing the request, which can be used for abuse detection.
 [[inference-example-openai]]
 ==== OpenAI service example
 
-The following example shows how to create an {infer} endpoint called
-`openai-embeddings` to perform a `text_embedding` task type.
+The following example shows how to create an {infer} endpoint called `openai-embeddings` to perform a `text_embedding` task type.
+The embeddings created by requests to this endpoint will have 128 dimensions.
 
 [source,console]
 ------------------------------------------------------------
@@ -144,14 +150,14 @@ PUT _inference/text_embedding/openai-embeddings
     "service": "openai",
     "service_settings": {
         "api_key": "<api_key>",
-        "model_id": "text-embedding-ada-002"
+        "model_id": "text-embedding-3-small",
+        "dimensions": 128
     }
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-The next example shows how to create an {infer} endpoint called
-`openai-completion` to perform a `completion` task type.
+The next example shows how to create an {infer} endpoint called `openai-completion` to perform a `completion` task type.
 
 [source,console]
 ------------------------------------------------------------

diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -232,8 +232,8 @@ it will be set to the length of the first vector added to the field.
 
 `index`::
 (Optional, Boolean)
-If `true`, you can search this field using the <<knn-search-api, kNN search
-API>>. Defaults to `true`.
+If `true`, you can search this field using the <<query-dsl-knn-query, knn query>>
+or <<search-api-knn, knn in _search>> . Defaults to `true`.
 
 [[dense-vector-similarity]]
 `similarity`::

diff --git a/docs/reference/migration/migrate_9_0.asciidoc b/docs/reference/migration/migrate_9_0.asciidoc
@@ -244,6 +244,25 @@ The deprecated highlighting `force_source` parameter is no longer supported.
 Users should remove usages of the `force_source` parameter from their search requests.
 ====
 
+[discrete]
+[[breaking_90_transforms_changes]]
+==== {transforms-cap} changes
+
+[[updating_deprecated_transform_roles]]
+.Updating deprecated {transform} roles (`data_frame_transforms_admin` and `data_frame_transforms_user`)
+[%collapsible]
+====
+*Details* +
+The `data_frame_transforms_admin` and `data_frame_transforms_user` {transform} roles have been deprecated.
+
+*Impact* +
+Users must update any existing {transforms} that use deprecated {transform} roles (`data_frame_transforms_admin` or `data_frame_transforms_user`) to use the new equivalent {transform} roles (`transform_admin` or `transform_user`).
+To update the {transform} roles:
+
+1. Switch to a user with the `transform_admin` role (to replace `data_frame_transforms_admin`) or the `transform_user` role (to replace `data_frame_transforms_user`).
+2. Call the <<update-transform, update {transforms} API>> with that user.
+====
+
 
 [discrete]
 [[deprecated-9.0]]

diff --git a/docs/reference/migration/migrate_9_0/transforms-migration-guide.asciidoc b/docs/reference/migration/migrate_9_0/transforms-migration-guide.asciidoc
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
@@ -1942,3 +1942,8 @@ Refer to <<get-ip-location-database-api>>.
 === Delete geoip database configuration API
 
 Refer to <<delete-ip-location-database-api>>.
+
+[role="exclude",id="knn-search-api"]
+=== Delete _knn_search API
+
+Refer to <<search-api-knn>>.
diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc
@@ -50,8 +50,6 @@ include::search/async-search.asciidoc[]
 
 include::search/point-in-time-api.asciidoc[]
 
-include::search/knn-search.asciidoc[]
-
 include::search/retriever.asciidoc[]
 
 include::search/rrf.asciidoc[]