Merge branch 'main' into feature/knn-vector-rescore-query

# Conflicts:
#	server/src/main/java/org/elasticsearch/TransportVersions.java

.buildkite/pipelines/periodic-packaging.template.yml

@@ -7,19 +7,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8

.buildkite/pipelines/periodic-packaging.yml

@@ -8,19 +8,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8

.buildkite/pipelines/periodic-platform-support.yml

@@ -7,19 +7,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8
@@ -90,7 +85,6 @@ steps:
           setup:
             image:
               - amazonlinux-2023
-              - amazonlinux-2
         agents:
           provider: aws
           imagePrefix: elasticsearch-{{matrix.image}}

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -10,19 +10,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -13,8 +13,8 @@
  * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch.
  */
 public enum DockerBase {
-    // "latest" here is intentional, since the image name specifies "8"
-    DEFAULT("docker.elastic.co/ubi8/ubi-minimal:latest", "", "microdnf"),
+    // "latest" here is intentional, since the image name specifies "9"
+    DEFAULT("docker.elastic.co/ubi9/ubi-minimal:latest", "", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.gradle.test.GradleTestPolicySetupPlugin;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.gradle.api.Action;
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
@@ -112,7 +113,6 @@ public void execute(Task t) {
             test.jvmArgs(
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
-                "-Djava.security.manager=allow",
                 "-Dtests.testfeatures.enabled=true",
                 "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
@@ -127,6 +127,13 @@ public void execute(Task t) {
             );
 
             test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir));
+            test.getJvmArgumentProviders().add(() -> {
+                if (test.getJavaVersion().compareTo(JavaVersion.VERSION_23) <= 0) {
+                    return List.of("-Djava.security.manager=allow");
+                } else {
+                    return List.of();
+                }
+            });
 
             String argline = System.getProperty("tests.jvm.argline");
             if (argline != null) {

build-tools-internal/version.properties

@@ -35,7 +35,7 @@ commonscodec      = 1.15
 protobuf          = 3.25.5
 
 # test dependencies
-randomizedrunner  = 2.8.0
+randomizedrunner  = 2.8.2
 junit             = 4.13.2
 junit5            = 5.7.1
 hamcrest          = 2.1

build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java

@@ -9,11 +9,14 @@
 
 package org.elasticsearch.gradle.test;
 
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.invocation.Gradle;
 import org.gradle.api.tasks.testing.Test;
 
+import java.util.List;
+
 public class GradleTestPolicySetupPlugin implements Plugin<Project> {
 
     @Override
@@ -23,8 +26,13 @@ public void apply(Project project) {
             test.systemProperty("tests.gradle", true);
             test.systemProperty("tests.task", test.getPath());
 
-            // Flag is required for later Java versions since our tests use a custom security manager
-            test.jvmArgs("-Djava.security.manager=allow");
+            test.getJvmArgumentProviders().add(() -> {
+                if (test.getJavaVersion().compareTo(JavaVersion.VERSION_23) <= 0) {
+                    return List.of("-Djava.security.manager=allow");
+                } else {
+                    return List.of();
+                }
+            });
 
             SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider();
             // don't track these as inputs since they contain absolute paths and break cache relocatability

dev-tools/publish_zstd_binaries.sh

@@ -79,8 +79,8 @@ build_linux_jar() {
 }
 
 echo 'Building Linux jars...'
-LINUX_ARM_JAR=$(build_linux_jar "linux/amd64" "x86-64")
-LINUX_X86_JAR=$(build_linux_jar "linux/arm64" "aarch64")
+LINUX_ARM_JAR=$(build_linux_jar "linux/arm64" "aarch64")
+LINUX_X86_JAR=$(build_linux_jar "linux/amd64" "x86-64")
 
 build_windows_jar() {
   ARTIFACT="$TEMP/zstd-$VERSION-windows-x86-64.jar"

distribution/build.gradle

@@ -70,10 +70,10 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) {
   // Explicitly add the dependency on the RHEL UBI Docker base image
   String[] rhelUbiFields = [
     'Red Hat Universal Base Image minimal',
-    '8',
-    'https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8',
+    '9',
+    'https://catalog.redhat.com/software/containers/ubi9-minimal/61832888c0d15aff4912fe0d',
     'Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf',
-    'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/8/ubi-minimal-8-source.tar.gz'
+    'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/9/ubi-minimal-9-source.tar.gz'
   ]
   additionalLines << rhelUbiFields.join(',')
 }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,6 +11,8 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.jdk.RuntimeVersionFeature;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -137,9 +139,13 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         return Stream.of();
     }
 
+    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
     private static Stream<String> maybeAllowSecurityManager() {
-        // Will become conditional on useEntitlements once entitlements can run without SM
-        return Stream.of("-Djava.security.manager=allow");
+        if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
+            // Will become conditional on useEntitlements once entitlements can run without SM
+            return Stream.of("-Djava.security.manager=allow");
+        }
+        return Stream.of();
     }
 
     private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {

docs/changelog/118025.yaml

@@ -0,0 +1,5 @@
+pr: 118025
+summary: Update sparse text embeddings API route for Inference Service
+area: Inference
+type: enhancement
+issues: []

docs/changelog/118104.yaml

@@ -0,0 +1,12 @@
+pr: 118104
+summary: Remove old `_knn_search` tech preview API in v9
+area: Vector Search
+type: breaking
+issues: []
+breaking:
+  title: Remove old `_knn_search` tech preview API in v9
+  area: REST API
+  details: The original, tech-preview api for vector search, `_knn_search`, has been removed in v9. For all vector search
+    operations, you should utilize the `_search` endpoint.
+  impact: The `_knn_search` API is now inaccessible without providing a compatible-with flag for v8.
+  notable: false

docs/changelog/118114.yaml

@@ -0,0 +1,5 @@
+pr: 118114
+summary: Enable physical plan verification
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/118177.yaml

@@ -0,0 +1,6 @@
+pr: 118177
+summary: Fixing bedrock event executor terminated cache issue
+area: Machine Learning
+type: bug
+issues:
+ - 117916

docs/changelog/118267.yaml

@@ -0,0 +1,5 @@
+pr: 118267
+summary: Adding get migration reindex status
+area: Data streams
+type: enhancement
+issues: []

docs/reference/inference/service-openai.asciidoc

@@ -76,6 +76,12 @@ https://platform.openai.com/api-keys[API keys section].
 include::inference-shared.asciidoc[tag=api-key-admonition]
 --
 
+`dimensions`:::
+(Optional, integer)
+The number of dimensions the resulting output embeddings should have.
+Only supported in `text-embedding-3` and later models.
+If not set the OpenAI defined default for the model is used.
+
 `model_id`:::
 (Required, string)
 The name of the model to use for the {infer} task.
@@ -134,8 +140,8 @@ Specifies the user issuing the request, which can be used for abuse detection.
 [[inference-example-openai]]
 ==== OpenAI service example
 
-The following example shows how to create an {infer} endpoint called
-`openai-embeddings` to perform a `text_embedding` task type.
+The following example shows how to create an {infer} endpoint called `openai-embeddings` to perform a `text_embedding` task type.
+The embeddings created by requests to this endpoint will have 128 dimensions.
 
 [source,console]
 ------------------------------------------------------------
@@ -144,14 +150,14 @@ PUT _inference/text_embedding/openai-embeddings
     "service": "openai",
     "service_settings": {
         "api_key": "<api_key>",
-        "model_id": "text-embedding-ada-002"
+        "model_id": "text-embedding-3-small",
+        "dimensions": 128
     }
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-The next example shows how to create an {infer} endpoint called
-`openai-completion` to perform a `completion` task type.
+The next example shows how to create an {infer} endpoint called `openai-completion` to perform a `completion` task type.
 
 [source,console]
 ------------------------------------------------------------

docs/reference/mapping/types/dense-vector.asciidoc

@@ -232,8 +232,8 @@ it will be set to the length of the first vector added to the field.
 
 `index`::
 (Optional, Boolean)
-If `true`, you can search this field using the <<knn-search-api, kNN search
-API>>. Defaults to `true`.
+If `true`, you can search this field using the <<query-dsl-knn-query, knn query>>
+or <<search-api-knn, knn in _search>> . Defaults to `true`.
 
 [[dense-vector-similarity]]
 `similarity`::

docs/reference/migration/migrate_9_0.asciidoc

@@ -244,6 +244,25 @@ The deprecated highlighting `force_source` parameter is no longer supported.
 Users should remove usages of the `force_source` parameter from their search requests.
 ====
 
+[discrete]
+[[breaking_90_transforms_changes]]
+==== {transforms-cap} changes
+
+[[updating_deprecated_transform_roles]]
+.Updating deprecated {transform} roles (`data_frame_transforms_admin` and `data_frame_transforms_user`)
+[%collapsible]
+====
+*Details* +
+The `data_frame_transforms_admin` and `data_frame_transforms_user` {transform} roles have been deprecated.
+
+*Impact* +
+Users must update any existing {transforms} that use deprecated {transform} roles (`data_frame_transforms_admin` or `data_frame_transforms_user`) to use the new equivalent {transform} roles (`transform_admin` or `transform_user`).
+To update the {transform} roles:
+
+1. Switch to a user with the `transform_admin` role (to replace `data_frame_transforms_admin`) or the `transform_user` role (to replace `data_frame_transforms_user`).
+2. Call the <<update-transform, update {transforms} API>> with that user.
+====
+
 
 [discrete]
 [[deprecated-9.0]]

docs/reference/redirects.asciidoc

@@ -1942,3 +1942,8 @@ Refer to <<get-ip-location-database-api>>.
 === Delete geoip database configuration API
 
 Refer to <<delete-ip-location-database-api>>.
+
+[role="exclude",id="knn-search-api"]
+=== Delete _knn_search API
+
+Refer to <<search-api-knn>>.

docs/reference/search.asciidoc

@@ -50,8 +50,6 @@ include::search/async-search.asciidoc[]
 
 include::search/point-in-time-api.asciidoc[]
 
-include::search/knn-search.asciidoc[]
-
 include::search/retriever.asciidoc[]
 
 include::search/rrf.asciidoc[]