Merge branch 'elastic:main' into endpoint_validation

.ci/scripts/packaging-test.sh

@@ -39,7 +39,7 @@ if [ -f "/etc/os-release" ] ; then
         # Work around incorrect lintian version
         #  https://github.com/elastic/elasticsearch/issues/48573
         if [ $VERSION_ID == 10 ] ; then
-            sudo apt-get update -y
+            sudo apt-get update -y || true
             sudo apt-get install -y --allow-downgrades lintian=2.15.0
         fi
     fi

docs/changelog/111684.yaml

@@ -0,0 +1,5 @@
+pr: 111684
+summary: Write downloaded model parts async
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/113051.yaml

@@ -0,0 +1,5 @@
+pr: 113051
+summary: Add Search Inference ID To Semantic Text Mapping
+area: Mapping
+type: enhancement
+issues: []

docs/reference/index.asciidoc

@@ -6,10 +6,10 @@ include::links.asciidoc[]
 
 include::landing-page.asciidoc[]
 
-include::intro.asciidoc[]
-
 include::release-notes/highlights.asciidoc[]
 
+include::intro.asciidoc[]
+
 include::quickstart/index.asciidoc[]
 
 include::setup.asciidoc[]

docs/reference/indices/index-templates.asciidoc

@@ -44,6 +44,7 @@ following index patterns:
 - `metrics-*-*`
 - `synthetics-*-*`
 - `profiling-*`
+- `security_solution-*-*`
 // end::built-in-index-template-patterns[]
 
 {fleet-guide}/fleet-overview.html[{agent}] uses these templates to create

docs/reference/landing-page.asciidoc

@@ -62,7 +62,7 @@
       Elasticsearch is the search and analytics engine that powers the Elastic Stack.
     </p>
     <p>
-      <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started.html">
+      <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/elasticsearch-intro.html">
         <button class="btn btn-primary">Get started</button>
       </a>
     </p>

docs/reference/quickstart/getting-started.asciidoc

@@ -1,47 +1,20 @@
 [[getting-started]]
-== Quick start guide
+== Quick start: Add data using Elasticsearch APIs
+++++
+<titleabbrev>Basics: Add data using APIs</titleabbrev>
+++++
 
-This guide helps you learn how to:
+In this quick start guide, you'll learn how to do the following tasks:
 
-* Run {es} and {kib} (using {ecloud} or in a local Docker dev environment),
-* add simple (non-timestamped) dataset to {es},
-* run basic searches.
-
-[TIP]
-====
-If you're interested in using {es} with Python, check out Elastic Search Labs. This is the best place to explore AI-powered search use cases, such as working with embeddings, vector search, and retrieval augmented generation (RAG).
-
-* https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[Tutorial]: this walks you through building a complete search solution with {es}, from the ground up.
-* https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs` repository]: it contains a range of Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps].
-====
-
-[discrete]
-[[run-elasticsearch]]
-=== Run {es}
-
-The simplest way to set up {es} is to create a managed deployment with {ess} on
-{ecloud}. If you prefer to manage your own test environment, install and
-run {es} using Docker.
-
-include::{es-ref-dir}/tab-widgets/code.asciidoc[]
-include::{es-ref-dir}/tab-widgets/quick-start-install-widget.asciidoc[]
-
-[discrete]
-[[send-requests-to-elasticsearch]]
-=== Send requests to {es}
-
-You send data and other requests to {es} using REST APIs. This lets you interact
-with {es} using any client that sends HTTP requests, such as
-https://curl.se[curl]. You can also use {kib}'s Console to send requests to
-{es}.
-
-include::{es-ref-dir}/tab-widgets/api-call-widget.asciidoc[]
+* Add a small, non-timestamped dataset to {es} using Elasticsearch REST APIs.
+* Run basic searches.
 
 [discrete]
 [[add-data]]
 === Add data
 
-You add data to {es} as JSON objects called documents. {es} stores these
+You add data to {es} as JSON objects called documents.
+{es} stores these
 documents in searchable indices.
 
 [discrete]
@@ -58,6 +31,13 @@ The request automatically creates the index.
 PUT books
 ----
 // TESTSETUP
+
+[source,console]
+--------------------------------------------------
+DELETE books
+--------------------------------------------------
+// TEARDOWN
+
 ////
 
 [source,console]
@@ -236,10 +216,11 @@ JSON object submitted during indexing.
 [[qs-match-query]]
 ==== `match` query
 
-You can use the `match` query to search for documents that contain a specific value in a specific field.
+You can use the <<query-dsl-match-query,`match` query>> to search for documents that contain a specific value in a specific field.
 This is the standard query for performing full-text search, including fuzzy matching and phrase searches.
 
 Run the following command to search the `books` index for documents containing `brave` in the `name` field:
+
 [source,console]
 ----
 GET books/_search
@@ -251,34 +232,4 @@ GET books/_search
   }
 }
 ----
-// TEST[continued]
-
-[discrete]
-[[whats-next]]
-=== Next steps
-
-Now that {es} is up and running and you've learned the basics, you'll probably want to test out larger datasets, or index your own data.
-
-[discrete]
-[[whats-next-search-learn-more]]
-==== Learn more about search queries
-
-* <<search-with-elasticsearch>>. Jump here to learn about exact value search, full-text search, vector search, and more, using the <<search-search,search API>>.
-
-[discrete]
-[[whats-next-more-data]]
-==== Add more data
-
-* Learn how to {kibana-ref}/sample-data.html[install sample data] using {kib}. This is a quick way to test out {es} on larger workloads.
-* Learn how to use the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[upload data UI] in {kib} to add your own CSV, TSV, or JSON files.
-* Use the https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[bulk API] to ingest your own datasets to {es}.
-
-[discrete]
-[[whats-next-client-libraries]]
-==== {es} programming language clients
-
-* Check out our https://www.elastic.co/guide/en/elasticsearch/client/index.html[client library] to work with your {es} instance in your preferred programming language.
-* If you're using Python, check out https://www.elastic.co/search-labs[Elastic Search Labs] for a range of examples that use the {es} Python client. This is the best place to explore AI-powered search use cases, such as working with embeddings, vector search, and retrieval augmented generation (RAG).
-** This extensive, hands-on https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[tutorial] 
-walks you through building a complete search solution with {es}, from the ground up.
-** https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs`] contains a range of executable Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps].
+// TEST[continued]

docs/reference/quickstart/index.asciidoc

@@ -1,10 +1,29 @@
 [[quickstart]]
-= Quickstart
+= Quick starts
 
-Get started quickly with {es}.
+Use these quick starts to get hands-on experience with the {es} APIs.
+Unless otherwise noted, these examples will use queries written in <<query-dsl,Query DSL>> syntax.
 
-* Learn how to run {es} (and {kib}) for <<run-elasticsearch-locally,local development>>.
-* Follow our <<getting-started,Quickstart guide>> to add data to {es} and query it.
+[discrete]
+[[quickstart-requirements]]
+== Requirements
 
-include::run-elasticsearch-locally.asciidoc[]
-include::getting-started.asciidoc[]
+You'll need a running {es} cluster, together with {kib} to use the Dev Tools API Console.
+Get started <<run-elasticsearch-locally,locally in Docker>> , or see our <<elasticsearch-intro-deploy,other deployment options>>.
+
+[discrete]
+[[quickstart-list]]
+== Hands-on quick starts
+
+* <<getting-started,Basics: Add data using APIs>>. Learn how to add data to {es} and perform basic searches.
+
+[discrete]
+[[quickstart-python-links]]
+== Working in Python
+
+If you're interested in using {es} with Python, check out Elastic Search Labs:
+
+* https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs` repository]: Contains a range of Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps].
+* https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[Tutorial]: This walks you through building a complete search solution with {es} from the ground up using Flask.
+
+include::getting-started.asciidoc[]

docs/reference/quickstart/run-elasticsearch-locally.asciidoc → docs/reference/run-elasticsearch-locally.asciidoc

@@ -1,32 +1,21 @@
 [[run-elasticsearch-locally]]
-== Run {es} locally in Docker (without security)
+== Run {es} locally in Docker
 ++++
-<titleabbrev>Local dev setup (Docker)</titleabbrev>
+<titleabbrev>Run {es} locally</titleabbrev>
 ++++
 
 [WARNING]
 ====
 *DO NOT USE THESE INSTRUCTIONS FOR PRODUCTION DEPLOYMENTS*
 
 The instructions on this page are for *local development only*. Do not use these instructions for production deployments, because they are not secure.
-While this approach is convenient for experimenting and learning, you should never run the service in this way in a production environment.
+While this approach is convenient for experimenting and learning, you should never run Elasticsearch in this way in a production environment.
 ====
 
-The following commands help you very quickly spin up a single-node {es} cluster, together with {kib} in Docker.
-Note that if you don't need the {kib} UI, you can skip those instructions.
+Follow this tutorial if you want to quickly set up {es} in Docker for local development or testing.
 
-[discrete]
-[[local-dev-why]]
-=== When would I use this setup?
-
-Use this setup if you want to quickly spin up {es} (and {kib}) for local development or testing.
-
-For example you might:
-
-* Want to run a quick test to see how a feature works.
-* Follow a tutorial or guide that requires an {es} cluster, like our <<getting-started,quick start guide>>.
-* Experiment with the {es} APIs using different tools, like the Dev Tools Console, cURL, or an Elastic programming language client.
-* Quickly spin up an {es} cluster to test an executable https://github.com/elastic/elasticsearch-labs/tree/main/notebooks#readme[Python notebook] locally.
+This tutorial also includes instructions for installing {kib}. 
+ If you don't need access to the {kib} UI, then you can skip those instructions.
 
 [discrete]
 [[local-dev-prerequisites]]
@@ -118,12 +107,12 @@ When you access {kib}, use `elastic` as the username and the password you set ea
 
 [NOTE]
 ====
-The service is started with a trial license. The trial license enables all features of Elasticsearch for a trial period of 30 days. After the trial period expires, the license is downgraded to a basic license, which is free forever. If you prefer to skip the trial and use the basic license, set the value of the `xpack.license.self_generated.type` variable to basic instead. For a detailed feature comparison between the different licenses, refer to our https://www.elastic.co/subscriptions[subscriptions page].
+The service is started with a trial license. The trial license enables all features of Elasticsearch for a trial period of 30 days. After the trial period expires, the license is downgraded to a basic license, which is free forever.
 ====
 
 [discrete]
 [[local-dev-connecting-clients]]
-== Connecting to {es} with language clients
+=== Connect to {es} with language clients
 
 To connect to the {es} cluster from a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable.
 
@@ -172,12 +161,11 @@ curl -u elastic:$ELASTIC_PASSWORD \
 [[local-dev-next-steps]]
 === Next steps
 
-Use our <<getting-started,quick start guide>> to learn the basics of {es}: how to add data and query it.
+Use our <<quickstart,quick start guides>> to learn the basics of {es}.
 
 [discrete]
 [[local-dev-production]]
 === Moving to production
 
-This setup is not suitable for production use. For production deployments, we recommend using our managed service on Elastic Cloud. https://cloud.elastic.co/registration[Sign up for a free trial] (no credit card required).
-
-Otherwise, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html[Install {es}] to learn about the various options for installing {es} in a self-managed production environment, including using Docker.
+This setup is not suitable for production use.
+Refer to <<elasticsearch-intro-deploy, deployment options>> for more information.

docs/reference/setup.asciidoc

@@ -27,6 +27,8 @@ the only resource-intensive application on the host or container. For example,
 you might run {metricbeat} alongside {es} for cluster statistics, but a
 resource-heavy {ls} deployment should be on its own host.
 
+include::run-elasticsearch-locally.asciidoc[]
+
 include::setup/install.asciidoc[]
 
 include::setup/configuration.asciidoc[]

docs/reference/tab-widgets/api-call.asciidoc

@@ -1,5 +1,5 @@
 // tag::cloud[]
-**Use {kib}**
+**Option 1: Use {kib}**
 
 //tag::kibana-api-ex[]
 . Open {kib}'s main menu ("*☰*" near Elastic logo) and go to **Dev Tools > Console**.
@@ -16,17 +16,17 @@ GET /
 
 //end::kibana-api-ex[]
 
-**Use curl**
+**Option 2: Use `curl`**
 
-To communicate with {es} using curl or another client, you need your cluster's
+To communicate with {es} using `curl` or another client, you need your cluster's
 endpoint.
 
 . Open {kib}'s main menu and click **Manage this deployment**.
 
 . From your deployment menu, go to the **Elasticsearch** page. Click **Copy
 endpoint**.
 
-. To submit an example API request, run the following curl command in a new
+. To submit an example API request, run the following `curl` command in a new
 terminal session. Replace `<password>` with the password for the `elastic` user.
 Replace `<elasticsearch_endpoint>` with your endpoint.
 +

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java

@@ -502,7 +502,7 @@ static void putComposableIndexTemplate(String id, List<String> patterns, @Nullab
         request.indexTemplate(
             ComposableIndexTemplate.builder()
                 .indexPatterns(patterns)
-                .template(new Template(settings, null, null, null))
+                .template(Template.builder().settings(settings))
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
                 .build()
         );

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -2441,7 +2441,13 @@ static void putComposableIndexTemplate(
         request.indexTemplate(
             ComposableIndexTemplate.builder()
                 .indexPatterns(patterns)
-                .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), aliases, lifecycle))
+                .template(
+                    Template.builder()
+                        .settings(settings)
+                        .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings))
+                        .aliases(aliases)
+                        .lifecycle(lifecycle)
+                )
                 .metadata(metadata)
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore))
                 .build()

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java

@@ -453,7 +453,7 @@ void putComposableIndexTemplate(Client client, String id, List<String> patterns,
         request.indexTemplate(
             ComposableIndexTemplate.builder()
                 .indexPatterns(patterns)
-                .template(new Template(null, null, aliases, null))
+                .template(Template.builder().aliases(aliases))
                 .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
                 .build()
         );

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java

@@ -201,12 +201,10 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
                         ComposableIndexTemplate.builder()
                             .indexPatterns(List.of(".test-data-stream"))
                             .template(
-                                new Template(
-                                    Settings.EMPTY,
-                                    mappings,
-                                    null,
-                                    DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build()
-                                )
+                                Template.builder()
+                                    .settings(Settings.EMPTY)
+                                    .mappings(mappings)
+                                    .lifecycle(DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build())
                             )
                             .dataStreamTemplate(new DataStreamTemplate())
                             .build(),