Merge branch '8.16' into backport/8.16/pr-199548

.buildkite/scripts/steps/checks.sh

@@ -13,6 +13,4 @@ fi
 .buildkite/scripts/steps/checks/saved_objects_definition_change.sh
 .buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
-.buildkite/scripts/steps/openapi_bundling/security_solution_openapi_bundling.sh
 .buildkite/scripts/steps/code_generation/osquery_codegen.sh
-.buildkite/scripts/steps/openapi_bundling/final_merge.sh

.buildkite/scripts/steps/checks/capture_oas_snapshot.sh

@@ -24,3 +24,6 @@ run_check() {
 retry 5 15 run_check
 
 check_for_changed_files "$cmd" true
+
+.buildkite/scripts/steps/openapi_bundling/security_solution_openapi_bundling.sh
+.buildkite/scripts/steps/openapi_bundling/final_merge.sh

.buildkite/scripts/steps/openapi_bundling/final_merge.sh

@@ -7,6 +7,5 @@ source .buildkite/scripts/common/util.sh
 echo --- Merge Kibana OpenAPI specs
 
 (cd oas_docs && make api-docs && make api-docs-lint)
-(cd oas_docs && make api-docs-staging && make api-docs-lint)
 
-check_for_changed_files "make api-docs && make api-docs-staging" true
+check_for_changed_files "make api-docs" true

docs/management/action-types.asciidoc

@@ -8,6 +8,12 @@ Actions are instantiations of a connector that are linked to rules and run as ba
 [cols="2"]
 |===
 
+// ifeval::["featureAIConnector"=="true"]
+// a| <<inference-action-type,{infer}>>
+
+// | Send a request to {infer}.
+// endif::[]
+
 a| <<bedrock-action-type,{bedrock}>>
 
 | Send a request to {bedrock}.
@@ -28,10 +34,6 @@ a| <<gemini-action-type,{gemini}>>
 
 | Send a request to {gemini}.
 
-a| <<inference-action-type,{infer}>>
-
-| Send a request to {infer}.
-
 a| <<email-action-type,Email>>
 
 | Send email from your server.

docs/management/connectors/action-types/inference.asciidoc

@@ -1,13 +1,14 @@
 [[inference-action-type]]
-== {infer-cap} connector and action
+== AI connector and action
 ++++
-<titleabbrev>{infer-cap}</titleabbrev>
+<titleabbrev>AI</titleabbrev>
 ++++
 :frontmatter-description: Add a connector that can send requests to {inference}.
 :frontmatter-tags-products: [kibana] 
 :frontmatter-tags-content-type: [how-to] 
 :frontmatter-tags-user-goals: [configure]
 
+coming::[]
 
 The {infer} connector uses the {es} client to send requests to an {infer} service.
 The connector uses the <<execute-connector-api,run connector API>> to send the request.
@@ -16,6 +17,7 @@ The connector uses the <<execute-connector-api,run connector API>> to send the r
 [[define-inference-ui]]
 === Create connectors in {kib}
 
+// TBD After you set the `xpack.stack_connectors.enableExperimental` to include `inferenceConnectorOn`,
 You can create connectors in *{stack-manage-app} > {connectors-ui}*. For example:
 
 [role="screenshot"]

docs/management/connectors/index.asciidoc

@@ -4,7 +4,9 @@ include::action-types/crowdstrike.asciidoc[leveloffset=+1]
 include::action-types/d3security.asciidoc[leveloffset=+1]
 include::action-types/email.asciidoc[leveloffset=+1]
 include::action-types/gemini.asciidoc[leveloffset=+1]
-include::action-types/inference.asciidoc[leveloffset=+1]
+// ifeval::["featureAIConnector"=="true"]
+// include::action-types/inference.asciidoc[leveloffset=+1]
+// endif::[]
 include::action-types/resilient.asciidoc[leveloffset=+1]
 include::action-types/index.asciidoc[leveloffset=+1]
 include::action-types/jira.asciidoc[leveloffset=+1]

docs/search/index.asciidoc

@@ -10,7 +10,7 @@ The *Search* space in the {kib} UI contains the following GUI features:
 * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-application-overview.html[Search Applications]
 * https://www.elastic.co/guide/en/elasticsearch/reference/current/behavioral-analytics-overview.html[Behavioral Analytics]
 * <<inference-endpoints,Inference Endpoints UI>>
-* <<search-assistant,AI Assistant for Search>>
+* <<search-ai-assistant,AI Assistant for Search>>
 * Dev Tools <<console-kibana, Console>>
 
 [float]

docs/search/inference-endpoints/index.asciidoc

@@ -2,4 +2,14 @@
 [[inference-endpoints]]
 == Inference endpoints UI
 
-(coming in 8.16.0)
+Inference endpoints streamline the deployment and management of machine learning models in
+{es}. The *Inference endpoints* page in {kib} provides an interface for displaying inference 
+endpoints that have been created using the {ref}/put-inference-api.html[Inference API].
+
+[role="screenshot"]
+image::images/inference-endpoints-ui.png[Inference endpoints UI]
+
+Available actions:
+
+* Copy the inference endpoint ID
+* Delete endpoints

docs/search/search-ai-assistant/index.asciidoc

@@ -1,5 +1,143 @@
 [role="xpack"]
-[[search-assistant]]
-== Search AI Assistant
+[[search-ai-assistant]]
+== AI Assistant
 
-(coming in 8.16.0)
+[TIP]
+====
+Don't confuse AI Assistant with <<playground,Playground>>! Use Playground to chat with your data, test and tweak different {es} queries in the Playground UI, and download the code to integrate into your own RAG application.
+
+Use AI Assistant to get help with Elasticsearch and Kibana tasks directly in the UI.
+====
+
+.Observability use cases
+****
+Refer to the {observability-guide}/obs-ai-assistant.html[Observability documentation] for more information on how to use AI Assistant in Observability contexts.
+****
+
+*AI Assistant for Observability and Search* uses generative AI to help you with a variety of tasks related to Elasticsearch and Kibana, including:
+
+1. *Constructing Queries*: Assists you in building queries to search and analyze your data.
+2. *Indexing Data*: Guides you on how to index data into Elasticsearch.
+3. *Searching Data*: Helps you search for specific data within your Elasticsearch indices.
+4. *Using Elasticsearch APIs*: Calls Elasticsearch APIs on your behalf if you need specific operations performed.
+5. *Generating Sample Data*: Helps you create sample data for testing and development purposes.
+6. *Visualizing and Analyzing Data*: Assists you in creating visualizations and analyzing your data using Kibana.
+7. *Explaining ES|QL*: Explains how ES|QL works and help you convert queries from other languages to {ref}/esql.html[ES|QL.]
+
+[discrete]
+[[ai-assistant-requirements]]
+=== Requirements
+
+To use AI Assistant in *Search* contexts, you must have the following:
+
+* Elastic Stack version 8.16.0, or an Elasticsearch Serverless project.
+* A <<action-types,generative AI connector>> to connect to a LLM provider, or a local model.
+** You need an account with a third-party generative AI provider, which AI Assistant uses to generate responses, or else you need to host your own local model.
+** To set up AI Assistant, you need the `Actions and Connectors : All` <<kibana-privileges,privilege>>.
+* To use AI Assistant, you need at least the `Elastic AI Assistant : All` and `Actions and Connectors : Read` <<kibana-privileges,privilege>>.
+* AI Assistant requires {ml-docs}/ml-nlp-elser.html[ELSER], Elastic's proprietary semantic search model.
+
+[discrete]
+[[ai-assistant-data-information]]
+=== Your data and AI Assistant
+
+Elastic does not use customer data for model training. This includes anything you send the model, such as alert or event data, detection rule configurations, queries, and prompts. However, any data you provide to AI Assistant will be processed by the third-party provider you chose when setting up the generative AI connector as part of the assistant setup.
+
+Elastic does not control third-party tools, and assumes no responsibility or liability for their content, operation, or use, nor for any loss or damage that may arise from your using such tools. Please exercise caution when using AI tools with personal, sensitive, or confidential information. Any data you submit may be used by the provider for AI training or other purposes. There is no guarantee that the provider will keep any information you provide secure or confidential. You should familiarize yourself with the privacy practices and terms of use of any generative AI tools prior to use.
+
+[discrete]
+[[ai-assistant-using]]
+=== Using AI Assistant
+
+To open AI Assistant, select the **AI Assistant** button in the top toolbar in the UI.
+You can also use the global search field in the UI to find AI Assistant.
+// <<kibana-concepts-finding-your-apps-objects,global search field>>
+// TODO link will be available once https://github.com/elastic/kibana/pull/199352 is merged.
+
+[role="screenshot"]
+image::images/ai-assistant-button.png[AI Assistant button,50]
+
+This opens the AI Assistant chat interface flyout.
+
+[role="screenshot]
+image::images/ai-assistant-welcome-chat.png[AI Assistant Welcome chat,450]
+
+You can get started by selecting *✨ Suggest* to get some example prompts, or by typing into the chat field.
+
+[discrete]
+[[ai-assistant-add-custom-data]]
+=== Add data to the AI Assistant knowledge base
+
+[NOTE]
+====
+This functionality is not available on Elastic Cloud Serverless projects.
+====
+
+You can improve the relevance of AI Assistant’s responses by indexing your own data into AI Assistant's knowledge base.
+AI Assistant uses {ml-docs}/ml-nlp-elser.html[ELSER], Elastic's proprietary semantic search model, to power its search capabilities.
+
+[discrete]
+[[search-ai-assistant-use-the-ui]]
+==== Use the UI
+
+To add external data to the knowledge base in UI:
+
+. In the AI Assistant UI, select the **Settings** icon: `⋮`.
+. Under *Actions*, click **Manage knowledge base**.
+. Click the **New entry** button, and choose either:
++
+** **Single entry**: Write content for a single entry in the UI.
+** **Bulk import**: Upload a newline delimited JSON (`ndjson`) file containing a list of entries to add to the knowledge base.
+Each object should conform to the following format:
++
+[source,json]
+----
+{
+  "id": "a_unique_human_readable_id",
+  "text": "Contents of item",
+}
+----
+
+[discrete]
+[[observability-ai-assistant-add-data-to-kb]]
+==== Use Search connectors
+
+// Will be updated to mention reindex option for arbitrary indices
+// Need to consolidate docs with obs team first
+
+[NOTE]
+====
+This functionality is not available on Elastic Cloud Serverless projects.
+====
+
+You can ingest external data (GitHub issues, Markdown files, Jira tickets, text files, etc.) into {es} using {ref}/es-connectors.html[Search Connectors]. Connectors sync third party data sources to {es}. 
+
+Supported service types include {ref}/es-connectors-github.html[GitHub], {ref}/es-connectors-slack.html[Slack], {ref}/es-connectors-jira.html[Jira], and more. These can be Elastic managed or self-managed on your own infrastructure.
+
+To create a connector and make its content available to the AI Assistant knowledge base, follow these steps:
+
+. *In {kib} UI, go to _Search -> Content -> Connectors_ and follow the instructions to create a new connector.*
++
+For example, if you create a {ref}/es-connectors-github.html[GitHub connector] you must set a `name`, attach it to a new or existing `index`, add your `personal access token` and include the `list of repositories` to synchronize.
++
+TIP: Learn more about configuring and {ref}/es-connectors-usage.html[using connectors] in the Elasticsearch documentation.
++
+. *Create a pipeline and process the data with ELSER.*
++
+To process connector data using {ml-docs}/ml-nlp-elser.html[ELSER], you must create an *ML Inference Pipeline*:
++
+.. Open the previously created connector and select the *Pipelines* tab.
+.. Select *Copy and customize* button at the `Unlock your custom pipelines` box.
+.. Select *Add Inference Pipeline* button at the `Machine Learning Inference Pipelines` box.
+.. Select *ELSER (Elastic Learned Sparse EncodeR)* ML model to add the necessary embeddings to the data.
+.. Select the fields that need to be evaluated as part of the inference pipeline.
+.. Test and save the inference pipeline and the overall pipeline.
+. *Sync data.*
++
+Once the pipeline is set up, perform a *Full Content Sync* of the connector. The inference pipeline will process the data as follows:
++
+* As data comes in, the ELSER model processes the data, creating sparse embeddings for each document.
+* If you inspect the ingested documents, you can see how the weights and tokens are added to the `predicted_value` field.
+. *Confirm AI Assistant can access the index.*
++
+Ask the AI Assistant a specific question to confirm that the data is available for the AI Assistant knowledge base.

docs/settings/alert-action-settings.asciidoc

@@ -283,15 +283,16 @@ A configuration URL that varies by connector:
 --
 * For an <<bedrock-action-type,{bedrock} connector>>, specifies the {bedrock} request URL.
 * For an <<gemini-action-type,{gemini} connector>>, specifies the {gemini} request URL.
-* For an <<inference-action-type,{inference} connector>>, specifies the Elastic {inference} request.
 * For a <<openai-action-type,OpenAI connector>>, specifies the OpenAI request URL.
 * For a <<resilient-action-type,{ibm-r} connector>>, specifies the {ibm-r} instance URL.
 * For a <<jira-action-type,Jira connector>>, specifies the Jira instance URL.
 * For an <<opsgenie-action-type,{opsgenie} connector>>, specifies the {opsgenie} URL. For example, `https://api.opsgenie.com` or `https://api.eu.opsgenie.com`.
 * For a <<pagerduty-action-type,PagerDuty connector>>, specifies the PagerDuty event URL. Defaults to `https://events.pagerduty.com/v2/enqueue`.
 * For a <<servicenow-action-type,{sn-itsm}>>, <<servicenow-sir-action-type,{sn-sir}>>, or <<servicenow-itom-action-type,{sn-itom} connector>> specifies the ServiceNow instance URL.
 * For a <<swimlane-action-type,{swimlane} connector>>, specifies the {swimlane} instance URL.
-
+// ifeval::["featureAIConnector"=="true"]
+// * For an <<inference-action-type,AI connector>>, specifies the Elastic {inference} request.
+// endif::[]
 NOTE: If you are using the `xpack.actions.allowedHosts` setting, make sure the hostname in the URL is added to the allowed hosts.
 --
 

docs/user/dashboard/lens.asciidoc

@@ -96,13 +96,37 @@ All columns that belong to the same layer pane group are sorted in the table.
 
 * *Text alignment* — Aligns the values in the cell to the *Left*, *Center*, or *Right*.
 
+* *Color by value* — Applies color to the cell or text values. To change the color, click the *Edit colors* icon. 
+
 * *Hide column* — Hides the column for the field.
 
 * *Directly filter on click* — Turns column values into clickable links that allow you to filter or drill down into the data.
 
 * *Summary row* — Adds a row that displays the summary value. When specified, allows you to enter a *Summary label*.
 
-* *Color by value* — Applies color to the cell or text values. To change the color, click *Edit*. 
+[float]
+[[assign-colors-to-terms]]
+===== Assign colors to terms
+
+preview::[]
+
+For term-based metrics, assign a color to each term with color mapping. 
+
+. Create a custom table.
+
+. In the layer pane, select a *Rows* or *Metrics* field. 
+
+. In the *Color by value* option, select *Cell* or *Text*.
+
+. Click the *Edit colors* icon.
+
+. Toggle the button to use the Color Mapping feature.
+
+. Select a color palette and mode.
+
+. Click *Add assignment* to assign a color to a specific term, or click *Add all unassigned terms* to assign colors to all terms. Assigning colors to dates is unsupported.
+
+. Configure color assignments. You can also select whether unassigned terms should be mapped to the selected color palette or a single color. 
 
 [float]
 [[drag-and-drop-keyboard-navigation]]

fleet_packages.json

@@ -56,6 +56,6 @@
   },
   {
     "name": "security_detection_engine",
-    "version": "8.15.9"
+    "version": "8.16.1"
   }
 ]

oas_docs/README.md

@@ -45,8 +45,7 @@ Besides the scripts in the `oas_docs/scripts` folder, there is an `oas_docs/make
 | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `api-docs`                 | Builds ESS Kibana OpenAPI bundle                                                                                                                                                                                   |
 | `api-docs-serverless`      | Builds Serverless Kibana OpenAPI bundle                                                                                                                                                                            |
-| `api-docs-lint`            | Lints built result bundles                                                                                                                                                                                         |
-| `api-docs-lint-errs`       | Lints built result bundles for errors                                                                                                                                                                              |
+| `api-docs-lint`            | Lints built result bundles              |
 | `api-docs-preview`         | Generates (ESS + Serverless) Kibana OpenAPI bundles preview                                                                                                                                                        |
 | `api-docs-overlay`         | Applies [overlays](https://docs.bump.sh/help/specification-support/overlays/) from `overlays` folder to the Kibana OpenAPI bundles and generate `*.new.yaml` files. Overlays help to fine tune the result bundles. |
 | `api-docs-overlay-preview` | Generates a preview for bundles produced by `api-docs-overlay`                                                                                                                                                     |

oas_docs/examples/get_connector_types_generativeai_response.yaml

@@ -31,14 +31,3 @@ value:
     supported_feature_ids:
       - generativeAIForSecurity
     is_system_action_type: false
-  - id: .inference
-    name: Inference API
-    enabled: true
-    enabled_in_config: true
-    enabled_in_license: true
-    minimum_license_required: enterprise
-    supported_feature_ids:
-      - generativeAIForSecurity
-      - generativeAIForObservability
-      - generativeAIForSearchPlayground
-    is_system_action_type: false