Merge branch 'main' into poc_decouple_consumers_feature_ids

.buildkite/ftr_platform_stateful_configs.yml

@@ -375,3 +375,4 @@ enabled:
   - x-pack/test/custom_branding/config.ts
   # stateful config files that run deployment-agnostic tests
   - x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.stateful.config.ts
+  - x-pack/test/api_integration/apis/cloud/config.ts

.buildkite/pipelines/fips.yml

@@ -40,14 +40,15 @@ steps:
       machineType: n2-standard-2
       preemptible: true
 
-  - command: .buildkite/scripts/steps/fips/smoke_test.sh
-    label: 'Pick Smoke Test Group Run Order'
+  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+    label: 'Pick Test Group Run Order'
     depends_on: build
     timeout_in_minutes: 10
     env:
       FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
       FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
-      LIMIT_CONFIG_TYPE: 'functional'
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
     retry:
       automatic:
         - exit_status: '*'

.buildkite/scripts/common/env.sh

@@ -146,7 +146,7 @@ if [[ "${KBN_ENABLE_FIPS:-}" == "true" ]] || is_pr_with_label "ci:enable-fips-ag
   fi
 
   if [[ -f "$KIBANA_DIR/config/kibana.yml" ]]; then
-    echo -e '\nxpack.security.experimental.fipsMode.enabled: true' >>"$KIBANA_DIR/config/kibana.yml"
+    echo -e '\nxpack.security.fipsMode.enabled: true' >>"$KIBANA_DIR/config/kibana.yml"
   fi
 fi
 

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -60,7 +60,14 @@ while read -r config; do
   # --trace-warnings to debug
   # Node.js process-warning detected:
   # Warning: Closing file descriptor 24 on garbage collection
-  cmd="NODE_OPTIONS=\"--max-old-space-size=12288 --trace-warnings\" node ./scripts/jest --config=\"$config\" $parallelism --coverage=false --passWithNoTests"
+  cmd="NODE_OPTIONS=\"--max-old-space-size=12288 --trace-warnings"
+
+  if [ "${KBN_ENABLE_FIPS:-}" == "true" ]; then
+    cmd=$cmd" --enable-fips --openssl-config=$HOME/nodejs.cnf"
+  fi
+
+  cmd=$cmd"\" node ./scripts/jest --config=\"$config\" $parallelism --coverage=false --passWithNoTests"
+
   echo "actual full command is:"
   echo "$cmd"
   echo ""

.devcontainer/scripts/env.sh

@@ -9,7 +9,7 @@ setup_fips() {
   fi
 
   if [ -n "$FIPS" ] && [ "$FIPS" = "1" ]; then
-    sed -i '/xpack.security.experimental.fipsMode.enabled:/ {s/.*/xpack.security.experimental.fipsMode.enabled: true/; t}; $a\xpack.security.experimental.fipsMode.enabled: true' "$KBN_CONFIG_FILE"
+    sed -i '/xpack.security.fipsMode.enabled:/ {s/.*/xpack.security.fipsMode.enabled: true/; t}; $a\xpack.security.fipsMode.enabled: true' "$KBN_CONFIG_FILE"
 
     # Patch node_modules so we can start Kibana in dev mode
     sed -i 's/hashType = hashType || '\''md5'\'';/hashType = hashType || '\''sha1'\'';/g' "${KBN_DIR}/node_modules/file-loader/node_modules/loader-utils/lib/getHashDigest.js"
@@ -21,7 +21,7 @@ setup_fips() {
     echo "FIPS mode enabled"
     echo "If manually bootstrapping in FIPS mode use: NODE_OPTIONS='' yarn kbn bootstrap"
   else
-    sed -i '/xpack.security.experimental.fipsMode.enabled:/ {s/.*/xpack.security.experimental.fipsMode.enabled: false/; t}; $a\xpack.security.experimental.fipsMode.enabled: false' "$KBN_CONFIG_FILE"
+    sed -i '/xpack.security.fipsMode.enabled:/ {s/.*/xpack.security.fipsMode.enabled: false/; t}; $a\xpack.security.fipsMode.enabled: false' "$KBN_CONFIG_FILE"
   fi
 }
 

docs/developer/plugin-list.asciidoc

@@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana.
 using the CURL scripts in the scripts folder.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks]
+|This plugin contains various LLM tasks.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess]
 |Exposes services to access logs data.
 
@@ -767,6 +771,10 @@ Elastic.
 |This plugin helps users learn how to use the Painless scripting language.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase]
+|This plugin contains the product documentation base service.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling]
 |Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views.
 

docs/upgrade-notes.asciidoc

@@ -49,6 +49,32 @@ For Elastic Security release information, refer to {security-guide}/release-note
 [float]
 ==== Kibana APIs
 
+[discrete]
+[[breaking-199656]]
+.Removed all security v1 endpoints (9.0.0)
+[%collapsible]
+====
+*Details* +
+All `v1` Kibana security HTTP endpoints have been removed.
+
+`GET /api/security/v1/logout` has been replaced by `GET /api/security/logout`
+`GET /api/security/v1/oidc/implicit` has been replaced by `GET /api/security/oidc/implicit`
+`GET /api/security/v1/oidc` has been replaced by GET `/api/security/oidc/callback`
+`POST /api/security/v1/oidc` has been replaced by POST `/api/security/oidc/initiate_login`
+`POST /api/security/v1/saml` has been replaced by POST `/api/security/saml/callback`
+`GET /api/security/v1/me` has been removed with no replacement.
+
+For more information, refer to {kibana-pull}199656[#199656].
+
+*Impact* + 
+Any HTTP API calls to the `v1` Kibana security endpoints will fail with a 404 status code starting from version 9.0.0.
+Third party OIDC and SAML identity providers configured with `v1` endpoints will no longer work.
+
+*Action* +
+Update any OIDC and SAML identity providers to reference the corresponding replacement endpoint listed above.
+Remove references to the `/api/security/v1/me` endpoint from any automations, applications, tooling, and scripts.
+====
+
 [discrete]
 [[breaking-193792]]
 .Access to all internal APIs is blocked (9.0.0)
@@ -814,18 +840,6 @@ The legacy audit logger has been removed. For more information, refer to {kibana
 Audit logs will be written to the default location in the new ECS format. To change the output file, filter events, and more, use the <<audit-logging-settings, audit logging settings>>.
 ====
 
-[discrete]
-[[breaking-47929]]
-.[Security] Removed `/api/security/v1/saml` route. (8.0)
-[%collapsible]
-====
-*Details* +
-The `/api/security/v1/saml` route has been removed and is reflected in the kibana.yml `server.xsrf.whitelist` setting, {es}, and the Identity Provider SAML settings. For more information, refer to {kibana-pull}47929[#47929]
-
-*Impact* +
-Use the `/api/security/saml/callback` route, or wait to upgrade to 8.0.0-alpha2 when the `/api/security/saml/callback` route breaking change is reverted.
-====
-
 [discrete]
 [[breaking-41700]]
 .[Security] Legacy browsers rejected by default. (8.0)

docs/user/security/audit-logging.asciidoc

@@ -148,6 +148,9 @@ Refer to the corresponding {es} logs for potential write errors.
 | `success` | Creating trained model.
 | `failure` | Failed to create trained model.
 
+.1+| `product_documentation_create`
+| `unknown` | User requested to install the product documentation for use in AI Assistants.
+
 3+a|
 ====== Type: change
 
@@ -334,6 +337,9 @@ Refer to the corresponding {es} logs for potential write errors.
 | `success` | Updating trained model deployment.
 | `failure` | Failed to update trained model deployment.
 
+.1+| `product_documentation_update`
+| `unknown` | User requested to update the product documentation for use in AI Assistants.
+
 3+a|
 ====== Type: deletion
 
@@ -425,6 +431,9 @@ Refer to the corresponding {es} logs for potential write errors.
 | `success` | Deleting trained model.
 | `failure` | Failed to delete trained model.
 
+.1+| `product_documentation_delete`
+| `unknown` | User requested to delete the product documentation for use in AI Assistants.
+
 3+a|
 ====== Type: access
 

docs/user/security/fips-140-2.asciidoc

@@ -29,7 +29,7 @@ For {kib}, adherence to FIPS 140-2 is ensured by:
 
 ==== Configuring {kib} for FIPS 140-2
 
-Apart from setting `xpack.security.experimental.fipsMode.enabled` to `true` in your {kib} config, a number of security related 
+Apart from setting `xpack.security.fipsMode.enabled` to `true` in your {kib} config, a number of security related 
 settings need to be reviewed and configured in order to run {kib} successfully in a FIPS 140-2 compliant Node.js 
 environment.
 
@@ -56,8 +56,3 @@ As an example, avoid PKCS#12 specific settings such as:
 * `server.ssl.truststore.path`
 * `elasticsearch.ssl.keystore.path`
 * `elasticsearch.ssl.truststore.path`
-
-===== Limitations
-
-Configuring {kib} to run in FIPS mode is still considered to be experimental. Not all features are guaranteed to
-function as expected.

oas_docs/output/kibana.serverless.yaml

@@ -7395,6 +7395,43 @@ paths:
       tags:
         - Security Endpoint Management API
       x-beta: true
+  /api/entity_store/enable:
+    post:
+      operationId: InitEntityStore
+      requestBody:
+        content:
+          application/json; Elastic-Api-Version=2023-10-31:
+            schema:
+              type: object
+              properties:
+                fieldHistoryLength:
+                  default: 10
+                  description: The number of historical values to keep for each field.
+                  type: integer
+                filter:
+                  type: string
+                indexPattern:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        description: Schema for the entity store initialization
+        required: true
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  succeeded:
+                    type: boolean
+          description: Successful response
+      summary: Initialize the Entity Store
+      tags:
+        - Security Entity Analytics API
+      x-beta: true
   /api/entity_store/engines:
     get:
       operationId: ListEntityEngines
@@ -7713,6 +7750,27 @@ paths:
       tags:
         - Security Entity Analytics API
       x-beta: true
+  /api/entity_store/status:
+    get:
+      operationId: GetEntityStoreStatus
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  status:
+                    $ref: '#/components/schemas/Security_Entity_Analytics_API_StoreStatus'
+          description: Successful response
+      summary: Get the status of the Entity Store
+      tags:
+        - Security Entity Analytics API
+      x-beta: true
   /api/exception_lists:
     delete:
       description: Delete an exception list using the `id` or `list_id` field.
@@ -45880,6 +45938,14 @@ components:
         - index
         - description
         - category
+    Security_Entity_Analytics_API_StoreStatus:
+      enum:
+        - not_installed
+        - installing
+        - running
+        - stopped
+        - error
+      type: string
     Security_Entity_Analytics_API_TaskManagerUnavailableResponse:
       description: Task manager is unavailable
       type: object

oas_docs/output/kibana.yaml

@@ -10282,6 +10282,42 @@ paths:
       summary: Create or update a protection updates note
       tags:
         - Security Endpoint Management API
+  /api/entity_store/enable:
+    post:
+      operationId: InitEntityStore
+      requestBody:
+        content:
+          application/json; Elastic-Api-Version=2023-10-31:
+            schema:
+              type: object
+              properties:
+                fieldHistoryLength:
+                  default: 10
+                  description: The number of historical values to keep for each field.
+                  type: integer
+                filter:
+                  type: string
+                indexPattern:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        description: Schema for the entity store initialization
+        required: true
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  succeeded:
+                    type: boolean
+          description: Successful response
+      summary: Initialize the Entity Store
+      tags:
+        - Security Entity Analytics API
   /api/entity_store/engines:
     get:
       operationId: ListEntityEngines
@@ -10591,6 +10627,26 @@ paths:
       summary: List Entity Store Entities
       tags:
         - Security Entity Analytics API
+  /api/entity_store/status:
+    get:
+      operationId: GetEntityStoreStatus
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  status:
+                    $ref: '#/components/schemas/Security_Entity_Analytics_API_StoreStatus'
+          description: Successful response
+      summary: Get the status of the Entity Store
+      tags:
+        - Security Entity Analytics API
   /api/exception_lists:
     delete:
       description: Delete an exception list using the `id` or `list_id` field.
@@ -53601,6 +53657,14 @@ components:
         - index
         - description
         - category
+    Security_Entity_Analytics_API_StoreStatus:
+      enum:
+        - not_installed
+        - installing
+        - running
+        - stopped
+        - error
+      type: string
     Security_Entity_Analytics_API_TaskManagerUnavailableResponse:
       description: Task manager is unavailable
       type: object