Merge branch 'main' of github.com:Omolola-Akinleye/kibana into compli…

…ance-dashboard-data-api

.buildkite/disabled_jest_configs.json

@@ -1 +1,3 @@
-[]
+[
+  "x-pack/plugins/index_management/jest.config.js"
+]

.buildkite/ftr_configs.yml

@@ -14,8 +14,11 @@ disabled:
   - x-pack/test/api_integration/config.ts
   - x-pack/test/fleet_api_integration/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
-
+  - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
+  - x-pack/test/security_solution_endpoint/config.base.ts
+  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -163,8 +166,10 @@ enabled:
   - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group3/config_with_schedule_circuit_breaker.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group4/config_non_dedicated_task_runner.ts
   - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/config.ts
   - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group2/config.ts
   - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/config.ts
@@ -221,22 +226,10 @@ enabled:
   - x-pack/test/cases_api_integration/spaces_only/config.ts
   - x-pack/test/cloud_security_posture_functional/config.ts
   - x-pack/test/cloud_security_posture_api/config.ts
+  - x-pack/test/dataset_quality_api_integration/basic/config.ts
   - x-pack/test/detection_engine_api_integration/basic/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group2/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group3/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group4/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group6/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group7/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group8/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group9/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/prebuilt_rules/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/bundled_prebuilt_rules_package/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/large_prebuilt_rules_package/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/update_prebuilt_rules_package/config.ts
   - x-pack/test/disable_ems/config.ts
   - x-pack/test/encrypted_saved_objects_api_integration/config.ts
   - x-pack/test/examples/config.ts
@@ -327,6 +320,7 @@ enabled:
   - x-pack/test/functional/apps/transform/feature_controls/config.ts
   - x-pack/test/functional/apps/upgrade_assistant/config.ts
   - x-pack/test/functional/apps/uptime/config.ts
+  - x-pack/test/functional/apps/user_profiles/config.ts
   - x-pack/test/functional/apps/visualize/config.ts
   - x-pack/test/functional/apps/watcher/config.ts
   - x-pack/test/functional/config_security_basic.ts
@@ -341,7 +335,6 @@ enabled:
   - x-pack/test/monitoring_api_integration/config.ts
   - x-pack/test/observability_api_integration/basic/config.ts
   - x-pack/test/observability_api_integration/trial/config.ts
-  - x-pack/test/observability_api_integration/apis/config.ts
   - x-pack/test/observability_functional/with_rac_write.config.ts
   - x-pack/test/observability_onboarding_api_integration/basic/config.ts
   - x-pack/test/observability_onboarding_api_integration/cloud/config.ts
@@ -392,8 +385,11 @@ enabled:
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
   - x-pack/test/security_solution_endpoint_api_int/config.ts
+  - x-pack/test/security_solution_endpoint_api_int/serverless.config.ts
   - x-pack/test/security_solution_endpoint/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -418,6 +414,8 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group3.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group4.ts
+  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
+  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
   - x-pack/test_serverless/functional/test_suites/search/config.ts
   - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
@@ -426,13 +424,17 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group3.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
+  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
+  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/security/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group3.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group4.ts
+  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts
+  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group6.ts
   - x-pack/performance/journeys/ecommerce_dashboard.ts
   - x-pack/performance/journeys/ecommerce_dashboard_map_only.ts
   - x-pack/performance/journeys/flight_dashboard.ts
@@ -452,9 +454,40 @@ enabled:
   - x-pack/performance/journeys/apm_service_inventory.ts
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/serverless.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/ess.config.ts
-
-
-
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/keyword/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/long/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/long/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/text/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_execution_logic/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/user_roles/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/user_roles/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/ess.config.ts
 

.buildkite/pipeline-utils/ci-stats/client.ts

@@ -201,7 +201,7 @@ export class CiStatsClient {
           headers: this.defaultHeaders,
         });
       } catch (error) {
-        console.error('CI Stats request error:', error);
+        console.error('CI Stats request error:', error?.response?.data?.message);
 
         if (attempt < maxAttempts) {
           const sec = attempt * 3;
@@ -210,7 +210,7 @@ export class CiStatsClient {
           continue;
         }
 
-        throw error;
+        throw new Error('Failed to connect to CI Stats.');
       }
     }
   }

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -273,7 +273,12 @@ export async function pickTestGroupRunOrder() {
           ]
         : []),
       // if we are running on a external job, like kibana-code-coverage-main, try finding times that are specific to that job
-      ...(!prNumber && pipelineSlug !== 'kibana-on-merge'
+      // kibana-elasticsearch-serverless-verify-and-promote is not necessarily run in commit order -
+      // using kibana-on-merge groups will provide a closer approximation, with a failure mode -
+      // of too many ftr groups instead of potential timeouts.
+      ...(!prNumber &&
+      pipelineSlug !== 'kibana-on-merge' &&
+      pipelineSlug !== 'kibana-elasticsearch-serverless-verify-and-promote'
         ? [
             {
               branch: ownBranch,

.buildkite/pipelines/artifacts.yml

@@ -71,6 +71,16 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
     soft_fail:

.buildkite/pipelines/es_serverless/emergency_relelease_branch_testing.yml

@@ -0,0 +1,17 @@
+# https://buildkite.com/elastic/kibana-serverless-emergency-release-branch-testing
+
+## Triggers the artifacts container image build for emergency releases
+agents:
+  queue: kibana-default
+
+notify:
+  - slack: "#kibana-mission-control"
+    if: "build.state == 'passed' || build.state == 'failed' || build.state == 'scheduled'"
+
+steps:
+  - trigger: "kibana-artifacts-container-image"
+    label: ":docker: Build Kibana Artifacts Container Image"
+    build:
+      branch: $BUILDKITE_BRANCH
+      commit: $BUILDKITE_COMMIT
+      message: Running PR build for $BUILDKITE_BRANCH

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -63,7 +63,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 2
+        parallelism: 6
         retry:
           automatic:
             - exit_status: '*'
@@ -76,7 +76,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 2
+        parallelism: 4
         retry:
           automatic:
             - exit_status: '*'
@@ -89,7 +89,33 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 4
+        parallelism: 8
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+        label: 'Defend Workflows Cypress Tests on Serverless'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-virt
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 6
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
+        label: 'Serverless Osquery Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 6
         retry:
           automatic:
             - exit_status: '*'

.buildkite/pipelines/flaky_tests/groups.json

@@ -24,6 +24,14 @@
       "key": "cypress/security_serverless_explore",
       "name": "[Serverless] Security Solution Explore - Cypress"
     },
+    {
+      "key": "cypress/defend_workflows",
+      "name": "Security Solution Defend Workflows - Cypress"
+    },
+    {
+      "key": "cypress/defend_workflows_serverless",
+      "name": "[Serverless] Security Solution Defend Workflows - Cypress"
+    },
     {
       "key": "cypress/osquery_cypress",
       "name": "Osquery - Cypress"

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -162,10 +162,11 @@ for (const testSuite of testSuites) {
           `Group configuration was not found in groups.json for the following cypress suite: {${suiteName}}.`
         );
       }
+      const agentQueue = suiteName.includes('defend_workflows') ? 'n2-4-virt' : 'n2-4-spot';
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: { queue: 'n2-4-spot' },
+        agents: { queue: agentQueue },
         depends_on: 'build',
         parallelism: testSuite.count,
         concurrency,

.buildkite/pipelines/on_merge.yml

@@ -85,7 +85,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '*'
@@ -97,7 +97,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'
@@ -109,7 +109,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 4
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '*'
@@ -157,7 +157,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 10
     retry:
       automatic:
         - exit_status: '*'
@@ -168,8 +168,8 @@ steps:
     agents:
       queue: n2-4-virt
     depends_on: build
-    timeout_in_minutes: 120
-    parallelism: 2
+    timeout_in_minutes: 60
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '*'
@@ -180,7 +180,7 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
     parallelism: 2
     retry:
       automatic:
@@ -192,7 +192,19 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 50
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
+    label: 'Serverless Osquery Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
     parallelism: 6
     retry:
       automatic:
@@ -242,7 +254,7 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: n2-16-spot
+      queue: n2-4-spot
     timeout_in_minutes: 60
     retry:
       automatic: