Merge branch 'main' into telemetry/replace-settings-field-usages

.buildkite/ftr_configs.yml

@@ -91,9 +91,6 @@ disabled:
   - x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
 
-  # Failing: See https://github.com/elastic/kibana/issues/174020
-  - x-pack/test/api_integration/apis/asset_manager/config_when_enabled.ts
-
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -188,6 +185,7 @@ enabled:
   - x-pack/test/api_integration/config_security_trial.ts
   - x-pack/test/api_integration/apis/aiops/config.ts
   - x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
+  - x-pack/test/api_integration/apis/asset_manager/config_when_enabled.ts
   - x-pack/test/api_integration/apis/cases/config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts
   - x-pack/test/api_integration/apis/console/config.ts
@@ -216,6 +214,7 @@ enabled:
   - x-pack/test/api_integration/apis/stats/config.ts
   - x-pack/test/api_integration/apis/status/config.ts
   - x-pack/test/api_integration/apis/synthetics/config.ts
+  - x-pack/test/api_integration/apis/slos/config.ts
   - x-pack/test/api_integration/apis/telemetry/config.ts
   - x-pack/test/api_integration/apis/transform/config.ts
   - x-pack/test/api_integration/apis/upgrade_assistant/config.ts
@@ -423,6 +422,7 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.screenshots.ts
   - x-pack/test_serverless/functional/test_suites/search/config.ts
   - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
@@ -458,6 +458,7 @@ enabled:
   - x-pack/performance/journeys/tags_listing_page.ts
   - x-pack/performance/journeys/cloud_security_dashboard.ts
   - x-pack/performance/journeys/apm_service_inventory.ts
+  - x-pack/performance/journeys/infra_hosts_view.ts
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts
@@ -496,6 +497,8 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/rule_management/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/rule_management/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts
@@ -513,4 +516,4 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_read/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_read/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/serverless.config.ts

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -143,11 +143,9 @@ for (const testSuite of testSuites) {
       },
       depends_on: 'build',
       timeout_in_minutes: 150,
+      cancel_on_build_failing: true,
       retry: {
-        automatic: [
-          { exit_status: '-1', limit: 3 },
-          // { exit_status: '*', limit: 1 },
-        ],
+        automatic: [{ exit_status: '-1', limit: 3 }],
       },
     });
     continue;
@@ -168,10 +166,15 @@ for (const testSuite of testSuites) {
         label: group.name,
         agents: { queue: agentQueue },
         depends_on: 'build',
+        timeout_in_minutes: 150,
         parallelism: testSuite.count,
         concurrency,
         concurrency_group: process.env.UUID,
         concurrency_method: 'eager',
+        cancel_on_build_failing: true,
+        retry: {
+          automatic: [{ exit_status: '-1', limit: 3 }],
+        },
         env: {
           // disable split of test cases between parallel jobs when running them in flaky test runner
           // by setting chunks vars to value 1, which means all test will run in one job

.buildkite/pipelines/pipeline.kibana-serverless-release.yaml

@@ -7,9 +7,6 @@ steps:
       env:
         SERVICE_COMMIT_HASH: "${BUILDKITE_COMMIT:0:12}"
         REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/config.yaml
-        SERVICE: kibana-controller
-        NAMESPACE: kibana-ci
-        IMAGE_NAME: kibana-serverless
 
 notify:
   - slack: "#kibana-mission-control"

.buildkite/pipelines/pull_request/base.yml

@@ -57,54 +57,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
-    label: 'Threat Intelligence Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 2
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
-    label: 'Osquery Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-    label: 'Serverless Osquery Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  # status_exception: Native role management is not enabled in this Elasticsearch instance
-  # - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
-  #   label: 'Serverless Security Defend Workflows Cypress Tests'
-  #   agents:
-  #     queue: n2-4-spot
-  #   depends_on: build
-  #   timeout_in_minutes: 60
-  #   retry:
-  #     automatic:
-  #       - exit_status: '*'
-  #         limit: 1
-
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:

.buildkite/pipelines/pull_request/security_solution/defend_workflows.yml

@@ -22,3 +22,15 @@ steps:
       automatic:
         - exit_status: '*'
           limit: 1
+
+  # status_exception: Native role management is not enabled in this Elasticsearch instance
+  # - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
+  #   label: 'Serverless Security Defend Workflows Cypress Tests'
+  #   agents:
+  #     queue: n2-4-spot
+  #   depends_on: build
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '*'
+  #         limit: 1

.buildkite/pipelines/pull_request/security_solution/osquery_cypress.yml

@@ -0,0 +1,24 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
+    label: 'Osquery Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
+    label: 'Serverless Osquery Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/security_solution/threat_intelligence.yml

@@ -0,0 +1,12 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
+    label: 'Threat Intelligence Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml

@@ -27,6 +27,11 @@ steps:
   - label: ":cookie: 24h bake time before continuing promotion"
     if: build.env("ENVIRONMENT") == "production-canary"
     command: "sleep 86400"
+    soft_fail:
+      # A manual cancel of that step produces return code 255.
+      # We're treating this case as a soft fail to allow manual bake time skipping.
+      # To stop the promotion entirely, instead click the "Cancel" button at the top of the page
+      - exit_status: 255
     agents:
       # How long can this agent live for in minutes - 25 hours
       instanceMaxAge: 1500

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -265,6 +265,58 @@ const uploadPipeline = (pipelineContent: string | object) => {
       );
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^package.json/,
+        /^src\/plugins\/data/,
+        /^src\/plugins\/kibana_utils/,
+        /^src\/plugins\/inspector/,
+        /^src\/plugins\/data_views/,
+        /^src\/core/,
+        /^packages\/kbn-securitysolution-.*/,
+        /^packages\/kbn-es-query/,
+        /^packages\/kbn-securitysolution-io-ts-list-types/,
+        /^packages\/kbn-i18n-react/,
+        /^packages\/kbn-i18n/,
+        /^packages\/shared-ux/,
+        /^packages\/kbn-doc-links/,
+        /^packages\/kbn-securitysolution-io-ts-list-types/,
+        /^x-pack\/plugins\/threat_intelligence/,
+        /^x-pack\/packages\/security-solution/,
+        /^x-pack\/test\/threat_intelligence_cypress/,
+        /^x-pack\/plugins\/cases/,
+        /^x-pack\/plugins\/timelines/,
+        /^x-pack\/plugins\/triggers_actions_ui/,
+        /^x-pack\/plugins\/rule_registry/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(
+        getPipeline('.buildkite/pipelines/pull_request/security_solution/threat_intelligence.yml')
+      );
+    }
+
+    if (
+      (await doAnyChangesMatch([
+        /^src\/plugins\/controls/,
+        /^packages\/kbn-securitysolution-.*/,
+        /^x-pack\/plugins\/lists/,
+        /^x-pack\/plugins\/security_solution/,
+        /^x-pack\/plugins\/timelines/,
+        /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
+        /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/alerts_table/,
+        /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/connectors_context\.tsx/,
+        /^x-pack\/test\/defend_workflows_cypress/,
+        /^x-pack\/test\/security_solution_cypress/,
+        /^fleet_packages\.json/, // It contains reference to prebuilt detection rules, we want to run security solution tests if it changes
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(
+        getPipeline('.buildkite/pipelines/pull_request/security_solution/osquery_cypress.yml')
+      );
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     // remove duplicated steps

.buildkite/scripts/serverless/create_deploy_tag/info_sections/so_snapshot_comparison.ts

@@ -11,12 +11,15 @@ import { readFileSync } from 'fs';
 import { exec } from '../shared';
 import { BuildkiteClient, getKibanaDir } from '#pipeline-utils';
 
+type VersionChanges = Record<string, { from: number; to: number; emoji: string }>;
+
 export function compareSOSnapshots(
   previousSha: string,
   selectedSha: string
 ): null | {
   hasChanges: boolean;
   changed: string[];
+  versionChanges: VersionChanges;
   command: string;
 } {
   assertValidSha(previousSha);
@@ -33,9 +36,15 @@ export function compareSOSnapshots(
     const buildkite = new BuildkiteClient({ exec });
     buildkite.uploadArtifacts(outputPath);
 
+    const versionChanges = Object.keys(soComparisonResult.changes).reduce((changes, pluginId) => {
+      changes[pluginId] = soComparisonResult.changes[pluginId].versionChange;
+      return changes;
+    }, {} as VersionChanges);
+
     return {
       hasChanges: soComparisonResult.hasChanges,
       changed: soComparisonResult.changed,
+      versionChanges,
       command,
     };
   } catch (ex) {
@@ -47,14 +56,24 @@ export function compareSOSnapshots(
 export function makeSOComparisonBlockHtml(comparisonResult: {
   hasChanges: boolean;
   changed: string[];
+  versionChanges: VersionChanges;
   command: string;
 }): string {
   if (comparisonResult.hasChanges) {
+    const { versionChanges, changed, command } = comparisonResult;
     return `<div>
-<h4>Plugin Saved Object migration changes: *yes, ${comparisonResult.changed.length} plugin(s)*</h4>
-<div>Changed plugins: <strong>${comparisonResult.changed.join(', ')}</strong></div>
+<h4>Plugin Saved Object migration changes: *yes, ${changed.length} plugin(s)*</h4>
+<div>Changed plugins:</div>
+ <div>
+    <ul>
+${Object.keys(versionChanges).map((key) => {
+  const { from, to, emoji } = versionChanges[key];
+  return `<li>${emoji} ${key}: ${from} => ${to}</li>`;
+})}
+    </ul>
+ </div>
 <i>Find detailed info in the archived artifacts, or run the command yourself: </i>
-<div><pre>${comparisonResult.command}</pre></div>
+<div><pre>${command}</pre></div>
 </div>`;
   } else {
     return `<div>

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -119,9 +119,7 @@ steps:
     build:
       env:
         SERVICE_COMMIT_HASH: "$GIT_ABBREV_COMMIT"
-        SERVICE: kibana-controller
-        NAMESPACE: kibana-ci
-        IMAGE_NAME: kibana-serverless
+        SERVICE: kibana
         REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/dev.yaml
 EOF
 

.buildkite/scripts/steps/checks.sh

@@ -23,6 +23,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/ftr_configs.sh
 .buildkite/scripts/steps/checks/saved_objects_compat_changes.sh
 .buildkite/scripts/steps/checks/saved_objects_definition_change.sh
+.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
 .buildkite/scripts/steps/code_generation/osquery_codegen.sh
 .buildkite/scripts/steps/checks/yarn_deduplicate.sh

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -62,7 +62,7 @@ fi
 
 echo "--- Create Deployment"
 CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
-if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
+if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; then
   jq '
     .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
     .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" |

.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Elastic Assistant OpenAPI Code Generation
+
+(cd x-pack/plugins/elastic_assistant && yarn openapi:generate)
+check_for_changed_files "yarn openapi:generate" true

.buildkite/scripts/steps/serverless/build_and_deploy.sh

@@ -59,6 +59,11 @@ deploy() {
       -XPOST -d "$PROJECT_CREATE_CONFIGURATION" &>> $DEPLOY_LOGS
 
     PROJECT_ID=$(jq -r --slurp '.[1].id' $DEPLOY_LOGS)
+    if [ -z "${PROJECT_ID}" ] || [ "$PROJECT_ID" = 'null' ]; then
+      echo "Failed to create project. Deploy logs:"
+      cat $DEPLOY_LOGS
+      exit 1
+    fi
 
     echo "Get credentials..."
     curl -s -XPOST -H "Authorization: ApiKey $PROJECT_API_KEY" \

.eslintignore

@@ -29,7 +29,6 @@ snapshots.js
 /x-pack/plugins/reporting/server/export_types/printable_pdf/server/lib/pdf/assets/**
 /x-pack/plugins/reporting/server/export_types/printable_pdf_v2/server/lib/pdf/assets/**
 /x-pack/plugins/cloud_integrations/cloud_full_story/server/assets/**
-/x-pack/plugins/cloud_integrations/cloud_gain_sight/server/assets/**
 
 # package overrides
 /packages/kbn-eslint-config