Merge branch 'main' into coverage-overview-duplication-bug

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -273,7 +273,12 @@ export async function pickTestGroupRunOrder() {
           ]
         : []),
       // if we are running on a external job, like kibana-code-coverage-main, try finding times that are specific to that job
-      ...(!prNumber && pipelineSlug !== 'kibana-on-merge'
+      // kibana-elasticsearch-serverless-verify-and-promote is not necessarily run in commit order -
+      // using kibana-on-merge groups will provide a closer approximation, with a failure mode -
+      // of too many ftr groups instead of potential timeouts.
+      ...(!prNumber &&
+      pipelineSlug !== 'kibana-on-merge' &&
+      pipelineSlug !== 'kibana-elasticsearch-serverless-verify-and-promote'
         ? [
             {
               branch: ownBranch,

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -63,7 +63,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 4
+        parallelism: 6
         retry:
           automatic:
             - exit_status: '*'

.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml

@@ -46,7 +46,7 @@ steps:
         env:
           USE_GROUP_LABEL: true
         agents:
-          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.4"
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
 
   - group: "Observability"
     steps:
@@ -56,4 +56,4 @@ steps:
           NOTIFICATION_APPENDIX: "<!subteam^S060GJFKGGP> please execute your manual testing plan."
           USE_GROUP_LABEL: true
         agents:
-          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.4"
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -23,7 +23,9 @@ steps:
 
   - wait: ~
 
-  - label: ":judge::seedling: Trigger Manual Tests Phase"
-    command: "make -C /agent trigger-manual-verification-phase"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.4"
+  - group: "Kibana Release Manager"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase"
+        command: "make -C /agent trigger-manual-verification-phase"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/security_solution/base.yml

@@ -0,0 +1,36 @@
+steps:
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless
+    label: 'Serverless MKI QA Security Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
+    label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
+    label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/lifecycle/post_command.sh

@@ -14,6 +14,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-security-solution/**/management/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-osquery/**/*.png'
   buildkite-agent artifact upload 'target/kibana-osquery/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'

.buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [ -z "$1" ]
+  then
+    echo "No target script from the package.json file, is supplied"
+    exit 1
+fi
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-security-solution-chrome
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+cd x-pack/test/security_solution_cypress
+set +e
+
+QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
+
+CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh

@@ -1,19 +1,5 @@
 #!/bin/bash
-set -euo pipefail
-
-source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
-.buildkite/scripts/bootstrap.sh
-
-export JOB=kibana-security-solution-chrome
 
-buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
-
-echo "--- Serverless Security Second Quality Gate"
-cd x-pack/test/security_solution_cypress
-set +e
-
-VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key)
-ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY)
+set -euo pipefail
 
-CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status
+ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+
+const getPipeline = (filename: string, removeSteps = true) => {
+  const str = fs.readFileSync(filename).toString();
+  return removeSteps ? str.replace(/^steps:/, '') : str;
+};
+
+const uploadPipeline = (pipelineContent: string | object) => {
+  const str =
+    typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
+
+  execSync('buildkite-agent pipeline upload', {
+    input: str,
+    stdio: ['pipe', 'inherit', 'inherit'],
+  });
+};
+
+(async () => {
+  try {
+    const pipeline = [];
+
+    pipeline.push(getPipeline('.buildkite/pipelines/security_solution/base.yml', false));
+    // remove duplicated steps
+    uploadPipeline([...new Set(pipeline)].join('\n'));
+  } catch (ex) {
+    console.error('PR pipeline generation error', ex.message);
+    process.exit(1);
+  }
+})();

.buildkite/scripts/steps/checks.sh

@@ -24,4 +24,5 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/saved_objects_compat_changes.sh
 .buildkite/scripts/steps/checks/saved_objects_definition_change.sh
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
+.buildkite/scripts/steps/code_generation/osquery_codegen.sh
 .buildkite/scripts/steps/checks/yarn_deduplicate.sh

.buildkite/scripts/steps/code_generation/osquery_codegen.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Osquery OpenAPI Code Generation
+
+(cd x-pack/plugins/osquery && yarn openapi:generate)
+check_for_changed_files "yarn openapi:generate" true

.github/CODEOWNERS

@@ -78,9 +78,7 @@ x-pack/test/cloud_integration/plugins/saml_provider @elastic/kibana-core
 x-pack/plugins/cloud_integrations/cloud_links @elastic/kibana-core
 x-pack/plugins/cloud @elastic/kibana-core
 x-pack/plugins/cloud_security_posture @elastic/kibana-cloud-security-posture
-packages/shared-ux/code_editor/impl @elastic/appex-sharedux
-packages/shared-ux/code_editor/mocks @elastic/appex-sharedux
-packages/shared-ux/code_editor/types @elastic/appex-sharedux
+packages/shared-ux/code_editor @elastic/appex-sharedux
 packages/kbn-coloring @elastic/kibana-visualizations
 packages/kbn-config @elastic/kibana-core
 packages/kbn-config-mocks @elastic/kibana-core
@@ -676,9 +674,7 @@ examples/share_examples @elastic/appex-sharedux
 src/plugins/share @elastic/appex-sharedux
 packages/kbn-shared-svg @elastic/apm-ui
 packages/shared-ux/avatar/solution @elastic/appex-sharedux
-packages/shared-ux/button/exit_full_screen/impl @elastic/appex-sharedux
-packages/shared-ux/button/exit_full_screen/mocks @elastic/appex-sharedux
-packages/shared-ux/button/exit_full_screen/types @elastic/appex-sharedux
+packages/shared-ux/button/exit_full_screen @elastic/appex-sharedux
 packages/shared-ux/button_toolbar @elastic/appex-sharedux
 packages/shared-ux/card/no_data/impl @elastic/appex-sharedux
 packages/shared-ux/card/no_data/mocks @elastic/appex-sharedux
@@ -956,6 +952,7 @@ x-pack/plugins/infra/server/lib/alerting @elastic/actionable-observability
 /x-pack/test/fleet_cypress @elastic/fleet
 /x-pack/test/fleet_functional @elastic/fleet
 /src/dev/build/tasks/bundle_fleet_packages.ts @elastic/fleet @elastic/kibana-operations
+/x-pack/plugins/fleet/server/services/elastic_agent_manifest.ts @elastic/fleet @elastic/obs-cloudnative-monitoring
 
 # APM
 /x-pack/test/functional/apps/apm/  @elastic/apm-ui
@@ -1364,14 +1361,10 @@ x-pack/plugins/security_solution/server/usage/ @elastic/security-data-analytics
 x-pack/plugins/security_solution/server/lib/telemetry/ @elastic/security-data-analytics
 
 ## Security Solution sub teams - security-engineering-productivity
-x-pack/test/security_solution_cypress/cypress/README.md @elastic/security-engineering-productivity
-x-pack/test/security_solution_cypress/es_archives @elastic/security-engineering-productivity
-x-pack/test/security_solution_cypress/cli_config.ts @elastic/security-engineering-productivity
-x-pack/test/security_solution_cypress/config.ts @elastic/security-engineering-productivity
-x-pack/test/security_solution_cypress/runner.ts @elastic/security-engineering-productivity
-x-pack/test/security_solution_cypress/serverless_config.ts @elastic/security-engineering-productivity
-x-pack/test/security_solution_cypress/cypress/tags.ts @elastic/security-engineering-productivity
-x-pack/plugins/security_solution/scripts/run_cypress @MadameSheema @patrykkopycinski @oatkiller @maximpn @banderror
+/x-pack/test/security_solution_cypress/* @elastic/security-engineering-productivity
+/x-pack/test/security_solution_cypress/cypress/* @elastic/security-engineering-productivity
+/x-pack/test/security_solution_cypress/es_archives @elastic/security-engineering-productivity
+/x-pack/plugins/security_solution/scripts/run_cypress @MadameSheema @patrykkopycinski @oatkiller @maximpn @banderror
 
 ## Security Solution sub teams - adaptive-workload-protection
 x-pack/plugins/security_solution/public/common/components/sessions_viewer @elastic/kibana-cloud-security-posture

api_docs/actions.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the actions plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 ---
 import actionsObj from './actions.devdocs.json';

api_docs/advanced_settings.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the advancedSettings plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 ---
 import advancedSettingsObj from './advanced_settings.devdocs.json';

api_docs/aiops.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops
 title: "aiops"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiops plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
 ---
 import aiopsObj from './aiops.devdocs.json';

api_docs/alerting.devdocs.json

@@ -109,8 +109,8 @@
           "trackAdoption": false,
           "references": [
             {
-              "plugin": "stackAlerts",
-              "path": "x-pack/plugins/stack_alerts/public/rule_types/es_query/index.ts"
+              "plugin": "ml",
+              "path": "x-pack/plugins/ml/public/alerting/register_ml_alerts.ts"
             },
             {
               "plugin": "stackAlerts",
@@ -125,8 +125,8 @@
               "path": "x-pack/plugins/stack_alerts/public/rule_types/es_query/index.ts"
             },
             {
-              "plugin": "ml",
-              "path": "x-pack/plugins/ml/public/alerting/register_ml_alerts.ts"
+              "plugin": "stackAlerts",
+              "path": "x-pack/plugins/stack_alerts/public/rule_types/es_query/index.ts"
             }
           ],
           "children": [

api_docs/alerting.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/alerting
 title: "alerting"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the alerting plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'alerting']
 ---
 import alertingObj from './alerting.devdocs.json';

api_docs/apm.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apm
 title: "apm"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apm plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apm']
 ---
 import apmObj from './apm.devdocs.json';

api_docs/apm_data_access.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apmDataAccess
 title: "apmDataAccess"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apmDataAccess plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apmDataAccess']
 ---
 import apmDataAccessObj from './apm_data_access.devdocs.json';

api_docs/asset_manager.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/assetManager
 title: "assetManager"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the assetManager plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'assetManager']
 ---
 import assetManagerObj from './asset_manager.devdocs.json';

api_docs/banners.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/banners
 title: "banners"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the banners plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'banners']
 ---
 import bannersObj from './banners.devdocs.json';

api_docs/bfetch.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/bfetch
 title: "bfetch"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the bfetch plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'bfetch']
 ---
 import bfetchObj from './bfetch.devdocs.json';

api_docs/canvas.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/canvas
 title: "canvas"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the canvas plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'canvas']
 ---
 import canvasObj from './canvas.devdocs.json';

api_docs/cases.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/cases
 title: "cases"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the cases plugin
-date: 2023-11-01
+date: 2023-11-06
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'cases']
 ---
 import casesObj from './cases.devdocs.json';