Merge branch 'refactor-suppression-fe' of https://github.com/WafaaNas…

…r/kibana into refactor-suppression-fe

.buildkite/ftr_configs.yml

@@ -291,6 +291,7 @@ enabled:
   - x-pack/test/functional/apps/lens/open_in_lens/dashboard/config.ts
   - x-pack/test/functional/apps/license_management/config.ts
   - x-pack/test/functional/apps/logstash/config.ts
+  - x-pack/test/functional/apps/managed_content/config.ts
   - x-pack/test/functional/apps/management/config.ts
   - x-pack/test/functional/apps/maps/group1/config.ts
   - x-pack/test/functional/apps/maps/group2/config.ts
@@ -415,6 +416,7 @@ enabled:
   - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.saved_objects_management.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group3.ts
@@ -426,6 +428,7 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/config.ts
   - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.saved_objects_management.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group3.ts
@@ -435,6 +438,7 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/security/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group3.ts
@@ -516,4 +520,4 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_read/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_read/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/serverless.config.ts

.buildkite/pipelines/on_merge.yml

@@ -337,6 +337,16 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/quick_checks.sh
+    label: 'Quick Checks'
+    agents:
+      queue: n2-2-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:

.buildkite/pipelines/performance/daily.yml

@@ -21,11 +21,11 @@ steps:
       queue: kb-static-ubuntu
     depends_on: build
     key: tests
-    timeout_in_minutes: 60
+    timeout_in_minutes: 90
     retry:
       automatic:
         - exit_status: '-1'
-          limit: 3
+          limit: 2
         - exit_status: '*'
           limit: 1
 

.buildkite/pipelines/pull_request/base.yml

@@ -19,17 +19,18 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/build_api_docs.sh
-    label: 'Check Types and Build API Docs'
+  - command: .buildkite/scripts/steps/quick_checks.sh
+    label: 'Quick Checks'
     agents:
-      queue: n2-4-spot
-    key: build_api_docs
+      queue: n2-2-spot
     timeout_in_minutes: 60
     retry:
       automatic:
         - exit_status: '-1'
           limit: 3
 
+  - wait
+
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
     agents:
@@ -88,3 +89,14 @@ steps:
       automatic:
         - exit_status: '-1'
           limit: 3
+
+  - command: .buildkite/scripts/steps/build_api_docs.sh
+    label: 'Check Types and Build API Docs'
+    agents:
+      queue: n2-4-spot
+    key: build_api_docs
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/scripts/common/util.sh

@@ -171,15 +171,23 @@ download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
 
+# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
+  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+  VAULT_KV_PREFIX="secret/kibana-issues/dev"
+  IS_LEGACY_VAULT_ADDR=true
+else
+  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
+  IS_LEGACY_VAULT_ADDR=false
+fi
+export IS_LEGACY_VAULT_ADDR
 
 vault_get() {
   key_path=$1
   field=$2
 
-  fullPath="secret/ci/elastic-kibana/$key_path"
-  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    fullPath="secret/kibana-issues/dev/$key_path"
-  fi
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
 
   if [[ -z "${2:-}" || "${2:-}" =~ ^-.* ]]; then
     retry 5 5 vault read "$fullPath" "${@:2}"
@@ -193,11 +201,17 @@ vault_set() {
   shift
   fields=("$@")
 
-  fullPath="secret/ci/elastic-kibana/$key_path"
-  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    fullPath="secret/kibana-issues/dev/$key_path"
-  fi
+
+  fullPath="$VAULT_PATH_PREFIX/$key_path"
 
   # shellcheck disable=SC2068
   retry 5 5 vault write "$fullPath" ${fields[@]}
 }
+
+vault_kv_set() {
+  kv_path=$1
+  shift
+  fields=("$@")
+
+  vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
+}

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -300,20 +300,9 @@ const uploadPipeline = (pipelineContent: string | object) => {
     }
 
     if (
-      (await doAnyChangesMatch([
-        /^src\/plugins\/controls/,
-        /^packages\/kbn-securitysolution-.*/,
-        /^x-pack\/plugins\/lists/,
-        /^x-pack\/plugins\/security_solution/,
-        /^x-pack\/plugins\/timelines/,
-        /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
-        /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/alerts_table/,
-        /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/connectors_context\.tsx/,
-        /^x-pack\/test\/defend_workflows_cypress/,
-        /^x-pack\/test\/security_solution_cypress/,
-        /^fleet_packages\.json/, // It contains reference to prebuilt detection rules, we want to run security solution tests if it changes
-      ])) ||
-      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+      ((await doAnyChangesMatch([/^x-pack\/plugins\/osquery/, /^x-pack\/test\/osquery_cypress/])) ||
+        GITHUB_PR_LABELS.includes('ci:all-cypress-suites')) &&
+      !GITHUB_PR_LABELS.includes('ci:skip-cypress-osquery')
     ) {
       pipeline.push(
         getPipeline('.buildkite/pipelines/pull_request/security_solution/osquery_cypress.yml')

.buildkite/scripts/pipelines/security_solution_ess/pipeline.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "Inside the security solution pipeline"

.buildkite/scripts/steps/checks.sh

@@ -5,25 +5,8 @@ set -euo pipefail
 export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/bootstrap.sh
 
-.buildkite/scripts/steps/checks/precommit_hook.sh
-.buildkite/scripts/steps/checks/ts_projects.sh
-.buildkite/scripts/steps/checks/packages.sh
-.buildkite/scripts/steps/checks/bazel_packages.sh
-.buildkite/scripts/steps/checks/verify_notice.sh
-.buildkite/scripts/steps/checks/plugin_list_docs.sh
-.buildkite/scripts/steps/checks/event_log.sh
-.buildkite/scripts/steps/checks/telemetry.sh
-.buildkite/scripts/steps/checks/jest_configs.sh
-.buildkite/scripts/steps/checks/bundle_limits.sh
-.buildkite/scripts/steps/checks/i18n.sh
-.buildkite/scripts/steps/checks/file_casing.sh
-.buildkite/scripts/steps/checks/licenses.sh
-.buildkite/scripts/steps/checks/test_projects.sh
-.buildkite/scripts/steps/checks/test_hardening.sh
-.buildkite/scripts/steps/checks/ftr_configs.sh
 .buildkite/scripts/steps/checks/saved_objects_compat_changes.sh
 .buildkite/scripts/steps/checks/saved_objects_definition_change.sh
 .buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
 .buildkite/scripts/steps/code_generation/osquery_codegen.sh
-.buildkite/scripts/steps/checks/yarn_deduplicate.sh

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -86,7 +86,13 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
   VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
   VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
   retry 5 30 vault login -no-print "$VAULT_TOKEN"
-  vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+
+  # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+  if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  else
+    vault_kv_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  fi
 
   echo "Enabling Stack Monitoring..."
   jq '
@@ -121,10 +127,11 @@ fi
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
 CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
 
-if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+  VAULT_READ_COMMAND="vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
 else
-  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+  VAULT_READ_COMMAND="vault kv get $VAULT_KV_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
 fi
 
 cat << EOF | buildkite-agent annotate --style "info" --context cloud
@@ -134,7 +141,7 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
 
   Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
 
-  Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
+  Credentials: \`$VAULT_READ_COMMAND\`
 
   Kibana image: \`$KIBANA_CLOUD_IMAGE\`
 

.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh

@@ -6,5 +6,5 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Elastic Assistant OpenAPI Code Generation
 
-(cd x-pack/plugins/elastic_assistant && yarn openapi:generate)
+(cd x-pack/packages/kbn-elastic-assistant-common && yarn openapi:generate)
 check_for_changed_files "yarn openapi:generate" true

.buildkite/scripts/steps/package_testing/test.sh

@@ -5,6 +5,9 @@ set -euo pipefail
 source "$(dirname "$0")/../../common/util.sh"
 .buildkite/scripts/bootstrap.sh
 
+# temporary adding this to get screenshots
+is_test_execution_step
+
 echo "--- Package Testing for $TEST_PACKAGE"
 
 mkdir -p target

.buildkite/scripts/steps/quick_checks.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export DISABLE_BOOTSTRAP_VALIDATION=false
+.buildkite/scripts/bootstrap.sh
+
+.buildkite/scripts/steps/checks/precommit_hook.sh
+.buildkite/scripts/steps/checks/ts_projects.sh
+.buildkite/scripts/steps/checks/packages.sh
+.buildkite/scripts/steps/checks/bazel_packages.sh
+.buildkite/scripts/steps/checks/verify_notice.sh
+.buildkite/scripts/steps/checks/plugin_list_docs.sh
+.buildkite/scripts/steps/checks/event_log.sh
+.buildkite/scripts/steps/checks/telemetry.sh
+.buildkite/scripts/steps/checks/jest_configs.sh
+.buildkite/scripts/steps/checks/bundle_limits.sh
+.buildkite/scripts/steps/checks/i18n.sh
+.buildkite/scripts/steps/checks/file_casing.sh
+.buildkite/scripts/steps/checks/licenses.sh
+.buildkite/scripts/steps/checks/test_projects.sh
+.buildkite/scripts/steps/checks/test_hardening.sh
+.buildkite/scripts/steps/checks/ftr_configs.sh
+.buildkite/scripts/steps/checks/yarn_deduplicate.sh

.buildkite/scripts/steps/serverless/build_and_deploy.sh

@@ -77,7 +77,14 @@ deploy() {
     VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
     VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
     retry 5 30 vault login -no-print "$VAULT_TOKEN"
-    vault_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+
+    # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+    if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+      vault_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+    else
+      vault_kv_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+    fi
+
   else
     echo "Updating project..."
     curl -s \
@@ -91,10 +98,11 @@ deploy() {
   PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
   PROJECT_ELASTICSEARCH_URL=$(jq -r --slurp '.[1].endpoints.elasticsearch' $DEPLOY_LOGS)
 
-  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+  # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
+  if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    VAULT_READ_COMMAND="vault read $VAULT_PATH_PREFIX/cloud-deploy/$PROJECT_NAME"
   else
-    VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+    VAULT_READ_COMMAND="vault kv get $VAULT_KV_PREFIX/cloud-deploy/$PROJECT_NAME"
   fi
 
   cat << EOF | buildkite-agent annotate --style "info" --context "project-$PROJECT_TYPE"
@@ -104,7 +112,7 @@ Kibana: $PROJECT_KIBANA_LOGIN_URL
 
 Elasticsearch: $PROJECT_ELASTICSEARCH_URL
 
-Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$PROJECT_NAME\`
+Credentials: \`$VAULT_READ_COMMAND\`
 
 Kibana image: \`$KIBANA_IMAGE\`
 EOF

.eslintrc.js

@@ -1121,6 +1121,7 @@ module.exports = {
         'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
+        'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
       ],
       plugins: ['eslint-plugin-node', 'react'],
       env: {
@@ -1218,6 +1219,8 @@ module.exports = {
             'x-pack/plugins/security_solution_ess/**/*.{js,mjs,ts,tsx}',
             'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
             'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
+            'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}',
+            'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
           ],
           rules: {
             '@typescript-eslint/consistent-type-imports': 'error',