diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index b17a63170f907..b792825bd9dcf 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -322,6 +322,7 @@ enabled: - x-pack/test/functional/apps/security/config.ts - x-pack/test/functional/apps/snapshot_restore/config.ts - x-pack/test/functional/apps/spaces/config.ts + - x-pack/test/functional/apps/spaces/in_solution_navigation/config.ts - x-pack/test/functional/apps/status_page/config.ts - x-pack/test/functional/apps/transform/creation/index_pattern/config.ts - x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/config.ts diff --git a/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml b/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml new file mode 100644 index 0000000000000..5368b60314e47 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml @@ -0,0 +1,51 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-api-docs-daily + description: Builds api_docs daily and creates a PR with the changes + links: + - url: 'https://buildkite.com/elastic/kibana-api-docs-daily' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:kibana-operations' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / api-docs / daily + description: Builds api_docs daily and creates a PR with the changes + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main + cancel_intermediate_builds: true + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/build_api_docs.yml + skip_intermediate_builds: false + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + schedules: + Daily build: + cronline: 0 0 * * * America/New_York + message: Daily build + branch: main diff --git a/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml b/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml new file mode 100644 index 0000000000000..32f78b24e8d23 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml @@ -0,0 +1,38 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-fips-daily + description: Run Kibana FIPS smoke tests + links: + - title: Pipeline link + url: https://buildkite.com/elastic/kibana-fips +spec: + type: buildkite-pipeline + owner: group:kibana-operations + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / fips + description: Run Kibana FIPS smoke tests + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts" + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true" + repository: elastic/kibana + branch_configuration: main + default_branch: main + pipeline_file: ".buildkite/pipelines/fips.yml" + provider_settings: + trigger_mode: none + schedules: + daily: + branch: main + cronline: 0 9 * * * America/New_York + teams: + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY diff --git a/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml b/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml new file mode 100644 index 0000000000000..8805fef47f914 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-fleet-packages + description: Installs all fleet packages into Kibana to ensure the install step works + links: + - url: 'https://buildkite.com/elastic/kibana-fleet-packages' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:kibana-operations' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / fleet-packages + description: Installs all fleet packages into Kibana to ensure the install step works + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#fleet-notifications' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/fleet/packages_daily.yml + skip_intermediate_builds: false + provider_settings: + trigger_mode: none + publish_commit_status: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + fleet: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + schedules: + Single user daily test: + cronline: 0 9 * * * America/New_York + message: Single user daily test + env: {} + branch: main diff --git a/.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml b/.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml new file mode 100644 index 0000000000000..3e8f53b101626 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml @@ -0,0 +1,46 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-serverless-secsol-qg-api-integration + description: Runs the serverless security solution api integration tests for the Quality Gate + links: + - url: 'https://buildkite.com/elastic/kibana-serverless-security-solution-quality-gate-api-integration' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:kibana-operations' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / serverless / security-solution-quality-gate / api-integration + description: Runs the serverless security solution api integration tests for the Quality Gate + spec: + env: {} + allow_rebuilds: true + branch_configuration: '' + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/security_solution/api_integration.yml + skip_intermediate_builds: false + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + security-engineering-productivity: + access_level: MANAGE_BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml index 00e7c5b8fb91e..2cc1439713e48 100644 --- a/.buildkite/pipeline-resource-definitions/locations.yml +++ b/.buildkite/pipeline-resource-definitions/locations.yml @@ -7,20 +7,24 @@ metadata: spec: type: url targets: - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml' - - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml' + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-security-solution-quality-gate-api-integration.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml diff --git a/.buildkite/pipelines/build_api_docs.yml b/.buildkite/pipelines/build_api_docs.yml index b2ab95b701d57..ef1f875861387 100644 --- a/.buildkite/pipelines/build_api_docs.yml +++ b/.buildkite/pipelines/build_api_docs.yml @@ -1,13 +1,46 @@ env: PUBLISH_API_DOCS_CHANGES: 'true' steps: - - command: .buildkite/scripts/steps/build_api_docs.sh + - command: .buildkite/scripts/steps/api_docs/type_check_with_status.sh + label: Run scripts/type_check + key: type_check + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 50 + + - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh label: 'Build API Docs' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true key: build_api_docs - timeout_in_minutes: 60 + timeout_in_minutes: 50 + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - wait: ~ + + - command: .buildkite/scripts/steps/api_docs/publish_api_docs.sh + label: 'Publish API Docs' + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true + key: publish_api_docs + timeout_in_minutes: 50 retry: automatic: - exit_status: '-1' limit: 3 + if: env('PUBLISH_API_DOCS_CHANGES') == 'true' diff --git a/.buildkite/pipelines/fips.yml b/.buildkite/pipelines/fips.yml new file mode 100644 index 0000000000000..44e44ac69d8a4 --- /dev/null +++ b/.buildkite/pipelines/fips.yml @@ -0,0 +1,32 @@ +env: + DISABLE_CI_STATS_SHIPPING: "true" +steps: + - command: .buildkite/scripts/steps/build_kibana.sh + label: Build Kibana Distribution and Plugins + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-16 + preemptible: true + key: build + if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" + timeout_in_minutes: 60 + retry: + automatic: + - exit_status: "-1" + limit: 3 + + - wait + + - command: TEST_PACKAGE=fips .buildkite/scripts/steps/package_testing/test.sh + label: "Smoke testing for FIPS" + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 + timeout_in_minutes: 600 diff --git a/.buildkite/pipelines/fleet/packages_daily.yml b/.buildkite/pipelines/fleet/packages_daily.yml index cecd88a948ddc..12d7b85f27919 100644 --- a/.buildkite/pipelines/fleet/packages_daily.yml +++ b/.buildkite/pipelines/fleet/packages_daily.yml @@ -3,14 +3,21 @@ steps: label: Pre-Build timeout_in_minutes: 10 agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-2 - wait - command: .buildkite/scripts/steps/fleet/install_all_packages.sh label: Install All Packages agents: - queue: n2-2 + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + localSsds: 1 + machineType: n2-standard-2 env: # ensure that the FTR logs all output for these tests DISABLE_CI_LOG_OUTPUT_CAPTURE: 'true' @@ -24,4 +31,7 @@ steps: - label: Post-Build command: .buildkite/scripts/lifecycle/post_build.sh agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-2 diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index c9d755d5d94c8..5a4d7afcaa128 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -91,8 +91,8 @@ steps: - exit_status: '-1' limit: 3 - - command: .buildkite/scripts/steps/build_api_docs.sh - label: 'Check Types and Build API Docs' + - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh + label: 'Build API Docs' agents: queue: n2-4-spot key: build_api_docs diff --git a/.buildkite/pipelines/security_solution/api_integration.yml b/.buildkite/pipelines/security_solution/api_integration.yml index 046d9fafe3873..399a8b996c88b 100644 --- a/.buildkite/pipelines/security_solution/api_integration.yml +++ b/.buildkite/pipelines/security_solution/api_integration.yml @@ -3,7 +3,11 @@ steps: label: 'Upload runtime info' key: upload_runtime_info agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 300 retry: automatic: @@ -13,66 +17,90 @@ steps: - group: 'Execute Tests' key: test_execution steps: - - label: Running exception_workflows:qa:serverless - command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_workflows:qa:serverless - key: exception_workflows:qa:serverless + - label: Running exception_workflows:essentials:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_workflows:essentials:qa:serverless + key: exception_workflows:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: - exit_status: '*' limit: 2 - - label: Running exception_operators_date_numeric_types:qa:serverless - command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_date_numeric_types:qa:serverless - key: exception_operators_date_numeric_types:qa:serverless + - label: Running exception_operators_date_numeric_types:essentials:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_date_numeric_types:essentials:qa:serverless + key: exception_operators_date_numeric_types:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: - exit_status: '*' limit: 2 - - label: Running exception_operators_keyword:qa:serverless - command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_keyword:qa:serverless - key: exception_operators_keyword:qa:serverless + - label: Running exception_operators_keyword:essentials:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_keyword:essentials:qa:serverless + key: exception_operators_keyword:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: - exit_status: '*' limit: 2 - - label: Running exception_operators_ips:qa:serverless - command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_ips:qa:serverless - key: exception_operators_ips:qa:serverless + - label: Running exception_operators_ips:essentials:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_ips:essentials:qa:serverless + key: exception_operators_ips:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: - exit_status: '*' limit: 2 - - label: Running exception_operators_long:qa:serverless - command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_long:qa:serverless - key: exception_operators_long:qa:serverless + - label: Running exception_operators_long:essentials:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_long:essentials:qa:serverless + key: exception_operators_long:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: - exit_status: '1' limit: 2 - - label: Running exception_operators_text:qa:serverless - command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_text:qa:serverless - key: exception_operators_text:qa:serverless + - label: Running exception_operators_text:essentials:qa:serverless + command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_operators_text:essentials:qa:serverless + key: exception_operators_text:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -83,7 +111,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh alerts:qa:serverless key: alerts:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -94,7 +126,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh alerts:essentials:qa:serverless key: alerts:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -105,7 +141,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh actions:qa:serverless key: actions:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -116,7 +156,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh genai:qa:serverless key: genai:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -127,7 +171,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_execution_logic:qa:serverless key: rule_execution_logic:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -138,7 +186,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_patch:qa:serverless key: rule_patch:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -149,7 +201,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_patch:essentials:qa:serverless key: rule_patch:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -160,7 +216,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_update:qa:serverless key: rule_update:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -171,7 +231,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_update:essentials:qa:serverless key: rule_update:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -182,7 +246,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rules_management:essentials:qa:serverless key: rules_management:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -193,7 +261,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_management:qa:serverless key: prebuilt_rules_management:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -204,7 +276,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_bundled_prebuilt_rules_package:qa:serverless key: prebuilt_rules_bundled_prebuilt_rules_package:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -215,7 +291,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_large_prebuilt_rules_package:qa:serverless key: prebuilt_rules_large_prebuilt_rules_package:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -226,7 +306,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_update_prebuilt_rules_package:qa:serverless key: prebuilt_rules_update_prebuilt_rules_package:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -237,7 +321,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_bulk_actions:qa:serverless key: rule_bulk_actions:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -248,7 +336,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_read:qa:serverless key: rule_read:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -259,7 +351,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_import_export:essentials:qa:serverless key: rule_import_export:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -270,7 +366,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_import_export:qa:serverless key: rule_import_export:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -282,7 +382,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_management:qa:serverless key: rule_management:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -293,7 +397,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_read:essentials:qa:serverless key: rule_read:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -304,7 +412,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_creation:qa:serverless key: rule_creation:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -315,7 +427,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_creation:essentials:qa:serverless key: rule_creation:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -326,7 +442,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_delete:qa:serverless key: rule_delete:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -337,7 +457,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_delete:essentials:qa:serverless key: rule_delete:essentials:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -348,7 +472,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh exception_lists_items:qa:serverless key: exception_lists_items:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -359,7 +487,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh lists_items:qa:serverless key: lists_items:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -370,7 +502,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh user_roles:qa:serverless key: user_roles:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -381,7 +517,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh telemetry:qa:serverless key: telemetry:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: @@ -391,7 +531,11 @@ steps: command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh entity_analytics:qa:serverless key: entity_analytics:qa:serverless agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 120 retry: automatic: diff --git a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml index 974514f47c101..87906082ec5aa 100644 --- a/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml +++ b/.buildkite/pipelines/security_solution_quality_gate/mki_security_solution_defend_workflows.yml @@ -14,6 +14,20 @@ steps: - exit_status: "*" limit: 1 - - command: "echo 'Running the defend worklows tests in this step" + - command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh cypress:dw:qa:serverless:run + label: 'Serverless MKI QA Defend Workflows Cypress Tests on Serverless' key: test_defend_workflows - label: "Serverless MKI QA Defend Workflows - Security Solution Cypress Tests" + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 + timeout_in_minutes: 300 + parallelism: 6 + retry: + automatic: + - exit_status: '*' + limit: 1 diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh index 8b0ab8dcae33a..6b2706c10067b 100755 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh @@ -17,7 +17,6 @@ set +e # Generate a random 5-digit number random_number=$((10000 + $RANDOM % 90000)) -PROXY_URL="https://cloud-handler-test-r344edqiza-uc.a.run.app" # Check the healthcheck of the proxy service response=$(curl -s -o /dev/null -w "%{http_code}" "$PROXY_URL/healthcheck") echo "Proxy Healthcheck Response code: $response" diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh new file mode 100755 index 0000000000000..fe384fa0a6281 --- /dev/null +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -euo pipefail + +if [ -z "$1" ] + then + echo "No target script from the package.json file, is supplied" + exit 1 +fi + + +source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/functional/common_cypress.sh +.buildkite/scripts/bootstrap.sh + +export JOB=kibana-defend-workflows-serverless-cypress + +buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true" + +source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh + +cd x-pack/plugins/security_solution +set +e + +export BK_ANALYTICS_API_KEY=$(vault_get security-solution-quality-gate serverless-cypress-defend-workflows) + +echo "--- Running the tests for target $1" +BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status diff --git a/.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh b/.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh index 85f361752f6f6..a7f1529e1fb9b 100644 --- a/.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh +++ b/.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh @@ -13,6 +13,6 @@ vault_get security-quality-gate/role-users/sec-sol-auto-03 data -format=json > . export CLOUD_QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key) export QA_CONSOLE_URL=$(vault_get security-solution-quality-gate qa_console_url) # The vault entries relevant to the Proxy service (Cloud Handler) -export PROXY_URL=$(vault_get security-solution-quality-gate-proxy proxy_url_test) +export PROXY_URL=$(vault_get security-solution-quality-gate-proxy proxy_url_prod) export PROXY_CLIENT_ID=$(vault_get security-solution-quality-gate-proxy client_id) export PROXY_SECRET=$(vault_get security-solution-quality-gate-proxy secret) diff --git a/.buildkite/scripts/steps/api_docs/build_api_docs.sh b/.buildkite/scripts/steps/api_docs/build_api_docs.sh new file mode 100755 index 0000000000000..247e6a2079922 --- /dev/null +++ b/.buildkite/scripts/steps/api_docs/build_api_docs.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +.buildkite/scripts/bootstrap.sh + +echo "--- Build API Docs" +node --max-old-space-size=12000 scripts/build_api_docs + +if [[ "${PUBLISH_API_DOCS_CHANGES:-}" == "true" ]]; then + echo "--- Store API Docs changes in Buildkite" + + git diff > api_docs_changes.diff + buildkite-agent artifact upload api_docs_changes.diff + + echo "API Docs changes uploaded" +fi diff --git a/.buildkite/scripts/steps/api_docs/publish_api_docs.sh b/.buildkite/scripts/steps/api_docs/publish_api_docs.sh new file mode 100755 index 0000000000000..77f6fb0888fb3 --- /dev/null +++ b/.buildkite/scripts/steps/api_docs/publish_api_docs.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "--- Publish API Docs" + +buildkite-agent artifact download "api_docs_changes.diff" . + +if [[ -s "api_docs_changes.diff" ]]; then + echo "Changes detected in API Docs" + git apply api_docs_changes.diff + rm api_docs_changes.diff +else + echo "No changes detected in API Docs" + exit 0 +fi + +git config --global user.name kibanamachine +git config --global user.email '42973632+kibanamachine@users.noreply.github.com' + +branch="api_docs_$(date +%F_%H-%M-%S)" +git checkout -b "$branch" +git add ./*.docnav.json +git add api_docs +git commit -m "[api-docs] Daily api_docs build" + +git push origin "$branch" + +prUrl=$(gh pr create --repo elastic/kibana --base main --head "$branch" --title "[api-docs] $(date +%F) Daily api_docs build" --body "Generated by $BUILDKITE_BUILD_URL" --label "release_note:skip" --label "docs") +echo "Opened PR: $prUrl" +gh pr merge --repo elastic/kibana --auto --squash "$prUrl" + +GH_TOKEN="$KIBANA_CI_GITHUB_TOKEN" gh pr review --repo elastic/kibana --approve -b "Automated review from $BUILDKITE_BUILD_URL" "$prUrl" diff --git a/.buildkite/scripts/steps/api_docs/type_check_with_status.sh b/.buildkite/scripts/steps/api_docs/type_check_with_status.sh new file mode 100755 index 0000000000000..188c02dbc525c --- /dev/null +++ b/.buildkite/scripts/steps/api_docs/type_check_with_status.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +.buildkite/scripts/bootstrap.sh + +echo --- Check Types +set +e +node scripts/type_check +TYPE_CHECK_RESULT=$? +echo Type check exited with status $TYPE_CHECK_RESULT +set -e + +if [[ $TYPE_CHECK_RESULT -ne 0 ]]; then + echo "Type check failed - setting status to failure" + buildkite-agent meta-data set "type_check_status" "failure" +else + echo "Type check passed - setting status to success" + buildkite-agent meta-data set "type_check_status" "success" +fi + +exit $TYPE_CHECK_RESULT diff --git a/.buildkite/scripts/steps/artifacts/docker_image.sh b/.buildkite/scripts/steps/artifacts/docker_image.sh index 790c52c212faf..beb6f04e6e9d8 100755 --- a/.buildkite/scripts/steps/artifacts/docker_image.sh +++ b/.buildkite/scripts/steps/artifacts/docker_image.sh @@ -99,16 +99,9 @@ gsutil -m cp -r "$CDN_ASSETS_FOLDER/*" "gs://$GCS_SA_CDN_BUCKET/$GIT_ABBREV_COMM gcloud auth revoke "$GCS_SA_CDN_EMAIL" echo "--- Validate CDN assets" -( - shopt -s globstar - cd $CDN_ASSETS_FOLDER - for CDN_ASSET in **/*; do - if [[ -f "$CDN_ASSET" ]]; then - echo -n "Testing $CDN_ASSET..." - curl --retry 10 --retry-max-time 600 --connect-timeout 120 --max-time 120 -I --write-out '%{http_code}\n' --fail --silent --output /dev/null "$GCS_SA_CDN_URL/$CDN_ASSET" - fi - done -) +ts-node "$(git rev-parse --show-toplevel)/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts" \ + "$GCS_SA_CDN_URL" \ + "$CDN_ASSETS_FOLDER" echo "--- Upload archives" buildkite-agent artifact upload "kibana-$BASE_VERSION-linux-x86_64.tar.gz" diff --git a/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts b/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts new file mode 100644 index 0000000000000..61b2d8774f4ac --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/validate_cdn_assets.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as glob from 'glob'; +import axios from 'axios'; + +const CDN_URL_PREFIX = process.argv[2]; +const CDN_ASSETS_FOLDER = process.argv[3]; + +async function main() { + const allAssets = glob.sync('**/*', { cwd: CDN_ASSETS_FOLDER, nodir: true }); + const totalAssetCount = allAssets.length; + let assetsProcessed = 0; + let assetsFound = 0; + + const batchSize = 50; + + console.log(`Starting CDN asset validation for ${totalAssetCount} assets...`); + + while (assetsProcessed < totalAssetCount) { + const batch = allAssets.slice(assetsProcessed, assetsProcessed + batchSize); + const results = await Promise.all(batch.map((url) => headAssetUrlWithRetry(url))); + assetsProcessed += results.length; + results.forEach((result) => { + if (result.status === 200) { + console.log(`Testing ${result.assetPath}...${result.status}`); + assetsFound++; + } else { + console.error(`Testing ${result.assetPath}...${result.status} (${result.testUrl})`); + } + }); + } + + return { + totalAssetCount, + assetsProcessed, + assetsFound, + assetsNotFound: totalAssetCount - assetsFound, + }; +} + +async function headAssetUrl(assetPath: string) { + const testUrl = `${CDN_URL_PREFIX}/${assetPath}`; + const response = await axios.head(testUrl, { + timeout: 1000, + }); + return { + status: response.status, + testUrl, + assetPath, + }; +} + +async function headAssetUrlWithRetry( + assetPath: string, + retries = 5 +): Promise<{ + status: number; + testUrl: string; + assetPath: string; +}> { + const result = await headAssetUrl(assetPath); + if (result.status === 200) { + return result; + } else if (retries > 0) { + console.log(`Retrying ${assetPath}...(retries left: ${retries})`); + await new Promise((resolve) => setTimeout(resolve, 1000)); + return headAssetUrlWithRetry(assetPath, retries - 1); + } else { + return { + status: result.status || 0, + testUrl: result.testUrl, + assetPath, + }; + } +} + +main() + .then(({ totalAssetCount, assetsNotFound }) => { + if (assetsNotFound) { + console.error(`Couldn't find ${assetsNotFound} assets on CDN.`); + process.exit(1); + } else { + console.log(`All ${totalAssetCount} found on CDN.`); + process.exit(0); + } + }) + .catch((error) => { + console.error(error.message); + process.exit(1); + }); diff --git a/.buildkite/scripts/steps/build_api_docs.sh b/.buildkite/scripts/steps/build_api_docs.sh deleted file mode 100755 index f86032c902d1a..0000000000000 --- a/.buildkite/scripts/steps/build_api_docs.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -.buildkite/scripts/bootstrap.sh - -echo "--- Run scripts/type_check to ensure that all build available" -node scripts/type_check - -echo "--- Build API Docs" -node --max-old-space-size=12000 scripts/build_api_docs - -if [[ "${PUBLISH_API_DOCS_CHANGES:-}" == "true" ]]; then - echo "--- Publish API Docs" - - git config --global user.name kibanamachine - git config --global user.email '42973632+kibanamachine@users.noreply.github.com' - - branch="api_docs_$(date +%F_%H-%M-%S)" - git checkout -b "$branch" - git add ./*.docnav.json - git add api_docs - git commit -m "[api-docs] Daily api_docs build" - - git remote add kibanamachine https://github.com/kibanamachine/kibana.git - git push -u kibanamachine "$branch" - prUrl=$(gh pr create --repo elastic/kibana --title "[api-docs] $(date +%F) Daily api_docs build" --body "Generated by $BUILDKITE_BUILD_URL" --label "release_note:skip" --label "docs") - echo "Opened PR: $prUrl" - gh pr merge --repo elastic/kibana --auto --squash "$prUrl" - - GH_TOKEN="$KIBANA_CI_GITHUB_TOKEN" gh pr review --repo elastic/kibana --approve -b "Automated review from $BUILDKITE_BUILD_URL" "$prUrl" -fi diff --git a/.buildkite/scripts/steps/fips/build.sh b/.buildkite/scripts/steps/fips/build.sh old mode 100644 new mode 100755 diff --git a/.buildkite/scripts/steps/fips/smoke_test.sh b/.buildkite/scripts/steps/fips/smoke_test.sh new file mode 100755 index 0000000000000..7026c7bab941c --- /dev/null +++ b/.buildkite/scripts/steps/fips/smoke_test.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +if [ -z "$KIBANA_BUILD_LOCATION" ]; then + export KIBANA_BUILD_LOCATION="/usr/share/kibana" +fi + +# a FTR failure will result in the script returning an exit code of 10 +exitCode=0 + +configs=( + "x-pack/test/reporting_functional/reporting_and_security.config.ts" + "x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts" + "x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts" + "x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts" + "x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts" + "x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts" + "x-pack/test/functional/apps/saved_objects_management/config.ts" + "x-pack/test/functional/apps/user_profiles/config.ts" + "x-pack/test/functional/apps/security/config.ts" +) + +cd /home/vagrant/kibana + +for config in "${configs[@]}"; do + set +e + node /home/vagrant/kibana/scripts/functional_tests \ + --bail \ + --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ + --config="$config" + lastCode=$? + set -e + + if [ $lastCode -ne 0 ]; then + exitCode=10 + echo "FTR exited with code $lastCode" + echo "^^^ +++" + + if [[ "$failedConfigs" ]]; then + failedConfigs="${failedConfigs}"$'\n'"$config" + else + failedConfigs="$config" + fi + fi +done + +if [[ "$failedConfigs" ]]; then + echo "$failedConfigs" >/home/vagrant/ftr_failed_configs +fi + +echo "--- FIPS smoke test complete" + +exit $exitCode diff --git a/.buildkite/scripts/steps/package_testing/test.sh b/.buildkite/scripts/steps/package_testing/test.sh index c16d5cf98b5f5..99750529815c8 100755 --- a/.buildkite/scripts/steps/package_testing/test.sh +++ b/.buildkite/scripts/steps/package_testing/test.sh @@ -21,17 +21,25 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then elif [[ "$TEST_PACKAGE" == "docker" ]]; then download_artifact "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" KIBANA_IP_ADDRESS="192.168.56.7" +elif [[ "$TEST_PACKAGE" == "fips" ]]; then + download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" fi cd .. export VAGRANT_CWD=$PWD/test/package -vagrant up "$TEST_PACKAGE" --no-provision -node scripts/es snapshot \ - -E network.bind_host=127.0.0.1,192.168.56.1 \ - -E discovery.type=single-node \ - --license=trial & -while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done +if [[ "$TEST_PACKAGE" == "fips" ]]; then + vagrant up "$TEST_PACKAGE" +else + vagrant up "$TEST_PACKAGE" --no-provision + + node scripts/es snapshot \ + -E network.bind_host=127.0.0.1,192.168.56.1 \ + -E discovery.type=single-node \ + --license=trial & + while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done +fi function echoKibanaLogs { if [[ "$TEST_PACKAGE" == "deb" ]] || [[ "$TEST_PACKAGE" == "rpm" ]]; then @@ -47,13 +55,25 @@ function echoKibanaLogs { } trap "echoKibanaLogs" EXIT -vagrant provision "$TEST_PACKAGE" +if [[ "$TEST_PACKAGE" == "fips" ]]; then + set +e + vagrant ssh $TEST_PACKAGE -t -c "/home/vagrant/kibana/.buildkite/scripts/steps/fips/smoke_test.sh" + vagrant ssh $TEST_PACKAGE -t -c "cat /home/vagrant/ftr_failed_configs 2>/dev/null" >ftr_failed_configs + set -e + + if [ -s ftr_failed_configs ]; then + buildkite-agent meta-data set "ftr-failed-configs" <./ftr_failed_configs + fi +else + vagrant provision "$TEST_PACKAGE" + + export TEST_BROWSER_HEADLESS=1 + export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601" + export TEST_ES_URL="http://elastic:changeme@192.168.56.1:9200" -export TEST_BROWSER_HEADLESS=1 -export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601" -export TEST_ES_URL="http://elastic:changeme@192.168.56.1:9200" + echo "--- FTR - Reporting" -cd x-pack + cd x-pack -echo "--- FTR - Reporting" -node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet + node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet +fi diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0f58444069968..e85da3abf6ec2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1138,6 +1138,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql /x-pack/test/functional/apps/dataset_quality @elastic/obs-ux-logs-team /x-pack/test_serverless/functional/test_suites/observability/dataset_quality @elastic/obs-ux-logs-team /x-pack/test_serverless/functional/test_suites/observability/ @elastic/obs-ux-logs-team +/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview @elastic/obs-ux-logs-team # Observability onboarding tour /x-pack/plugins/observability_solution/observability_shared/public/components/tour @elastic/platform-onboarding diff --git a/.github/paths-labeller.yml b/.github/paths-labeller.yml index 68a00be960c4b..49c2f6dba53b0 100644 --- a/.github/paths-labeller.yml +++ b/.github/paths-labeller.yml @@ -27,3 +27,9 @@ - 'x-pack/plugins/observability_solution/observability_ai_assistant_*/**/*.*' - 'x-pack/test/observability_ai_assistant_api_integration/**/*.*' - 'x-pack/test/observability_ai_assistant_functional/**/*.*' +- 'ci:project-deploy-observability': + - 'packages/kbn-apm-*/**/*.*' + - 'x-pack/plugins/observability_solution/**/*.*' + - 'x-pack/plugins/serverless_observability/**/*.*' + - 'x-pack/test/apm_api_integration/**/*.*' + - 'x-pack/test/observability_*/**/*.*' diff --git a/.github/workflows/oblt-github-commands.yml b/.github/workflows/oblt-github-commands.yml index 4388901e2fa0e..d3f4bd61b817e 100644 --- a/.github/workflows/oblt-github-commands.yml +++ b/.github/workflows/oblt-github-commands.yml @@ -50,8 +50,7 @@ jobs: Just comment with: - \`/oblt-deploy\` : Deploy a Kibana instance using the Observability test environments. - - \`run\` \`elasticsearch-ci/docs\` : Re-trigger the docs validation. (use unformatted text in the comment!) - + - \`run\` \`docs-build\` : Re-trigger the docs validation. (use unformatted text in the comment!)
`.replace(/ +/g, '') diff --git a/api_docs/actions.mdx b/api_docs/actions.mdx index 402680307651d..7ba5642e5020b 100644 --- a/api_docs/actions.mdx +++ b/api_docs/actions.mdx @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions title: "actions" image: https://source.unsplash.com/400x175/?github description: API docs for the actions plugin -date: 2024-04-25 +date: 2024-04-29 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions'] --- import actionsObj from './actions.devdocs.json'; diff --git a/api_docs/advanced_settings.mdx b/api_docs/advanced_settings.mdx index bbd25c4d494da..d5827a9796e8d 100644 --- a/api_docs/advanced_settings.mdx +++ b/api_docs/advanced_settings.mdx @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings title: "advancedSettings" image: https://source.unsplash.com/400x175/?github description: API docs for the advancedSettings plugin -date: 2024-04-25 +date: 2024-04-29 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings'] --- import advancedSettingsObj from './advanced_settings.devdocs.json'; diff --git a/api_docs/ai_assistant_management_selection.mdx b/api_docs/ai_assistant_management_selection.mdx index 0980bd159eab3..50f210af42c31 100644 --- a/api_docs/ai_assistant_management_selection.mdx +++ b/api_docs/ai_assistant_management_selection.mdx @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiAssistantManagementSelection title: "aiAssistantManagementSelection" image: https://source.unsplash.com/400x175/?github description: API docs for the aiAssistantManagementSelection plugin -date: 2024-04-25 +date: 2024-04-29 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiAssistantManagementSelection'] --- import aiAssistantManagementSelectionObj from './ai_assistant_management_selection.devdocs.json'; diff --git a/api_docs/aiops.mdx b/api_docs/aiops.mdx index b66931a062e7e..34b84d611cc44 100644 --- a/api_docs/aiops.mdx +++ b/api_docs/aiops.mdx @@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops title: "aiops" image: https://source.unsplash.com/400x175/?github description: API docs for the aiops plugin -date: 2024-04-25 +date: 2024-04-29 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops'] --- import aiopsObj from './aiops.devdocs.json'; diff --git a/api_docs/alerting.devdocs.json b/api_docs/alerting.devdocs.json index 73f246b95136b..085b4432f4fac 100644 --- a/api_docs/alerting.devdocs.json +++ b/api_docs/alerting.devdocs.json @@ -3528,6 +3528,17 @@ "deprecated": false, "trackAdoption": false }, + { + "parentPluginId": "alerting", + "id": "def-server.RuleExecutorOptions.startedAtOverridden", + "type": "boolean", + "tags": [], + "label": "startedAtOverridden", + "description": [], + "path": "x-pack/plugins/alerting/server/types.ts", + "deprecated": false, + "trackAdoption": false + }, { "parentPluginId": "alerting", "id": "def-server.RuleExecutorOptions.state", @@ -3598,7 +3609,8 @@ "label": "getTimeRange", "description": [], "signature": [ - "(timeWindow?: string | undefined) => { dateStart: string; dateEnd: string; }" + "(timeWindow?: string | undefined) => ", + "GetTimeRangeResult" ], "path": "x-pack/plugins/alerting/server/types.ts", "deprecated": false, @@ -4859,6 +4871,21 @@ "trackAdoption": false, "initialIsOpen": false }, + { + "parentPluginId": "alerting", + "id": "def-server.API_KEY_PENDING_INVALIDATION_TYPE", + "type": "string", + "tags": [], + "label": "API_KEY_PENDING_INVALIDATION_TYPE", + "description": [], + "signature": [ + "\"api_key_pending_invalidation\"" + ], + "path": "x-pack/plugins/alerting/server/saved_objects/index.ts", + "deprecated": false, + "trackAdoption": false, + "initialIsOpen": false + }, { "parentPluginId": "alerting", "id": "def-server.BulkEditOperation", @@ -5361,7 +5388,7 @@ }, ">, \"id\" | \"snoozeSchedule\">; version?: string | undefined; }) => Promise