From 26ebd71558229f6cfa5dfc3cc50827739df1d0f4 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 12 Jun 2024 14:26:31 +0200 Subject: [PATCH] [8.13][BK] Migrate kibana on merge 8.13 (#186075) ## Summary Backport of #184391 to 8.13 Test run successful, outside of a widely failing flaky FTR: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/124 --- .buildkite/pipelines/on_merge.yml | 169 +++++++++++++++++++++++++----- 1 file changed, 140 insertions(+), 29 deletions(-) diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 81103ac677c8f..35957146f772c 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -6,7 +6,10 @@ steps: label: Pre-Build timeout_in_minutes: 10 agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-2 retry: automatic: - exit_status: '*' @@ -16,7 +19,11 @@ steps: - label: 'Triggering changes-based pipelines' branches: main agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-qa + provider: gcp + machineType: n2-standard-2 + # TODO: this can probably be deleted after the migration https://github.com/elastic/kibana-operations/issues/15 plugins: - chronotc/monorepo-diff#v2.0.4: watch: @@ -26,12 +33,19 @@ steps: command: 'ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-buildkite-pipelines-deploy main' label: 'Trigger pipeline deploy' agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh label: Build Kibana Distribution and Plugins agents: - queue: n2-16-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-16 + preemptible: true key: build timeout_in_minutes: 60 retry: @@ -42,7 +56,11 @@ steps: - command: .buildkite/scripts/steps/quick_checks.sh label: 'Quick Checks' agents: - queue: n2-2-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + preemptible: true key: quick_checks timeout_in_minutes: 60 retry: @@ -56,7 +74,11 @@ steps: label: Check Public API Docs key: public-api-docs agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 80 retry: automatic: @@ -68,7 +90,10 @@ steps: - command: .buildkite/scripts/steps/ci_stats_ready.sh label: Mark CI Stats as ready agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 timeout_in_minutes: 10 depends_on: - build @@ -81,7 +106,10 @@ steps: - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh label: 'Pick Test Group Run Order' agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 timeout_in_minutes: 10 env: JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh' @@ -95,7 +123,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh label: 'Rule Management - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 8 @@ -107,7 +139,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 6 @@ -119,7 +155,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh label: 'Detection Engine - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 8 @@ -131,7 +171,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh label: 'Detection Engine - Exceptions - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 6 @@ -143,7 +187,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh label: 'AI Assistant - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 1 @@ -155,7 +203,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_entity_analytics.sh label: 'Entity Analytics - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 2 @@ -167,7 +219,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_explore.sh label: 'Explore - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 4 @@ -179,7 +235,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh label: 'Investigations - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 8 @@ -191,7 +251,11 @@ steps: - command: .buildkite/scripts/steps/functional/threat_intelligence.sh label: 'Threat Intelligence Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 2 @@ -203,7 +267,11 @@ steps: - command: .buildkite/scripts/steps/functional/osquery_cypress.sh label: 'Osquery Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: build timeout_in_minutes: 60 parallelism: 6 @@ -215,7 +283,13 @@ steps: - command: .buildkite/scripts/steps/functional/defend_workflows.sh label: 'Defend Workflows Cypress Tests' agents: - queue: n2-4-virt + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 depends_on: - build timeout_in_minutes: 60 @@ -231,12 +305,19 @@ steps: depends_on: - build agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - command: .buildkite/scripts/steps/lint.sh label: 'Linting' agents: - queue: n2-8-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true key: linting timeout_in_minutes: 60 retry: @@ -247,7 +328,11 @@ steps: - command: .buildkite/scripts/steps/lint_with_types.sh label: 'Linting (with types)' agents: - queue: n2-16-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-16 + preemptible: true key: linting_with_types timeout_in_minutes: 90 retry: @@ -258,7 +343,11 @@ steps: - command: .buildkite/scripts/steps/checks.sh label: 'Checks' agents: - queue: n2-2-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + preemptible: true timeout_in_minutes: 60 retry: automatic: @@ -268,7 +357,11 @@ steps: - command: .buildkite/scripts/steps/check_types.sh label: 'Check Types' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 60 retry: automatic: @@ -278,7 +371,11 @@ steps: - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh label: 'Build Storybooks' agents: - queue: n2-8-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true key: storybooks timeout_in_minutes: 80 retry: @@ -289,7 +386,11 @@ steps: - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh label: 'Populate local dev bazel cache (Linux)' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 30 retry: automatic: @@ -299,7 +400,11 @@ steps: - command: .buildkite/scripts/steps/archive_so_migration_snapshot.sh target/plugin_so_types_snapshot.json label: 'Extract Saved Object migration plugin types' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true artifact_paths: "target/plugin_so_types_snapshot.json" @@ -310,7 +415,10 @@ steps: label: Post-Build timeout_in_minutes: 10 agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - wait @@ -318,4 +426,7 @@ steps: label: Trigger container image build timeout_in_minutes: 10 agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2