[8.13][BK] Migrate kibana on merge 8.13 (#186075)

## Summary Backport of #184391 to 8.13 Test run successful, outside of a widely failing flaky FTR: https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/124
elastic · Jun 12, 2024 · 26ebd71 · 26ebd71
1 parent 0edd5d0
commit 26ebd71
Showing 1 changed file with 140 additions and 29 deletions.
diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -6,7 +6,10 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
     retry:
       automatic:
         - exit_status: '*'
@@ -16,7 +19,11 @@ steps:
   - label: 'Triggering changes-based pipelines'
     branches: main
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
+    # TODO: this can probably be deleted after the migration https://github.com/elastic/kibana-operations/issues/15
     plugins:
       - chronotc/monorepo-diff#v2.0.4:
           watch:
@@ -26,12 +33,19 @@ steps:
                 command: 'ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-buildkite-pipelines-deploy main'
                 label: 'Trigger pipeline deploy'
                 agents:
-                  queue: 'kibana-default'
+                  image: family/kibana-ubuntu-2004
+                  imageProject: elastic-images-prod
+                  provider: gcp
+                  machineType: n2-standard-2
 
   - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     key: build
     timeout_in_minutes: 60
     retry:
@@ -42,7 +56,11 @@ steps:
   - command: .buildkite/scripts/steps/quick_checks.sh
     label: 'Quick Checks'
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
     key: quick_checks
     timeout_in_minutes: 60
     retry:
@@ -56,7 +74,11 @@ steps:
     label: Check Public API Docs
     key: public-api-docs
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     timeout_in_minutes: 80
     retry:
       automatic:
@@ -68,7 +90,10 @@ steps:
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -81,7 +106,10 @@ steps:
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
@@ -95,7 +123,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh
     label: 'Rule Management - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 8
@@ -107,7 +139,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh
     label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 6
@@ -119,7 +155,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh
     label: 'Detection Engine - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 8
@@ -131,7 +171,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
     label: 'Detection Engine - Exceptions - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 6
@@ -143,7 +187,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
     label: 'AI Assistant - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 1
@@ -155,7 +203,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_entity_analytics.sh
     label: 'Entity Analytics - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 2
@@ -167,7 +219,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_explore.sh
     label: 'Explore - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 4
@@ -179,7 +235,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
     label: 'Investigations - Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 8
@@ -191,7 +251,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
     label: 'Threat Intelligence Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 2
@@ -203,7 +267,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
     label: 'Osquery Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 6
@@ -215,7 +283,13 @@ steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows.sh
     label: 'Defend Workflows Cypress Tests'
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on:
       - build
     timeout_in_minutes: 60
@@ -231,12 +305,19 @@ steps:
     depends_on:
       - build
     agents:
-      queue: 'kibana-default'
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
 
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
-      queue: n2-8-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+      preemptible: true
     key: linting
     timeout_in_minutes: 60
     retry:
@@ -247,7 +328,11 @@ steps:
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     key: linting_with_types
     timeout_in_minutes: 90
     retry:
@@ -258,7 +343,11 @@ steps:
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -268,7 +357,11 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -278,7 +371,11 @@ steps:
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'
     agents:
-      queue: n2-8-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+      preemptible: true
     key: storybooks
     timeout_in_minutes: 80
     retry:
@@ -289,7 +386,11 @@ steps:
   - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
     label: 'Populate local dev bazel cache (Linux)'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -299,7 +400,11 @@ steps:
   - command: .buildkite/scripts/steps/archive_so_migration_snapshot.sh target/plugin_so_types_snapshot.json
     label: 'Extract Saved Object migration plugin types'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     artifact_paths:
       "target/plugin_so_types_snapshot.json"
 
@@ -310,12 +415,18 @@ steps:
     label: Post-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
 
   - wait
 
   - command: '.buildkite/scripts/steps/artifacts/docker_image_trigger.sh'
     label: Trigger container image build
     timeout_in_minutes: 10
     agents:
-      queue: 'kibana-default'
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2