From 0e0cfbdb769829a7812a52cc578f3f310ea7e032 Mon Sep 17 00:00:00 2001 From: Jon Date: Wed, 6 Nov 2024 13:59:47 -0600 Subject: [PATCH] [ci] Remove plugin build (#197125) The platform plugin builds were used when functional tests were, at times, run from source. This is mostly no longer a requirement. There are two remaining cypress scripts that I updated to use the build instead. With the time saved I'm dropping the number of vCPUs from 16 to 8. These are mostly underutilized by this step, with the exception of the distribution plugin build. --- .../es_serverless/verify_es_serverless_image.yml | 4 ++-- .buildkite/pipelines/es_snapshots/verify.yml | 4 ++-- .buildkite/pipelines/fips.yml | 4 ++-- .buildkite/pipelines/flaky_tests/pipeline.ts | 2 +- .buildkite/pipelines/on_merge.yml | 4 ++-- .buildkite/pipelines/on_merge_unsupported_ftrs.yml | 4 ++-- .buildkite/pipelines/performance/daily.yml | 4 ++-- .../performance/data_set_extraction_daily.yml | 4 ++-- .buildkite/pipelines/pull_request/base.yml | 4 ++-- .../scalability/api_capacity_testing_daily.yml | 4 ++-- .buildkite/scripts/build_kibana_plugins.sh | 7 ------- .buildkite/scripts/download_build_artifacts.sh | 5 +---- .buildkite/scripts/post_build_kibana_plugins.sh | 13 ------------- .buildkite/scripts/steps/build_kibana.sh | 2 -- .../steps/functional/defend_workflows_burn.sh | 5 +---- .../functional/defend_workflows_serverless_burn.sh | 5 +---- .../scripts/steps/on_merge_build_and_metrics.sh | 2 -- .../synthetics/.buildkite/pipelines/flaky.js | 4 ++-- .../uptime/.buildkite/pipelines/flaky.js | 4 ++-- .../ux/.buildkite/pipelines/flaky.js | 4 ++-- 20 files changed, 28 insertions(+), 61 deletions(-) delete mode 100755 .buildkite/scripts/build_kibana_plugins.sh delete mode 100755 .buildkite/scripts/post_build_kibana_plugins.sh diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml index 85ce702580dbb..dd2fd8b7fb3b6 100644 --- a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml +++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml @@ -31,13 +31,13 @@ steps: provider: gcp machineType: n2-standard-2 - - label: "Build Kibana Distribution and Plugins" + - label: "Build Kibana Distribution" command: .buildkite/scripts/steps/build_kibana.sh agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-prod provider: gcp - machineType: n2-standard-16 + machineType: n2-standard-8 key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" depends_on: pre-build diff --git a/.buildkite/pipelines/es_snapshots/verify.yml b/.buildkite/pipelines/es_snapshots/verify.yml index d91af527c8266..e8c9899e9aed7 100755 --- a/.buildkite/pipelines/es_snapshots/verify.yml +++ b/.buildkite/pipelines/es_snapshots/verify.yml @@ -23,12 +23,12 @@ steps: - wait - command: .buildkite/scripts/steps/build_kibana.sh - label: Build Kibana Distribution and Plugins + label: Build Kibana Distribution agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-prod provider: gcp - machineType: n2-standard-16 + machineType: n2-standard-8 preemptible: true key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" diff --git a/.buildkite/pipelines/fips.yml b/.buildkite/pipelines/fips.yml index 09ae10496456f..f4a5b3623bbcc 100644 --- a/.buildkite/pipelines/fips.yml +++ b/.buildkite/pipelines/fips.yml @@ -17,9 +17,9 @@ steps: - wait - command: .buildkite/scripts/steps/build_kibana.sh - label: Build Kibana Distribution and Plugins + label: Build Kibana Distribution agents: - machineType: n2-standard-16 + machineType: n2-standard-8 preemptible: true key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" diff --git a/.buildkite/pipelines/flaky_tests/pipeline.ts b/.buildkite/pipelines/flaky_tests/pipeline.ts index ec1b4994beae3..0d6c3b5c40f7d 100644 --- a/.buildkite/pipelines/flaky_tests/pipeline.ts +++ b/.buildkite/pipelines/flaky_tests/pipeline.ts @@ -122,7 +122,7 @@ const pipeline = { steps.push({ command: '.buildkite/scripts/steps/build_kibana.sh', - label: 'Build Kibana Distribution and Plugins', + label: 'Build Kibana Distribution', agents: expandAgentQueue('c2-8'), key: 'build', if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''", diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index a7825e01a3b22..c98f84b9e54a2 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -17,12 +17,12 @@ steps: - wait - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh - label: Build Kibana Distribution and Plugins + label: Build Kibana Distribution agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-prod provider: gcp - machineType: n2-standard-16 + machineType: n2-standard-8 preemptible: true key: build timeout_in_minutes: 60 diff --git a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml index cf1a523be5a32..a7800fcc92dce 100644 --- a/.buildkite/pipelines/on_merge_unsupported_ftrs.yml +++ b/.buildkite/pipelines/on_merge_unsupported_ftrs.yml @@ -13,12 +13,12 @@ steps: limit: 1 - command: .buildkite/scripts/steps/build_kibana.sh - label: Build Kibana Distribution and Plugins + label: Build Kibana Distribution agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-prod provider: gcp - machineType: n2-standard-16 + machineType: n2-standard-8 preemptible: true key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" diff --git a/.buildkite/pipelines/performance/daily.yml b/.buildkite/pipelines/performance/daily.yml index bbbdde955df76..1eeb044304d22 100644 --- a/.buildkite/pipelines/performance/daily.yml +++ b/.buildkite/pipelines/performance/daily.yml @@ -9,13 +9,13 @@ steps: - wait - - label: '🧑‍🏭 Build Kibana Distribution and Plugins' + - label: '🧑‍🏭 Build Kibana Distribution' command: .buildkite/scripts/steps/build_kibana.sh agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-prod provider: gcp - machineType: c2-standard-16 + machineType: n2-standard-8 key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" diff --git a/.buildkite/pipelines/performance/data_set_extraction_daily.yml b/.buildkite/pipelines/performance/data_set_extraction_daily.yml index bb2c33f7a2d33..dcb7355318960 100644 --- a/.buildkite/pipelines/performance/data_set_extraction_daily.yml +++ b/.buildkite/pipelines/performance/data_set_extraction_daily.yml @@ -10,13 +10,13 @@ steps: - wait - - label: ':building_construction: Build Kibana Distribution and Plugins' + - label: ':building_construction: Build Kibana Distribution ' command: .buildkite/scripts/steps/build_kibana.sh agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-prod provider: gcp - machineType: c2-standard-16 + machineType: n2-standard-8 key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index e7b593f464b54..f56159882b2af 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -8,9 +8,9 @@ steps: - wait - command: .buildkite/scripts/steps/build_kibana.sh - label: Build Kibana Distribution and Plugins + label: Build Kibana Distribution agents: - machineType: n2-standard-16 + machineType: n2-standard-8 preemptible: true key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" diff --git a/.buildkite/pipelines/scalability/api_capacity_testing_daily.yml b/.buildkite/pipelines/scalability/api_capacity_testing_daily.yml index 785726bf95174..909329234c328 100644 --- a/.buildkite/pipelines/scalability/api_capacity_testing_daily.yml +++ b/.buildkite/pipelines/scalability/api_capacity_testing_daily.yml @@ -9,13 +9,13 @@ steps: - wait - - label: 'Build Kibana Distribution and Plugins' + - label: 'Build Kibana Distribution' command: .buildkite/scripts/steps/build_kibana.sh agents: image: family/kibana-ubuntu-2004 imageProject: elastic-images-prod provider: gcp - machineType: n2-standard-16 + machineType: n2-standard-8 preemptible: true key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" diff --git a/.buildkite/scripts/build_kibana_plugins.sh b/.buildkite/scripts/build_kibana_plugins.sh deleted file mode 100755 index 4e566008b3351..0000000000000 --- a/.buildkite/scripts/build_kibana_plugins.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Build Platform Plugins" -THREADS=$(grep -c ^processor /proc/cpuinfo) -node scripts/build_kibana_platform_plugins --examples --test-plugins --workers "$THREADS" --no-inspect-workers --no-progress diff --git a/.buildkite/scripts/download_build_artifacts.sh b/.buildkite/scripts/download_build_artifacts.sh index 1b52e82c2d66b..497efa181018e 100755 --- a/.buildkite/scripts/download_build_artifacts.sh +++ b/.buildkite/scripts/download_build_artifacts.sh @@ -6,12 +6,11 @@ source "$(dirname "$0")/common/util.sh" if [[ "${KIBANA_BUILD_ID:-}" != "false" ]]; then if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then - echo '--- Downloading Distribution and Plugin artifacts' + echo '--- Downloading Distribution' cd "$WORKSPACE" download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" - download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" mkdir -p "$KIBANA_BUILD_LOCATION" tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1 @@ -24,7 +23,5 @@ if [[ "${KIBANA_BUILD_ID:-}" != "false" ]]; then fi cd "$KIBANA_DIR" - - tar -xzf ../kibana-default-plugins.tar.gz fi fi diff --git a/.buildkite/scripts/post_build_kibana_plugins.sh b/.buildkite/scripts/post_build_kibana_plugins.sh deleted file mode 100755 index 288fd262cafa9..0000000000000 --- a/.buildkite/scripts/post_build_kibana_plugins.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Archive built plugins" -shopt -s globstar -tar -zcf \ - target/kibana-default-plugins.tar.gz \ - x-pack/plugins/**/target/public \ - x-pack/test/**/target/public \ - examples/**/target/public \ - x-pack/examples/**/target/public \ - test/**/target/public diff --git a/.buildkite/scripts/steps/build_kibana.sh b/.buildkite/scripts/steps/build_kibana.sh index bf4a9ff243d02..4bad332dd073f 100755 --- a/.buildkite/scripts/steps/build_kibana.sh +++ b/.buildkite/scripts/steps/build_kibana.sh @@ -4,6 +4,4 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh .buildkite/scripts/build_kibana.sh -.buildkite/scripts/build_kibana_plugins.sh -.buildkite/scripts/post_build_kibana_plugins.sh .buildkite/scripts/post_build_kibana.sh diff --git a/.buildkite/scripts/steps/functional/defend_workflows_burn.sh b/.buildkite/scripts/steps/functional/defend_workflows_burn.sh index 6a97ba4e82b33..44d87ba4a6635 100644 --- a/.buildkite/scripts/steps/functional/defend_workflows_burn.sh +++ b/.buildkite/scripts/steps/functional/defend_workflows_burn.sh @@ -4,10 +4,7 @@ set -euo pipefail source .buildkite/scripts/steps/functional/common.sh -.buildkite/scripts/bootstrap.sh -.buildkite/scripts/copy_es_snapshot_cache.sh -node scripts/build_kibana_platform_plugins.js - +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} export JOB=kibana-defend-workflows-cypress buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false' diff --git a/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh b/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh index 4bebee15953e6..a092ffbd4186b 100644 --- a/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh +++ b/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh @@ -4,10 +4,7 @@ set -euo pipefail source .buildkite/scripts/steps/functional/common.sh -.buildkite/scripts/bootstrap.sh -.buildkite/scripts/copy_es_snapshot_cache.sh -node scripts/build_kibana_platform_plugins.js - +export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} export JOB=kibana-defend-workflows-serverless-cypress buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false' diff --git a/.buildkite/scripts/steps/on_merge_build_and_metrics.sh b/.buildkite/scripts/steps/on_merge_build_and_metrics.sh index fb05bb99b0c54..1f1e492f87bec 100755 --- a/.buildkite/scripts/steps/on_merge_build_and_metrics.sh +++ b/.buildkite/scripts/steps/on_merge_build_and_metrics.sh @@ -4,7 +4,5 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh .buildkite/scripts/build_kibana.sh -.buildkite/scripts/build_kibana_plugins.sh -.buildkite/scripts/post_build_kibana_plugins.sh .buildkite/scripts/post_build_kibana.sh .buildkite/scripts/saved_object_field_metrics.sh diff --git a/x-pack/plugins/observability_solution/synthetics/.buildkite/pipelines/flaky.js b/x-pack/plugins/observability_solution/synthetics/.buildkite/pipelines/flaky.js index 7a1ee5c582d7d..ca024e9b42286 100644 --- a/x-pack/plugins/observability_solution/synthetics/.buildkite/pipelines/flaky.js +++ b/x-pack/plugins/observability_solution/synthetics/.buildkite/pipelines/flaky.js @@ -63,8 +63,8 @@ console.log(JSON.stringify(pipeline, null, 2)); function getBuildJob() { return { command: '.buildkite/scripts/steps/build_kibana.sh', - label: 'Build Kibana Distribution and Plugins', - agents: { queue: 'c2-8' }, + label: 'Build Kibana Distribution', + agents: { queue: 'n2-8' }, key: BUILD_UUID, if: `build.env('${KIBANA_BUILD_ID}') == null || build.env('${KIBANA_BUILD_ID}') == ''`, }; diff --git a/x-pack/plugins/observability_solution/uptime/.buildkite/pipelines/flaky.js b/x-pack/plugins/observability_solution/uptime/.buildkite/pipelines/flaky.js index 7a1ee5c582d7d..ca024e9b42286 100644 --- a/x-pack/plugins/observability_solution/uptime/.buildkite/pipelines/flaky.js +++ b/x-pack/plugins/observability_solution/uptime/.buildkite/pipelines/flaky.js @@ -63,8 +63,8 @@ console.log(JSON.stringify(pipeline, null, 2)); function getBuildJob() { return { command: '.buildkite/scripts/steps/build_kibana.sh', - label: 'Build Kibana Distribution and Plugins', - agents: { queue: 'c2-8' }, + label: 'Build Kibana Distribution', + agents: { queue: 'n2-8' }, key: BUILD_UUID, if: `build.env('${KIBANA_BUILD_ID}') == null || build.env('${KIBANA_BUILD_ID}') == ''`, }; diff --git a/x-pack/plugins/observability_solution/ux/.buildkite/pipelines/flaky.js b/x-pack/plugins/observability_solution/ux/.buildkite/pipelines/flaky.js index 6a01e8c52d002..b9a1782820aeb 100644 --- a/x-pack/plugins/observability_solution/ux/.buildkite/pipelines/flaky.js +++ b/x-pack/plugins/observability_solution/ux/.buildkite/pipelines/flaky.js @@ -64,8 +64,8 @@ console.log(JSON.stringify(pipeline, null, 2)); function getBuildJob() { return { command: '.buildkite/scripts/steps/build_kibana.sh', - label: 'Build Kibana Distribution and Plugins', - agents: { queue: 'c2-8' }, + label: 'Build Kibana Distribution', + agents: { queue: 'n2-8' }, key: BUILD_UUID, if: `build.env('${KIBANA_BUILD_ID}') == null || build.env('${KIBANA_BUILD_ID}') == ''`, };