From 71144eded7f56705e54c768c3741e5fcf1b62f9a Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Tue, 24 Dec 2024 10:58:37 +0100 Subject: [PATCH] [Security Solution] Unskip tests for the prebuilt rules bootstrap endpoint (#205106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Fixes: https://github.com/elastic/kibana/issues/197108** **Fixes: https://github.com/elastic/kibana/issues/202037** **Fixes: https://github.com/elastic/kibana/issues/203632** ## Summary Builds on top of https://github.com/elastic/kibana/pull/203799 and: - simplifies the package deletion code (1 [API call](https://www.elastic.co/docs/api/doc/kibana/operation/operation-delete-fleet-epm-packages-pkgname-pkgversion) instead of 2, per package) - adds retry logic - refactors the testing utils a bit Flaky test runs: - [ESS](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7632) - 190 runs - [Serverless](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7633) - 100 runs ## Details Here's how the test logs look like: ``` info Starting tests └-: Rules Management - Prebuilt Rules - Prebuilt Rules Management └-> "before all" hook: beforeTestSuite.trigger in "Rules Management - Prebuilt Rules - Prebuilt Rules Management" └-: @ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules └-> "before all" hook: beforeTestSuite.trigger for "should install fleet packages required for detection engine to function" └-> should install fleet packages required for detection engine to function └-> "before each" hook: global before each for "should install fleet packages required for detection engine to function" └-> "before each" hook for "should install fleet packages required for detection engine to function" │ debg Deleting security_detection_engine package │ debg Deleted security_detection_engine package (was not installed) { │ statusCode: 400, │ error: 'Bad Request', │ message: 'security_detection_engine is not installed' │ } │ debg Deleting endpoint package │ debg Deleted endpoint package (was not installed) { │ statusCode: 400, │ error: 'Bad Request', │ message: 'endpoint is not installed' │ } └- ✓ pass (21.6s) └-> should skip installing fleet packages if they are already installed └-> "before each" hook: global before each for "should skip installing fleet packages if they are already installed" └-> "before each" hook for "should skip installing fleet packages if they are already installed" │ debg Deleting security_detection_engine package │ debg Deleted security_detection_engine package (was installed) │ debg Deleting endpoint package │ debg Deleted endpoint package (was installed) └- ✓ pass (22.4s) └-> "after all" hook: afterTestSuite.trigger for "should skip installing fleet packages if they are already installed" └-> "after all" hook: afterTestSuite.trigger in "Rules Management - Prebuilt Rules - Prebuilt Rules Management" 2 passing (1.0m) ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed --- .../prerelease_packages.ts | 6 +- .../bootstrap_prebuilt_rules.ts | 18 ++-- .../fleet_integration.ts | 8 +- .../delete_endpoint_fleet_package.ts | 30 ------- .../prebuilt_rules/delete_fleet_packages.ts | 87 +++++++++++++++++++ .../delete_prebuilt_rules_fleet_package.ts | 30 ------- .../utils/rules/prebuilt_rules/index.ts | 2 +- 7 files changed, 107 insertions(+), 74 deletions(-) delete mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_endpoint_fleet_package.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_fleet_packages.ts delete mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts index edae7dd4eb014..9f7809b16b98d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts @@ -22,7 +22,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); - const retry = getService('retry'); + const retryService = getService('retry'); /* This test makes use of the mock packages created in the '/fleet_bundled_packages' folder, /* in order to assert that, in production environments, the latest stable version of the package @@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext): void => { beforeEach(async () => { await deleteAllRules(supertest, log); await deleteAllPrebuiltRuleAssets(es, log); - await deletePrebuiltRulesFleetPackage(supertest); + await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService }); }); it('should install latest stable version and ignore prerelease packages', async () => { @@ -49,7 +49,7 @@ export default ({ getService }: FtrProviderContext): void => { const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI( es, supertest, - retry + retryService ); expect(fleetPackageInstallationResponse.items.length).toBe(1); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/bootstrap_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/bootstrap_prebuilt_rules.ts index a437561ef0a04..d98765a07a190 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/bootstrap_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/bootstrap_prebuilt_rules.ts @@ -11,18 +11,24 @@ import { } from '@kbn/security-solution-plugin/common/detection_engine/constants'; import expect from 'expect'; import { FtrProviderContext } from '../../../../../../ftr_provider_context'; -import { deletePrebuiltRulesFleetPackage } from '../../../../utils'; -import { deleteEndpointFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_endpoint_fleet_package'; +import { + deleteAllPrebuiltRuleAssets, + deleteEndpointFleetPackage, + deletePrebuiltRulesFleetPackage, +} from '../../../../utils'; export default ({ getService }: FtrProviderContext): void => { + const es = getService('es'); + const log = getService('log'); const supertest = getService('supertest'); + const retryService = getService('retry'); const securitySolutionApi = getService('securitySolutionApi'); - // FLAKY: https://github.com/elastic/kibana/issues/203632 - describe.skip('@ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules', () => { + describe('@ess @serverless @skipInServerlessMKI Bootstrap Prebuilt Rules', () => { beforeEach(async () => { - await deletePrebuiltRulesFleetPackage(supertest); - await deleteEndpointFleetPackage(supertest); + await deleteAllPrebuiltRuleAssets(es, log); + await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService }); + await deleteEndpointFleetPackage({ supertest, es, log, retryService }); }); it('should install fleet packages required for detection engine to function', async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts index cb317870d3773..291ed2d7ea51a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts @@ -13,18 +13,18 @@ import { import { deleteAllRules } from '../../../../../../../common/utils/security_solution'; import { deleteAllPrebuiltRuleAssets } from '../../../../utils/rules/prebuilt_rules/delete_all_prebuilt_rule_assets'; import { deleteAllTimelines } from '../../../../utils/rules/prebuilt_rules/delete_all_timelines'; -import { deletePrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package'; +import { deletePrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/delete_fleet_packages'; import { installPrebuiltRulesFleetPackage } from '../../../../utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package'; export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); - const retry = getService('retry'); + const retryService = getService('retry'); describe('@ess @serverless @skipInServerlessMKI install_prebuilt_rules_from_real_package', () => { beforeEach(async () => { - await deletePrebuiltRulesFleetPackage(supertest); + await deletePrebuiltRulesFleetPackage({ supertest, es, log, retryService }); await deleteAllRules(supertest, log); await deleteAllTimelines(es, log); await deleteAllPrebuiltRuleAssets(es, log); @@ -48,7 +48,7 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, overrideExistingPackage: true, - retryService: retry, + retryService, }); // Verify that status is updated after package installation diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_endpoint_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_endpoint_fleet_package.ts deleted file mode 100644 index cea8363d9085c..0000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_endpoint_fleet_package.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { epmRouteService } from '@kbn/fleet-plugin/common'; -import { ENDPOINT_PACKAGE_NAME } from '@kbn/security-solution-plugin/common/detection_engine/constants'; -import type SuperTest from 'supertest'; - -/** - * Delete the endpoint package using fleet API. - * - * @param supertest Supertest instance - */ -export async function deleteEndpointFleetPackage(supertest: SuperTest.Agent) { - const resp = await supertest - .get(epmRouteService.getInfoPath(ENDPOINT_PACKAGE_NAME)) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .send(); - - if (resp.status === 200 && resp.body.item.status === 'installed') { - await supertest - .delete(epmRouteService.getRemovePath(ENDPOINT_PACKAGE_NAME, resp.body.item.version)) - .set('kbn-xsrf', 'true') - .send({ force: true }); - } -} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_fleet_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_fleet_packages.ts new file mode 100644 index 0000000000000..df6f25dd0b1a1 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_fleet_packages.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { epmRouteService } from '@kbn/fleet-plugin/common'; +import { RetryService } from '@kbn/ftr-common-functional-services'; +import { + ENDPOINT_PACKAGE_NAME, + PREBUILT_RULES_PACKAGE_NAME, +} from '@kbn/security-solution-plugin/common/detection_engine/constants'; +import { ToolingLog } from '@kbn/tooling-log'; +import type SuperTest from 'supertest'; +import { refreshSavedObjectIndices } from '../../refresh_index'; + +interface DeleteFleetPackageDeps { + supertest: SuperTest.Agent; + retryService: RetryService; + log: ToolingLog; + es: Client; +} + +interface DeleteFleetPackageArgs { + packageName: string; + dependencies: DeleteFleetPackageDeps; +} + +/** + * Delete the security_detection_engine package using fleet API. + */ +export async function deletePrebuiltRulesFleetPackage(params: DeleteFleetPackageDeps) { + await deleteFleetPackage({ + packageName: PREBUILT_RULES_PACKAGE_NAME, + dependencies: params, + }); +} + +/** + * Delete the endpoint package using fleet API. + */ +export async function deleteEndpointFleetPackage(params: DeleteFleetPackageDeps) { + await deleteFleetPackage({ + packageName: ENDPOINT_PACKAGE_NAME, + dependencies: params, + }); +} + +async function deleteFleetPackage(params: DeleteFleetPackageArgs): Promise { + const { packageName, dependencies } = params; + const { supertest, retryService, log, es } = dependencies; + + await retryService.tryWithRetries( + 'deleteFleetPackage', + async () => { + log.debug(`Deleting ${packageName} package`); + + const response = await supertest + .delete(epmRouteService.getRemovePath(packageName)) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ force: true }); + + if (response.status === 200) { + log.debug(`Deleted ${packageName} package (was installed)`); + return; + } else if ( + response.status === 400 && + response.body.message === `${packageName} is not installed` + ) { + log.debug(`Deleted ${packageName} package (was not installed)`, response.body); + return; + } else { + log.warning(`Error deleting ${packageName} package`, response.body); + throw response.error; + } + }, + { + retryCount: 2, // overall max 3 attempts + timeout: 3 * 60000, // total timeout applied to all attempts altogether + } + ); + + await refreshSavedObjectIndices(es); +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package.ts deleted file mode 100644 index 084d59c55df0d..0000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/delete_prebuilt_rules_fleet_package.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { epmRouteService } from '@kbn/fleet-plugin/common'; -import { PREBUILT_RULES_PACKAGE_NAME } from '@kbn/security-solution-plugin/common/detection_engine/constants'; -import type SuperTest from 'supertest'; - -/** - * Delete the security_detection_engine package using fleet API. - * - * @param supertest Supertest instance - */ -export async function deletePrebuiltRulesFleetPackage(supertest: SuperTest.Agent) { - const resp = await supertest - .get(epmRouteService.getInfoPath(PREBUILT_RULES_PACKAGE_NAME)) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .send(); - - if (resp.status === 200 && resp.body.item.status === 'installed') { - await supertest - .delete(epmRouteService.getRemovePath(PREBUILT_RULES_PACKAGE_NAME, resp.body.item.version)) - .set('kbn-xsrf', 'true') - .send({ force: true }); - } -} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/index.ts index fabd3df2f2d16..fc437bc14f349 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/index.ts @@ -7,7 +7,7 @@ export * from './create_prebuilt_rule_saved_objects'; export * from './delete_all_prebuilt_rule_assets'; export * from './delete_all_timelines'; -export * from './delete_prebuilt_rules_fleet_package'; +export * from './delete_fleet_packages'; export * from './get_installed_rules'; export * from './get_prebuilt_rules_and_timelines_status'; export * from './get_prebuilt_rules_status';