[Security Solution] Migrate Prebuilt rules API integration tests to `…

…security_solution_api_integration` folder (elastic#169951) Addresses partially: elastic#151902 ## Summary - Migrates Prebuilt rules-related API integration tests to `security_solution_api_integration` folder. - Moves tests from `x-pack/test/detection_engine_api_integration/security_and_spaces` into `x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license`. - PR moves following folders: - `/prebuilt_rules` - `/bundled_prebuilt_rules_package` - `/large_prebuilt_rules_package` - `/update_prebuilt_rules_package` - Duplicates or completely moves needed utils into: - `x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules`, depending if they are no longer used in the original folder or they still are in the remaining test (should be moved shortly as well) - All tests run on both **ESS** and **Serverless** ## Flaky test runner - [ ] [`/prebuilt_rules`](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3888) 🟢 - [ ] [`/bundled_prebuilt_rules_package`](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3916)🟢 - [ ] [`/large_prebuilt_rules_package`](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3917)🟢 - [ ] [`/update_prebuilt_rules_package`](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3920)🟢 [Link to all for PR](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds?branch=refs%2Fpull%2F169951%2Fhead) ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <[email protected]>
ashokaditya · Nov 10, 2023 · 4b28ec4 · 4b28ec4
1 parent 7c32757
commit 4b28ec4
Show file tree

Hide file tree

Showing 43 changed files with 401 additions and 150 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -228,10 +228,6 @@ enabled:
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/prebuilt_rules/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/bundled_prebuilt_rules_package/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/large_prebuilt_rules_package/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/update_prebuilt_rules_package/config.ts
   - x-pack/test/disable_ems/config.ts
   - x-pack/test/encrypted_saved_objects_api_integration/config.ts
   - x-pack/test/examples/config.ts
@@ -467,8 +463,11 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/ess.config.ts
 
-
-
-
-
-
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/ess.config.ts
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1299,6 +1299,7 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
 /x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_details @elastic/security-detection-rule-management
 /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules @elastic/security-detection-rule-management
 /x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/rule_management @elastic/security-detection-rule-management
+/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules @elastic/security-detection-rule-management
 
 /x-pack/plugins/security_solution/public/common/components/health_truncate_text @elastic/security-detection-rule-management
 /x-pack/plugins/security_solution/public/common/components/links_to_docs @elastic/security-detection-rule-management

diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts
@@ -93,12 +93,6 @@ export * from './wait_for_rule_status';
 export * from './wait_for_signals_to_be_present';
 export * from './prebuilt_rules/create_prebuilt_rule_saved_objects';
 export * from './prebuilt_rules/delete_all_prebuilt_rule_assets';
-export * from './prebuilt_rules/delete_prebuilt_rules_fleet_package';
-export * from './prebuilt_rules/get_prebuilt_rules_status';
-export * from './prebuilt_rules/get_prebuilt_rules_and_timelines_status';
-export * from './prebuilt_rules/install_prebuilt_rules_fleet_package';
-export * from './prebuilt_rules/install_prebuilt_rules';
-export * from './prebuilt_rules/upgrade_prebuilt_rules';
 export * from './prebuilt_rules/install_mock_prebuilt_rules';
 export * from './prebuilt_rules/install_prebuilt_rules_and_timelines';
 export * from './get_legacy_action_so';
diff --git a/...on_engine_api_integration/utils/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts b/...on_engine_api_integration/utils/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts
diff --git a/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts b/x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
@@ -8,6 +8,8 @@ import { FtrConfigProviderContext } from '@kbn/test';
 export interface CreateTestConfigOptions {
   testFiles: string[];
   junit: { reportName: string };
+  kbnTestServerArgs?: string[];
+  kbnTestServerEnv?: Record<string, string>;
 }
 
 export function createTestConfig(options: CreateTestConfigOptions) {
@@ -20,7 +22,15 @@ export function createTestConfig(options: CreateTestConfigOptions) {
       ...svlSharedConfig.getAll(),
       kbnTestServer: {
         ...svlSharedConfig.get('kbnTestServer'),
-        serverArgs: [...svlSharedConfig.get('kbnTestServer.serverArgs'), '--serverless=security'],
+        serverArgs: [
+          ...svlSharedConfig.get('kbnTestServer.serverArgs'),
+          '--serverless=security',
+          ...(options.kbnTestServerArgs || []),
+        ],
+        env: {
+          ...svlSharedConfig.get('kbnTestServer.env'),
+          ...options.kbnTestServerEnv,
+        },
       },
       testFiles: options.testFiles,
       junit: options.junit,

diff --git a/x-pack/test/security_solution_api_integration/package.json b/x-pack/test/security_solution_api_integration/package.json
@@ -48,6 +48,26 @@
     "entity_analytics:runner:serverless": "npm run run-tests:ea:default risk_engine serverless serverlessEnv",
     "entity_analytics:qa:serverless": "npm run run-tests:ea:default risk_engine serverless qaEnv",
     "entity_analytics:server:ess": "npm run initialize-server:ea:default risk_engine ess",
-    "entity_analytics:runner:ess": "npm run run-tests:ea:default risk_engine ess essEnv"
+    "entity_analytics:runner:ess": "npm run run-tests:ea:default risk_engine ess essEnv",
+    "prebuilt_rules_management:server:serverless": "npm run initialize-server:dr:default prebuilt_rules/management serverless",
+    "prebuilt_rules_management:runner:serverless": "npm run run-tests:dr:default prebuilt_rules/management serverless serverlessEnv",
+    "prebuilt_rules_management:qa:serverless": "npm run run-tests:dr:default prebuilt_rules/management serverless qaEnv",
+    "prebuilt_rules_management:server:ess": "npm run initialize-server:dr:default prebuilt_rules/management ess",
+    "prebuilt_rules_management:runner:ess": "npm run run-tests:dr:default prebuilt_rules/management ess essEnv",
+    "prebuilt_rules_bundled_prebuilt_rules_package:server:serverless": "npm run initialize-server:dr:default prebuilt_rules/bundled_prebuilt_rules_package serverless",
+    "prebuilt_rules_bundled_prebuilt_rules_package:runner:serverless": "npm run run-tests:dr:default prebuilt_rules/bundled_prebuilt_rules_package serverless serverlessEnv",
+    "prebuilt_rules_bundled_prebuilt_rules_package:qa:serverless": "npm run run-tests:dr:default prebuilt_rules/bundled_prebuilt_rules_package serverless qaEnv",
+    "prebuilt_rules_bundled_prebuilt_rules_package:server:ess": "npm run initialize-server:dr:default prebuilt_rules/bundled_prebuilt_rules_package ess",
+    "prebuilt_rules_bundled_prebuilt_rules_package:runner:ess": "npm run run-tests:dr:default prebuilt_rules/bundled_prebuilt_rules_package ess essEnv",
+    "prebuilt_rules_large_prebuilt_rules_package:server:serverless": "npm run initialize-server:dr:default prebuilt_rules/large_prebuilt_rules_package serverless",
+    "prebuilt_rules_large_prebuilt_rules_package:runner:serverless": "npm run run-tests:dr:default prebuilt_rules/large_prebuilt_rules_package serverless serverlessEnv",
+    "prebuilt_rules_large_prebuilt_rules_package:qa:serverless": "npm run run-tests:dr:default prebuilt_rules/large_prebuilt_rules_package serverless qaEnv",
+    "prebuilt_rules_large_prebuilt_rules_package:server:ess": "npm run initialize-server:dr:default prebuilt_rules/large_prebuilt_rules_package ess",
+    "prebuilt_rules_large_prebuilt_rules_package:runner:ess": "npm run run-tests:dr:default prebuilt_rules/large_prebuilt_rules_package ess essEnv",
+    "prebuilt_rules_update_prebuilt_rules_package:server:serverless": "npm run initialize-server:dr:default prebuilt_rules/update_prebuilt_rules_package serverless",
+    "prebuilt_rules_update_prebuilt_rules_package:runner:serverless": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package serverless serverlessEnv",
+    "prebuilt_rules_update_prebuilt_rules_package:qa:serverless": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package serverless qaEnv",
+    "prebuilt_rules_update_prebuilt_rules_package:server:ess": "npm run initialize-server:dr:default prebuilt_rules/update_prebuilt_rules_package ess",
+    "prebuilt_rules_update_prebuilt_rules_package:runner:ess": "npm run run-tests:dr:default prebuilt_rules/update_prebuilt_rules_package ess essEnvs"
   }
 }
diff --git a/.../bundled_prebuilt_rules_package/config.ts → ...built_rules_package/configs/ess.config.ts b/.../bundled_prebuilt_rules_package/config.ts → ...built_rules_package/configs/ess.config.ts
@@ -4,24 +4,26 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
 export const BUNDLED_PACKAGE_DIR = path.join(
   path.dirname(__filename),
-  './fleet_bundled_packages/fixtures'
+  './../fleet_bundled_packages/fixtures'
 );
 
-// eslint-disable-next-line import/no-default-export
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
-  const functionalConfig = await readConfigFile(require.resolve('../config.base.ts'));
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../../../../config/ess/config.base.trial')
+  );
 
   return {
     ...functionalConfig.getAll(),
-    testFiles: [
-      require.resolve('./prerelease_packages.ts'),
-      require.resolve('./install_latest_bundled_prebuilt_rules.ts'),
-    ],
+    testFiles: [require.resolve('..')],
+    junit: {
+      reportName: 'Detection Engine ESS / Bundled Prebuilt Rules Package API Integration Tests',
+    },
     kbnTestServer: {
       ...functionalConfig.get('kbnTestServer'),
       serverArgs: [

diff --git a/...efault_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/serverless.config.ts b/...efault_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/serverless.config.ts
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import path from 'path';
+import { createTestConfig } from '../../../../../../config/serverless/config.base';
+
+export const BUNDLED_PACKAGE_DIR = path.join(
+  path.dirname(__filename),
+  './../fleet_bundled_packages/fixtures'
+);
+export default createTestConfig({
+  testFiles: [require.resolve('..')],
+  junit: {
+    reportName:
+      'Detection Engine Serverless / Bundled Prebuilte Rules Package API Integration Tests',
+  },
+  kbnTestServerArgs: [
+    /*  Tests in this directory simulate an air-gapped environment in which the instance doesn't have access to EPR.
+     *  To do that, we point the Fleet url to an invalid URL, and instruct Fleet to fetch bundled packages at the
+     *  location defined in BUNDLED_PACKAGE_DIR.
+     */
+    `--xpack.fleet.registryUrl=http://invalidURL:8080`,
+    `--xpack.fleet.developer.bundledPackageLocation=${BUNDLED_PACKAGE_DIR}`,
+  ],
+});
diff --git a/...ures/security_detection_engine-99.0.0.zip → ...ures/security_detection_engine-99.0.0.zip b/...ures/security_detection_engine-99.0.0.zip → ...ures/security_detection_engine-99.0.0.zip
diff --git a/...curity_detection_engine-99.0.1-beta.1.zip → ...curity_detection_engine-99.0.1-beta.1.zip b/...curity_detection_engine-99.0.1-beta.1.zip → ...curity_detection_engine-99.0.1-beta.1.zip
diff --git a/...etections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/index.ts b/...etections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/index.ts
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default ({ loadTestFile }: FtrProviderContext): void => {
+  describe('Detection Engine API - Bundled Prebuilt Rules Package', function () {
+    loadTestFile(require.resolve('./install_latest_bundled_prebuilt_rules'));
+    loadTestFile(require.resolve('./prerelease_packages'));
+  });
+};
diff --git a/.../install_latest_bundled_prebuilt_rules.ts → .../install_latest_bundled_prebuilt_rules.ts b/.../install_latest_bundled_prebuilt_rules.ts → .../install_latest_bundled_prebuilt_rules.ts
@@ -6,18 +6,18 @@
  */
 import fs from 'fs/promises';
 import path from 'path';
-// @ts-expect-error we have to check types with "allowJs: false" for now, causing this import to fail
 import { REPO_ROOT } from '@kbn/repo-info';
 import JSON5 from 'json5';
 import expect from 'expect';
 import { PackageSpecManifest } from '@kbn/fleet-plugin/common';
 import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
-import { deleteAllPrebuiltRuleAssets, deleteAllRules } from '../../utils';
-import { getPrebuiltRulesStatus } from '../../utils/prebuilt_rules/get_prebuilt_rules_status';
-import { installPrebuiltRulesPackageByVersion } from '../../utils/prebuilt_rules/install_fleet_package_by_url';
-
-// eslint-disable-next-line import/no-default-export
+import { FtrProviderContext } from '../../../../../ftr_provider_context';
+import {
+  deleteAllRules,
+  deleteAllPrebuiltRuleAssets,
+  getPrebuiltRulesStatus,
+  installPrebuiltRulesPackageByVersion,
+} from '../../../utils';
 export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
@@ -28,7 +28,7 @@ export default ({ getService }: FtrProviderContext): void => {
   /* attempt to install it from the local file system. The API response from EPM provides
   /* us with the information of whether the package was installed from the registry or
   /* from a package that was bundled with Kibana */
-  describe('install_bundled_prebuilt_rules', () => {
+  describe('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
       await deleteAllPrebuiltRuleAssets(es);

diff --git a/...uilt_rules_package/prerelease_packages.ts → ...uilt_rules_package/prerelease_packages.ts b/...uilt_rules_package/prerelease_packages.ts → ...uilt_rules_package/prerelease_packages.ts
@@ -5,14 +5,19 @@
  * 2.0.
  */
 import expect from 'expect';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
-import { deleteAllPrebuiltRuleAssets, deleteAllRules } from '../../utils';
-import { getInstalledRules } from '../../utils/prebuilt_rules/get_installed_rules';
-import { getPrebuiltRulesStatus } from '../../utils/prebuilt_rules/get_prebuilt_rules_status';
-import { installPrebuiltRulesPackageViaFleetAPI } from '../../utils/prebuilt_rules/install_fleet_package_by_url';
-import { installPrebuiltRules } from '../../utils/prebuilt_rules/install_prebuilt_rules';
 
-// eslint-disable-next-line import/no-default-export
+import { FtrProviderContext } from '../../../../../ftr_provider_context';
+import {
+  deleteAllPrebuiltRuleAssets,
+  deleteAllRules,
+  deletePrebuiltRulesFleetPackage,
+  getInstalledRules,
+  getPrebuiltRulesFleetPackage,
+  getPrebuiltRulesStatus,
+  installPrebuiltRules,
+  installPrebuiltRulesPackageViaFleetAPI,
+} from '../../../utils';
+
 export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
@@ -25,10 +30,11 @@ export default ({ getService }: FtrProviderContext): void => {
   /* (We use high mock version numbers to prevent clashes with real packages downloaded in other tests.)
   /* To do assertions on which packages have been installed, 99.0.0 has a single rule to install,
   /* while 99.0.1-beta.1 has 2 rules to install. Also, both packages have the version as part of the rule names. */
-  describe('prerelease_packages', () => {
+  describe('@ess @serverless @skipInQA prerelease_packages', () => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
       await deleteAllPrebuiltRuleAssets(es);
+      await deletePrebuiltRulesFleetPackage(supertest);
     });
 
     it('should install latest stable version and ignore prerelease packages', async () => {
@@ -38,13 +44,27 @@ export default ({ getService }: FtrProviderContext): void => {
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0);
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);
 
-      await installPrebuiltRulesPackageViaFleetAPI(es, supertest);
+      // Install package without specifying version to check if latest stable version is installed
+      const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI(
+        es,
+        supertest
+      );
+
+      expect(fleetPackageInstallationResponse.items.length).toBe(1);
+      expect(fleetPackageInstallationResponse.items[0].id).toBe('rule_99.0.0'); // Name of the rule in package 99.0.0
+
+      // Get the installed package and check if the version is 99.0.0
+      const prebuiltRulesFleetPackage = await getPrebuiltRulesFleetPackage(supertest);
+      expect(prebuiltRulesFleetPackage.body.item.version).toBe('99.0.0');
+      expect(prebuiltRulesFleetPackage.status).toBe(200);
 
+      // Get status of our prebuilt rules (nothing should be instaled yet)
       const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest);
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0);
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBe(1); // 1 rule in package 99.0.0
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);
 
+      // Install prebuilt rules
       await installPrebuiltRules(es, supertest);
 
       // Verify that status is updated after package installation

diff --git a/...es/large_prebuilt_rules_package/config.ts → ...built_rules_package/configs/ess.config.ts b/...es/large_prebuilt_rules_package/config.ts → ...built_rules_package/configs/ess.config.ts
@@ -4,21 +4,27 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import { FtrConfigProviderContext } from '@kbn/test';
 import path from 'path';
 
 export const BUNDLED_PACKAGE_DIR = path.join(
   path.dirname(__filename),
-  './fleet_bundled_packages/fixtures'
+  './../fleet_bundled_packages/fixtures'
 );
 
-// eslint-disable-next-line import/no-default-export
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
-  const functionalConfig = await readConfigFile(require.resolve('../config.base.ts'));
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../../../../config/ess/config.base.trial')
+  );
 
   return {
     ...functionalConfig.getAll(),
-    testFiles: [require.resolve('./install_large_prebuilt_rules_package.ts')],
+    testFiles: [require.resolve('..')],
+    junit: {
+      reportName:
+        'Detection Engine ESS / Large Prebuilt Rules Package Installation API Integration Tests',
+    },
     kbnTestServer: {
       ...functionalConfig.get('kbnTestServer'),
       serverArgs: [
@@ -36,7 +42,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         /*  Limit the heap memory to the lowest amount with which Kibana doesn't crash with an out of memory error
          *  when installing the large package.
          */
-        NODE_OPTIONS: '--max-old-space-size=700',
+        NODE_OPTIONS: '--max-old-space-size=800',
       },
     },
   };