From 75f078b7f2bb2215f35dc910d8f11894d166033e Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 16 Oct 2024 10:10:34 +1100 Subject: [PATCH] [8.x] [Security Solution] Allow exporting of prebuilt rules via the API (#194498) (#196447) # Backport This will backport the following commits from `main` to `8.x`: - [[Security Solution] Allow exporting of prebuilt rules via the API (#194498)](https://github.com/elastic/kibana/pull/194498) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Ryland Herrick --- .../api/rules/bulk_actions/route.ts | 3 +- .../api/rules/export_rules/route.ts | 30 +- .../logic/export/get_export_all.ts | 10 +- .../logic/export/get_export_by_object_ids.ts | 15 +- .../trial_license_complete_tier/index.ts | 1 + .../rules_export.ts | 335 ++++++++++++++++++ 6 files changed, 379 insertions(+), 15 deletions(-) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/rules_export.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts index 4d31bd457a3e9..658a9b193e0a2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts @@ -281,7 +281,8 @@ export const performBulkActionRoute = ( rules.map(({ params }) => params.ruleId), exporter, request, - actionsClient + actionsClient, + config.experimentalFeatures.prebuiltRulesCustomizationEnabled ); const responseBody = `${exported.rulesNdjson}${exported.exceptionLists}${exported.actionConnectors}${exported.exportDetails}`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts index 478a0ce02cc96..3c770c714334c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts @@ -15,7 +15,10 @@ import { } from '../../../../../../../common/api/detection_engine/rule_management'; import type { SecuritySolutionPluginRouter } from '../../../../../../types'; import type { ConfigType } from '../../../../../../config'; -import { getNonPackagedRulesCount } from '../../../logic/search/get_existing_prepackaged_rules'; +import { + getNonPackagedRulesCount, + getRulesCount, +} from '../../../logic/search/get_existing_prepackaged_rules'; import { getExportByObjectIds } from '../../../logic/export/get_export_by_object_ids'; import { getExportAll } from '../../../logic/export/get_export_all'; import { buildSiemResponse } from '../../../../routes/utils'; @@ -57,6 +60,8 @@ export const exportRulesRoute = ( const client = getClient({ includedHiddenTypes: ['action'] }); const actionsExporter = getExporter(client); + const { prebuiltRulesCustomizationEnabled } = config.experimentalFeatures; + try { const exportSizeLimit = config.maxRuleImportExportSize; if (request.body?.objects != null && request.body.objects.length > exportSizeLimit) { @@ -65,10 +70,19 @@ export const exportRulesRoute = ( body: `Can't export more than ${exportSizeLimit} rules`, }); } else { - const nonPackagedRulesCount = await getNonPackagedRulesCount({ - rulesClient, - }); - if (nonPackagedRulesCount > exportSizeLimit) { + let rulesCount = 0; + + if (prebuiltRulesCustomizationEnabled) { + rulesCount = await getRulesCount({ + rulesClient, + filter: '', + }); + } else { + rulesCount = await getNonPackagedRulesCount({ + rulesClient, + }); + } + if (rulesCount > exportSizeLimit) { return siemResponse.error({ statusCode: 400, body: `Can't export more than ${exportSizeLimit} rules`, @@ -84,14 +98,16 @@ export const exportRulesRoute = ( request.body.objects.map((obj) => obj.rule_id), actionsExporter, request, - actionsClient + actionsClient, + prebuiltRulesCustomizationEnabled ) : await getExportAll( rulesClient, exceptionsClient, actionsExporter, request, - actionsClient + actionsClient, + prebuiltRulesCustomizationEnabled ); const responseBody = request.query.exclude_export_details diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts index cdf8c6333e595..4407a15622cd6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts @@ -11,7 +11,7 @@ import type { ISavedObjectsExporter, KibanaRequest } from '@kbn/core/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { ActionsClient } from '@kbn/actions-plugin/server'; -import { getNonPackagedRules } from '../search/get_existing_prepackaged_rules'; +import { getNonPackagedRules, getRules } from '../search/get_existing_prepackaged_rules'; import { getExportDetailsNdjson } from './get_export_details_ndjson'; import { transformAlertsToRules } from '../../utils/utils'; import { getRuleExceptionsForExport } from './get_export_rule_exceptions'; @@ -23,14 +23,18 @@ export const getExportAll = async ( exceptionsClient: ExceptionListClient | undefined, actionsExporter: ISavedObjectsExporter, request: KibanaRequest, - actionsClient: ActionsClient + actionsClient: ActionsClient, + prebuiltRulesCustomizationEnabled?: boolean ): Promise<{ rulesNdjson: string; exportDetails: string; exceptionLists: string | null; actionConnectors: string; + prebuiltRulesCustomizationEnabled?: boolean; }> => { - const ruleAlertTypes = await getNonPackagedRules({ rulesClient }); + const ruleAlertTypes = prebuiltRulesCustomizationEnabled + ? await getRules({ rulesClient, filter: '' }) + : await getNonPackagedRules({ rulesClient }); const rules = transformAlertsToRules(ruleAlertTypes); const exportRules = rules.map((r) => transformRuleToExportableFormat(r)); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts index 7c3142aed85f6..02355d39e7e6d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts @@ -29,15 +29,21 @@ export const getExportByObjectIds = async ( ruleIds: string[], actionsExporter: ISavedObjectsExporter, request: KibanaRequest, - actionsClient: ActionsClient + actionsClient: ActionsClient, + prebuiltRulesCustomizationEnabled?: boolean ): Promise<{ rulesNdjson: string; exportDetails: string; exceptionLists: string | null; actionConnectors: string; + prebuiltRulesCustomizationEnabled?: boolean; }> => withSecuritySpan('getExportByObjectIds', async () => { - const rulesAndErrors = await fetchRulesByIds(rulesClient, ruleIds); + const rulesAndErrors = await fetchRulesByIds( + rulesClient, + ruleIds, + prebuiltRulesCustomizationEnabled + ); const { rules, missingRuleIds } = rulesAndErrors; // Retrieve exceptions @@ -76,7 +82,8 @@ interface FetchRulesResult { const fetchRulesByIds = async ( rulesClient: RulesClient, - ruleIds: string[] + ruleIds: string[], + prebuiltRulesCustomizationEnabled?: boolean ): Promise => { // It's important to avoid too many clauses in the request otherwise ES will fail to process the request // with `too_many_clauses` error (see https://github.com/elastic/kibana/issues/170015). The clauses limit @@ -110,7 +117,7 @@ const fetchRulesByIds = async ( return matchingRule != null && hasValidRuleType(matchingRule) && - matchingRule.params.immutable !== true + (prebuiltRulesCustomizationEnabled || matchingRule.params.immutable !== true) ? { rule: transformRuleToExportableFormat(internalRuleToAPIResponse(matchingRule)), } diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts index 76a461d438463..4324ce4602d72 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts @@ -10,5 +10,6 @@ import { FtrProviderContext } from '../../../../../../ftr_provider_context'; export default ({ loadTestFile }: FtrProviderContext): void => { describe('Rules Management - Prebuilt Rules - Update Prebuilt Rules Package', function () { loadTestFile(require.resolve('./is_customized_calculation')); + loadTestFile(require.resolve('./rules_export')); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/rules_export.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/rules_export.ts new file mode 100644 index 0000000000000..e49a23f6138a3 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/rules_export.ts @@ -0,0 +1,335 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; + +import { + BulkActionEditTypeEnum, + BulkActionTypeEnum, +} from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; +import { deleteAllRules } from '../../../../../../../common/utils/security_solution'; +import { + binaryToString, + createPrebuiltRuleAssetSavedObjects, + createRuleAssetSavedObject, + deleteAllPrebuiltRuleAssets, + getCustomQueryRuleParams, + installPrebuiltRules, +} from '../../../../utils'; + +const parseNdJson = (ndJson: Buffer): unknown[] => + ndJson + .toString() + .split('\n') + .filter((line) => !!line) + .map((line) => JSON.parse(line)); + +export default ({ getService }: FtrProviderContext): void => { + const es = getService('es'); + const supertest = getService('supertest'); + const securitySolutionApi = getService('securitySolutionApi'); + const log = getService('log'); + + /** + * This test suite is skipped in Serverless MKI environments due to reliance on the + * feature flag for prebuilt rule customization. + */ + describe('@ess @serverless @skipInServerlessMKI Exporting Rules with Prebuilt Rule Customization', () => { + beforeEach(async () => { + await deleteAllPrebuiltRuleAssets(es, log); + await deleteAllRules(supertest, log); + }); + + it('exports a set of custom installed rules via the _export API', async () => { + await securitySolutionApi + .bulkCreateRules({ + body: [ + getCustomQueryRuleParams({ rule_id: 'rule-id-1' }), + getCustomQueryRuleParams({ rule_id: 'rule-id-2' }), + ], + }) + .expect(200); + + const { body: exportResult } = await securitySolutionApi + .exportRules({ query: {}, body: null }) + .expect(200) + .parse(binaryToString); + + const ndJson = parseNdJson(exportResult); + + expect(ndJson).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: 'rule-id-1', + rule_source: { + type: 'internal', + }, + }), + expect.objectContaining({ + rule_id: 'rule-id-2', + rule_source: { + type: 'internal', + }, + }), + ]) + ); + }); + + describe('with prebuilt rules installed', () => { + let ruleAssets: Array>; + + beforeEach(async () => { + ruleAssets = [ + createRuleAssetSavedObject({ + rule_id: '000047bb-b27a-47ec-8b62-ef1a5d2c9e19', + tags: ['test-tag'], + }), + createRuleAssetSavedObject({ + rule_id: '60b88c41-c45d-454d-945c-5809734dfb34', + tags: ['test-tag-2'], + }), + ]; + await createPrebuiltRuleAssetSavedObjects(es, ruleAssets); + await installPrebuiltRules(es, supertest); + }); + + it('exports a set of prebuilt installed rules via the _export API', async () => { + const { body: exportResult } = await securitySolutionApi + .exportRules({ query: {}, body: null }) + .expect(200) + .parse(binaryToString); + + const parsedExportResult = parseNdJson(exportResult); + + expect(parsedExportResult).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: ruleAssets[0]['security-rule'].rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + expect.objectContaining({ + rule_id: ruleAssets[1]['security-rule'].rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + ]) + ); + + const [firstExportedRule, secondExportedRule] = parsedExportResult as Array<{ + id: string; + rule_id: string; + }>; + + const { body: bulkEditResult } = await securitySolutionApi + .performRulesBulkAction({ + query: {}, + body: { + ids: [firstExportedRule.id], + action: BulkActionTypeEnum.edit, + [BulkActionTypeEnum.edit]: [ + { + type: BulkActionEditTypeEnum.add_tags, + value: ['new-tag'], + }, + ], + }, + }) + .expect(200); + + expect(bulkEditResult.attributes.summary).toEqual({ + failed: 0, + skipped: 0, + succeeded: 1, + total: 1, + }); + expect(bulkEditResult.attributes.results.updated[0].rule_source.is_customized).toEqual( + true + ); + + const { body: secondExportResult } = await securitySolutionApi + .exportRules({ query: {}, body: null }) + .expect(200) + .parse(binaryToString); + + expect(parseNdJson(secondExportResult)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: firstExportedRule.rule_id, + rule_source: { + type: 'external', + is_customized: true, + }, + }), + expect.objectContaining({ + rule_id: secondExportedRule.rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + ]) + ); + }); + + it('exports a set of custom and prebuilt installed rules via the _export API', async () => { + await securitySolutionApi + .bulkCreateRules({ + body: [ + getCustomQueryRuleParams({ rule_id: 'rule-id-1' }), + getCustomQueryRuleParams({ rule_id: 'rule-id-2' }), + ], + }) + .expect(200); + + const { body: exportResult } = await securitySolutionApi + .exportRules({ query: {}, body: null }) + .expect(200) + .parse(binaryToString); + + const exportJson = parseNdJson(exportResult); + expect(exportJson).toHaveLength(5); // 2 prebuilt rules + 2 custom rules + 1 stats object + + expect(exportJson).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: ruleAssets[0]['security-rule'].rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + expect.objectContaining({ + rule_id: ruleAssets[1]['security-rule'].rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + expect.objectContaining({ + rule_id: 'rule-id-1', + rule_source: { + type: 'internal', + }, + }), + expect.objectContaining({ + rule_id: 'rule-id-2', + rule_source: { + type: 'internal', + }, + }), + ]) + ); + }); + + it('exports both custom and prebuilt rules when rule_ids are specified via the _export API', async () => { + await securitySolutionApi + .bulkCreateRules({ + body: [ + getCustomQueryRuleParams({ rule_id: 'rule-id-1' }), + getCustomQueryRuleParams({ rule_id: 'rule-id-2' }), + ], + }) + .expect(200); + + const { body: exportResult } = await securitySolutionApi + .exportRules({ + query: {}, + body: { + objects: [ + { rule_id: ruleAssets[1]['security-rule'].rule_id }, + { rule_id: 'rule-id-2' }, + ], + }, + }) + .expect(200) + .parse(binaryToString); + + const exportJson = parseNdJson(exportResult); + expect(exportJson).toHaveLength(3); // 1 prebuilt rule + 1 custom rule + 1 stats object + + expect(exportJson).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: ruleAssets[1]['security-rule'].rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + expect.objectContaining({ + rule_id: 'rule-id-2', + rule_source: { + type: 'internal', + }, + }), + ]) + ); + }); + + it('exports a set of custom and prebuilt installed rules via the bulk_actions API', async () => { + await securitySolutionApi + .bulkCreateRules({ + body: [ + getCustomQueryRuleParams({ rule_id: 'rule-id-1' }), + getCustomQueryRuleParams({ rule_id: 'rule-id-2' }), + ], + }) + .expect(200); + + const { body: exportResult } = await securitySolutionApi + .performRulesBulkAction({ + body: { query: '', action: BulkActionTypeEnum.export }, + query: {}, + }) + .expect(200) + .expect('Content-Type', 'application/ndjson') + .expect('Content-Disposition', 'attachment; filename="rules_export.ndjson"') + .parse(binaryToString); + + const exportJson = parseNdJson(exportResult); + expect(exportJson).toHaveLength(5); // 2 prebuilt rules + 2 custom rules + 1 stats object + + expect(exportJson).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: ruleAssets[0]['security-rule'].rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + expect.objectContaining({ + rule_id: ruleAssets[1]['security-rule'].rule_id, + rule_source: { + type: 'external', + is_customized: false, + }, + }), + expect.objectContaining({ + rule_id: 'rule-id-1', + rule_source: { + type: 'internal', + }, + }), + expect.objectContaining({ + rule_id: 'rule-id-2', + rule_source: { + type: 'internal', + }, + }), + ]) + ); + }); + }); + }); +};