diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts index 2b36645363edc..6807f50d3f6b1 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { RuleToImport } from './rule_to_import'; +import type { RuleToImport, ValidatedRuleToImport } from './rule_to_import'; export const getImportRulesSchemaMock = (rewrites?: Partial): RuleToImport => ({ @@ -15,12 +15,18 @@ export const getImportRulesSchemaMock = (rewrites?: Partial): Rule severity: 'high', type: 'query', risk_score: 55, - language: 'kuery', rule_id: 'rule-1', immutable: false, ...rewrites, } as RuleToImport); +export const getValidatedRuleToImportMock = ( + overrides?: Partial +): ValidatedRuleToImport => ({ + version: 1, + ...getImportRulesSchemaMock(overrides), +}); + export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): RuleToImport => ({ id: '6afb8ce1-ea94-4790-8653-fd0b021d2113', description: 'some description', @@ -29,7 +35,6 @@ export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): RuleToImport severity: 'high', type: 'query', risk_score: 55, - language: 'kuery', rule_id: ruleId, immutable: false, }); @@ -63,7 +68,6 @@ export const getImportThreatMatchRulesSchemaMock = ( severity: 'high', type: 'threat_match', risk_score: 55, - language: 'kuery', rule_id: 'rule-1', threat_index: ['index-123'], threat_mapping: [{ entries: [{ field: 'host.name', type: 'mapping', value: 'host.name' }] }], diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts index 359dad3235475..e945683fc5019 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts @@ -550,7 +550,7 @@ describe('RuleToImport', () => { ); }); - test('You cannot set the immutable to a number when trying to create a rule', () => { + test('You cannot set immutable to a number', () => { const payload = getImportRulesSchemaMock({ // @ts-expect-error assign unsupported value immutable: 5, @@ -560,11 +560,11 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"immutable: Invalid literal value, expected false"` + `"immutable: Expected boolean, received number"` ); }); - test('You can optionally set the immutable to be false', () => { + test('You can optionally set immutable to false', () => { const payload: RuleToImportInput = getImportRulesSchemaMock({ immutable: false, }); @@ -574,32 +574,14 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('You cannot set the immutable to be true', () => { + test('You can optionally set immutable to true', () => { const payload = getImportRulesSchemaMock({ - // @ts-expect-error assign unsupported value immutable: true, }); const result = RuleToImport.safeParse(payload); - expectParseError(result); - - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"immutable: Invalid literal value, expected false"` - ); - }); - - test('You cannot set the immutable to be a number', () => { - const payload = getImportRulesSchemaMock({ - // @ts-expect-error assign unsupported value - immutable: 5, - }); - const result = RuleToImport.safeParse(payload); - expectParseError(result); - - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"immutable: Invalid literal value, expected false"` - ); + expectParseSuccess(result); }); test('You cannot set the risk_score to 101', () => { @@ -1091,5 +1073,16 @@ describe('RuleToImport', () => { expectParseSuccess(result); expect(result.data).toEqual(payload); }); + + describe('backwards compatibility', () => { + it('allows version to be absent', () => { + const payload = getImportRulesSchemaMock(); + delete payload.version; + + const result = RuleToImport.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual(payload); + }); + }); }); }); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index b4b18a90b548c..3372d04d06652 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -12,13 +12,14 @@ import { RequiredFieldInput, RuleSignatureId, TypeSpecificCreateProps, + RuleVersion, } from '../../model/rule_schema'; /** * Differences from this and the createRulesSchema are * - rule_id is required * - id is optional (but ignored in the import code - rule_id is exclusively used for imports) - * - immutable is optional but if it is any value other than false it will be rejected + * - immutable is optional (but ignored in the import code) * - created_at is optional (but ignored in the import code) * - updated_at is optional (but ignored in the import code) * - created_by is optional (but ignored in the import code) @@ -29,7 +30,6 @@ export type RuleToImportInput = z.input; export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, - immutable: z.literal(false).default(false), /* Overriding `required_fields` from ResponseFields because in ResponseFields `required_fields` has the output type, @@ -40,3 +40,19 @@ export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( required_fields: z.array(RequiredFieldInput).optional(), }) ); + +/** + * This type represents new rules being imported once the prebuilt rule + * customization work is complete. In order to provide backwards compatibility + * with existing rules, and not change behavior, we now validate `version` in + * the route as opposed to the type itself. + * + * It differs from RuleToImport in that it requires a `version` field. + */ +export type ValidatedRuleToImport = z.infer; +export type ValidatedRuleToImportInput = z.input; +export const ValidatedRuleToImport = RuleToImport.and( + z.object({ + version: RuleVersion, + }) +); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts index de21ac3a7964c..cef6d0fa03685 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { RuleToImport } from './rule_to_import'; +import type { RuleToImport, ValidatedRuleToImport } from './rule_to_import'; /** * Additional validation that is implemented outside of the schema itself. @@ -55,3 +55,6 @@ const validateThreshold = (rule: RuleToImport): string[] => { } return errors; }; + +export const ruleToImportHasVersion = (rule: RuleToImport): rule is ValidatedRuleToImport => + !!rule.version; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts index 0dbfd8a230a5a..76625966a1eae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts @@ -20,7 +20,7 @@ const MAX_PREBUILT_RULES_COUNT = 10_000; export interface IPrebuiltRuleAssetsClient { fetchLatestAssets: () => Promise; - fetchLatestVersions(): Promise; + fetchLatestVersions(ruleIds?: string[]): Promise; fetchAssetsByVersion(versions: RuleVersionSpecifier[]): Promise; } @@ -72,8 +72,12 @@ export const createPrebuiltRuleAssetsClient = ( }); }, - fetchLatestVersions: (): Promise => { + fetchLatestVersions: (ruleIds: string[] = []): Promise => { return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestVersions', async () => { + const filter = ruleIds + .map((ruleId) => `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id: ${ruleId}`) + .join(' OR '); + const findResult = await savedObjectsClient.find< PrebuiltRuleAsset, { @@ -83,6 +87,7 @@ export const createPrebuiltRuleAssetsClient = ( } >({ type: PREBUILT_RULE_ASSETS_SO_TYPE, + filter, aggs: { rules: { terms: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index 123b39a588c59..89fee3201e009 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -15,7 +15,7 @@ import { import { getRulesSchemaMock } from '../../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import type { requestMock } from '../../../../routes/__mocks__'; -import { createMockConfig, requestContextMock, serverMock } from '../../../../routes/__mocks__'; +import { configMock, requestContextMock, serverMock } from '../../../../routes/__mocks__'; import { buildHapiStream } from '../../../../routes/__mocks__/utils'; import { getImportRulesRequest, @@ -26,15 +26,22 @@ import { getBasicEmptySearchResponse, } from '../../../../routes/__mocks__/request_responses'; -import * as createRulesAndExceptionsStreamFromNdJson from '../../../logic/import/create_rules_stream_from_ndjson'; +import * as createPromiseFromRuleImportStream from '../../../logic/import/create_promise_from_rule_import_stream'; import { getQueryRuleParams } from '../../../../rule_schema/mocks'; import { importRulesRoute } from './route'; import { HttpAuthzError } from '../../../../../machine_learning/validation'; +import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from '../../../../prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client'; jest.mock('../../../../../machine_learning/authz'); +let mockPrebuiltRuleAssetsClient: ReturnType; + +jest.mock('../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client', () => ({ + createPrebuiltRuleAssetsClient: () => mockPrebuiltRuleAssetsClient, +})); + describe('Import rules route', () => { - let config: ReturnType; + let config: ReturnType; let server: ReturnType; let request: ReturnType; let { clients, context } = requestContextMock.createTools(); @@ -42,7 +49,7 @@ describe('Import rules route', () => { beforeEach(() => { server = serverMock.create(); ({ clients, context } = requestContextMock.createTools()); - config = createMockConfig(); + config = configMock.createDefault(); const hapiStream = buildHapiStream(ruleIdsToNdJsonString(['rule-1'])); request = getImportRulesRequest(hapiStream); @@ -54,6 +61,7 @@ describe('Import rules route', () => { context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise(getBasicEmptySearchResponse()) ); + mockPrebuiltRuleAssetsClient = createPrebuiltRuleAssetsClientMock(); importRulesRoute(server.router, config); }); @@ -112,9 +120,9 @@ describe('Import rules route', () => { }); }); - test('returns error if createRulesAndExceptionsStreamFromNdJson throws error', async () => { + test('returns error if createPromiseFromRuleImportStream throws error', async () => { const transformMock = jest - .spyOn(createRulesAndExceptionsStreamFromNdJson, 'createRulesAndExceptionsStreamFromNdJson') + .spyOn(createPromiseFromRuleImportStream, 'createPromiseFromRuleImportStream') .mockImplementation(() => { throw new Error('Test error'); }); @@ -133,6 +141,30 @@ describe('Import rules route', () => { expect(response.status).toEqual(400); expect(response.body).toEqual({ message: 'Invalid file extension .html', status_code: 400 }); }); + + describe('with prebuilt rules customization enabled', () => { + beforeEach(() => { + clients.detectionRulesClient.importRules.mockResolvedValueOnce([]); + server = serverMock.create(); // old server already registered this route + config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); + + importRulesRoute(server.router, config); + }); + + test('returns 500 if importing fails', async () => { + clients.detectionRulesClient.importRules + .mockReset() + .mockRejectedValue(new Error('test error')); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(500); + expect(response.body).toMatchObject({ + message: 'test error', + status_code: 500, + }); + }); + }); }); describe('single rule import', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 6407909ed8bf5..e9131050d9629 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -8,8 +8,7 @@ import { schema } from '@kbn/config-schema'; import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { createPromiseFromStreams } from '@kbn/utils'; -import { chunk } from 'lodash/fp'; +import { chunk, partition } from 'lodash/fp'; import { extname } from 'path'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { @@ -19,13 +18,22 @@ import { import { DETECTION_ENGINE_RULES_URL } from '../../../../../../../common/constants'; import type { ConfigType } from '../../../../../../config'; import type { HapiReadableStream, SecuritySolutionPluginRouter } from '../../../../../../types'; -import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; -import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; +import type { ImportRuleResponse } from '../../../../routes/utils'; +import { + buildSiemResponse, + createBulkErrorObject, + isBulkError, + isImportRegular, +} from '../../../../routes/utils'; +import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; -import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; -import type { RuleExceptionsPromiseFromStreams } from '../../../logic/import/import_rules_utils'; -import { importRules as importRulesHelper } from '../../../logic/import/import_rules_utils'; +import { createRuleSourceImporter } from '../../../logic/import/rule_source_importer'; +import { importRules } from '../../../logic/import/import_rules'; +// eslint-disable-next-line no-restricted-imports +import { importRulesLegacy } from '../../../logic/import/import_rules_legacy'; +import { createPromiseFromRuleImportStream } from '../../../logic/import/create_promise_from_rule_import_stream'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; +import { isRuleToImport } from '../../../logic/import/utils'; import { getTupleDuplicateErrorsAndUniqueRules, migrateLegacyActionsIds, @@ -73,6 +81,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C 'licensing', ]); + const { prebuiltRulesCustomizationEnabled } = config.experimentalFeatures; const detectionRulesClient = ctx.securitySolution.getDetectionRulesClient(); const actionsClient = ctx.actions.getActionsClient(); const actionSOClient = ctx.core.savedObjects.getClient({ @@ -95,10 +104,9 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C const objectLimit = config.maxRuleImportExportSize; // parse file to separate out exceptions from rules - const readAllStream = createRulesAndExceptionsStreamFromNdJson(objectLimit); - const [{ exceptions, rules, actionConnectors }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([request.body.file as HapiReadableStream, ...readAllStream]); + const [{ exceptions, rules, actionConnectors }] = await createPromiseFromRuleImportStream( + { stream: request.body.file as HapiReadableStream, objectLimit } + ); // import exceptions, includes validation const { @@ -138,22 +146,53 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C // rulesWithMigratedActions: Is returned only in case connectors were exported from different namespace and the // original rules actions' ids were replaced with new destinationIds - const parsedRules = actionConnectorErrors.length + const parsedRuleStream = actionConnectorErrors.length ? [] : rulesWithMigratedActions || migratedParsedObjectsWithoutDuplicateErrors; - const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); - - const importRuleResponse: ImportRuleResponse[] = await importRulesHelper({ - ruleChunks: chunkParseObjects, - rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], - overwriteRules: request.query.overwrite, - detectionRulesClient, - allowMissingConnectorSecrets: !!actionConnectors.length, - savedObjectsClient, + const ruleSourceImporter = createRuleSourceImporter({ + config, + context: ctx.securitySolution, + prebuiltRuleAssetsClient: createPrebuiltRuleAssetsClient(savedObjectsClient), }); - const errorsResp = importRuleResponse.filter((resp) => isBulkError(resp)) as BulkError[]; + const [parsedRules, parsedRuleErrors] = partition(isRuleToImport, parsedRuleStream); + const ruleChunks = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); + + let importRuleResponse: ImportRuleResponse[] = []; + + if (prebuiltRulesCustomizationEnabled) { + importRuleResponse = await importRules({ + ruleChunks, + overwriteRules: request.query.overwrite, + allowMissingConnectorSecrets: !!actionConnectors.length, + ruleSourceImporter, + detectionRulesClient, + }); + } else { + importRuleResponse = await importRulesLegacy({ + ruleChunks, + overwriteRules: request.query.overwrite, + allowMissingConnectorSecrets: !!actionConnectors.length, + detectionRulesClient, + savedObjectsClient, + }); + } + + const parseErrors = parsedRuleErrors.map((error) => + createBulkErrorObject({ + statusCode: 400, + message: error.message, + }) + ); + const importErrors = importRuleResponse.filter(isBulkError); + const errors = [ + ...parseErrors, + ...actionConnectorErrors, + ...duplicateIdErrors, + ...importErrors, + ]; + const successes = importRuleResponse.filter((resp) => { if (isImportRegular(resp)) { return resp.status_code === 200; @@ -161,11 +200,12 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C return false; } }); - const importRules: ImportRulesResponse = { - success: errorsResp.length === 0, + + const importRulesResponse: ImportRulesResponse = { + success: errors.length === 0, success_count: successes.length, rules_count: rules.length, - errors: errorsResp, + errors, exceptions_errors: exceptionsErrors, exceptions_success: exceptionsSuccess, exceptions_success_count: exceptionsSuccessCount, @@ -175,7 +215,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C action_connectors_warnings: actionConnectorWarnings, }; - return response.ok({ body: ImportRulesResponse.parse(importRules) }); + return response.ok({ body: ImportRulesResponse.parse(importRulesResponse) }); } catch (err) { const error = transformError(err); return siemResponse.error({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts index b6d14a307801e..19d028bb9e666 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts @@ -18,6 +18,7 @@ const createDetectionRulesClientMock = () => { deleteRule: jest.fn(), upgradePrebuiltRule: jest.fn(), importRule: jest.fn(), + importRules: jest.fn(), }; return mocked; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts index 38e40ab67611f..890f8a6bad7ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts @@ -12,6 +12,9 @@ import type { BaseRuleParams } from '../../../../rule_schema'; import { migrateLegacyInvestigationFields } from '../../../utils/utils'; import type { NormalizedRuleParams } from './normalize_rule_params'; +/** + * @deprecated Use convertObjectKeysToSnakeCase instead + */ export const commonParamsCamelToSnake = (params: BaseRuleParams) => { return { description: params.description, @@ -42,7 +45,7 @@ export const commonParamsCamelToSnake = (params: BaseRuleParams) => { version: params.version, exceptions_list: params.exceptionsList, immutable: params.immutable, - rule_source: convertObjectKeysToSnakeCase(params.ruleSource), + rule_source: params.ruleSource ? convertObjectKeysToSnakeCase(params.ruleSource) : undefined, related_integrations: params.relatedIntegrations ?? [], required_fields: params.requiredFields ?? [], response_actions: params.responseActions?.map(transformAlertToRuleResponseAction), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.test.ts new file mode 100644 index 0000000000000..98e5b235159e8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { convertPrebuiltRuleAssetToRuleResponse } from './convert_prebuilt_rule_asset_to_rule_response'; +import { getPrebuiltRuleMock } from '../../../../prebuilt_rules/mocks'; + +describe('convertPrebuiltRuleAssetToRuleResponse', () => { + it('converts a valid prebuilt asset (without a language field) to valid rule response (with a language field)', () => { + const ruleAssetWithoutLanguage = getPrebuiltRuleMock({ language: undefined }); + + expect(convertPrebuiltRuleAssetToRuleResponse(ruleAssetWithoutLanguage)).toMatchObject({ + language: 'kuery', + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts index f7a5d78798880..1e87721557214 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts @@ -6,10 +6,9 @@ */ import { v4 as uuidv4 } from 'uuid'; -import { addEcsToRequiredFields } from '../../../../../../../common/detection_engine/rule_management/utils'; import { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; -import { RULE_DEFAULTS } from '../mergers/apply_rule_defaults'; +import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; export const convertPrebuiltRuleAssetToRuleResponse = ( prebuiltRuleAsset: PrebuiltRuleAsset @@ -30,10 +29,10 @@ export const convertPrebuiltRuleAssetToRuleResponse = ( revision: 1, }; + const ruleWithDefaults = applyRuleDefaults(prebuiltRuleAsset); + return RuleResponse.parse({ - ...RULE_DEFAULTS, - ...prebuiltRuleAsset, - required_fields: addEcsToRequiredFields(prebuiltRuleAsset.required_fields), + ...ruleWithDefaults, ...ruleResponseSpecificFields, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts index 0c2edf5535f35..52aac47447df1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts @@ -73,7 +73,7 @@ export const convertRuleResponseToAlertingRule = ( from: rule.from, investigationFields: rule.investigation_fields, immutable: rule.immutable, - ruleSource: convertObjectKeysToCamelCase(rule.rule_source), + ruleSource: rule.rule_source ? convertObjectKeysToCamelCase(rule.rule_source) : undefined, license: rule.license, outputIndex: rule.output_index ?? '', timelineId: rule.timeline_id, @@ -122,7 +122,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific timestampField: params.timestamp_field, eventCategoryOverride: params.event_category_override, tiebreakerField: params.tiebreaker_field, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'esql': { @@ -130,7 +132,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific type: params.type, language: params.language, query: params.query, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'threat_match': { @@ -150,7 +154,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific threatIndicatorPath: params.threat_indicator_path, concurrentSearches: params.concurrent_searches, itemsPerSearch: params.items_per_search, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'query': { @@ -162,7 +168,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific query: params.query ?? '', filters: params.filters, savedId: params.saved_id, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'saved_query': { @@ -174,7 +182,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific filters: params.filters, savedId: params.saved_id, dataViewId: params.data_view_id, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'threshold': { @@ -197,7 +207,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific type: params.type, anomalyThreshold: params.anomaly_threshold, machineLearningJobId: normalizeMachineLearningJobIds(params.machine_learning_job_id), - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'new_terms': { @@ -210,7 +222,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific filters: params.filters, language: params.language ?? 'kuery', dataViewId: params.data_view_id, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } default: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts index b8b5db137583b..8398bab4253f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { normalizeRuleSource } from './normalize_rule_params'; +import { normalizeRuleSource, normalizeRuleParams } from './normalize_rule_params'; import type { BaseRuleParams } from '../../../../rule_schema'; describe('normalizeRuleSource', () => { @@ -53,3 +53,14 @@ describe('normalizeRuleSource', () => { }); }); }); + +describe('normalizeRuleParams', () => { + it('migrates legacy investigation fields', () => { + const params = { + investigationFields: ['field_1', 'field_2'], + } as BaseRuleParams; + const result = normalizeRuleParams(params); + + expect(result.investigationFields).toMatchObject({ field_names: ['field_1', 'field_2'] }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts index 8d5793c04f22b..7917bc0a10b22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts @@ -5,6 +5,7 @@ * 2.0. */ import type { BaseRuleParams, RuleSourceCamelCased } from '../../../../rule_schema'; +import { migrateLegacyInvestigationFields } from '../../../utils/utils'; interface NormalizeRuleSourceParams { immutable: BaseRuleParams['immutable']; @@ -41,12 +42,20 @@ export const normalizeRuleSource = ({ }; export const normalizeRuleParams = (params: BaseRuleParams): NormalizedRuleParams => { + const investigationFields = migrateLegacyInvestigationFields(params.investigationFields); + const ruleSource = normalizeRuleSource({ + immutable: params.immutable, + ruleSource: params.ruleSource, + }); + return { ...params, + // These fields are typed as optional in the data model, but they are required in our domain + setup: params.setup ?? '', + relatedIntegrations: params.relatedIntegrations ?? [], + requiredFields: params.requiredFields ?? [], // Fields to normalize - ruleSource: normalizeRuleSource({ - immutable: params.immutable, - ruleSource: params.ruleSource, - }), + investigationFields, + ruleSource, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts index 5a2f7ba0d3548..35d8efa430233 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts @@ -26,7 +26,9 @@ export const typeSpecificCamelToSnake = ( timestamp_field: params.timestampField, event_category_override: params.eventCategoryOverride, tiebreaker_field: params.tiebreakerField, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'esql': { @@ -34,7 +36,9 @@ export const typeSpecificCamelToSnake = ( type: params.type, language: params.language, query: params.query, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'threat_match': { @@ -54,7 +58,9 @@ export const typeSpecificCamelToSnake = ( threat_indicator_path: params.threatIndicatorPath, concurrent_searches: params.concurrentSearches, items_per_search: params.itemsPerSearch, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'query': { @@ -66,7 +72,9 @@ export const typeSpecificCamelToSnake = ( query: params.query, filters: params.filters, saved_id: params.savedId, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'saved_query': { @@ -78,7 +86,9 @@ export const typeSpecificCamelToSnake = ( filters: params.filters, saved_id: params.savedId, data_view_id: params.dataViewId, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'threshold': { @@ -101,7 +111,9 @@ export const typeSpecificCamelToSnake = ( type: params.type, anomaly_threshold: params.anomalyThreshold, machine_learning_job_id: params.machineLearningJobId, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'new_terms': { @@ -114,7 +126,9 @@ export const typeSpecificCamelToSnake = ( filters: params.filters, language: params.language, data_view_id: params.dataViewId, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } default: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts index e3b922fa831a6..4300b17b3be80 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts @@ -9,10 +9,7 @@ import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; import type { ActionsClient } from '@kbn/actions-plugin/server'; import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { - getCreateRulesSchemaMock, - getRulesSchemaMock, -} from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; import { buildMlAuthz } from '../../../../machine_learning/authz'; import { throwAuthzError } from '../../../../machine_learning/validation'; import { getRuleMock } from '../../../routes/__mocks__/request_responses'; @@ -20,6 +17,7 @@ import { getQueryRuleParams } from '../../../rule_schema/mocks'; import { createDetectionRulesClient } from './detection_rules_client'; import type { IDetectionRulesClient } from './detection_rules_client_interface'; import { getRuleByRuleId } from './methods/get_rule_by_rule_id'; +import { getValidatedRuleToImportMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; jest.mock('../../../../machine_learning/authz'); jest.mock('../../../../machine_learning/validation'); @@ -33,14 +31,12 @@ describe('DetectionRulesClient.importRule', () => { const mlAuthz = (buildMlAuthz as jest.Mock)(); let actionsClient: jest.Mocked; - const immutable = false as const; // Can only take value of false const allowMissingConnectorSecrets = true; const ruleToImport = { - ...getCreateRulesSchemaMock(), + ...getValidatedRuleToImportMock(), tags: ['import-tag'], rule_id: 'rule-id', version: 1, - immutable, }; const existingRule = getRulesSchemaMock(); existingRule.rule_id = ruleToImport.rule_id; @@ -72,9 +68,10 @@ describe('DetectionRulesClient.importRule', () => { name: ruleToImport.name, tags: ruleToImport.tags, params: expect.objectContaining({ - immutable, + immutable: ruleToImport.immutable, ruleId: ruleToImport.rule_id, version: ruleToImport.version, + ruleSource: { type: 'internal' }, }), }), allowMissingConnectorSecrets, @@ -115,8 +112,11 @@ describe('DetectionRulesClient.importRule', () => { name: ruleToImport.name, tags: ruleToImport.tags, params: expect.objectContaining({ - index: ruleToImport.index, description: ruleToImport.description, + immutable: ruleToImport.immutable, + ruleId: ruleToImport.rule_id, + version: ruleToImport.version, + ruleSource: { type: 'internal' }, }), }), id: existingRule.id, @@ -227,6 +227,39 @@ describe('DetectionRulesClient.importRule', () => { ); }); + it('preserves the passed "rule_source" and "immutable" values', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + overrideFields: { + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + immutable: true, + ruleSource: { + isCustomized: true, + type: 'external', + }, + }), + }), + }) + ); + }); + it('rejects when overwriteRules is false', async () => { (getRuleByRuleId as jest.Mock).mockResolvedValue(existingRule); await expect( @@ -237,11 +270,159 @@ describe('DetectionRulesClient.importRule', () => { }) ).rejects.toMatchObject({ error: { - status_code: 409, + ruleId: ruleToImport.rule_id, + type: 'conflict', message: `rule_id: "${ruleToImport.rule_id}" already exists`, }, - rule_id: ruleToImport.rule_id, }); }); + + it("always uses the existing rule's 'id' value", async () => { + const rule = { + ...getValidatedRuleToImportMock(), + id: 'some-id', + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + }) + ); + }); + + it("uses the existing rule's 'version' value if not unspecified", async () => { + const rule = { + ...getValidatedRuleToImportMock(), + version: undefined, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + version: existingRule.version, + }), + }), + }) + ); + }); + + it("uses the specified 'version' value", async () => { + const rule = { + ...getValidatedRuleToImportMock(), + version: 42, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + version: rule.version, + }), + }), + }) + ); + }); + }); + + describe('when importing a new rule', () => { + beforeEach(() => { + (getRuleByRuleId as jest.Mock).mockReset().mockResolvedValueOnce(null); + }); + + it('preserves the passed "rule_source" and "immutable" values', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + overrideFields: { + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + immutable: true, + ruleSource: { + isCustomized: true, + type: 'external', + }, + }), + }), + }) + ); + }); + + it('preserves the passed "enabled" value', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + enabled: true, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + enabled: true, + }), + }) + ); + }); + + it('defaults defaultable values', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + actions: [], + }), + }) + ); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts new file mode 100644 index 0000000000000..58b1385dda09c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; +import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/actions_client.mock'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; + +import { buildMlAuthz } from '../../../../machine_learning/__mocks__/authz'; +import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { ruleSourceImporterMock } from '../import/rule_source_importer/rule_source_importer.mock'; +import { createDetectionRulesClient } from './detection_rules_client'; +import { importRule } from './methods/import_rule'; +import { createRuleImportErrorObject } from '../import/errors'; +import { checkRuleExceptionReferences } from '../import/check_rule_exception_references'; + +jest.mock('./methods/import_rule'); +jest.mock('../import/check_rule_exception_references'); + +describe('detectionRulesClient.importRules', () => { + let subject: ReturnType; + let ruleToImport: ReturnType; + let mockRuleSourceImporter: ReturnType; + + beforeEach(() => { + subject = createDetectionRulesClient({ + actionsClient: actionsClientMock.create(), + rulesClient: rulesClientMock.create(), + mlAuthz: buildMlAuthz(), + savedObjectsClient: savedObjectsClientMock.create(), + }); + + (checkRuleExceptionReferences as jest.Mock).mockReturnValue([[], []]); + (importRule as jest.Mock).mockResolvedValue(getRulesSchemaMock()); + + ruleToImport = getImportRulesSchemaMock(); + mockRuleSourceImporter = ruleSourceImporterMock.create(); + mockRuleSourceImporter.calculateRuleSource.mockReturnValue({ + ruleSource: { type: 'internal' }, + immutable: false, + }); + }); + + it('returns imported rules as RuleResponses if import was successful', async () => { + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + ruleSourceImporter: mockRuleSourceImporter, + rules: [ruleToImport, ruleToImport], + }); + + expect(result).toEqual([getRulesSchemaMock(), getRulesSchemaMock()]); + }); + + it('returns an import error if rule import throws an import error', async () => { + const importError = createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'an error occurred', + }); + (importRule as jest.Mock).mockReset().mockRejectedValueOnce(importError); + + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + ruleSourceImporter: mockRuleSourceImporter, + rules: [ruleToImport], + }); + + expect(result).toEqual([importError]); + }); + + it('returns a generic error if rule import throws unexpectedly', async () => { + const genericError = new Error('an unexpected error occurred'); + (importRule as jest.Mock).mockReset().mockRejectedValueOnce(genericError); + + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + ruleSourceImporter: mockRuleSourceImporter, + rules: [ruleToImport], + }); + + expect(result).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ + message: 'an unexpected error occurred', + ruleId: ruleToImport.rule_id, + type: 'unknown', + }), + }), + ]); + }); + + describe('when rule has no exception list references', () => { + beforeEach(() => { + (checkRuleExceptionReferences as jest.Mock).mockReset().mockReturnValueOnce([ + [ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'list not found', + }), + ], + [], + ]); + }); + + it('returns both exception list reference errors and the imported rule if import succeeds', async () => { + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + ruleSourceImporter: mockRuleSourceImporter, + rules: [ruleToImport], + }); + + expect(result).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ + message: 'list not found', + ruleId: 'rule-id', + type: 'unknown', + }), + }), + getRulesSchemaMock(), + ]); + }); + + it('returns both exception list reference errors and the imported rule if import throws an error', async () => { + const importError = createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'an error occurred', + }); + (importRule as jest.Mock).mockReset().mockRejectedValueOnce(importError); + + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + ruleSourceImporter: mockRuleSourceImporter, + rules: [ruleToImport], + }); + + expect(result).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ + message: 'list not found', + ruleId: 'rule-id', + type: 'unknown', + }), + }), + expect.objectContaining({ + error: expect.objectContaining({ + message: 'an error occurred', + ruleId: 'rule-id', + type: 'unknown', + }), + }), + ]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index 4753df0ffe411..fb6f5c4a03c1f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -13,12 +13,14 @@ import type { RuleResponse } from '../../../../../../common/api/detection_engine import { withSecuritySpan } from '../../../../../utils/with_security_span'; import type { MlAuthz } from '../../../../machine_learning/authz'; import { createPrebuiltRuleAssetsClient } from '../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; +import type { RuleImportErrorObject } from '../import/errors'; import type { CreateCustomRuleArgs, CreatePrebuiltRuleArgs, DeleteRuleArgs, IDetectionRulesClient, ImportRuleArgs, + ImportRulesArgs, PatchRuleArgs, UpdateRuleArgs, UpgradePrebuiltRuleArgs, @@ -29,6 +31,7 @@ import { importRule } from './methods/import_rule'; import { patchRule } from './methods/patch_rule'; import { updateRule } from './methods/update_rule'; import { upgradePrebuiltRule } from './methods/upgrade_prebuilt_rule'; +import { importRules } from './methods/import_rules'; interface DetectionRulesClientParams { actionsClient: ActionsClient; @@ -131,5 +134,15 @@ export const createDetectionRulesClient = ({ }); }); }, + + async importRules(args: ImportRulesArgs): Promise> { + return withSecuritySpan('DetectionRulesClient.importRules', async () => { + return importRules({ + ...args, + detectionRulesClient: this, + savedObjectsClient, + }); + }); + }, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index d7b45f83e8bf8..53933fa93a4a3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -10,9 +10,12 @@ import type { RuleUpdateProps, RulePatchProps, RuleObjectId, - RuleToImport, RuleResponse, + RuleToImport, + RuleSource, } from '../../../../../../common/api/detection_engine'; +import type { IRuleSourceImporter } from '../import/rule_source_importer'; +import type { RuleImportErrorObject } from '../import/errors'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; export interface IDetectionRulesClient { @@ -23,6 +26,7 @@ export interface IDetectionRulesClient { deleteRule: (args: DeleteRuleArgs) => Promise; upgradePrebuiltRule: (args: UpgradePrebuiltRuleArgs) => Promise; importRule: (args: ImportRuleArgs) => Promise; + importRules: (args: ImportRulesArgs) => Promise>; } export interface CreateCustomRuleArgs { @@ -51,6 +55,14 @@ export interface UpgradePrebuiltRuleArgs { export interface ImportRuleArgs { ruleToImport: RuleToImport; + overrideFields?: { rule_source: RuleSource; immutable: boolean }; overwriteRules?: boolean; allowMissingConnectorSecrets?: boolean; } + +export interface ImportRulesArgs { + rules: RuleToImport[]; + overwriteRules: boolean; + ruleSourceImporter: IRuleSourceImporter; + allowMissingConnectorSecrets?: boolean; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts index 40f0b3eca3b98..8c91149bd5fa0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts @@ -44,7 +44,9 @@ export const RULE_DEFAULTS = { version: 1, }; -export function applyRuleDefaults(rule: RuleCreateProps & { immutable?: boolean }) { +export function applyRuleDefaults( + rule: RuleCreateProps & { immutable?: boolean; rule_source?: RuleSource } +) { const typeSpecificParams = setTypeSpecificDefaults(rule); const immutable = rule.immutable ?? false; @@ -54,7 +56,7 @@ export function applyRuleDefaults(rule: RuleCreateProps & { immutable?: boolean ...typeSpecificParams, rule_id: rule.rule_id ?? uuidv4(), immutable, - rule_source: convertImmutableToRuleSource(immutable), + rule_source: rule.rule_source ?? convertImmutableToRuleSource(immutable), required_fields: addEcsToRequiredFields(rule.required_fields), }; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts index 4066cb00849a3..dd57e66c41a64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts @@ -11,7 +11,6 @@ import type { ActionsClient } from '@kbn/actions-plugin/server'; import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import type { MlAuthz } from '../../../../../machine_learning/authz'; import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import { createBulkErrorObject } from '../../../../routes/utils'; import { convertAlertingRuleToRuleResponse } from '../converters/convert_alerting_rule_to_rule_response'; import { convertRuleResponseToAlertingRule } from '../converters/convert_rule_response_to_alerting_rule'; import type { ImportRuleArgs } from '../detection_rules_client_interface'; @@ -19,6 +18,7 @@ import { applyRuleUpdate } from '../mergers/apply_rule_update'; import { validateMlAuth, toggleRuleEnabledOnUpdate } from '../utils'; import { createRule } from './create_rule'; import { getRuleByRuleId } from './get_rule_by_rule_id'; +import { createRuleImportErrorObject } from '../../import/errors'; interface ImportRuleOptions { actionsClient: ActionsClient; @@ -35,29 +35,34 @@ export const importRule = async ({ prebuiltRuleAssetClient, mlAuthz, }: ImportRuleOptions): Promise => { - const { ruleToImport, overwriteRules, allowMissingConnectorSecrets } = importRulePayload; + const { ruleToImport, overwriteRules, overrideFields, allowMissingConnectorSecrets } = + importRulePayload; + // For backwards compatibility, immutable is false by default + const rule = { ...ruleToImport, immutable: false, ...overrideFields }; await validateMlAuth(mlAuthz, ruleToImport.type); const existingRule = await getRuleByRuleId({ rulesClient, - ruleId: ruleToImport.rule_id, + ruleId: rule.rule_id, }); if (existingRule && !overwriteRules) { - throw createBulkErrorObject({ + throw createRuleImportErrorObject({ ruleId: existingRule.rule_id, - statusCode: 409, + type: 'conflict', message: `rule_id: "${existingRule.rule_id}" already exists`, }); } if (existingRule && overwriteRules) { - const ruleWithUpdates = await applyRuleUpdate({ + let ruleWithUpdates = await applyRuleUpdate({ prebuiltRuleAssetClient, existingRule, - ruleUpdate: ruleToImport, + ruleUpdate: rule, }); + // applyRuleUpdate prefers the existing rule's values for `rule_source` and `immutable`, but we want to use the importing rule's calculated values + ruleWithUpdates = { ...ruleWithUpdates, ...overrideFields }; const updatedRule = await rulesClient.update({ id: existingRule.id, @@ -75,7 +80,7 @@ export const importRule = async ({ actionsClient, rulesClient, mlAuthz, - rule: ruleToImport, + rule, allowMissingConnectorSecrets, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts new file mode 100644 index 0000000000000..0a66813289290 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; + +import type { RuleResponse, RuleToImport } from '../../../../../../../common/api/detection_engine'; +import { ruleToImportHasVersion } from '../../../../../../../common/api/detection_engine/rule_management'; +import type { IRuleSourceImporter } from '../../import/rule_source_importer'; +import { + type RuleImportErrorObject, + createRuleImportErrorObject, + isRuleImportError, +} from '../../import/errors'; +import { checkRuleExceptionReferences } from '../../import/check_rule_exception_references'; +import { getReferencedExceptionLists } from '../../import/gather_referenced_exceptions'; +import type { IDetectionRulesClient } from '../detection_rules_client_interface'; + +/** + * Imports rules + */ + +export const importRules = async ({ + allowMissingConnectorSecrets, + detectionRulesClient, + overwriteRules, + ruleSourceImporter, + rules, + savedObjectsClient, +}: { + allowMissingConnectorSecrets?: boolean; + detectionRulesClient: IDetectionRulesClient; + overwriteRules: boolean; + ruleSourceImporter: IRuleSourceImporter; + rules: RuleToImport[]; + savedObjectsClient: SavedObjectsClientContract; +}): Promise> => { + const existingLists = await getReferencedExceptionLists({ + rules, + savedObjectsClient, + }); + await ruleSourceImporter.setup(rules); + + return Promise.all( + rules.map(async (rule) => { + const errors: RuleImportErrorObject[] = []; + + try { + if (!ruleSourceImporter.isPrebuiltRule(rule)) { + rule.version = rule.version ?? 1; + } + + if (!ruleToImportHasVersion(rule)) { + return createRuleImportErrorObject({ + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.cannotImportPrebuiltRuleWithoutVersion', + { + defaultMessage: + 'Prebuilt rules must specify a "version" to be imported. [rule_id: {ruleId}]', + values: { ruleId: rule.rule_id }, + } + ), + ruleId: rule.rule_id, + }); + } + + const { immutable, ruleSource } = ruleSourceImporter.calculateRuleSource(rule); + + const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ + rule, + existingLists, + }); + errors.push(...exceptionErrors); + + const importedRule = await detectionRulesClient.importRule({ + ruleToImport: { + ...rule, + exceptions_list: [...exceptions], + }, + overrideFields: { rule_source: ruleSource, immutable }, + overwriteRules, + allowMissingConnectorSecrets, + }); + + return [...errors, importedRule]; + } catch (err) { + const { error, message } = err; + + const caughtError = isRuleImportError(err) + ? err + : createRuleImportErrorObject({ + ruleId: rule.rule_id, + message: message ?? error?.message ?? 'unknown error', + }); + + return [...errors, caughtError]; + } + }) + ).then((results) => results.flat()); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts index 84352c1ea0f1e..08bfa95207555 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts @@ -373,7 +373,6 @@ describe('importRuleActionConnectors', () => { ], description: 'some description', immutable: false, - language: 'kuery', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', risk_score: 55, @@ -483,7 +482,6 @@ describe('importRuleActionConnectors', () => { ], description: 'some description', immutable: false, - language: 'kuery', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', risk_score: 55, @@ -502,7 +500,6 @@ describe('importRuleActionConnectors', () => { ], description: 'some description', immutable: false, - language: 'kuery', name: 'Query with a rule id', id: '0abc78e0-7031-11ed-b076-53cc4d57aaf1', rule_id: 'rule_2', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts new file mode 100644 index 0000000000000..e3bfb75c6a88d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; +import { getPrebuiltRuleMock } from '../../../prebuilt_rules/mocks'; +import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; + +describe('calculateRuleSourceForImport', () => { + it('calculates as internal if no asset is found', () => { + const result = calculateRuleSourceForImport({ + rule: getRulesSchemaMock(), + prebuiltRuleAssetsByRuleId: {}, + isKnownPrebuiltRule: false, + }); + + expect(result).toEqual({ + ruleSource: { + type: 'internal', + }, + immutable: false, + }); + }); + + it('calculates as modified external type if an asset is found without a matching version', () => { + const rule = getRulesSchemaMock(); + rule.rule_id = 'rule_id'; + + const result = calculateRuleSourceForImport({ + rule, + prebuiltRuleAssetsByRuleId: {}, + isKnownPrebuiltRule: true, + }); + + expect(result).toEqual({ + ruleSource: { + type: 'external', + is_customized: true, + }, + immutable: true, + }); + }); + + it('calculates as external with customizations if a matching asset/version is found', () => { + const rule = getRulesSchemaMock(); + rule.rule_id = 'rule_id'; + const prebuiltRuleAssetsByRuleId = { rule_id: getPrebuiltRuleMock({ rule_id: 'rule_id' }) }; + + const result = calculateRuleSourceForImport({ + rule, + prebuiltRuleAssetsByRuleId, + isKnownPrebuiltRule: true, + }); + + expect(result).toEqual({ + ruleSource: { + type: 'external', + is_customized: true, + }, + immutable: true, + }); + }); + + it('calculates as external without customizations if an exact match is found', () => { + const rule = getRulesSchemaMock(); + rule.rule_id = 'rule_id'; + const prebuiltRuleAssetsByRuleId = { rule_id: getPrebuiltRuleMock(rule) }; + + const result = calculateRuleSourceForImport({ + rule, + prebuiltRuleAssetsByRuleId, + isKnownPrebuiltRule: true, + }); + + expect(result).toEqual({ + ruleSource: { + type: 'external', + is_customized: false, + }, + immutable: true, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts new file mode 100644 index 0000000000000..133566a7b776b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleSource, + ValidatedRuleToImport, +} from '../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; +import { convertRuleToImportToRuleResponse } from './converters/convert_rule_to_import_to_rule_response'; + +/** + * Calculates the rule_source field for a rule being imported + * + * @param rule The rule to be imported + * @param prebuiltRuleAssets A list of prebuilt rule assets, which may include + * the installed version of the specified prebuilt rule. + * @param isKnownPrebuiltRule {boolean} Whether the rule's rule_id is available as a + * prebuilt asset (independent of the specified version). + * + * @returns The calculated rule_source and immutable fields for the rule + */ +export const calculateRuleSourceForImport = ({ + rule, + prebuiltRuleAssetsByRuleId, + isKnownPrebuiltRule, +}: { + rule: ValidatedRuleToImport; + prebuiltRuleAssetsByRuleId: Record; + isKnownPrebuiltRule: boolean; +}): { ruleSource: RuleSource; immutable: boolean } => { + const assetWithMatchingVersion = prebuiltRuleAssetsByRuleId[rule.rule_id]; + // We convert here so that RuleSource calculation can + // continue to deal only with RuleResponses. The fields missing from the + // incoming rule are not actually needed for the calculation, but only to + // satisfy the type system. + const ruleResponseForImport = convertRuleToImportToRuleResponse(rule); + const ruleSource = calculateRuleSourceFromAsset({ + rule: ruleResponseForImport, + assetWithMatchingVersion, + isKnownPrebuiltRule, + }); + + return { + ruleSource, + immutable: ruleSource.type === 'external', + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts new file mode 100644 index 0000000000000..9a2f68479fdea --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { getPrebuiltRuleMock } from '../../../prebuilt_rules/mocks'; + +describe('calculateRuleSourceFromAsset', () => { + it('calculates as internal if no asset is found', () => { + const result = calculateRuleSourceFromAsset({ + rule: getRulesSchemaMock(), + assetWithMatchingVersion: undefined, + isKnownPrebuiltRule: false, + }); + + expect(result).toEqual({ + type: 'internal', + }); + }); + + it('calculates as customized external type if an asset is found matching rule_id but not version', () => { + const ruleToImport = getRulesSchemaMock(); + const result = calculateRuleSourceFromAsset({ + rule: ruleToImport, + assetWithMatchingVersion: undefined, + isKnownPrebuiltRule: true, + }); + + expect(result).toEqual({ + type: 'external', + is_customized: true, + }); + }); + + describe('matching rule_id and version is found', () => { + it('calculates as customized external type if the imported rule has all fields unchanged from the asset', () => { + const ruleToImport = getRulesSchemaMock(); + const result = calculateRuleSourceFromAsset({ + rule: getRulesSchemaMock(), // version 1 + assetWithMatchingVersion: getPrebuiltRuleMock({ + ...ruleToImport, + version: 1, // version 1 (same version as imported rule) + // no other overwrites -> no differences + }), + isKnownPrebuiltRule: true, + }); + + expect(result).toEqual({ + type: 'external', + is_customized: false, + }); + }); + + it('calculates as non-customized external type the imported rule has fields which differ from the asset', () => { + const ruleToImport = getRulesSchemaMock(); + const result = calculateRuleSourceFromAsset({ + rule: getRulesSchemaMock(), // version 1 + assetWithMatchingVersion: getPrebuiltRuleMock({ + ...ruleToImport, + version: 1, // version 1 (same version as imported rule) + name: 'Customized name', // mock a customization + }), + isKnownPrebuiltRule: true, + }); + + expect(result).toEqual({ + type: 'external', + is_customized: true, + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts new file mode 100644 index 0000000000000..4f0caf9b10056 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse, RuleSource } from '../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { calculateIsCustomized } from '../detection_rules_client/mergers/rule_source/calculate_is_customized'; + +/** + * Calculates rule_source for a rule based on two pieces of information: + * 1. The prebuilt rule asset that matches the specified rule_id and version + * 2. Whether a prebuilt rule with the specified rule_id is currently installed + * + * @param rule The rule for which rule_source is being calculated + * @param assetWithMatchingVersion The prebuilt rule asset that matches the specified rule_id and version + * @param isKnownPrebuiltRule Whether a prebuilt rule with the specified rule_id is currently installed + * + * @returns The calculated rule_source + */ +export const calculateRuleSourceFromAsset = ({ + rule, + assetWithMatchingVersion, + isKnownPrebuiltRule, +}: { + rule: RuleResponse; + assetWithMatchingVersion: PrebuiltRuleAsset | undefined; + isKnownPrebuiltRule: boolean; +}): RuleSource => { + if (!isKnownPrebuiltRule) { + return { + type: 'internal', + }; + } + + if (assetWithMatchingVersion == null) { + return { + type: 'external', + is_customized: true, + }; + } + + const isCustomized = calculateIsCustomized(assetWithMatchingVersion, rule); + + return { + type: 'external', + is_customized: isCustomized, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts index 2a249e7d9383a..b6f9c8959fb77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts @@ -63,11 +63,11 @@ describe('checkRuleExceptionReferences', () => { [ { error: { + ruleId: 'rule-1', message: 'Rule with rule_id: "rule-1" references a non existent exception list of list_id: "my-list". Reference has been removed.', - status_code: 400, + type: 'unknown', }, - rule_id: 'rule-1', }, ], [], @@ -94,11 +94,11 @@ describe('checkRuleExceptionReferences', () => { [ { error: { + ruleId: 'rule-1', message: 'Rule with rule_id: "rule-1" references a non existent exception list of list_id: "my-list". Reference has been removed.', - status_code: 400, + type: 'unknown', }, - rule_id: 'rule-1', }, ], [], @@ -127,9 +127,9 @@ describe('checkRuleExceptionReferences', () => { error: { message: 'Rule with rule_id: "rule-1" references a non existent exception list of list_id: "my-list". Reference has been removed.', - status_code: 400, + ruleId: 'rule-1', + type: 'unknown', }, - rule_id: 'rule-1', }, ], [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts index efa6026d875bf..2d89fbf956536 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts @@ -7,8 +7,7 @@ import type { ListArray, ExceptionListSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; -import type { BulkError } from '../../../routes/utils'; -import { createBulkErrorObject } from '../../../routes/utils'; +import { type RuleImportErrorObject, createRuleImportErrorObject } from './errors'; /** * Helper to check if all the exception lists referenced on a @@ -27,9 +26,9 @@ export const checkRuleExceptionReferences = ({ }: { rule: RuleToImport; existingLists: Record; -}): [BulkError[], ListArray] => { +}): [RuleImportErrorObject[], ListArray] => { let ruleExceptions: ListArray = []; - let errors: BulkError[] = []; + let errors: RuleImportErrorObject[] = []; const { rule_id: ruleId } = rule; const exceptionLists = rule.exceptions_list ?? []; @@ -54,9 +53,8 @@ export const checkRuleExceptionReferences = ({ // this error to notify a user of the action taken. errors = [ ...errors, - createBulkErrorObject({ + createRuleImportErrorObject({ ruleId, - statusCode: 400, message: `Rule with rule_id: "${ruleId}" references a non existent exception list of list_id: "${exceptionList.list_id}". Reference has been removed.`, }), ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.test.ts new file mode 100644 index 0000000000000..bd486764576de --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getImportRulesSchemaMock, + getValidatedRuleToImportMock, +} from '../../../../../../../common/api/detection_engine/rule_management/mocks'; +import { convertRuleToImportToRuleResponse } from './convert_rule_to_import_to_rule_response'; + +describe('convertRuleToImportToRuleResponse', () => { + it('converts a valid RuleToImport (without a language field) to valid RuleResponse (with a language field)', () => { + const ruleToImportWithoutLanguage = getImportRulesSchemaMock({ language: undefined }); + + expect(convertRuleToImportToRuleResponse(ruleToImportWithoutLanguage)).toMatchObject({ + language: 'kuery', + rule_id: ruleToImportWithoutLanguage.rule_id, + }); + }); + + it('converts a ValidatedRuleToImport and preserves its version', () => { + const ruleToImport = getValidatedRuleToImportMock({ version: 99 }); + + expect(convertRuleToImportToRuleResponse(ruleToImport)).toMatchObject({ + version: 99, + language: 'kuery', + rule_id: ruleToImport.rule_id, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.ts new file mode 100644 index 0000000000000..cbc8826c4b078 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidv4 } from 'uuid'; +import { applyRuleDefaults } from '../../detection_rules_client/mergers/apply_rule_defaults'; +import { RuleResponse, type RuleToImport } from '../../../../../../../common/api/detection_engine'; + +export const convertRuleToImportToRuleResponse = (ruleToImport: RuleToImport): RuleResponse => { + const ruleResponseSpecificFields = { + id: uuidv4(), + updated_at: new Date().toISOString(), + updated_by: '', + created_at: new Date().toISOString(), + created_by: '', + revision: 1, + }; + const ruleWithDefaults = applyRuleDefaults(ruleToImport); + + return RuleResponse.parse({ + ...ruleResponseSpecificFields, + ...ruleWithDefaults, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts new file mode 100644 index 0000000000000..cc5ffc7d48bd9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts @@ -0,0 +1,385 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Readable } from 'stream'; +import { BadRequestError } from '@kbn/securitysolution-es-utils'; + +import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; +import { + getOutputDetailsSample, + getSampleDetailsAsNdjson, +} from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import type { InvestigationFields } from '../../../../../../common/api/detection_engine'; +import { createPromiseFromRuleImportStream } from './create_promise_from_rule_import_stream'; + +export const getOutputSample = (): Partial => ({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', +}); + +export const getSampleAsNdjson = (sample: Partial): string => { + return `${JSON.stringify(sample)}\n`; +}; + +describe('createPromiseFromRuleImportStream', () => { + test('transforms an ndjson stream into a stream of rule objects', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); + + test('throws an error when the number of rules in the stream is equal to the limit', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + + await expect( + createPromiseFromRuleImportStream({ stream: ndJsonStream, objectLimit: 2 }) + ).rejects.toThrowError("Can't import more than 2 rules"); + }); + + test('throws an error when the number of rules in the stream is larger than the limit', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + + await expect( + createPromiseFromRuleImportStream({ stream: ndJsonStream, objectLimit: 1 }) + ).rejects.toThrowError("Can't import more than 1 rules"); + }); + + test('skips empty lines', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(''); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); + + test('filters the export details entry from the stream', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + const details = getOutputDetailsSample({ totalCount: 1, rulesCount: 1 }); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(getSampleDetailsAsNdjson(details)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); + + test('handles non parsable JSON strings and inserts the error as part of the return array', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('{,,,,\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + const resultOrError = result as Error[]; + expect(resultOrError[0]).toEqual({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + expect(resultOrError[1].message).toEqual(`Expected property name or '}' in JSON at position 1`); + expect(resultOrError[2]).toEqual({ + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + }); + + test('handles non-validated data', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(`{}\n`); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + const resultOrError = result as BadRequestError[]; + expect(resultOrError[0]).toEqual({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + expect(resultOrError[1].message).toContain( + `name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more` + ); + expect(resultOrError[2]).toEqual({ + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + }); + + test('non validated data is an instanceof BadRequestError', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(`{}\n`); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + const resultOrError = result as BadRequestError[]; + expect(resultOrError[1] instanceof BadRequestError).toEqual(true); + }); + + test('migrates investigation_fields', async () => { + const sample1 = { + ...getOutputSample(), + investigation_fields: ['foo', 'bar'] as unknown as InvestigationFields, + }; + const sample2 = { + ...getOutputSample(), + rule_id: 'rule-2', + investigation_fields: [] as unknown as InvestigationFields, + }; + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + investigation_fields: { + field_names: ['foo', 'bar'], + }, + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts new file mode 100644 index 0000000000000..d0e1019819ac2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Readable } from 'stream'; +import { createPromiseFromStreams } from '@kbn/utils'; +import type { SavedObject } from '@kbn/core/server'; +import type { + ImportExceptionsListSchema, + ImportExceptionListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; + +import { createRulesAndExceptionsStreamFromNdJson } from './create_rules_stream_from_ndjson'; +import type { RuleFromImportStream } from './utils'; + +export interface RuleImportStreamResult { + rules: RuleFromImportStream[]; + exceptions: Array; + actionConnectors: SavedObject[]; +} + +/** + * Utility for generating a promise from a Readable stream corresponding to an + * NDJSON file. Used during rule import. + */ +export const createPromiseFromRuleImportStream = ({ + objectLimit, + stream, +}: { + objectLimit: number; + stream: Readable; +}): Promise => { + const readAllStream = createRulesAndExceptionsStreamFromNdJson(objectLimit); + + return createPromiseFromStreams([stream, ...readAllStream]); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts deleted file mode 100644 index 5e37f161c3dde..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Readable } from 'stream'; -import { createPromiseFromStreams } from '@kbn/utils'; -import { createRulesAndExceptionsStreamFromNdJson } from './create_rules_stream_from_ndjson'; -import { BadRequestError } from '@kbn/securitysolution-es-utils'; - -import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; -import { - getOutputDetailsSample, - getSampleDetailsAsNdjson, -} from '../../../../../../common/api/detection_engine/rule_management/mocks'; -import type { RuleExceptionsPromiseFromStreams } from './import_rules_utils'; -import type { InvestigationFields } from '../../../../../../common/api/detection_engine'; - -export const getOutputSample = (): Partial => ({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', -}); - -export const getSampleAsNdjson = (sample: Partial): string => { - return `${JSON.stringify(sample)}\n`; -}; - -describe('create_rules_stream_from_ndjson', () => { - describe('createRulesAndExceptionsStreamFromNdJson', () => { - test('transforms an ndjson stream into a stream of rule objects', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }, - ]); - }); - - // TODO - Yara - there's a integration test testing this, but causing timeoutes here - test.skip('returns error when ndjson stream is larger than limit', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(2); - await expect( - createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]) - ).rejects.toThrowError("Can't import more than 1 rules"); - }); - - test('skips empty lines', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push('\n'); - this.push(getSampleAsNdjson(sample2)); - this.push(''); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }, - ]); - }); - - test('filters the export details entry from the stream', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - const details = getOutputDetailsSample({ totalCount: 1, rulesCount: 1 }); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push(getSampleDetailsAsNdjson(details)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }, - ]); - }); - - test('handles non parsable JSON strings and inserts the error as part of the return array', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push('{,,,,\n'); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - const resultOrError = result as Error[]; - expect(resultOrError[0]).toEqual({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }); - expect(resultOrError[1].message).toEqual( - `Expected property name or '}' in JSON at position 1` - ); - expect(resultOrError[2]).toEqual({ - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }); - }); - - test('handles non-validated data', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(`{}\n`); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - const resultOrError = result as BadRequestError[]; - expect(resultOrError[0]).toEqual({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }); - expect(resultOrError[1].message).toContain( - `name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more` - ); - expect(resultOrError[2]).toEqual({ - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }); - }); - - test('non validated data is an instanceof BadRequestError', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(`{}\n`); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - const resultOrError = result as BadRequestError[]; - expect(resultOrError[1] instanceof BadRequestError).toEqual(true); - }); - - test('migrates investigation_fields', async () => { - const sample1 = { - ...getOutputSample(), - investigation_fields: ['foo', 'bar'] as unknown as InvestigationFields, - }; - const sample2 = { - ...getOutputSample(), - rule_id: 'rule-2', - investigation_fields: [] as unknown as InvestigationFields, - }; - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - investigation_fields: { - field_names: ['foo', 'bar'], - }, - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - immutable: false, - }, - ]); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts new file mode 100644 index 0000000000000..77945aa8a3fa5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { has } from 'lodash'; + +export type RuleImportErrorType = 'conflict' | 'unknown'; + +/** + * Generic interface representing a server-side failure during rule import. + * Used by utilities that import rules or related entities. + * + * NOTE that this does not inherit from Error + */ +export interface RuleImportErrorObject { + error: { + ruleId: string; + message: string; + type: RuleImportErrorType; + }; +} + +export const createRuleImportErrorObject = ({ + ruleId, + message, + type, +}: { + ruleId: string; + message: string; + type?: RuleImportErrorType; +}): RuleImportErrorObject => ({ + error: { + ruleId, + message, + type: type ?? 'unknown', + }, +}); + +export const isRuleImportError = (obj: unknown): obj is RuleImportErrorObject => + has(obj, 'error') && + has(obj, 'error.ruleId') && + has(obj, 'error.type') && + has(obj, 'error.message'); + +export const isRuleConflictError = (error: RuleImportErrorObject): boolean => + error.error.type === 'conflict'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts new file mode 100644 index 0000000000000..a6e4e8bad297d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; + +import { importRules } from './import_rules'; +import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; +import { detectionRulesClientMock } from '../detection_rules_client/__mocks__/detection_rules_client'; +import { ruleSourceImporterMock } from './rule_source_importer/rule_source_importer.mock'; +import { createRuleImportErrorObject } from './errors'; + +describe('importRules', () => { + let ruleToImport: ReturnType; + + let detectionRulesClient: jest.Mocked; + let mockRuleSourceImporter: ReturnType; + + beforeEach(() => { + jest.clearAllMocks(); + + detectionRulesClient = detectionRulesClientMock.create(); + detectionRulesClient.importRules.mockResolvedValue([]); + ruleToImport = getImportRulesSchemaMock(); + mockRuleSourceImporter = ruleSourceImporterMock.create(); + }); + + it('returns an empty rules response if no rules to import', async () => { + const result = await importRules({ + ruleChunks: [], + overwriteRules: false, + detectionRulesClient, + ruleSourceImporter: mockRuleSourceImporter, + }); + + expect(result).toEqual([]); + }); + + it('returns 400 errors if client import returns generic errors', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import error', + }), + createRuleImportErrorObject({ + ruleId: 'rule-id-2', + message: 'import error', + }), + ]); + + const result = await importRules({ + ruleChunks: [[ruleToImport]], + overwriteRules: false, + detectionRulesClient, + ruleSourceImporter: mockRuleSourceImporter, + }); + + expect(result).toEqual([ + { + error: { + message: 'import error', + status_code: 400, + }, + rule_id: 'rule-id', + }, + { + error: { + message: 'import error', + status_code: 400, + }, + rule_id: 'rule-id-2', + }, + ]); + }); + + it('returns multiple errors for the same rule if client import returns generic errors', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import error', + }), + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import error 2', + }), + ]); + + const result = await importRules({ + ruleChunks: [[ruleToImport]], + overwriteRules: false, + detectionRulesClient, + ruleSourceImporter: mockRuleSourceImporter, + }); + + expect(result).toEqual([ + { + error: { + message: 'import error', + status_code: 400, + }, + rule_id: 'rule-id', + }, + { + error: { + message: 'import error 2', + status_code: 400, + }, + rule_id: 'rule-id', + }, + ]); + }); + + it('returns 409 errors if client import returns conflict errors', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import conflict', + type: 'conflict', + }), + createRuleImportErrorObject({ + ruleId: 'rule-id-2', + message: 'import conflict', + type: 'conflict', + }), + ]); + + const result = await importRules({ + ruleChunks: [[ruleToImport]], + overwriteRules: false, + detectionRulesClient, + ruleSourceImporter: mockRuleSourceImporter, + }); + + expect(result).toEqual([ + { + error: { + message: 'import conflict', + status_code: 409, + }, + rule_id: 'rule-id', + }, + { + error: { + message: 'import conflict', + status_code: 409, + }, + rule_id: 'rule-id-2', + }, + ]); + }); + + it('returns a combination of 200s and 4xxs if some rules were imported and some errored', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'parse error', + }), + getRulesSchemaMock(), + createRuleImportErrorObject({ + ruleId: 'rule-id-2', + message: 'import conflict', + type: 'conflict', + }), + getRulesSchemaMock(), + ]); + const successfulRuleId = getRulesSchemaMock().rule_id; + + const result = await importRules({ + ruleChunks: [[ruleToImport]], + overwriteRules: false, + detectionRulesClient, + ruleSourceImporter: mockRuleSourceImporter, + }); + + expect(result).toEqual([ + { + error: { + message: 'parse error', + status_code: 400, + }, + rule_id: 'rule-id', + }, + { rule_id: successfulRuleId, status_code: 200 }, + { + error: { + message: 'import conflict', + status_code: 409, + }, + rule_id: 'rule-id-2', + }, + { rule_id: successfulRuleId, status_code: 200 }, + ]); + }); + + it('returns 200s if all rules were imported successfully', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + getRulesSchemaMock(), + getRulesSchemaMock(), + ]); + const successfulRuleId = getRulesSchemaMock().rule_id; + + const result = await importRules({ + ruleChunks: [[ruleToImport]], + overwriteRules: false, + detectionRulesClient, + ruleSourceImporter: mockRuleSourceImporter, + }); + + expect(result).toEqual([ + { rule_id: successfulRuleId, status_code: 200 }, + { rule_id: successfulRuleId, status_code: 200 }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts new file mode 100644 index 0000000000000..012c64c0ad8a4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleToImport } from '../../../../../../common/api/detection_engine'; +import { type ImportRuleResponse, createBulkErrorObject } from '../../../routes/utils'; +import type { IRuleSourceImporter } from './rule_source_importer'; +import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; +import { isRuleConflictError, isRuleImportError } from './errors'; + +/** + * Takes a stream of rules to be imported and either creates or updates rules + * based on user overwrite preferences + * @param ruleChunks {@link RuleToImport} - rules being imported + * @param overwriteRules {boolean} - whether to overwrite existing rules + * with imported rules if their rule_id matches + * @param detectionRulesClient {object} + * @returns {Promise} an array of error and success messages from import + */ +export const importRules = async ({ + ruleChunks, + overwriteRules, + detectionRulesClient, + ruleSourceImporter, + allowMissingConnectorSecrets, +}: { + ruleChunks: RuleToImport[][]; + overwriteRules: boolean; + detectionRulesClient: IDetectionRulesClient; + ruleSourceImporter: IRuleSourceImporter; + allowMissingConnectorSecrets?: boolean; +}): Promise => { + const response: ImportRuleResponse[] = []; + + if (ruleChunks.length === 0) { + return response; + } + + for (const rules of ruleChunks) { + const importedRulesResponse = await detectionRulesClient.importRules({ + allowMissingConnectorSecrets, + overwriteRules, + ruleSourceImporter, + rules, + }); + + const importResponses = importedRulesResponse.map((rule) => { + if (isRuleImportError(rule)) { + return createBulkErrorObject({ + message: rule.error.message, + statusCode: isRuleConflictError(rule) ? 409 : 400, + ruleId: rule.error.ruleId, + }); + } + + return { + rule_id: rule.rule_id, + status_code: 200, + }; + }); + + response.push(...importResponses); + } + + return response; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.test.ts similarity index 74% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.test.ts index 9af3dc37a140e..0d8fe8118471b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.test.ts @@ -11,10 +11,12 @@ import { getImportRulesSchemaMock } from '../../../../../../common/api/detection import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import { requestContextMock } from '../../../routes/__mocks__'; -import { importRules } from './import_rules_utils'; -import { createBulkErrorObject } from '../../../routes/utils'; +import { createRuleImportErrorObject } from './errors'; -describe('importRules', () => { +// eslint-disable-next-line no-restricted-imports +import { importRulesLegacy } from './import_rules_legacy'; + +describe('importRulesLegacy', () => { const { clients, context } = requestContextMock.createTools(); const ruleToImport = getImportRulesSchemaMock(); @@ -27,9 +29,8 @@ describe('importRules', () => { }); it('returns an empty rules response if no rules to import', async () => { - const result = await importRules({ + const result = await importRulesLegacy({ ruleChunks: [], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, @@ -38,39 +39,18 @@ describe('importRules', () => { expect(result).toEqual([]); }); - it('returns 400 error if "ruleChunks" includes Error', async () => { - const result = await importRules({ - ruleChunks: [[new Error('error importing')]], - rulesResponseAcc: [], - overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - savedObjectsClient, - }); - - expect(result).toEqual([ - { - error: { - message: 'error importing', - status_code: 400, - }, - rule_id: '(unknown id)', - }, - ]); - }); - it('returns 409 error if DetectionRulesClient throws with 409 - existing rule', async () => { clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { - throw createBulkErrorObject({ + throw createRuleImportErrorObject({ ruleId: ruleToImport.rule_id, - statusCode: 409, + type: 'conflict', message: `rule_id: "${ruleToImport.rule_id}" already exists`, }); }); const ruleChunk = [ruleToImport]; - const result = await importRules({ + const result = await importRulesLegacy({ ruleChunks: [ruleChunk], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, @@ -94,9 +74,8 @@ describe('importRules', () => { }); const ruleChunk = [ruleToImport]; - const result = await importRules({ + const result = await importRulesLegacy({ ruleChunks: [ruleChunk], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, @@ -104,4 +83,28 @@ describe('importRules', () => { expect(result).toEqual([{ rule_id: ruleToImport.rule_id, status_code: 200 }]); }); + + it('rejects a prebuilt rule specifying an immutable value of true', async () => { + const prebuiltRuleToImport = { + ...getImportRulesSchemaMock(), + immutable: true, + version: 1, + }; + const result = await importRulesLegacy({ + ruleChunks: [[prebuiltRuleToImport]], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + savedObjectsClient, + }); + + expect(result).toEqual([ + { + error: { + message: `Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: ${prebuiltRuleToImport.rule_id}]`, + status_code: 400, + }, + rule_id: prebuiltRuleToImport.rule_id, + }, + ]); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts similarity index 63% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts index 3adb381c8ecce..384683ce1916e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts @@ -5,61 +5,45 @@ * 2.0. */ -import type { SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; -import type { - ImportExceptionsListSchema, - ImportExceptionListItemSchema, -} from '@kbn/securitysolution-io-ts-list-types'; +import { i18n } from '@kbn/i18n'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; +import type { RuleToImport } from '../../../../../../common/api/detection_engine'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { getReferencedExceptionLists } from './gather_referenced_exceptions'; - -export type PromiseFromStreams = RuleToImport | Error; -export interface RuleExceptionsPromiseFromStreams { - rules: PromiseFromStreams[]; - exceptions: Array; - actionConnectors: SavedObject[]; -} +import { isRuleConflictError, isRuleImportError } from './errors'; /** * Takes rules to be imported and either creates or updates rules - * based on user overwrite preferences - * @param ruleChunks {array} - rules being imported - * @param rulesResponseAcc {array} - the accumulation of success and - * error messages gathered through the rules import logic - * @param mlAuthz {object} + * based on user overwrite preferences. + * + * @deprecated Use {@link importRules} instead. + * @param ruleChunks {@link RuleToImport} - rules being imported * @param overwriteRules {boolean} - whether to overwrite existing rules * with imported rules if their rule_id matches * @param detectionRulesClient {object} - * @param existingLists {object} - all exception lists referenced by - * rules that were found to exist * @returns {Promise} an array of error and success messages from import */ -export const importRules = async ({ +export const importRulesLegacy = async ({ ruleChunks, - rulesResponseAcc, overwriteRules, detectionRulesClient, allowMissingConnectorSecrets, savedObjectsClient, }: { - ruleChunks: PromiseFromStreams[][]; - rulesResponseAcc: ImportRuleResponse[]; + ruleChunks: RuleToImport[][]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; -}) => { - let importRuleResponse: ImportRuleResponse[] = [...rulesResponseAcc]; +}): Promise => { + const response: ImportRuleResponse[] = []; - // If we had 100% errors and no successful rule could be imported we still have to output an error. - // otherwise we would output we are success importing 0 rules. if (ruleChunks.length === 0) { - return importRuleResponse; + return response; } while (ruleChunks.length) { @@ -72,15 +56,22 @@ export const importRules = async ({ batchParseObjects.reduce>>((accum, parsedRule) => { const importsWorkerPromise = new Promise(async (resolve, reject) => { try { - if (parsedRule instanceof Error) { - // If the JSON object had a validation or parse error then we return - // early with the error and an (unknown) for the ruleId + if (parsedRule.immutable) { resolve( createBulkErrorObject({ statusCode: 400, - message: parsedRule.message, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.importPrebuiltRulesUnsupported', + { + defaultMessage: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: {ruleId}]', + values: { ruleId: parsedRule.rule_id }, + } + ), + ruleId: parsedRule.rule_id, }) ); + return null; } @@ -90,7 +81,15 @@ export const importRules = async ({ existingLists, }); - importRuleResponse = [...importRuleResponse, ...exceptionErrors]; + const exceptionBulkErrors = exceptionErrors.map((error) => + createBulkErrorObject({ + ruleId: error.error.ruleId, + statusCode: 400, + message: error.error.message, + }) + ); + + response.push(...exceptionBulkErrors); const importedRule = await detectionRulesClient.importRule({ ruleToImport: { @@ -107,6 +106,17 @@ export const importRules = async ({ }); } catch (err) { const { error, statusCode, message } = err; + if (isRuleImportError(err)) { + resolve( + createBulkErrorObject({ + message: err.error.message, + statusCode: isRuleConflictError(err) ? 409 : 400, + ruleId: err.error.ruleId, + }) + ); + return null; + } + resolve( createBulkErrorObject({ ruleId: parsedRule.rule_id, @@ -122,8 +132,8 @@ export const importRules = async ({ return [...accum, importsWorkerPromise]; }, []) ); - importRuleResponse = [...importRuleResponse, ...newImportRuleResponse]; + response.push(...newImportRuleResponse); } - return importRuleResponse; + return response; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts new file mode 100644 index 0000000000000..616c3d28eae95 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './rule_source_importer_interface'; +export * from './rule_source_importer'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.mock.ts new file mode 100644 index 0000000000000..983043e904c60 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.mock.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSourceImporter } from './rule_source_importer'; + +const createRuleSourceImporterMock = (): jest.Mocked => + ({ + setup: jest.fn(), + calculateRuleSource: jest.fn(), + isPrebuiltRule: jest.fn(), + } as unknown as jest.Mocked); + +export const ruleSourceImporterMock = { + create: createRuleSourceImporterMock, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts new file mode 100644 index 0000000000000..39c937f4645a7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleToImport, + ValidatedRuleToImport, +} from '../../../../../../../common/api/detection_engine'; +import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from '../../../../prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client'; +import { configMock, createMockConfig, requestContextMock } from '../../../../routes/__mocks__'; +import { getPrebuiltRuleMock } from '../../../../prebuilt_rules/mocks'; +import { createRuleSourceImporter } from './rule_source_importer'; +import * as calculateRuleSourceModule from '../calculate_rule_source_for_import'; + +describe('ruleSourceImporter', () => { + let ruleAssetsClientMock: ReturnType; + let config: ReturnType; + let context: ReturnType['securitySolution']; + let ruleToImport: RuleToImport; + let subject: ReturnType; + + beforeEach(() => { + jest.clearAllMocks(); + config = createMockConfig(); + config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); + context = requestContextMock.create().securitySolution; + ruleAssetsClientMock = createPrebuiltRuleAssetsClientMock(); + ruleAssetsClientMock.fetchLatestAssets.mockResolvedValue([{}]); + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([]); + ruleAssetsClientMock.fetchAssetsByVersion.mockResolvedValue([]); + ruleToImport = { rule_id: 'rule-1', version: 1 } as RuleToImport; + + subject = createRuleSourceImporter({ + context, + config, + prebuiltRuleAssetsClient: ruleAssetsClientMock, + }); + }); + + it('should initialize correctly', () => { + expect(subject).toBeDefined(); + + expect(() => subject.isPrebuiltRule(ruleToImport)).toThrowErrorMatchingInlineSnapshot( + `"Rule rule-1 was not registered during setup."` + ); + }); + + describe('#setup()', () => { + it('fetches the rules package on the initial call', async () => { + await subject.setup([]); + + expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); + }); + + it('does not fetch the rules package on subsequent calls', async () => { + await subject.setup([]); + await subject.setup([]); + await subject.setup([]); + + expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); + }); + + it('throws an error if the ruleAsstClient does', async () => { + ruleAssetsClientMock.fetchLatestAssets.mockReset().mockRejectedValue(new Error('failed')); + + await expect(() => subject.setup([])).rejects.toThrowErrorMatchingInlineSnapshot(`"failed"`); + }); + }); + + describe('#isPrebuiltRule()', () => { + beforeEach(() => { + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); + }); + + it("returns false if the rule's rule_id doesn't match an available rule asset", async () => { + ruleAssetsClientMock.fetchLatestVersions.mockReset().mockResolvedValue([]); + await subject.setup([ruleToImport]); + + expect(subject.isPrebuiltRule(ruleToImport)).toBe(false); + }); + + it("returns true if the rule's rule_id matches an available rule asset", async () => { + await subject.setup([ruleToImport]); + + expect(subject.isPrebuiltRule(ruleToImport)).toBe(true); + }); + + it('returns true if the rule has no version, but its rule_id matches an available rule asset', async () => { + const ruleWithoutVersion = { ...ruleToImport, version: undefined }; + await subject.setup([ruleWithoutVersion]); + + expect(subject.isPrebuiltRule(ruleWithoutVersion)).toBe(true); + }); + + it('throws an error if the rule is not known to the calculator', async () => { + await subject.setup([ruleToImport]); + + expect(() => + subject.isPrebuiltRule({ rule_id: 'other-rule' } as RuleToImport) + ).toThrowErrorMatchingInlineSnapshot(`"Rule other-rule was not registered during setup."`); + }); + + it('throws an error if the calculator is not set up', () => { + expect(() => subject.isPrebuiltRule(ruleToImport)).toThrowErrorMatchingInlineSnapshot( + `"Rule rule-1 was not registered during setup."` + ); + }); + }); + + describe('#calculateRuleSource()', () => { + let rule: ValidatedRuleToImport; + let calculatorSpy: jest.SpyInstance; + + beforeEach(() => { + rule = { rule_id: 'validated-rule', version: 1 } as ValidatedRuleToImport; + ruleAssetsClientMock.fetchAssetsByVersion.mockResolvedValue([ + getPrebuiltRuleMock({ rule_id: 'rule-1' }), + ]); + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ + getPrebuiltRuleMock({ rule_id: 'rule-1' }), + getPrebuiltRuleMock({ rule_id: 'rule-2' }), + getPrebuiltRuleMock({ rule_id: 'validated-rule' }), + ]); + calculatorSpy = jest + .spyOn(calculateRuleSourceModule, 'calculateRuleSourceForImport') + .mockReturnValue({ ruleSource: { type: 'internal' }, immutable: false }); + }); + + it('invokes calculateRuleSourceForImport with the correct arguments', async () => { + await subject.setup([rule]); + await subject.calculateRuleSource(rule); + + expect(calculatorSpy).toHaveBeenCalledTimes(1); + expect(calculatorSpy).toHaveBeenCalledWith({ + rule, + prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, + isKnownPrebuiltRule: true, + }); + }); + + it('throws an error if the rule is not known to the calculator', async () => { + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); + await subject.setup([ruleToImport]); + + expect(() => subject.calculateRuleSource(rule)).toThrowErrorMatchingInlineSnapshot( + `"Rule validated-rule was not registered during setup."` + ); + }); + + it('throws an error if the calculator is not set up', async () => { + expect(() => subject.calculateRuleSource(rule)).toThrowErrorMatchingInlineSnapshot( + `"Rule validated-rule was not registered during setup."` + ); + }); + + describe('for rules set up without a version', () => { + it('invokes the calculator with the correct arguments', async () => { + await subject.setup([{ ...rule, version: undefined }]); + await subject.calculateRuleSource(rule); + + expect(calculatorSpy).toHaveBeenCalledTimes(1); + expect(calculatorSpy).toHaveBeenCalledWith({ + rule, + prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, + isKnownPrebuiltRule: true, + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts new file mode 100644 index 0000000000000..1f5c2c5aa543b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SecuritySolutionApiRequestHandlerContext } from '../../../../../../types'; +import type { ConfigType } from '../../../../../../config'; +import type { + RuleToImport, + ValidatedRuleToImport, +} from '../../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; +import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; +import { ensureLatestRulesPackageInstalled } from '../../../../prebuilt_rules/logic/ensure_latest_rules_package_installed'; +import { calculateRuleSourceForImport } from '../calculate_rule_source_for_import'; +import type { CalculatedRuleSource, IRuleSourceImporter } from './rule_source_importer_interface'; + +interface RuleSpecifier { + rule_id: string; + version: number | undefined; +} + +/** + * Retrieves the rule IDs (`rule_id`s) of available prebuilt rule assets matching those + * of the specified rules. This information can be used to determine whether + * the rule being imported is a custom rule or a prebuilt rule. + * + * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. + * @param ruleAssetsClient - the {@link IPrebuiltRuleAssetsClient} to use for fetching the available rule assets. + * + * @returns A list of the prebuilt rule asset IDs that are available. + * + */ +const fetchAvailableRuleAssetIds = async ({ + rules, + ruleAssetsClient, +}: { + rules: RuleSpecifier[]; + ruleAssetsClient: IPrebuiltRuleAssetsClient; +}): Promise => { + const incomingRuleIds = rules.map((rule) => rule.rule_id); + const availableRuleAssetSpecifiers = await ruleAssetsClient.fetchLatestVersions(incomingRuleIds); + + return availableRuleAssetSpecifiers.map((specifier) => specifier.rule_id); +}; + +/** + * Retrieves prebuilt rule assets for rules being imported. These + * assets can be compared to the incoming rules for the purposes of calculating + * appropriate `rule_source` values. + * + * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. + * + * @returns The prebuilt rule assets matching the specified prebuilt + * rules. Assets match the `rule_id` and `version` of the specified rules. + * Because of this, there may be less assets returned than specified rules. + */ +const fetchMatchingAssets = async ({ + rules, + ruleAssetsClient, +}: { + rules: RuleSpecifier[]; + ruleAssetsClient: IPrebuiltRuleAssetsClient; +}): Promise => { + const incomingRuleVersions = rules.flatMap((rule) => { + if (rule.version == null) { + return []; + } + return { + rule_id: rule.rule_id, + version: rule.version, + }; + }); + + return ruleAssetsClient.fetchAssetsByVersion(incomingRuleVersions); +}; + +/** + * + * This class contains utilities for assisting with the calculation of + * `rule_source` during import. It ensures that the system contains the + * necessary assets, and provides utilities for fetching information from them, + * necessary for said calculation. + */ +export class RuleSourceImporter implements IRuleSourceImporter { + private context: SecuritySolutionApiRequestHandlerContext; + private config: ConfigType; + private ruleAssetsClient: IPrebuiltRuleAssetsClient; + private latestPackagesInstalled: boolean = false; + private matchingAssetsByRuleId: Record = {}; + private knownRules: RuleSpecifier[] = []; + private availableRuleAssetIds: Set = new Set(); + + constructor({ + config, + context, + prebuiltRuleAssetsClient, + }: { + config: ConfigType; + context: SecuritySolutionApiRequestHandlerContext; + prebuiltRuleAssetsClient: IPrebuiltRuleAssetsClient; + }) { + this.ruleAssetsClient = prebuiltRuleAssetsClient; + this.context = context; + this.config = config; + } + + /** + * + * Prepares the importing of rules by ensuring the latest rules + * package is installed and fetching the associated prebuilt rule assets. + */ + public async setup(rules: RuleToImport[]): Promise { + if (!this.latestPackagesInstalled) { + await ensureLatestRulesPackageInstalled(this.ruleAssetsClient, this.config, this.context); + this.latestPackagesInstalled = true; + } + + this.knownRules = rules.map((rule) => ({ rule_id: rule.rule_id, version: rule.version })); + this.matchingAssetsByRuleId = await this.fetchMatchingAssetsByRuleId(); + this.availableRuleAssetIds = new Set(await this.fetchAvailableRuleAssetIds()); + } + + public isPrebuiltRule(rule: RuleToImport): boolean { + this.validateRuleInput(rule); + + return this.availableRuleAssetIds.has(rule.rule_id); + } + + public calculateRuleSource(rule: ValidatedRuleToImport): CalculatedRuleSource { + this.validateRuleInput(rule); + + return calculateRuleSourceForImport({ + rule, + prebuiltRuleAssetsByRuleId: this.matchingAssetsByRuleId, + isKnownPrebuiltRule: this.availableRuleAssetIds.has(rule.rule_id), + }); + } + + private async fetchMatchingAssetsByRuleId(): Promise> { + this.validateSetupState(); + const matchingAssets = await fetchMatchingAssets({ + rules: this.knownRules, + ruleAssetsClient: this.ruleAssetsClient, + }); + + return matchingAssets.reduce>((map, asset) => { + map[asset.rule_id] = asset; + return map; + }, {}); + } + + private async fetchAvailableRuleAssetIds(): Promise { + this.validateSetupState(); + + return fetchAvailableRuleAssetIds({ + rules: this.knownRules, + ruleAssetsClient: this.ruleAssetsClient, + }); + } + + /** + * Runtime sanity checks to ensure no one's calling this stateful instance in the wrong way. + * */ + private validateSetupState() { + if (!this.latestPackagesInstalled) { + throw new Error('Expected rules package to be installed'); + } + } + + private validateRuleInput(rule: RuleToImport) { + if ( + !this.knownRules.some( + (knownRule) => + knownRule.rule_id === rule.rule_id && + (knownRule.version === rule.version || knownRule.version == null) + ) + ) { + throw new Error(`Rule ${rule.rule_id} was not registered during setup.`); + } + } +} + +export const createRuleSourceImporter = ({ + config, + context, + prebuiltRuleAssetsClient, +}: { + config: ConfigType; + context: SecuritySolutionApiRequestHandlerContext; + prebuiltRuleAssetsClient: IPrebuiltRuleAssetsClient; +}): RuleSourceImporter => { + return new RuleSourceImporter({ config, context, prebuiltRuleAssetsClient }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts new file mode 100644 index 0000000000000..ea19672863eeb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleSource, + RuleToImport, + ValidatedRuleToImport, +} from '../../../../../../../common/api/detection_engine'; + +export interface CalculatedRuleSource { + ruleSource: RuleSource; + immutable: boolean; +} + +export interface IRuleSourceImporter { + setup: (rules: RuleToImport[]) => Promise; + isPrebuiltRule: (rule: RuleToImport) => boolean; + calculateRuleSource: (rule: ValidatedRuleToImport) => CalculatedRuleSource; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/utils.ts new file mode 100644 index 0000000000000..b25efe7d9390d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/utils.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; + +export type RuleFromImportStream = RuleToImport | Error; + +export const isRuleToImport = (rule: RuleFromImportStream): rule is RuleToImport => + !(rule instanceof Error); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts index 53e96dc627080..01cd024b86b0c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts @@ -7,7 +7,6 @@ import { partition } from 'lodash/fp'; import { Readable } from 'stream'; -import { createPromiseFromStreams } from '@kbn/utils'; import type { RuleAction, ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types'; import type { PartialRule } from '@kbn/alerting-plugin/server'; import type { ActionsClient } from '@kbn/actions-plugin/server'; @@ -37,8 +36,7 @@ import { createBulkErrorObject } from '../../routes/utils'; import type { RuleAlertType } from '../../rule_schema'; import { getMlRuleParams, getQueryRuleParams, getThreatRuleParams } from '../../rule_schema/mocks'; -import { createRulesAndExceptionsStreamFromNdJson } from '../logic/import/create_rules_stream_from_ndjson'; -import type { RuleExceptionsPromiseFromStreams } from '../logic/import/import_rules_utils'; +import { createPromiseFromRuleImportStream } from '../logic/import/create_promise_from_rule_import_stream'; import { internalRuleToAPIResponse } from '../logic/detection_rules_client/converters/internal_rule_to_api_response'; type PromiseFromStreams = RuleToImport | Error; @@ -50,10 +48,10 @@ const createMockImportRule = async (rule: ReturnType([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); return rules; }; @@ -433,10 +431,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); const isInstanceOfError = output[0] instanceof Error; @@ -454,10 +452,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); @@ -485,10 +483,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); const isInstanceOfError = output[0] instanceof Error; @@ -507,10 +505,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, true); @@ -528,10 +526,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); const isInstanceOfError = output[0] instanceof Error; @@ -823,10 +821,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([]); const [errors, output] = await getInvalidConnectors(rules, clients.actionsClient); @@ -854,10 +852,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([]); const [errors, output] = await getInvalidConnectors(rules, clients.actionsClient); expect(output.length).toEqual(0); @@ -890,10 +888,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -935,10 +933,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -995,10 +993,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -1062,10 +1060,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -1131,10 +1129,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -1241,10 +1239,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', diff --git a/x-pack/plugins/security_solution/server/utils/object_case_converters.ts b/x-pack/plugins/security_solution/server/utils/object_case_converters.ts index ebe9158fe9c99..6a32491767bb5 100644 --- a/x-pack/plugins/security_solution/server/utils/object_case_converters.ts +++ b/x-pack/plugins/security_solution/server/utils/object_case_converters.ts @@ -8,20 +8,10 @@ import camelcaseKeys from 'camelcase-keys'; import snakecaseKeys from 'snakecase-keys'; import type { CamelCasedPropertiesDeep, SnakeCasedPropertiesDeep } from 'type-fest'; -export const convertObjectKeysToCamelCase = >( - obj: T | undefined -) => { - if (obj == null) { - return obj; - } - return camelcaseKeys(obj, { deep: true }) as unknown as CamelCasedPropertiesDeep; +export const convertObjectKeysToCamelCase = >(obj: T) => { + return camelcaseKeys(obj, { deep: true }) as CamelCasedPropertiesDeep; }; -export const convertObjectKeysToSnakeCase = >( - obj: T | undefined -) => { - if (obj == null) { - return obj; - } +export const convertObjectKeysToSnakeCase = >(obj: T) => { return snakecaseKeys(obj, { deep: true }) as SnakeCasedPropertiesDeep; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts new file mode 100644 index 0000000000000..934ee6460a5e2 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; + +import { + SAMPLE_PREBUILT_RULES_WITH_HISTORICAL_VERSIONS, + combineArrayToNdJson, + createHistoricalPrebuiltRuleAssetSavedObjects, + deleteAllPrebuiltRuleAssets, + fetchRule, + getCustomQueryRuleParams, + getInstalledRules, +} from '../../../../utils'; +import { deleteAllRules } from '../../../../../../../common/utils/security_solution'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + const es = getService('es'); + const log = getService('log'); + const securitySolutionApi = getService('securitySolutionApi'); + + const importRules = async (rules: unknown[]) => { + const buffer = Buffer.from(combineArrayToNdJson(rules)); + + return securitySolutionApi + .importRules({ query: {} }) + .attach('file', buffer, 'rules.ndjson') + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + }; + + const prebuiltRules = SAMPLE_PREBUILT_RULES_WITH_HISTORICAL_VERSIONS.map( + (prebuiltRule) => prebuiltRule['security-rule'] + ); + const prebuiltRuleIds = [...new Set(prebuiltRules.map((rule) => rule.rule_id))]; + + describe('@ess @serverless @skipInServerlessMKI import_rules', () => { + before(async () => { + await deleteAllPrebuiltRuleAssets(es, log); + await createHistoricalPrebuiltRuleAssetSavedObjects( + es, + SAMPLE_PREBUILT_RULES_WITH_HISTORICAL_VERSIONS + ); + }); + + after(async () => { + await deleteAllPrebuiltRuleAssets(es, log); + await deleteAllRules(supertest, log); + }); + + beforeEach(async () => { + await deleteAllRules(supertest, log); + }); + + describe('calculation of rule customization fields', () => { + it('defaults a versionless custom rule to "version: 1"', async () => { + const rule = getCustomQueryRuleParams({ rule_id: 'custom-rule', version: undefined }); + const { body } = await importRules([rule]); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); + expect(importedRule).toMatchObject({ + rule_id: rule.rule_id, + version: 1, + rule_source: { type: 'internal' }, + immutable: false, + }); + }); + + it('preserves a custom rule with a specified version', async () => { + const rule = getCustomQueryRuleParams({ rule_id: 'custom-rule', version: 23 }); + const { body } = await importRules([rule]); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); + expect(importedRule).toMatchObject({ + rule_id: rule.rule_id, + version: 23, + rule_source: { type: 'internal' }, + immutable: false, + }); + }); + + it('rejects a versionless prebuilt rule', async () => { + const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: undefined }); + const { body } = await importRules([rule]); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { + message: `Prebuilt rules must specify a "version" to be imported. [rule_id: ${prebuiltRuleIds[0]}]`, + status_code: 400, + }, + }); + }); + + it('respects the version of a prebuilt rule', async () => { + const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[1], version: 9999 }); + const { body } = await importRules([rule]); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); + expect(importedRule).toMatchObject({ + rule_id: rule.rule_id, + version: 9999, + rule_source: { type: 'external', is_customized: true }, + immutable: true, + }); + }); + + it('imports a combination of prebuilt and custom rules', async () => { + const rules = [ + getCustomQueryRuleParams({ rule_id: 'custom-rule', version: 23 }), + getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: 1234 }), + getCustomQueryRuleParams({ rule_id: 'custom-rule-2', version: undefined }), + prebuiltRules[3], + ]; + const { body } = await importRules(rules); + + expect(body).toMatchObject({ + rules_count: 4, + success: true, + success_count: 4, + errors: [], + }); + + const { data: importedRules } = await getInstalledRules(supertest); + + expect(importedRules).toHaveLength(4); + expect(importedRules).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: 'custom-rule', + version: 23, + rule_source: { type: 'internal' }, + immutable: false, + }), + expect.objectContaining({ + rule_id: prebuiltRuleIds[0], + version: 1234, + rule_source: { type: 'external', is_customized: true }, + immutable: true, + }), + expect.objectContaining({ + rule_id: 'custom-rule-2', + version: 1, + rule_source: { type: 'internal' }, + immutable: false, + }), + expect.objectContaining({ + rule_id: prebuiltRules[3].rule_id, + version: prebuiltRules[3].version, + rule_source: { type: 'external', is_customized: false }, + immutable: true, + }), + ]) + ); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts index 4324ce4602d72..58904243e51ca 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts @@ -8,8 +8,9 @@ import { FtrProviderContext } from '../../../../../../ftr_provider_context'; export default ({ loadTestFile }: FtrProviderContext): void => { - describe('Rules Management - Prebuilt Rules - Update Prebuilt Rules Package', function () { + describe('Rules Management - Prebuilt Rules - Prebuilt Rule Customization', function () { loadTestFile(require.resolve('./is_customized_calculation')); + loadTestFile(require.resolve('./import_rules')); loadTestFile(require.resolve('./rules_export')); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index b197e8127ca2d..038ed1787843a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1570,5 +1570,118 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); }); }); + + describe('supporting prebuilt rule customization', () => { + describe('compatibility with prebuilt rule fields', () => { + it('rejects rules with "immutable: true" when the feature flag is disabled', async () => { + const rule = getCustomQueryRuleParams({ + rule_id: 'rule-immutable', + // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} + immutable: true, + }); + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body).toMatchObject({ + success: false, + errors: [ + { + rule_id: 'rule-immutable', + error: { + status_code: 400, + message: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: rule-immutable]', + }, + }, + ], + }); + }); + + it('imports custom rules alongside prebuilt rules when feature flag is disabled', async () => { + const ndjson = combineToNdJson( + getCustomQueryRuleParams({ + rule_id: 'rule-immutable', + // @ts-expect-error the API supports the 'immutable' param, but we only need it in {@link RuleToImport} + immutable: true, + }), + // @ts-expect-error the API supports the 'immutable' param, but we only need it in {@link RuleToImport} + getCustomQueryRuleParams({ rule_id: 'custom-rule', immutable: false }) + ); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body).toMatchObject({ + success: false, + success_count: 1, + errors: [ + { + rule_id: 'rule-immutable', + error: { + status_code: 400, + message: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: rule-immutable]', + }, + }, + ], + }); + }); + + it('allows (but ignores) rules with a value for rule_source', async () => { + const rule = getCustomQueryRuleParams({ + rule_id: 'with-rule-source', + // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} + rule_source: { + type: 'ignored', + }, + }); + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body).toMatchObject({ + success: true, + success_count: 1, + }); + + const importedRule = await fetchRule(supertest, { ruleId: 'with-rule-source' }); + + expect(importedRule.rule_source).toMatchObject({ type: 'internal' }); + }); + + it('rejects rules without a rule_id', async () => { + const rule = getCustomQueryRuleParams({}); + delete rule.rule_id; + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { message: 'rule_id: Required', status_code: 400 }, + }); + }); + }); + }); }); };