From 4f3bbe8d30a962ddb4e9cd5c2d207dabaa063ffb Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Mon, 18 Nov 2024 20:47:32 +0100 Subject: [PATCH] [SecuritySolution][SIEM migrations] Add macros and lookups support in the API (#199370) --- .../common/api/quickstart_client.gen.ts | 55 +++++- .../common/siem_migrations/constants.ts | 14 +- .../siem_migrations/model/api/common.gen.ts | 2 +- .../model/api/common.schema.yaml | 2 +- ...migration.gen.ts => rule_migration.gen.ts} | 57 +++++- ...schema.yaml => rule_migration.schema.yaml} | 85 ++++++++- .../model/rule_migration.gen.ts | 62 +++++- .../model/rule_migration.schema.yaml | 61 +++++- .../siem_migrations/rules/resources/index.ts | 22 +++ .../rules/resources/splunk_identifier.test.ts | 89 +++++++++ .../rules/resources/splunk_identifier.ts | 48 +++++ .../siem_migrations/rules/resources/types.ts | 11 ++ .../siem_migrations/rules/__mocks__/mocks.ts | 44 +---- .../lib/siem_migrations/rules/api/create.ts | 43 +++-- .../lib/siem_migrations/rules/api/get.ts | 17 +- .../lib/siem_migrations/rules/api/index.ts | 7 + .../rules/api/resources/get.ts | 61 ++++++ .../rules/api/resources/upsert.ts | 67 +++++++ .../lib/siem_migrations/rules/api/retry.ts | 88 +++++++++ .../lib/siem_migrations/rules/api/start.ts | 86 ++++----- .../lib/siem_migrations/rules/api/stats.ts | 35 ++-- .../siem_migrations/rules/api/stats_all.ts | 30 +-- .../lib/siem_migrations/rules/api/stop.ts | 39 ++-- .../rules/api/util/with_license.ts | 35 ++++ .../rules/data/__mocks__/mocks.ts | 55 ++++++ .../__mocks__/rule_migrations_data_client.ts} | 4 +- .../rule_migrations_data_resources_client.ts | 9 + .../rule_migrations_data_rules_client.ts | 9 + .../__mocks__/rule_migrations_data_service.ts | 9 + .../data/rule_migrations_data_base_client.ts | 45 +++++ .../rules/data/rule_migrations_data_client.ts | 47 +++++ .../rule_migrations_data_resources_client.ts | 72 +++++++ .../rule_migrations_data_rules_client.ts} | 174 ++++++++--------- .../data/rule_migrations_data_service.test.ts | 112 +++++++++++ .../data/rule_migrations_data_service.ts | 75 ++++++++ .../rule_migrations_field_maps.ts} | 15 +- .../rules/data_stream/__mocks__/mocks.ts | 15 -- .../rule_migrations_data_stream.test.ts | 139 -------------- .../rule_migrations_data_stream.ts | 75 -------- .../siem_rule_migrations_service.test.ts | 58 ++++-- .../rules/siem_rule_migrations_service.ts | 77 ++++---- .../rules/task/__mocks__/mocks.ts | 35 ++++ .../__mocks__/rule_migrations_task_client.ts} | 4 +- .../__mocks__/rule_migrations_task_service.ts | 9 + .../siem_migrations/rules/task/agent/graph.ts | 9 +- .../agent/nodes/translate_query/prompt.ts | 39 ---- .../prompts/esql_translation_prompt.ts | 46 +++++ .../prompts/replace_resources_prompt.ts | 83 ++++++++ .../nodes/translate_query/translate_query.ts | 24 ++- .../siem_migrations/rules/task/agent/state.ts | 2 +- .../siem_migrations/rules/task/agent/types.ts | 2 + ...nner.ts => rule_migrations_task_client.ts} | 147 +++++++------- .../task/rule_migrations_task_service.ts | 40 ++++ .../lib/siem_migrations/rules/task/types.ts | 26 +-- .../task/util/rule_resource_retriever.test.ts | 180 ++++++++++++++++++ .../task/util/rule_resource_retriever.ts | 100 ++++++++++ .../server/lib/siem_migrations/rules/types.ts | 54 +----- .../siem_migrations_service.ts | 15 +- .../server/lib/siem_migrations/types.ts | 3 - .../security_solution/server/plugin.ts | 2 +- .../plugins/security_solution/server/types.ts | 2 +- .../plugins/security_solution/tsconfig.json | 3 +- .../services/security_solution_api.gen.ts | 64 ++++++- 63 files changed, 2173 insertions(+), 766 deletions(-) rename x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/{rules_migration.gen.ts => rule_migration.gen.ts} (64%) rename x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/{rules_migration.schema.yaml => rule_migration.schema.yaml} (69%) create mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts create mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts create mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts create mode 100644 x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/retry.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_license.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/{__mocks__/siem_rule_migrations_client.ts => data/__mocks__/rule_migrations_data_client.ts} (68%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_resources_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_rules_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_service.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/{data_stream/rule_migrations_data_client.ts => data/rule_migrations_data_rules_client.ts} (62%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/{data_stream/rule_migrations_field_map.ts => data/rule_migrations_field_maps.ts} (76%) delete mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/{data_stream/__mocks__/rule_migrations_data_stream.ts => task/__mocks__/rule_migrations_task_client.ts} (66%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/rule_migrations_task_service.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/esql_translation_prompt.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/replace_resources_prompt.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/{rule_migrations_task_runner.ts => rule_migrations_task_client.ts} (64%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.ts diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index 2eaa85dbbd14..264d0eaa14fe 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -356,6 +356,9 @@ import type { GetAllStatsRuleMigrationResponse, GetRuleMigrationRequestParamsInput, GetRuleMigrationResponse, + GetRuleMigrationResourcesRequestQueryInput, + GetRuleMigrationResourcesRequestParamsInput, + GetRuleMigrationResourcesResponse, GetRuleMigrationStatsRequestParamsInput, GetRuleMigrationStatsResponse, StartRuleMigrationRequestParamsInput, @@ -363,7 +366,10 @@ import type { StartRuleMigrationResponse, StopRuleMigrationRequestParamsInput, StopRuleMigrationResponse, -} from '../siem_migrations/model/api/rules/rules_migration.gen'; + UpsertRuleMigrationResourcesRequestParamsInput, + UpsertRuleMigrationResourcesRequestBodyInput, + UpsertRuleMigrationResourcesResponse, +} from '../siem_migrations/model/api/rules/rule_migration.gen'; export interface ClientOptions { kbnClient: KbnClient; @@ -1405,6 +1411,26 @@ finalize it. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Retrieves resources for an existing SIEM rules migration + */ + async getRuleMigrationResources(props: GetRuleMigrationResourcesProps) { + this.log.info(`${new Date().toISOString()} Calling API GetRuleMigrationResources`); + return this.kbnClient + .request({ + path: replaceParams( + '/internal/siem_migrations/rules/{migration_id}/resources', + props.params + ), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + + query: props.query, + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Retrieves the stats of a SIEM rules migration using the migration id provided */ @@ -2056,6 +2082,25 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Creates or updates resources for an existing SIEM rules migration + */ + async upsertRuleMigrationResources(props: UpsertRuleMigrationResourcesProps) { + this.log.info(`${new Date().toISOString()} Calling API UpsertRuleMigrationResources`); + return this.kbnClient + .request({ + path: replaceParams( + '/internal/siem_migrations/rules/{migration_id}/resources', + props.params + ), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'POST', + body: props.body, + }) + .catch(catchAxiosErrorFormatAndThrow); + } } export interface AlertsMigrationCleanupProps { @@ -2221,6 +2266,10 @@ export interface GetRuleExecutionResultsProps { export interface GetRuleMigrationProps { params: GetRuleMigrationRequestParamsInput; } +export interface GetRuleMigrationResourcesProps { + query: GetRuleMigrationResourcesRequestQueryInput; + params: GetRuleMigrationResourcesRequestParamsInput; +} export interface GetRuleMigrationStatsProps { params: GetRuleMigrationStatsRequestParamsInput; } @@ -2322,3 +2371,7 @@ export interface UpdateRuleProps { export interface UploadAssetCriticalityRecordsProps { attachment: FormData; } +export interface UpsertRuleMigrationResourcesProps { + params: UpsertRuleMigrationResourcesRequestParamsInput; + body: UpsertRuleMigrationResourcesRequestBodyInput; +} diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index f2efc646a810..8a2d6cf3775c 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -9,13 +9,13 @@ export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const; export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const; export const SIEM_RULE_MIGRATIONS_ALL_STATS_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/stats` as const; -export const SIEM_RULE_MIGRATIONS_GET_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}` as const; -export const SIEM_RULE_MIGRATIONS_START_PATH = - `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/start` as const; -export const SIEM_RULE_MIGRATIONS_STATS_PATH = - `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stats` as const; -export const SIEM_RULE_MIGRATIONS_STOP_PATH = - `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stop` as const; +export const SIEM_RULE_MIGRATION_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}` as const; +export const SIEM_RULE_MIGRATION_START_PATH = `${SIEM_RULE_MIGRATION_PATH}/start` as const; +export const SIEM_RULE_MIGRATION_RETRY_PATH = `${SIEM_RULE_MIGRATION_PATH}/retry` as const; +export const SIEM_RULE_MIGRATION_STATS_PATH = `${SIEM_RULE_MIGRATION_PATH}/stats` as const; +export const SIEM_RULE_MIGRATION_STOP_PATH = `${SIEM_RULE_MIGRATION_PATH}/stop` as const; + +export const SIEM_RULE_MIGRATION_RESOURCES_PATH = `${SIEM_RULE_MIGRATION_PATH}/resources` as const; export enum SiemMigrationStatus { PENDING = 'pending', diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts index 620475a6eb73..788035492853 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.gen.ts @@ -10,7 +10,7 @@ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. * * info: - * title: Common SIEM Migrations Attributes + * title: SIEM Rule Migrations API common components * version: not applicable */ diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml index 97450d191f30..5782fa777201 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/common.schema.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - title: Common SIEM Migrations Attributes + title: SIEM Rule Migrations API common components version: 'not applicable' paths: {} components: diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts similarity index 64% rename from x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts rename to x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts index 120505ec43cb..7ea6314726da 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts @@ -10,17 +10,21 @@ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. * * info: - * title: SIEM Rules Migration API endpoint + * title: SIEM Rules Migration API * version: 1 */ import { z } from '@kbn/zod'; +import { ArrayFromString } from '@kbn/zod-helpers'; import { OriginalRule, RuleMigrationAllTaskStats, RuleMigration, RuleMigrationTaskStats, + RuleMigrationResourceData, + RuleMigrationResourceType, + RuleMigrationResource, } from '../../rule_migration.gen'; import { ConnectorId, LangSmithOptions } from '../common.gen'; @@ -47,6 +51,29 @@ export type GetRuleMigrationRequestParamsInput = z.input; export const GetRuleMigrationResponse = z.array(RuleMigration); +export type GetRuleMigrationResourcesRequestQuery = z.infer< + typeof GetRuleMigrationResourcesRequestQuery +>; +export const GetRuleMigrationResourcesRequestQuery = z.object({ + type: RuleMigrationResourceType.optional(), + names: ArrayFromString(z.string()).optional(), +}); +export type GetRuleMigrationResourcesRequestQueryInput = z.input< + typeof GetRuleMigrationResourcesRequestQuery +>; + +export type GetRuleMigrationResourcesRequestParams = z.infer< + typeof GetRuleMigrationResourcesRequestParams +>; +export const GetRuleMigrationResourcesRequestParams = z.object({ + migration_id: z.string(), +}); +export type GetRuleMigrationResourcesRequestParamsInput = z.input< + typeof GetRuleMigrationResourcesRequestParams +>; + +export type GetRuleMigrationResourcesResponse = z.infer; +export const GetRuleMigrationResourcesResponse = z.array(RuleMigrationResource); export type GetRuleMigrationStatsRequestParams = z.infer; export const GetRuleMigrationStatsRequestParams = z.object({ @@ -93,3 +120,31 @@ export const StopRuleMigrationResponse = z.object({ */ stopped: z.boolean(), }); + +export type UpsertRuleMigrationResourcesRequestParams = z.infer< + typeof UpsertRuleMigrationResourcesRequestParams +>; +export const UpsertRuleMigrationResourcesRequestParams = z.object({ + migration_id: z.string(), +}); +export type UpsertRuleMigrationResourcesRequestParamsInput = z.input< + typeof UpsertRuleMigrationResourcesRequestParams +>; + +export type UpsertRuleMigrationResourcesRequestBody = z.infer< + typeof UpsertRuleMigrationResourcesRequestBody +>; +export const UpsertRuleMigrationResourcesRequestBody = z.array(RuleMigrationResourceData); +export type UpsertRuleMigrationResourcesRequestBodyInput = z.input< + typeof UpsertRuleMigrationResourcesRequestBody +>; + +export type UpsertRuleMigrationResourcesResponse = z.infer< + typeof UpsertRuleMigrationResourcesResponse +>; +export const UpsertRuleMigrationResourcesResponse = z.object({ + /** + * The request has been processed correctly. + */ + acknowledged: z.boolean(), +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml similarity index 69% rename from x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml rename to x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml index 7b06c3d6a22a..bac82e5b0248 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml @@ -1,8 +1,11 @@ openapi: 3.0.3 info: - title: SIEM Rules Migration API endpoint + title: SIEM Rules Migration API version: '1' paths: + + # Rule migrations APIs + /internal/siem_migrations/rules: post: summary: Creates a new rule migration @@ -49,6 +52,8 @@ paths: schema: $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationAllTaskStats' + ## Specific rule migration APIs + /internal/siem_migrations/rules/{migration_id}: get: summary: Retrieves all the rules of a migration @@ -175,3 +180,81 @@ paths: description: Indicates the migration has been stopped. 204: description: Indicates the migration id was not found running. + + # Rule migration resources APIs + + /internal/siem_migrations/rules/{migration_id}/resources: + post: + summary: Creates or updates rule migration resources for a migration + operationId: UpsertRuleMigrationResources + x-codegen-enabled: true + description: Creates or updates resources for an existing SIEM rules migration + tags: + - SIEM Rule Migrations + - Resources + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to attach the resources + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceData' + responses: + 200: + description: Indicates migration resources have been created or updated correctly. + content: + application/json: + schema: + type: object + required: + - acknowledged + properties: + acknowledged: + type: boolean + description: The request has been processed correctly. + + get: + summary: Gets rule migration resources for a migration + operationId: GetRuleMigrationResources + x-codegen-enabled: true + description: Retrieves resources for an existing SIEM rules migration + tags: + - SIEM Rule Migrations + - Resources + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to attach the resources + - name: type + in: query + required: false + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResourceType' + - name: names + in: query + required: false + schema: + type: array + description: The names of the resource to retrieve + items: + type: string + responses: + 200: + description: Indicates migration resources have been retrieved correctly + content: + application/json: + schema: + type: array + items: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationResource' diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index fe00c4b4df1c..ac178610cee6 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -10,12 +10,18 @@ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. * * info: - * title: Common Splunk Rules Attributes + * title: SIEM Rule Migration common components * version: not applicable */ import { z } from '@kbn/zod'; +/** + * The original rule vendor identifier. + */ +export type OriginalRuleVendor = z.infer; +export const OriginalRuleVendor = z.literal('splunk'); + /** * The original rule to migrate. */ @@ -25,10 +31,7 @@ export const OriginalRule = z.object({ * The original rule id. */ id: z.string(), - /** - * The original rule vendor identifier. - */ - vendor: z.literal('splunk'), + vendor: OriginalRuleVendor, /** * The original rule name. */ @@ -178,3 +181,52 @@ export const RuleMigrationAllTaskStats = z.array( }) ) ); + +/** + * The type of the rule migration resource. + */ +export type RuleMigrationResourceType = z.infer; +export const RuleMigrationResourceType = z.enum(['macro', 'list']); +export type RuleMigrationResourceTypeEnum = typeof RuleMigrationResourceType.enum; +export const RuleMigrationResourceTypeEnum = RuleMigrationResourceType.enum; + +/** + * The rule migration resource data provided by the vendor. + */ +export type RuleMigrationResourceData = z.infer; +export const RuleMigrationResourceData = z.object({ + type: RuleMigrationResourceType, + /** + * The resource name identifier. + */ + name: z.string(), + /** + * The resource content value. + */ + content: z.string(), + /** + * The resource arbitrary metadata. + */ + metadata: z.object({}).optional(), +}); + +/** + * The rule migration resource document object. + */ +export type RuleMigrationResource = z.infer; +export const RuleMigrationResource = RuleMigrationResourceData.merge( + z.object({ + /** + * The migration id + */ + migration_id: z.string(), + /** + * The moment of the last update + */ + updated_at: z.string().optional(), + /** + * The user who last updated the resource + */ + updated_by: z.string().optional(), + }) +); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index c9841856a691..c16849cec278 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -1,11 +1,18 @@ openapi: 3.0.3 info: - title: Common Splunk Rules Attributes + title: SIEM Rule Migration common components version: 'not applicable' paths: {} components: x-codegen-enabled: true schemas: + + OriginalRuleVendor: + type: string + description: The original rule vendor identifier. + enum: + - splunk + OriginalRule: type: object description: The original rule to migrate. @@ -21,10 +28,7 @@ components: type: string description: The original rule id. vendor: - type: string - description: The original rule vendor identifier. - enum: - - splunk + $ref: '#/components/schemas/OriginalRuleVendor' title: type: string description: The original rule name. @@ -180,3 +184,50 @@ components: migration_id: type: string description: The migration id + +## Rule migration resources + + RuleMigrationResourceType: + type: string + description: The type of the rule migration resource. + enum: + - macro # Reusable part a query that can be customized and called from multiple rules + - list # A list of values that can be used inside queries reused in different rules + + RuleMigrationResourceData: + type: object + description: The rule migration resource data provided by the vendor. + required: + - type + - name + - content + properties: + type: + $ref: '#/components/schemas/RuleMigrationResourceType' + name: + type: string + description: The resource name identifier. + content: + type: string + description: The resource content value. + metadata: + type: object + description: The resource arbitrary metadata. + + RuleMigrationResource: + description: The rule migration resource document object. + allOf: + - $ref: '#/components/schemas/RuleMigrationResourceData' + - type: object + required: + - migration_id + properties: + migration_id: + type: string + description: The migration id + updated_at: + type: string + description: The moment of the last update + updated_by: + type: string + description: The user who last updated the resource diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts new file mode 100644 index 000000000000..ffe4b3aca407 --- /dev/null +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { OriginalRule, OriginalRuleVendor } from '../../model/rule_migration.gen'; +import type { QueryResourceIdentifier, RuleResourceCollection } from './types'; +import { splResourceIdentifier } from './splunk_identifier'; + +export const getRuleResourceIdentifier = (rule: OriginalRule): QueryResourceIdentifier => { + return ruleResourceIdentifiers[rule.vendor]; +}; + +export const identifyRuleResources = (rule: OriginalRule): RuleResourceCollection => { + return getRuleResourceIdentifier(rule)(rule.query); +}; + +const ruleResourceIdentifiers: Record = { + splunk: splResourceIdentifier, +}; diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts new file mode 100644 index 000000000000..2fa3be223aa6 --- /dev/null +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.test.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { splResourceIdentifier } from './splunk_identifier'; + +describe('splResourceIdentifier', () => { + it('should extract macros correctly', () => { + const query = + '`macro_zero`, `macro_one(arg1)`, some search command `macro_two(arg1, arg2)` another command `macro_three(arg1, arg2, arg3)`'; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual(['macro_zero', 'macro_one(1)', 'macro_two(2)', 'macro_three(3)']); + expect(result.list).toEqual([]); + }); + + it('should extract lookup tables correctly', () => { + const query = + 'search ... | lookup my_lookup_table field AS alias OUTPUT new_field | inputlookup other_lookup_list | lookup third_lookup'; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual([]); + expect(result.list).toEqual(['my_lookup_table', 'other_lookup_list', 'third_lookup']); + }); + + it('should extract both macros and lookup tables correctly', () => { + const query = + '`macro_one` some search command | lookup my_lookup_table field AS alias OUTPUT new_field | inputlookup other_lookup_list | lookup third_lookup'; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual(['macro_one']); + expect(result.list).toEqual(['my_lookup_table', 'other_lookup_list', 'third_lookup']); + }); + + it('should return empty arrays if no macros or lookup tables are found', () => { + const query = 'search | stats count'; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual([]); + expect(result.list).toEqual([]); + }); + + it('should handle queries with both macros and lookup tables mixed with other commands', () => { + const query = + 'search `macro_one` | `my_lookup_table` field AS alias myfakelookup new_field | inputlookup real_lookup_list | `third_macro`'; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual(['macro_one', 'my_lookup_table', 'third_macro']); + expect(result.list).toEqual(['real_lookup_list']); + }); + + it('should ignore macros or lookup tables inside string literals with double quotes', () => { + const query = + '`macro_one` | lookup my_lookup_table | search title="`macro_two` and lookup another_table"'; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual(['macro_one']); + expect(result.list).toEqual(['my_lookup_table']); + }); + + it('should ignore macros or lookup tables inside string literals with single quotes', () => { + const query = + "`macro_one` | lookup my_lookup_table | search title='`macro_two` and lookup another_table'"; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual(['macro_one']); + expect(result.list).toEqual(['my_lookup_table']); + }); + + it('should ignore macros or lookup tables inside comments wrapped by ```', () => { + const query = + '`macro_one` | ```this is a comment with `macro_two` and lookup another_table``` lookup my_lookup_table'; + + const result = splResourceIdentifier(query); + + expect(result.macro).toEqual(['macro_one']); + expect(result.list).toEqual(['my_lookup_table']); + }); +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts new file mode 100644 index 000000000000..fa46fff941c6 --- /dev/null +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/splunk_identifier.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Important: + * This library uses regular expressions that are executed against arbitrary user input, they need to be safe from ReDoS attacks. + * Please make sure to test them before using them in production. + * At the time of writing, this tool can be used to test it: https://devina.io/redos-checker + */ + +import type { QueryResourceIdentifier } from './types'; + +const listRegex = /\b(?:lookup|inputlookup)\s+([\w-]+)\b/g; // Captures only the lookup table name +const macrosRegex = /`([\w-]+)(?:\(([^`]*?)\))?`/g; // Captures only the macro name and arguments + +const commentRegex = /```.*```/g; +const doubleQuoteStrRegex = /".*"/g; +const singleQuoteStrRegex = /'.*'/g; + +export const splResourceIdentifier: QueryResourceIdentifier = (query) => { + // sanitize the query to avoid mismatching macro and list names inside comments or literal strings + const sanitizedQuery = query + .replaceAll(commentRegex, '') + .replaceAll(doubleQuoteStrRegex, '"literal"') + .replaceAll(singleQuoteStrRegex, "'literal'"); + + const macro = []; + let macroMatch; + while ((macroMatch = macrosRegex.exec(sanitizedQuery)) !== null) { + const macroName = macroMatch[1]; + const args = macroMatch[2]; // This captures the content inside the parentheses + const argCount = args ? args.split(',').length : 0; // Count arguments if present + const macroWithArgs = argCount > 0 ? `${macroName}(${argCount})` : macroName; + macro.push(macroWithArgs); + } + + const list = []; + let listMatch; + while ((listMatch = listRegex.exec(sanitizedQuery)) !== null) { + list.push(listMatch[1]); + } + + return { macro, list }; +}; diff --git a/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts new file mode 100644 index 000000000000..93f6f3ad3db1 --- /dev/null +++ b/x-pack/plugins/security_solution/common/siem_migrations/rules/resources/types.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleMigrationResourceType } from '../../model/rule_migration.gen'; + +export type RuleResourceCollection = Record; +export type QueryResourceIdentifier = (query: string) => RuleResourceCollection; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts index 8811a54195e2..5bc9d4e23bc6 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts @@ -5,42 +5,16 @@ * 2.0. */ -export const createRuleMigrationDataClient = jest.fn().mockImplementation(() => ({ - create: jest.fn().mockResolvedValue({ success: true }), - getRules: jest.fn().mockResolvedValue([]), - takePending: jest.fn().mockResolvedValue([]), - saveFinished: jest.fn().mockResolvedValue({ success: true }), - saveError: jest.fn().mockResolvedValue({ success: true }), - releaseProcessing: jest.fn().mockResolvedValue({ success: true }), - releaseProcessable: jest.fn().mockResolvedValue({ success: true }), - getStats: jest.fn().mockResolvedValue({ - status: 'done', - rules: { - total: 1, - finished: 1, - processing: 0, - pending: 0, - failed: 0, - }, - }), - getAllStats: jest.fn().mockResolvedValue([]), -})); +import { mockRuleMigrationsDataClient } from '../data/__mocks__/mocks'; +import { mockRuleMigrationsTaskClient } from '../task/__mocks__/mocks'; -export const createRuleMigrationTaskClient = () => ({ - start: jest.fn().mockResolvedValue({ started: true }), - stop: jest.fn().mockResolvedValue({ stopped: true }), - getStats: jest.fn().mockResolvedValue({ - status: 'done', - rules: { - total: 1, - finished: 1, - processing: 0, - pending: 0, - failed: 0, - }, - }), - getAllStats: jest.fn().mockResolvedValue([]), -}); +export const createRuleMigrationDataClient = jest + .fn() + .mockImplementation(() => mockRuleMigrationsDataClient); + +export const createRuleMigrationTaskClient = jest + .fn() + .mockImplementation(() => mockRuleMigrationsTaskClient); export const createRuleMigrationClient = () => ({ data: createRuleMigrationDataClient(), diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index e2505ca83bee..025c52da766a 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -8,11 +8,14 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { v4 as uuidV4 } from 'uuid'; -import type { CreateRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { CreateRuleMigrationRequestBody } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { + CreateRuleMigrationRequestBody, + type CreateRuleMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATIONS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import type { CreateRuleMigrationInput } from '../data_stream/rule_migrations_data_client'; +import type { CreateRuleMigrationInput } from '../data/rule_migrations_data_client'; +import { withLicense } from './util/with_license'; export const registerSiemRuleMigrationsCreateRoute = ( router: SecuritySolutionPluginRouter, @@ -31,26 +34,28 @@ export const registerSiemRuleMigrationsCreateRoute = ( request: { body: buildRouteValidationWithZod(CreateRuleMigrationRequestBody) }, }, }, - async (context, req, res): Promise> => { - const originalRules = req.body; - try { - const ctx = await context.resolve(['securitySolution']); - const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + withLicense( + async (context, req, res): Promise> => { + const originalRules = req.body; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const migrationId = uuidV4(); + const migrationId = uuidV4(); - const ruleMigrations = originalRules.map((originalRule) => ({ - migration_id: migrationId, - original_rule: originalRule, - })); + const ruleMigrations = originalRules.map((originalRule) => ({ + migration_id: migrationId, + original_rule: originalRule, + })); - await ruleMigrationsClient.data.create(ruleMigrations); + await ruleMigrationsClient.data.rules.create(ruleMigrations); - return res.ok({ body: { migration_id: migrationId } }); - } catch (err) { - logger.error(err); - return res.badRequest({ body: err.message }); + return res.ok({ body: { migration_id: migrationId } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } } - } + ) ); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts index 0efb6706918f..e6edb05b3a68 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts @@ -7,10 +7,13 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import type { GetRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { GetRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { SIEM_RULE_MIGRATIONS_GET_PATH } from '../../../../../common/siem_migrations/constants'; +import { + GetRuleMigrationRequestParams, + type GetRuleMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; export const registerSiemRuleMigrationsGetRoute = ( router: SecuritySolutionPluginRouter, @@ -18,7 +21,7 @@ export const registerSiemRuleMigrationsGetRoute = ( ) => { router.versioned .get({ - path: SIEM_RULE_MIGRATIONS_GET_PATH, + path: SIEM_RULE_MIGRATION_PATH, access: 'internal', security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) @@ -29,19 +32,19 @@ export const registerSiemRuleMigrationsGetRoute = ( request: { params: buildRouteValidationWithZod(GetRuleMigrationRequestParams) }, }, }, - async (context, req, res): Promise> => { + withLicense(async (context, req, res): Promise> => { const migrationId = req.params.migration_id; try { const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const migrationRules = await ruleMigrationsClient.data.getRules(migrationId); + const migrationRules = await ruleMigrationsClient.data.rules.get(migrationId); return res.ok({ body: migrationRules }); } catch (err) { logger.error(err); return res.badRequest({ body: err.message }); } - } + }) ); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index f37eb2108a8a..dfc4c2156fe2 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -13,6 +13,9 @@ import { registerSiemRuleMigrationsStartRoute } from './start'; import { registerSiemRuleMigrationsStatsRoute } from './stats'; import { registerSiemRuleMigrationsStopRoute } from './stop'; import { registerSiemRuleMigrationsStatsAllRoute } from './stats_all'; +import { registerSiemRuleMigrationsResourceUpsertRoute } from './resources/upsert'; +import { registerSiemRuleMigrationsResourceGetRoute } from './resources/get'; +import { registerSiemRuleMigrationsRetryRoute } from './retry'; export const registerSiemRuleMigrationsRoutes = ( router: SecuritySolutionPluginRouter, @@ -22,6 +25,10 @@ export const registerSiemRuleMigrationsRoutes = ( registerSiemRuleMigrationsStatsAllRoute(router, logger); registerSiemRuleMigrationsGetRoute(router, logger); registerSiemRuleMigrationsStartRoute(router, logger); + registerSiemRuleMigrationsRetryRoute(router, logger); registerSiemRuleMigrationsStatsRoute(router, logger); registerSiemRuleMigrationsStopRoute(router, logger); + + registerSiemRuleMigrationsResourceUpsertRoute(router, logger); + registerSiemRuleMigrationsResourceGetRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts new file mode 100644 index 000000000000..7f2cfc8743f0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/get.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { + GetRuleMigrationResourcesRequestParams, + GetRuleMigrationResourcesRequestQuery, + type GetRuleMigrationResourcesResponse, +} from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_RESOURCES_PATH } from '../../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { withLicense } from '../util/with_license'; + +export const registerSiemRuleMigrationsResourceGetRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATION_RESOURCES_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(GetRuleMigrationResourcesRequestParams), + query: buildRouteValidationWithZod(GetRuleMigrationResourcesRequestQuery), + }, + }, + }, + withLicense( + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + const { type, names } = req.query; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const resources = await ruleMigrationsClient.data.resources.get( + migrationId, + type, + names + ); + + return res.ok({ body: resources }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts new file mode 100644 index 000000000000..be1f3e84c46e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/resources/upsert.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { RuleMigrationResource } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { + UpsertRuleMigrationResourcesRequestBody, + UpsertRuleMigrationResourcesRequestParams, + type UpsertRuleMigrationResourcesResponse, +} from '../../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_RESOURCES_PATH } from '../../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { withLicense } from '../util/with_license'; + +export const registerSiemRuleMigrationsResourceUpsertRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .post({ + path: SIEM_RULE_MIGRATION_RESOURCES_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(UpsertRuleMigrationResourcesRequestParams), + body: buildRouteValidationWithZod(UpsertRuleMigrationResourcesRequestBody), + }, + }, + }, + withLicense( + async ( + context, + req, + res + ): Promise> => { + const resources = req.body; + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const ruleMigrations = resources.map((resource) => ({ + migration_id: migrationId, + ...resource, + })); + + await ruleMigrationsClient.data.resources.upsert(ruleMigrations); + + return res.ok({ body: { acknowledged: true } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/retry.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/retry.ts new file mode 100644 index 000000000000..4406afc4333e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/retry.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { APMTracer } from '@kbn/langchain/server/tracers/apm'; +import { getLangSmithTracer } from '@kbn/langchain/server/tracers/langsmith'; +import { + StartRuleMigrationRequestBody, + StartRuleMigrationRequestParams, + type StartRuleMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_RETRY_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; + +export const registerSiemRuleMigrationsRetryRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATION_RETRY_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(StartRuleMigrationRequestParams), + body: buildRouteValidationWithZod(StartRuleMigrationRequestBody), + }, + }, + }, + withLicense( + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + const { langsmith_options: langsmithOptions, connector_id: connectorId } = req.body; + + try { + const ctx = await context.resolve(['core', 'actions', 'alerting', 'securitySolution']); + + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const inferenceClient = ctx.securitySolution.getInferenceClient(); + const actionsClient = ctx.actions.getActionsClient(); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + + const invocationConfig = { + callbacks: [ + new APMTracer({ projectName: langsmithOptions?.project_name ?? 'default' }, logger), + ...getLangSmithTracer({ ...langsmithOptions, logger }), + ], + }; + + const { updated } = await ruleMigrationsClient.task.updateToRetry(migrationId); + if (!updated) { + return res.ok({ body: { started: false } }); + } + + const { exists, started } = await ruleMigrationsClient.task.start({ + migrationId, + connectorId, + invocationConfig, + inferenceClient, + actionsClient, + soClient, + rulesClient, + }); + + if (!exists) { + return res.noContent(); + } + return res.ok({ body: { started } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts index f97a4f2ce239..73ba2fd3cce7 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -9,13 +9,14 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { APMTracer } from '@kbn/langchain/server/tracers/apm'; import { getLangSmithTracer } from '@kbn/langchain/server/tracers/langsmith'; -import type { StartRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; import { StartRuleMigrationRequestBody, StartRuleMigrationRequestParams, -} from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { SIEM_RULE_MIGRATIONS_START_PATH } from '../../../../../common/siem_migrations/constants'; + type StartRuleMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_START_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; export const registerSiemRuleMigrationsStartRoute = ( router: SecuritySolutionPluginRouter, @@ -23,7 +24,7 @@ export const registerSiemRuleMigrationsStartRoute = ( ) => { router.versioned .put({ - path: SIEM_RULE_MIGRATIONS_START_PATH, + path: SIEM_RULE_MIGRATION_START_PATH, access: 'internal', security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) @@ -37,55 +38,46 @@ export const registerSiemRuleMigrationsStartRoute = ( }, }, }, - async (context, req, res): Promise> => { - const migrationId = req.params.migration_id; - const { langsmith_options: langsmithOptions, connector_id: connectorId } = req.body; + withLicense( + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + const { langsmith_options: langsmithOptions, connector_id: connectorId } = req.body; - try { - const ctx = await context.resolve([ - 'core', - 'actions', - 'alerting', - 'securitySolution', - 'licensing', - ]); - if (!ctx.licensing.license.hasAtLeast('enterprise')) { - return res.forbidden({ - body: 'You must have a trial or enterprise license to use this feature', - }); - } + try { + const ctx = await context.resolve(['core', 'actions', 'alerting', 'securitySolution']); - const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const inferenceClient = ctx.securitySolution.getInferenceClient(); - const actionsClient = ctx.actions.getActionsClient(); - const soClient = ctx.core.savedObjects.client; - const rulesClient = ctx.alerting.getRulesClient(); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const inferenceClient = ctx.securitySolution.getInferenceClient(); + const actionsClient = ctx.actions.getActionsClient(); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); - const invocationConfig = { - callbacks: [ - new APMTracer({ projectName: langsmithOptions?.project_name ?? 'default' }, logger), - ...getLangSmithTracer({ ...langsmithOptions, logger }), - ], - }; + const invocationConfig = { + callbacks: [ + new APMTracer({ projectName: langsmithOptions?.project_name ?? 'default' }, logger), + ...getLangSmithTracer({ ...langsmithOptions, logger }), + ], + }; - const { exists, started } = await ruleMigrationsClient.task.start({ - migrationId, - connectorId, - invocationConfig, - inferenceClient, - actionsClient, - soClient, - rulesClient, - }); + const { exists, started } = await ruleMigrationsClient.task.start({ + migrationId, + connectorId, + invocationConfig, + inferenceClient, + actionsClient, + soClient, + rulesClient, + }); - if (!exists) { - return res.noContent(); + if (!exists) { + return res.noContent(); + } + return res.ok({ body: { started } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); } - return res.ok({ body: { started } }); - } catch (err) { - logger.error(err); - return res.badRequest({ body: err.message }); } - } + ) ); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts index 8316e01fc6a9..5fb7d9e0525c 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts @@ -7,10 +7,13 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import type { GetRuleMigrationStatsResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { GetRuleMigrationStatsRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { SIEM_RULE_MIGRATIONS_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import { + GetRuleMigrationStatsRequestParams, + type GetRuleMigrationStatsResponse, +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_STATS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; export const registerSiemRuleMigrationsStatsRoute = ( router: SecuritySolutionPluginRouter, @@ -18,7 +21,7 @@ export const registerSiemRuleMigrationsStatsRoute = ( ) => { router.versioned .get({ - path: SIEM_RULE_MIGRATIONS_STATS_PATH, + path: SIEM_RULE_MIGRATION_STATS_PATH, access: 'internal', security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) @@ -29,19 +32,21 @@ export const registerSiemRuleMigrationsStatsRoute = ( request: { params: buildRouteValidationWithZod(GetRuleMigrationStatsRequestParams) }, }, }, - async (context, req, res): Promise> => { - const migrationId = req.params.migration_id; - try { - const ctx = await context.resolve(['securitySolution']); - const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + withLicense( + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const stats = await ruleMigrationsClient.task.getStats(migrationId); + const stats = await ruleMigrationsClient.task.getStats(migrationId); - return res.ok({ body: stats }); - } catch (err) { - logger.error(err); - return res.badRequest({ body: err.message }); + return res.ok({ body: stats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } } - } + ) ); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts index dd2f2f503e19..9ef83d7ab70c 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts @@ -6,9 +6,10 @@ */ import type { IKibanaResponse, Logger } from '@kbn/core/server'; -import type { GetAllStatsRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import type { GetAllStatsRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; import { SIEM_RULE_MIGRATIONS_ALL_STATS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; export const registerSiemRuleMigrationsStatsAllRoute = ( router: SecuritySolutionPluginRouter, @@ -21,19 +22,24 @@ export const registerSiemRuleMigrationsStatsAllRoute = ( security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) .addVersion( - { version: '1', validate: {} }, - async (context, req, res): Promise> => { - try { - const ctx = await context.resolve(['securitySolution']); - const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + { + version: '1', + validate: {}, + }, + withLicense( + async (context, req, res): Promise> => { + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const allStats = await ruleMigrationsClient.task.getAllStats(); + const allStats = await ruleMigrationsClient.task.getAllStats(); - return res.ok({ body: allStats }); - } catch (err) { - logger.error(err); - return res.badRequest({ body: err.message }); + return res.ok({ body: allStats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } } - } + ) ); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts index 476710691018..349afc66013b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts @@ -7,10 +7,13 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import type { StopRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { StopRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { SIEM_RULE_MIGRATIONS_STOP_PATH } from '../../../../../common/siem_migrations/constants'; +import { + StopRuleMigrationRequestParams, + type StopRuleMigrationResponse, +} from '../../../../../common/siem_migrations/model/api/rules/rule_migration.gen'; +import { SIEM_RULE_MIGRATION_STOP_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { withLicense } from './util/with_license'; export const registerSiemRuleMigrationsStopRoute = ( router: SecuritySolutionPluginRouter, @@ -18,7 +21,7 @@ export const registerSiemRuleMigrationsStopRoute = ( ) => { router.versioned .put({ - path: SIEM_RULE_MIGRATIONS_STOP_PATH, + path: SIEM_RULE_MIGRATION_STOP_PATH, access: 'internal', security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) @@ -29,22 +32,24 @@ export const registerSiemRuleMigrationsStopRoute = ( request: { params: buildRouteValidationWithZod(StopRuleMigrationRequestParams) }, }, }, - async (context, req, res): Promise> => { - const migrationId = req.params.migration_id; - try { - const ctx = await context.resolve(['securitySolution']); - const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + withLicense( + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const { exists, stopped } = await ruleMigrationsClient.task.stop(migrationId); + const { exists, stopped } = await ruleMigrationsClient.task.stop(migrationId); - if (!exists) { - return res.noContent(); + if (!exists) { + return res.noContent(); + } + return res.ok({ body: { stopped } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); } - return res.ok({ body: { stopped } }); - } catch (err) { - logger.error(err); - return res.badRequest({ body: err.message }); } - } + ) ); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_license.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_license.ts new file mode 100644 index 000000000000..1cacee0bbae7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/util/with_license.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RequestHandler, RouteMethod } from '@kbn/core/server'; +import { i18n } from '@kbn/i18n'; +import type { SecuritySolutionRequestHandlerContext } from '../../../../../types'; + +const LICENSE_ERROR_MESSAGE = i18n.translate('xpack.securitySolution.api.licenseError', { + defaultMessage: 'Your license does not support this feature.', +}); + +/** + * Wraps a request handler with a check for the license. If the license is not valid, it will + * return a 403 error with a message. + */ +export const withLicense = < + P = unknown, + Q = unknown, + B = unknown, + Method extends RouteMethod = never +>( + handler: RequestHandler +): RequestHandler => { + return async (context, req, res) => { + const { license } = await context.licensing; + if (!license.hasAtLeast('enterprise')) { + return res.forbidden({ body: LICENSE_ERROR_MESSAGE }); + } + return handler(context, req, res); + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts new file mode 100644 index 000000000000..34e68d8a4736 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/mocks.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleMigrationsDataRulesClient } from '../rule_migrations_data_rules_client'; + +// Rule migrations data rules client +export const mockRuleMigrationsDataRulesClient = { + create: jest.fn().mockResolvedValue(undefined), + get: jest.fn().mockResolvedValue([]), + takePending: jest.fn().mockResolvedValue([]), + saveCompleted: jest.fn().mockResolvedValue(undefined), + saveError: jest.fn().mockResolvedValue(undefined), + releaseProcessing: jest.fn().mockResolvedValue(undefined), + updateStatus: jest.fn().mockResolvedValue(undefined), + getStats: jest.fn().mockResolvedValue(undefined), + getAllStats: jest.fn().mockResolvedValue([]), +} as unknown as RuleMigrationsDataRulesClient; +export const MockRuleMigrationsDataRulesClient = jest + .fn() + .mockImplementation(() => mockRuleMigrationsDataRulesClient); + +// Rule migrations data resources client +export const mockRuleMigrationsDataResourcesClient = { + upsert: jest.fn().mockResolvedValue(undefined), + get: jest.fn().mockResolvedValue(undefined), +}; +export const MockRuleMigrationsDataResourcesClient = jest + .fn() + .mockImplementation(() => mockRuleMigrationsDataResourcesClient); + +// Rule migrations data client +export const mockRuleMigrationsDataClient = { + rules: mockRuleMigrationsDataRulesClient, + resources: mockRuleMigrationsDataResourcesClient, +}; + +export const MockRuleMigrationsDataClient = jest + .fn() + .mockImplementation(() => mockRuleMigrationsDataClient); + +// Rule migrations data service +export const mockIndexName = 'mocked_siem_rule_migrations_index_name'; +export const mockInstall = jest.fn().mockResolvedValue(undefined); +export const mockCreateClient = jest.fn().mockReturnValue(mockRuleMigrationsDataClient); + +export const MockRuleMigrationsDataService = jest.fn().mockImplementation(() => ({ + createAdapter: jest.fn(), + install: mockInstall, + createClient: mockCreateClient, + createIndexNameProvider: jest.fn().mockResolvedValue(mockIndexName), +})); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_client.ts similarity index 68% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_client.ts index 98032605ed23..0a5bf34c88f4 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_client.ts @@ -5,5 +5,5 @@ * 2.0. */ -import { MockSiemRuleMigrationsClient } from './mocks'; -export const SiemRuleMigrationsClient = MockSiemRuleMigrationsClient; +import { MockRuleMigrationsDataClient } from './mocks'; +export const RuleMigrationsDataClient = MockRuleMigrationsDataClient; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_resources_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_resources_client.ts new file mode 100644 index 000000000000..96fc5b47fb1c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_resources_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRuleMigrationsDataResourcesClient } from './mocks'; +export const RuleMigrationsDataResourcesClient = MockRuleMigrationsDataResourcesClient; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_rules_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_rules_client.ts new file mode 100644 index 000000000000..a7a6a29c17cb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_rules_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRuleMigrationsDataRulesClient } from './mocks'; +export const RuleMigrationsDataRulesClient = MockRuleMigrationsDataRulesClient; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_service.ts new file mode 100644 index 000000000000..2b2900e213bb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/__mocks__/rule_migrations_data_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRuleMigrationsDataService } from './mocks'; +export const RuleMigrationsDataService = MockRuleMigrationsDataService; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts new file mode 100644 index 000000000000..8b5a81e2bc99 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_base_client.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import assert from 'assert'; +import type { SearchHit, SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { Stored } from '../types'; +import type { IndexNameProvider } from './rule_migrations_data_client'; + +export class RuleMigrationsDataBaseClient { + constructor( + protected getIndexName: IndexNameProvider, + protected username: string, + protected esClient: ElasticsearchClient, + protected logger: Logger + ) {} + + protected processResponseHits( + response: SearchResponse, + override?: Partial + ): Array> { + return this.processHits(response.hits.hits, override); + } + + protected processHits( + hits: Array> = [], + override: Partial = {} + ): Array> { + return hits.map(({ _id, _source }) => { + assert(_id, 'document should have _id'); + assert(_source, 'document should have _source'); + return { ..._source, ...override, _id }; + }); + } + + protected getTotalHits(response: SearchResponse) { + return typeof response.hits.total === 'number' + ? response.hits.total + : response.hits.total?.value ?? 0; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts new file mode 100644 index 000000000000..fe682ceeec78 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_client.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { + RuleMigration, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import { RuleMigrationsDataRulesClient } from './rule_migrations_data_rules_client'; +import { RuleMigrationsDataResourcesClient } from './rule_migrations_data_resources_client'; +import type { AdapterId } from './rule_migrations_data_service'; + +export type CreateRuleMigrationInput = Omit; +export type RuleMigrationDataStats = Omit; +export type RuleMigrationAllDataStats = Array; + +export type IndexNameProvider = () => Promise; +export type IndexNameProviders = Record; + +export class RuleMigrationsDataClient { + public readonly rules: RuleMigrationsDataRulesClient; + public readonly resources: RuleMigrationsDataResourcesClient; + + constructor( + indexNameProviders: IndexNameProviders, + username: string, + esClient: ElasticsearchClient, + logger: Logger + ) { + this.rules = new RuleMigrationsDataRulesClient( + indexNameProviders.rules, + username, + esClient, + logger + ); + this.resources = new RuleMigrationsDataResourcesClient( + indexNameProviders.resources, + username, + esClient, + logger + ); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts new file mode 100644 index 000000000000..66b463da79cc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_resources_client.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { sha256 } from 'js-sha256'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { + RuleMigrationResource, + RuleMigrationResourceType, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { StoredRuleMigrationResource } from '../types'; +import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client'; + +/* BULK_MAX_SIZE defines the number to break down the bulk operations by. + * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. + */ +const BULK_MAX_SIZE = 500 as const; + +export class RuleMigrationsDataResourcesClient extends RuleMigrationsDataBaseClient { + public async upsert(resources: RuleMigrationResource[]): Promise { + const index = await this.getIndexName(); + + let resourcesSlice: RuleMigrationResource[]; + while ((resourcesSlice = resources.splice(0, BULK_MAX_SIZE)).length > 0) { + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: resourcesSlice.flatMap((resource) => [ + { update: { _id: this.createId(resource), _index: index } }, + { doc: resource, doc_as_upsert: true }, + ]), + }) + .catch((error) => { + this.logger.error(`Error upsert resources: ${error.message}`); + throw error; + }); + } + } + + public async get( + migrationId: string, + type?: RuleMigrationResourceType, + names?: string[] + ): Promise { + const index = await this.getIndexName(); + + const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; + if (type) { + filter.push({ term: { type } }); + } + if (names) { + filter.push({ terms: { name: names } }); + } + const query = { bool: { filter } }; + + return this.esClient + .search({ index, query }) + .then(this.processResponseHits.bind(this)) + .catch((error) => { + this.logger.error(`Error searching resources: ${error.message}`); + throw error; + }); + } + + private createId(resource: RuleMigrationResource): string { + const key = `${resource.migration_id}-${resource.type}-${resource.name}`; + return sha256.create().update(key).hex(); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts similarity index 62% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts index 83808901a0bd..feedff65343d 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_rules_client.ts @@ -5,16 +5,12 @@ * 2.0. */ -import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; -import assert from 'assert'; import type { AggregationsFilterAggregate, AggregationsMaxAggregate, AggregationsStringTermsAggregate, AggregationsStringTermsBucket, QueryDslQueryContainer, - SearchHit, - SearchResponse, } from '@elastic/elasticsearch/lib/api/types'; import type { StoredRuleMigration } from '../types'; import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; @@ -22,53 +18,56 @@ import type { RuleMigration, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import { RuleMigrationsDataBaseClient } from './rule_migrations_data_base_client'; export type CreateRuleMigrationInput = Omit; export type RuleMigrationDataStats = Omit; export type RuleMigrationAllDataStats = Array; -export class RuleMigrationsDataClient { - constructor( - private dataStreamNamePromise: Promise, - private currentUser: AuthenticatedUser, - private esClient: ElasticsearchClient, - private logger: Logger - ) {} +/* BULK_MAX_SIZE defines the number to break down the bulk operations by. + * The 500 number was chosen as a reasonable number to avoid large payloads. It can be adjusted if needed. + */ +const BULK_MAX_SIZE = 500 as const; +export class RuleMigrationsDataRulesClient extends RuleMigrationsDataBaseClient { /** Indexes an array of rule migrations to be processed */ async create(ruleMigrations: CreateRuleMigrationInput[]): Promise { - const index = await this.dataStreamNamePromise; - await this.esClient - .bulk({ - refresh: 'wait_for', - operations: ruleMigrations.flatMap((ruleMigration) => [ - { create: { _index: index } }, - { - ...ruleMigration, - '@timestamp': new Date().toISOString(), - status: SiemMigrationStatus.PENDING, - created_by: this.currentUser.username, - }, - ]), - }) - .catch((error) => { - this.logger.error(`Error creating rule migrations: ${error.message}`); - throw error; - }); + const index = await this.getIndexName(); + + let ruleMigrationsSlice: CreateRuleMigrationInput[]; + while ((ruleMigrationsSlice = ruleMigrations.splice(0, BULK_MAX_SIZE)).length) { + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: ruleMigrationsSlice.flatMap((ruleMigration) => [ + { create: { _index: index } }, + { + ...ruleMigration, + '@timestamp': new Date().toISOString(), + status: SiemMigrationStatus.PENDING, + created_by: this.username, + }, + ]), + }) + .catch((error) => { + this.logger.error(`Error creating rule migrations: ${error.message}`); + throw error; + }); + } } /** Retrieves an array of rule documents of a specific migrations */ - async getRules(migrationId: string): Promise { - const index = await this.dataStreamNamePromise; + async get(migrationId: string): Promise { + const index = await this.getIndexName(); const query = this.getFilterQuery(migrationId); const storedRuleMigrations = await this.esClient .search({ index, query, sort: '_doc' }) + .then(this.processResponseHits.bind(this)) .catch((error) => { - this.logger.error(`Error searching getting rule migrations: ${error.message}`); + this.logger.error(`Error searching rule migrations: ${error.message}`); throw error; - }) - .then((response) => this.processHits(response.hits.hits)); + }); return storedRuleMigrations; } @@ -79,30 +78,26 @@ export class RuleMigrationsDataClient { * - Multiple tasks should not process the same migration simultaneously. */ async takePending(migrationId: string, size: number): Promise { - const index = await this.dataStreamNamePromise; + const index = await this.getIndexName(); const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PENDING); const storedRuleMigrations = await this.esClient .search({ index, query, sort: '_doc', size }) + .then((response) => + this.processResponseHits(response, { status: SiemMigrationStatus.PROCESSING }) + ) .catch((error) => { - this.logger.error(`Error searching for rule migrations: ${error.message}`); + this.logger.error(`Error searching rule migrations: ${error.message}`); throw error; - }) - .then((response) => - this.processHits(response.hits.hits, { status: SiemMigrationStatus.PROCESSING }) - ); + }); await this.esClient .bulk({ refresh: 'wait_for', - operations: storedRuleMigrations.flatMap(({ _id, _index, status }) => [ - { update: { _id, _index } }, + operations: storedRuleMigrations.flatMap(({ _id, status }) => [ + { update: { _id, _index: index } }, { - doc: { - status, - updated_by: this.currentUser.username, - updated_at: new Date().toISOString(), - }, + doc: { status, updated_by: this.username, updated_at: new Date().toISOString() }, }, ]), }) @@ -117,65 +112,63 @@ export class RuleMigrationsDataClient { } /** Updates one rule migration with the provided data and sets the status to `completed` */ - async saveFinished({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + async saveCompleted({ _id, ...ruleMigration }: StoredRuleMigration): Promise { + const index = await this.getIndexName(); const doc = { ...ruleMigration, status: SiemMigrationStatus.COMPLETED, - updated_by: this.currentUser.username, + updated_by: this.username, updated_at: new Date().toISOString(), }; - await this.esClient - .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) - .catch((error) => { - this.logger.error(`Error updating rule migration status to completed: ${error.message}`); - throw error; - }); + await this.esClient.update({ index, id: _id, doc, refresh: 'wait_for' }).catch((error) => { + this.logger.error(`Error updating rule migration status to completed: ${error.message}`); + throw error; + }); } /** Updates one rule migration with the provided data and sets the status to `failed` */ - async saveError({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + async saveError({ _id, ...ruleMigration }: StoredRuleMigration): Promise { + const index = await this.getIndexName(); const doc = { ...ruleMigration, status: SiemMigrationStatus.FAILED, - updated_by: this.currentUser.username, + updated_by: this.username, updated_at: new Date().toISOString(), }; - await this.esClient - .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) - .catch((error) => { - this.logger.error(`Error updating rule migration status to completed: ${error.message}`); - throw error; - }); + await this.esClient.update({ index, id: _id, doc, refresh: 'wait_for' }).catch((error) => { + this.logger.error(`Error updating rule migration status to failed: ${error.message}`); + throw error; + }); } /** Updates all the rule migration with the provided id with status `processing` back to `pending` */ async releaseProcessing(migrationId: string): Promise { - const index = await this.dataStreamNamePromise; - const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PROCESSING); - const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; - await this.esClient.updateByQuery({ index, query, script, refresh: false }).catch((error) => { - this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); - throw error; - }); + return this.updateStatus( + migrationId, + SiemMigrationStatus.PROCESSING, + SiemMigrationStatus.PENDING + ); } - /** Updates all the rule migration with the provided id with status `processing` or `failed` back to `pending` */ - async releaseProcessable(migrationId: string): Promise { - const index = await this.dataStreamNamePromise; - const query = this.getFilterQuery(migrationId, [ - SiemMigrationStatus.PROCESSING, - SiemMigrationStatus.FAILED, - ]); - const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; - await this.esClient.updateByQuery({ index, query, script, refresh: true }).catch((error) => { - this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); + /** Updates all the rule migration with the provided id and with status `statusToQuery` to `statusToUpdate` */ + async updateStatus( + migrationId: string, + statusToQuery: SiemMigrationStatus | SiemMigrationStatus[] | undefined, + statusToUpdate: SiemMigrationStatus, + { refresh = false }: { refresh?: boolean } = {} + ): Promise { + const index = await this.getIndexName(); + const query = this.getFilterQuery(migrationId, statusToQuery); + const script = { source: `ctx._source['status'] = '${statusToUpdate}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh }).catch((error) => { + this.logger.error(`Error updating rule migrations status: ${error.message}`); throw error; }); } /** Retrieves the stats for the rule migrations with the provided id */ async getStats(migrationId: string): Promise { - const index = await this.dataStreamNamePromise; + const index = await this.getIndexName(); const query = this.getFilterQuery(migrationId); const aggregations = { pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } }, @@ -206,7 +199,7 @@ export class RuleMigrationsDataClient { /** Retrieves the stats for all the rule migrations aggregated by migration id */ async getAllStats(): Promise { - const index = await this.dataStreamNamePromise; + const index = await this.getIndexName(); const aggregations = { migrationIds: { terms: { field: 'migration_id' }, @@ -255,21 +248,4 @@ export class RuleMigrationsDataClient { } return { bool: { filter } }; } - - private processHits( - hits: Array>, - override: Partial = {} - ): StoredRuleMigration[] { - return hits.map(({ _id, _index, _source }) => { - assert(_id, 'RuleMigration document should have _id'); - assert(_source, 'RuleMigration document should have _source'); - return { ..._source, ...override, _id, _index }; - }); - } - - private getTotalHits(response: SearchResponse) { - return typeof response.hits.total === 'number' - ? response.hits.total - : response.hits.total?.value ?? 0; - } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts new file mode 100644 index 000000000000..e738bd85e2f1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.test.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { INDEX_PATTERN, RuleMigrationsDataService } from './rule_migrations_data_service'; +import { Subject } from 'rxjs'; +import type { InstallParams } from '@kbn/index-adapter'; +import { IndexPatternAdapter } from '@kbn/index-adapter'; +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { securityServiceMock } from '@kbn/core-security-server-mocks'; +import type { IndexNameProviders } from './rule_migrations_data_client'; + +jest.mock('@kbn/index-adapter'); + +// This mock is required to have a way to await the index pattern name promise +let mockIndexNameProviders: IndexNameProviders; +jest.mock('./rule_migrations_data_client', () => ({ + RuleMigrationsDataClient: jest.fn((indexNameProviders: IndexNameProviders) => { + mockIndexNameProviders = indexNameProviders; + }), +})); + +const MockedIndexPatternAdapter = IndexPatternAdapter as unknown as jest.MockedClass< + typeof IndexPatternAdapter +>; + +const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; + +describe('SiemRuleMigrationsDataService', () => { + const kibanaVersion = '8.16.0'; + const logger = loggingSystemMock.createLogger(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('constructor', () => { + it('should create IndexPatternAdapters', () => { + new RuleMigrationsDataService(logger, kibanaVersion); + expect(MockedIndexPatternAdapter).toHaveBeenCalledTimes(2); + }); + + it('should create component templates', () => { + new RuleMigrationsDataService(logger, kibanaVersion); + const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; + expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: `${INDEX_PATTERN}-rules` }) + ); + expect(indexPatternAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: `${INDEX_PATTERN}-resources` }) + ); + }); + + it('should create index templates', () => { + new RuleMigrationsDataService(logger, kibanaVersion); + const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; + expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: `${INDEX_PATTERN}-rules` }) + ); + expect(indexPatternAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: `${INDEX_PATTERN}-resources` }) + ); + }); + }); + + describe('install', () => { + it('should install index pattern', async () => { + const index = new RuleMigrationsDataService(logger, kibanaVersion); + const params: Omit = { + esClient, + pluginStop$: new Subject(), + }; + await index.install(params); + const [indexPatternAdapter] = MockedIndexPatternAdapter.mock.instances; + expect(indexPatternAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); + }); + }); + + describe('createClient', () => { + const currentUser = securityServiceMock.createMockAuthenticatedUser(); + const createClientParams = { spaceId: 'space1', currentUser, esClient }; + + it('should install space index pattern', async () => { + const index = new RuleMigrationsDataService(logger, kibanaVersion); + const params: InstallParams = { + esClient, + logger: loggerMock.create(), + pluginStop$: new Subject(), + }; + const [rulesIndexPatternAdapter, resourcesIndexPatternAdapter] = + MockedIndexPatternAdapter.mock.instances; + (rulesIndexPatternAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); + + await index.install(params); + index.createClient(createClientParams); + + await mockIndexNameProviders.rules(); + await mockIndexNameProviders.resources(); + + expect(rulesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); + expect(rulesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); + + expect(resourcesIndexPatternAdapter.createIndex).toHaveBeenCalledWith('space1'); + expect(resourcesIndexPatternAdapter.getIndexName).toHaveBeenCalledWith('space1'); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts new file mode 100644 index 000000000000..c19a89cefd81 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_data_service.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; +import { IndexPatternAdapter, type FieldMap, type InstallParams } from '@kbn/index-adapter'; +import { + ruleMigrationsFieldMap, + ruleMigrationResourcesFieldMap, +} from './rule_migrations_field_maps'; +import type { IndexNameProvider, IndexNameProviders } from './rule_migrations_data_client'; +import { RuleMigrationsDataClient } from './rule_migrations_data_client'; + +const TOTAL_FIELDS_LIMIT = 2500; +export const INDEX_PATTERN = '.kibana-siem-rule-migrations'; + +export type AdapterId = 'rules' | 'resources'; + +interface CreateClientParams { + spaceId: string; + currentUser: AuthenticatedUser; + esClient: ElasticsearchClient; +} + +export class RuleMigrationsDataService { + private readonly adapters: Record; + + constructor(private logger: Logger, private kibanaVersion: string) { + this.adapters = { + rules: this.createAdapter({ id: 'rules', fieldMap: ruleMigrationsFieldMap }), + resources: this.createAdapter({ id: 'resources', fieldMap: ruleMigrationResourcesFieldMap }), + }; + } + + private createAdapter({ id, fieldMap }: { id: AdapterId; fieldMap: FieldMap }) { + const name = `${INDEX_PATTERN}-${id}`; + const adapter = new IndexPatternAdapter(name, { + kibanaVersion: this.kibanaVersion, + totalFieldsLimit: TOTAL_FIELDS_LIMIT, + }); + adapter.setComponentTemplate({ name, fieldMap }); + adapter.setIndexTemplate({ name, componentTemplateRefs: [name] }); + return adapter; + } + + public async install(params: Omit): Promise { + await Promise.all([ + this.adapters.rules.install({ ...params, logger: this.logger }), + this.adapters.resources.install({ ...params, logger: this.logger }), + ]); + } + + public createClient({ spaceId, currentUser, esClient }: CreateClientParams) { + const indexNameProviders: IndexNameProviders = { + rules: this.createIndexNameProvider('rules', spaceId), + resources: this.createIndexNameProvider('resources', spaceId), + }; + + return new RuleMigrationsDataClient( + indexNameProviders, + currentUser.username, + esClient, + this.logger + ); + } + + private createIndexNameProvider(adapter: AdapterId, spaceId: string): IndexNameProvider { + return async () => { + await this.adapters[adapter].createIndex(spaceId); // This will resolve instantly when the index is already created + return this.adapters[adapter].getIndexName(spaceId); + }; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts similarity index 76% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts index a65cd45b832e..8dbccb61d535 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data/rule_migrations_field_maps.ts @@ -6,7 +6,10 @@ */ import type { FieldMap, SchemaFieldMapKeys } from '@kbn/data-stream-adapter'; -import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { + RuleMigration, + RuleMigrationResource, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; export const ruleMigrationsFieldMap: FieldMap> = { '@timestamp': { type: 'date', required: false }, @@ -34,3 +37,13 @@ export const ruleMigrationsFieldMap: FieldMap> updated_at: { type: 'date', required: false }, updated_by: { type: 'keyword', required: false }, }; + +export const ruleMigrationResourcesFieldMap: FieldMap> = { + migration_id: { type: 'keyword', required: true }, + type: { type: 'keyword', required: true }, + name: { type: 'keyword', required: true }, + content: { type: 'text', required: false }, + metadata: { type: 'object', required: false }, + updated_at: { type: 'date', required: false }, + updated_by: { type: 'keyword', required: false }, +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts deleted file mode 100644 index 1d9a181d2de5..000000000000 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const mockIndexName = 'mocked_data_stream_name'; -export const mockInstall = jest.fn().mockResolvedValue(undefined); -export const mockCreateClient = jest.fn().mockReturnValue({}); - -export const MockRuleMigrationsDataStream = jest.fn().mockImplementation(() => ({ - install: mockInstall, - createClient: mockCreateClient, -})); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts deleted file mode 100644 index 467d26a38094..000000000000 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { RuleMigrationsDataStream } from './rule_migrations_data_stream'; -import { Subject } from 'rxjs'; -import type { InstallParams } from '@kbn/data-stream-adapter'; -import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; -import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; -import { loggerMock } from '@kbn/logging-mocks'; -import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; -import { securityServiceMock } from '@kbn/core-security-server-mocks'; - -jest.mock('@kbn/data-stream-adapter'); - -// This mock is required to have a way to await the data stream name promise -const mockDataStreamNamePromise = jest.fn(); -jest.mock('./rule_migrations_data_client', () => ({ - RuleMigrationsDataClient: jest.fn((dataStreamNamePromise: Promise) => { - mockDataStreamNamePromise.mockReturnValue(dataStreamNamePromise); - }), -})); - -const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass< - typeof DataStreamSpacesAdapter ->; - -const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; - -describe('SiemRuleMigrationsDataStream', () => { - const kibanaVersion = '8.16.0'; - const logger = loggingSystemMock.createLogger(); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('constructor', () => { - it('should create DataStreamSpacesAdapter', () => { - new RuleMigrationsDataStream(logger, kibanaVersion); - expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1); - }); - - it('should create component templates', () => { - new RuleMigrationsDataStream(logger, kibanaVersion); - const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith( - expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) - ); - }); - - it('should create index templates', () => { - new RuleMigrationsDataStream(logger, kibanaVersion); - const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith( - expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) - ); - }); - }); - - describe('install', () => { - it('should install data stream', async () => { - const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); - const params: Omit = { - esClient, - pluginStop$: new Subject(), - }; - await dataStream.install(params); - const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); - }); - - it('should log error', async () => { - const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); - const params: Omit = { - esClient, - pluginStop$: new Subject(), - }; - const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - const error = new Error('test-error'); - (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); - - await dataStream.install(params); - expect(logger.error).toHaveBeenCalledWith(expect.any(String), error); - }); - }); - - describe('createClient', () => { - const currentUser = securityServiceMock.createMockAuthenticatedUser(); - const createClientParams = { spaceId: 'space1', currentUser, esClient }; - - it('should install space data stream', async () => { - const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); - const params: InstallParams = { - esClient, - logger: loggerMock.create(), - pluginStop$: new Subject(), - }; - const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - (dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); - - await dataStream.install(params); - dataStream.createClient(createClientParams); - await mockDataStreamNamePromise(); - - expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1'); - expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1'); - }); - - it('should not install space data stream if install not executed', async () => { - const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); - await expect(async () => { - dataStream.createClient(createClientParams); - await mockDataStreamNamePromise(); - }).rejects.toThrowError(); - }); - - it('should throw error if main install had error', async () => { - const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); - const params: InstallParams = { - esClient, - logger: loggerMock.create(), - pluginStop$: new Subject(), - }; - const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - const error = new Error('test-error'); - (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); - await dataStream.install(params); - - await expect(async () => { - dataStream.createClient(createClientParams); - await mockDataStreamNamePromise(); - }).rejects.toThrowError(error); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts deleted file mode 100644 index a5855cefb132..000000000000 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DataStreamSpacesAdapter, type InstallParams } from '@kbn/data-stream-adapter'; -import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; -import { ruleMigrationsFieldMap } from './rule_migrations_field_map'; -import { RuleMigrationsDataClient } from './rule_migrations_data_client'; - -const TOTAL_FIELDS_LIMIT = 2500; - -const DATA_STREAM_NAME = '.kibana.siem-rule-migrations'; - -interface RuleMigrationsDataStreamCreateClientParams { - spaceId: string; - currentUser: AuthenticatedUser; - esClient: ElasticsearchClient; -} - -export class RuleMigrationsDataStream { - private readonly dataStreamAdapter: DataStreamSpacesAdapter; - private installPromise?: Promise; - - constructor(private logger: Logger, kibanaVersion: string) { - this.dataStreamAdapter = new DataStreamSpacesAdapter(DATA_STREAM_NAME, { - kibanaVersion, - totalFieldsLimit: TOTAL_FIELDS_LIMIT, - }); - this.dataStreamAdapter.setComponentTemplate({ - name: DATA_STREAM_NAME, - fieldMap: ruleMigrationsFieldMap, - }); - - this.dataStreamAdapter.setIndexTemplate({ - name: DATA_STREAM_NAME, - componentTemplateRefs: [DATA_STREAM_NAME], - }); - } - - async install(params: Omit) { - try { - this.installPromise = this.dataStreamAdapter.install({ ...params, logger: this.logger }); - await this.installPromise; - } catch (err) { - this.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err); - } - } - - createClient({ - spaceId, - currentUser, - esClient, - }: RuleMigrationsDataStreamCreateClientParams): RuleMigrationsDataClient { - const dataStreamNamePromise = this.installSpace(spaceId); - return new RuleMigrationsDataClient(dataStreamNamePromise, currentUser, esClient, this.logger); - } - - // Installs the data stream for the specific space. it will only install if it hasn't been installed yet. - // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed. - private async installSpace(spaceId: string): Promise { - if (!this.installPromise) { - throw new Error('Siem rule migrations data stream not installed'); - } - // wait for install to complete, may reject if install failed, routes should handle this - await this.installPromise; - let dataStreamName = await this.dataStreamAdapter.getInstalledSpaceName(spaceId); - if (!dataStreamName) { - dataStreamName = await this.dataStreamAdapter.installSpace(spaceId); - } - return dataStreamName; - } -} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts index 5c611d85e046..2e701f0fc7b3 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts @@ -10,19 +10,21 @@ import { httpServerMock, securityServiceMock, } from '@kbn/core/server/mocks'; -import { SiemRuleMigrationsService } from './siem_rule_migrations_service'; +import { + SiemRuleMigrationsService, + type SiemRuleMigrationsCreateClientParams, +} from './siem_rule_migrations_service'; import { Subject } from 'rxjs'; import { - MockRuleMigrationsDataStream, + MockRuleMigrationsDataService, mockInstall, - mockCreateClient, -} from './data_stream/__mocks__/mocks'; -import type { SiemRuleMigrationsCreateClientParams } from './types'; + mockCreateClient as mockDataCreateClient, +} from './data/__mocks__/mocks'; +import { mockCreateClient as mockTaskCreateClient, mockStopAll } from './task/__mocks__/mocks'; +import { waitFor } from '@testing-library/dom'; -jest.mock('./data_stream/rule_migrations_data_stream'); -jest.mock('./task/rule_migrations_task_runner', () => ({ - RuleMigrationsTaskRunner: jest.fn(), -})); +jest.mock('./data/rule_migrations_data_service'); +jest.mock('./task/rule_migrations_task_service'); describe('SiemRuleMigrationsService', () => { let ruleMigrationsService: SiemRuleMigrationsService; @@ -30,16 +32,17 @@ describe('SiemRuleMigrationsService', () => { const esClusterClient = elasticsearchServiceMock.createClusterClient(); const currentUser = securityServiceMock.createMockAuthenticatedUser(); - const logger = loggingSystemMock.createLogger(); + const loggerFactory = loggingSystemMock.create(); + const logger = loggerFactory.get('siemRuleMigrations'); const pluginStop$ = new Subject(); beforeEach(() => { jest.clearAllMocks(); - ruleMigrationsService = new SiemRuleMigrationsService(logger, kibanaVersion); + ruleMigrationsService = new SiemRuleMigrationsService(loggerFactory, kibanaVersion); }); it('should instantiate the rule migrations data stream adapter', () => { - expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith(logger, kibanaVersion); + expect(MockRuleMigrationsDataService).toHaveBeenCalledWith(logger, kibanaVersion); }); describe('when setup is called', () => { @@ -51,6 +54,16 @@ describe('SiemRuleMigrationsService', () => { pluginStop$, }); }); + + it('should log error when data installation fails', async () => { + const error = 'Failed to install'; + mockInstall.mockRejectedValueOnce(error); + ruleMigrationsService.setup({ esClusterClient, pluginStop$ }); + + await waitFor(() => { + expect(logger.error).toHaveBeenCalledWith('Error installing data service.', error); + }); + }); }); describe('when createClient is called', () => { @@ -77,12 +90,20 @@ describe('SiemRuleMigrationsService', () => { ruleMigrationsService.setup({ esClusterClient, pluginStop$ }); }); - it('should call installSpace', () => { + it('should create data client', () => { ruleMigrationsService.createClient(createClientParams); - expect(mockCreateClient).toHaveBeenCalledWith({ + expect(mockDataCreateClient).toHaveBeenCalledWith({ spaceId: createClientParams.spaceId, currentUser: createClientParams.currentUser, - esClient: esClusterClient.asScoped().asCurrentUser, + esClient: esClusterClient.asInternalUser, + }); + }); + + it('should create task client', () => { + ruleMigrationsService.createClient(createClientParams); + expect(mockTaskCreateClient).toHaveBeenCalledWith({ + currentUser: createClientParams.currentUser, + dataClient: mockDataCreateClient(), }); }); @@ -93,5 +114,12 @@ describe('SiemRuleMigrationsService', () => { expect(client).toHaveProperty('task'); }); }); + + describe('stop', () => { + it('should call taskService stopAll', () => { + ruleMigrationsService.stop(); + expect(mockStopAll).toHaveBeenCalled(); + }); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index 1bf9dcf11fd9..d9f4a1c5249c 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -6,32 +6,54 @@ */ import assert from 'assert'; -import type { IClusterClient, Logger } from '@kbn/core/server'; -import { RuleMigrationsDataStream } from './data_stream/rule_migrations_data_stream'; +import type { Subject } from 'rxjs'; import type { - SiemRulesMigrationsSetupParams, - SiemRuleMigrationsCreateClientParams, - SiemRuleMigrationsClient, -} from './types'; -import { RuleMigrationsTaskRunner } from './task/rule_migrations_task_runner'; + AuthenticatedUser, + LoggerFactory, + IClusterClient, + KibanaRequest, + Logger, +} from '@kbn/core/server'; +import { RuleMigrationsDataService } from './data/rule_migrations_data_service'; +import type { RuleMigrationsDataClient } from './data/rule_migrations_data_client'; +import type { RuleMigrationsTaskClient } from './task/rule_migrations_task_client'; +import { RuleMigrationsTaskService } from './task/rule_migrations_task_service'; + +export interface SiemRulesMigrationsSetupParams { + esClusterClient: IClusterClient; + pluginStop$: Subject; + tasksTimeoutMs?: number; +} + +export interface SiemRuleMigrationsCreateClientParams { + request: KibanaRequest; + currentUser: AuthenticatedUser | null; + spaceId: string; +} + +export interface SiemRuleMigrationsClient { + data: RuleMigrationsDataClient; + task: RuleMigrationsTaskClient; +} export class SiemRuleMigrationsService { - private rulesDataStream: RuleMigrationsDataStream; + private dataService: RuleMigrationsDataService; private esClusterClient?: IClusterClient; - private taskRunner: RuleMigrationsTaskRunner; + private taskService: RuleMigrationsTaskService; + private logger: Logger; - constructor(private logger: Logger, kibanaVersion: string) { - this.rulesDataStream = new RuleMigrationsDataStream(this.logger, kibanaVersion); - this.taskRunner = new RuleMigrationsTaskRunner(this.logger); + constructor(logger: LoggerFactory, kibanaVersion: string) { + this.logger = logger.get('siemRuleMigrations'); + this.dataService = new RuleMigrationsDataService(this.logger, kibanaVersion); + this.taskService = new RuleMigrationsTaskService(this.logger); } setup({ esClusterClient, ...params }: SiemRulesMigrationsSetupParams) { this.esClusterClient = esClusterClient; const esClient = esClusterClient.asInternalUser; - this.rulesDataStream.install({ ...params, esClient }).catch((err) => { - this.logger.error(`Error installing data stream for rule migrations: ${err.message}`); - throw err; + this.dataService.install({ ...params, esClient }).catch((err) => { + this.logger.error('Error installing data service.', err); }); } @@ -43,29 +65,14 @@ export class SiemRuleMigrationsService { assert(currentUser, 'Current user must be authenticated'); assert(this.esClusterClient, 'ES client not available, please call setup first'); - const esClient = this.esClusterClient.asScoped(request).asCurrentUser; - const dataClient = this.rulesDataStream.createClient({ spaceId, currentUser, esClient }); + const esClient = this.esClusterClient.asInternalUser; + const dataClient = this.dataService.createClient({ spaceId, currentUser, esClient }); + const taskClient = this.taskService.createClient({ currentUser, dataClient }); - return { - data: dataClient, - task: { - start: (params) => { - return this.taskRunner.start({ ...params, currentUser, dataClient }); - }, - stop: (migrationId) => { - return this.taskRunner.stop({ migrationId, dataClient }); - }, - getStats: async (migrationId) => { - return this.taskRunner.getStats({ migrationId, dataClient }); - }, - getAllStats: async () => { - return this.taskRunner.getAllStats({ dataClient }); - }, - }, - }; + return { data: dataClient, task: taskClient }; } stop() { - this.taskRunner.stopAll(); + this.taskService.stopAll(); } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts new file mode 100644 index 000000000000..1f463c8417b9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/mocks.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const mockRuleMigrationsTaskClient = { + start: jest.fn().mockResolvedValue({ started: true }), + stop: jest.fn().mockResolvedValue({ stopped: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), +}; + +export const MockRuleMigrationsTaskClient = jest + .fn() + .mockImplementation(() => mockRuleMigrationsTaskClient); + +// Rule migrations task service +export const mockStopAll = jest.fn(); +export const mockCreateClient = jest.fn().mockReturnValue(mockRuleMigrationsTaskClient); + +export const MockRuleMigrationsTaskService = jest.fn().mockImplementation(() => ({ + createClient: mockCreateClient, + stopAll: mockStopAll, +})); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/rule_migrations_data_stream.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/rule_migrations_task_client.ts similarity index 66% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/rule_migrations_data_stream.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/rule_migrations_task_client.ts index eb04aec6c60e..b4eac7ccf246 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/rule_migrations_data_stream.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/rule_migrations_task_client.ts @@ -5,5 +5,5 @@ * 2.0. */ -import { MockRuleMigrationsDataStream } from './mocks'; -export const RuleMigrationsDataStream = MockRuleMigrationsDataStream; +import { MockRuleMigrationsTaskClient } from './mocks'; +export const RuleMigrationsTaskClient = MockRuleMigrationsTaskClient; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/rule_migrations_task_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/rule_migrations_task_service.ts new file mode 100644 index 000000000000..04da946c083c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/__mocks__/rule_migrations_task_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRuleMigrationsTaskService } from './mocks'; +export const RuleMigrationsTaskService = MockRuleMigrationsTaskService; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index a44197d82850..0db4bbe18fee 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -15,11 +15,18 @@ export function getRuleMigrationAgent({ model, inferenceClient, prebuiltRulesMap, + resourceRetriever, connectorId, logger, }: MigrateRuleGraphParams) { const matchPrebuiltRuleNode = getMatchPrebuiltRuleNode({ model, prebuiltRulesMap, logger }); - const translationNode = getTranslateQueryNode({ inferenceClient, connectorId, logger }); + const translationNode = getTranslateQueryNode({ + model, + inferenceClient, + resourceRetriever, + connectorId, + logger, + }); const translateRuleGraph = new StateGraph(migrateRuleState) // Nodes diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts deleted file mode 100644 index 0b97faf7dc96..000000000000 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { MigrateRuleState } from '../../types'; - -export const getEsqlTranslationPrompt = ( - state: MigrateRuleState -): string => `You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk to Elastic Security. -Below you will find Splunk rule information: the title, description and the SPL (Search Processing Language) query. -Your goal is to translate the SPL query into an equivalent Elastic Security Query Language (ES|QL) query. - -Guidelines: -- Start the translation process by analyzing the SPL query and identifying the key components. -- Always use logs* index pattern for the ES|QL translated query. -- If, in the SPL query, you find a lookup list or macro that, based only on its name, you can not translate with confidence to ES|QL, mention it in the summary and -add a placeholder in the query with the format [macro:(parameters)] or [lookup:] including the [] keys, example: [macro:my_macro(first_param,second_param)] or [lookup:my_lookup]. - -The output will be parsed and should contain: -- First, the ES|QL query inside an \`\`\`esql code block. -- At the end, the summary of the translation process followed in markdown, starting with "## Migration Summary". - -This is the Splunk rule information: - -<> -${state.original_rule.title} -<> - -<> -${state.original_rule.description} -<> - -<> -${state.original_rule.query} -<> -`; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/esql_translation_prompt.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/esql_translation_prompt.ts new file mode 100644 index 000000000000..05e3c5c63bbe --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/esql_translation_prompt.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MigrateRuleState } from '../../../types'; + +export const getEsqlTranslationPrompt = (state: MigrateRuleState, query: string): string => { + return `You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk to Elastic Security. +Your goal is to translate the SPL query into an equivalent Elastic Security Query Language (ES|QL) query. + +## Splunk rule Information provided: +- Below you will find Splunk rule information: the title (<>), the description (<<DESCRIPTION>>), and the SPL (Search Processing Language) query (<<SPL_QUERY>>). +- Use all the information to analyze the intent of the rule, in order to translate into an equivalent ES|QL rule. +- The fields in the Splunk query may not be the same as in the Elastic Common Schema (ECS), so you may need to map them accordingly. + +## Guidelines: +- Analyze the SPL query and identify the key components. +- Translate the SPL query into an equivalent ES|QL query using ECS (Elastic Common Schema) field names. +- Always use logs* index pattern for the ES|QL translated query. +- If, in the SPL query, you find a lookup list or macro call, mention it in the summary and add a placeholder in the query with the format [macro:<macro_name>(argumentCount)] or [lookup:<lookup_name>] including the [] keys, + - Examples: + - \`get_duration(firstDate,secondDate)\` -> [macro:get_duration(2)] + - lookup dns_domains.csv -> [lookup:dns_domains.csv]. + +## The output will be parsed and must contain: +- First, the ES|QL query inside an \`\`\`esql code block. +- At the end, the summary of the translation process followed in markdown, starting with "## Migration Summary". + +Find the Splunk rule information below: + +<<TITLE>> +${state.original_rule.title} +<> + +<> +${state.original_rule.description} +<> + +<> +${query} +<> +`; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/replace_resources_prompt.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/replace_resources_prompt.ts new file mode 100644 index 000000000000..45b7a36dd292 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompts/replace_resources_prompt.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleMigrationResources } from '../../../../util/rule_resource_retriever'; +import type { MigrateRuleState } from '../../../types'; + +const getResourcesContext = (resources: RuleMigrationResources): string => { + const resourcesContext = []; + if (resources.macro?.length) { + const macrosSummary = resources.macro + .map((macro) => `\`${macro.name}\`: ${macro.content}`) + .join('\n'); + resourcesContext.push('<>', macrosSummary, '<>'); + } + if (resources.list?.length) { + const lookupsSummary = resources.list + .map((list) => `lookup ${list.name}: ${list.content}`) + .join('\n'); + resourcesContext.push('<>', lookupsSummary, '<>'); + } + return resourcesContext.join('\n'); +}; + +export const getReplaceQueryResourcesPrompt = ( + state: MigrateRuleState, + resources: RuleMigrationResources +): string => { + const resourcesContext = getResourcesContext(resources); + return `You are an agent expert in Splunk SPL (Search Processing Language). +Your task is to inline a set of macros and lookup table values in a SPL query. + +## Guidelines: + +- You will be provided with a SPL query and also the resources reference with the values of macros and lookup tables. +- You have to replace the macros and lookup tables in the SPL query with their actual values. +- The original and modified queries must be equivalent. +- Macros names have the number of arguments in parentheses, e.g., \`macroName(2)\`. You must replace the correct macro accounting for the number of arguments. + +## Process: + +- Go through the SPL query and identify all the macros and lookup tables that are used. Two scenarios may arise: + - The macro or lookup table is provided in the resources: Replace the call by their actual value in the query. + - The macro or lookup table is not provided in the resources: Keep the call in the query as it is. + +## Example: + Having the following macros: + \`someSource\`: sourcetype="somesource" + \`searchTitle(1)\`: search title="$value$" + \`searchTitle\`: search title=* + \`searchType\`: search type=* + And the following SPL query: + \`\`\`spl + \`someSource\` \`someFilter\` + | \`searchTitle("sometitle")\` + | \`searchType("sometype")\` + | table * + \`\`\` + The correct replacement would be: + \`\`\`spl + sourcetype="somesource" \`someFilter\` + | search title="sometitle" + | \`searchType("sometype")\` + | table * + \`\`\` + +## Important: You must respond only with the modified query inside a \`\`\`spl code block, nothing else. + +# Find the macros and lookup tables below: + +${resourcesContext} + +# Find the SPL query below: + +\`\`\`spl +${state.original_rule.query} +\`\`\` + +`; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts index 00e1e60c7b5f..e12d3b96ceb3 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts @@ -7,26 +7,44 @@ import type { Logger } from '@kbn/core/server'; import type { InferenceClient } from '@kbn/inference-plugin/server'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import { isEmpty } from 'lodash/fp'; import type { GraphNode } from '../../types'; import { getEsqlKnowledgeBase } from './esql_knowledge_base_caller'; -import { getEsqlTranslationPrompt } from './prompt'; +import { getReplaceQueryResourcesPrompt } from './prompts/replace_resources_prompt'; +import { getEsqlTranslationPrompt } from './prompts/esql_translation_prompt'; import { SiemMigrationRuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; +import type { RuleResourceRetriever } from '../../../util/rule_resource_retriever'; +import type { ChatModel } from '../../../util/actions_client_chat'; interface GetTranslateQueryNodeParams { + model: ChatModel; inferenceClient: InferenceClient; + resourceRetriever: RuleResourceRetriever; connectorId: string; logger: Logger; } export const getTranslateQueryNode = ({ + model, inferenceClient, + resourceRetriever, connectorId, logger, }: GetTranslateQueryNodeParams): GraphNode => { const esqlKnowledgeBaseCaller = getEsqlKnowledgeBase({ inferenceClient, connectorId, logger }); return async (state) => { - const input = getEsqlTranslationPrompt(state); - const response = await esqlKnowledgeBaseCaller(input); + let query = state.original_rule.query; + + const resources = await resourceRetriever.getResources(state.original_rule); + if (!isEmpty(resources)) { + const replaceQueryResourcesPrompt = getReplaceQueryResourcesPrompt(state, resources); + const stringParser = new StringOutputParser(); + query = await model.pipe(stringParser).invoke(replaceQueryResourcesPrompt); + } + + const prompt = getEsqlTranslationPrompt(state, query); + const response = await esqlKnowledgeBaseCaller(prompt); const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; const summary = response.match(/## Migration Summary[\s\S]*$/)?.[0] ?? ''; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts index c1e510bdc052..512406d6577d 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -7,12 +7,12 @@ import type { BaseMessage } from '@langchain/core/messages'; import { Annotation, messagesStateReducer } from '@langchain/langgraph'; +import type { SiemMigrationRuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; import type { ElasticRule, OriginalRule, RuleMigration, } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { SiemMigrationRuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; export const migrateRuleState = Annotation.Root({ messages: Annotation({ diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts index 643d200e4b0b..975c03439842 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts @@ -10,6 +10,7 @@ import type { InferenceClient } from '@kbn/inference-plugin/server'; import type { migrateRuleState } from './state'; import type { ChatModel } from '../util/actions_client_chat'; import type { PrebuiltRulesMapByName } from '../util/prebuilt_rules'; +import type { RuleResourceRetriever } from '../util/rule_resource_retriever'; export type MigrateRuleState = typeof migrateRuleState.State; export type GraphNode = (state: MigrateRuleState) => Promise>; @@ -19,5 +20,6 @@ export interface MigrateRuleGraphParams { model: ChatModel; connectorId: string; prebuiltRulesMap: PrebuiltRulesMapByName; + resourceRetriever: RuleResourceRetriever; logger: Logger; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts similarity index 64% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts index 6ae7294fb525..98319a77a766 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_client.ts @@ -5,66 +5,60 @@ * 2.0. */ -import type { Logger } from '@kbn/core/server'; +import type { AuthenticatedUser, Logger } from '@kbn/core/server'; import { AbortError, abortSignalToPromise } from '@kbn/kibana-utils-plugin/server'; import type { RunnableConfig } from '@langchain/core/runnables'; +import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; import type { RuleMigrationAllTaskStats, RuleMigrationTaskStats, } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationDataStats } from '../data_stream/rule_migrations_data_client'; +import type { + RuleMigrationDataStats, + RuleMigrationsDataClient, +} from '../data/rule_migrations_data_client'; import type { RuleMigrationTaskStartParams, RuleMigrationTaskStartResult, - RuleMigrationTaskStatsParams, - RuleMigrationTaskStopParams, RuleMigrationTaskStopResult, RuleMigrationTaskPrepareParams, RuleMigrationTaskRunParams, MigrationAgent, - RuleMigrationAllTaskStatsParams, } from './types'; import { getRuleMigrationAgent } from './agent'; import type { MigrateRuleState } from './agent/types'; import { retrievePrebuiltRulesMap } from './util/prebuilt_rules'; import { ActionsClientChat } from './util/actions_client_chat'; - -interface TaskLogger { - info: (msg: string) => void; - debug: (msg: string) => void; - error: (msg: string, error: Error) => void; -} -const getTaskLogger = (logger: Logger): TaskLogger => { - const prefix = '[ruleMigrationsTask]: '; - return { - info: (msg) => logger.info(`${prefix}${msg}`), - debug: (msg) => logger.debug(`${prefix}${msg}`), - error: (msg, error) => logger.error(`${prefix}${msg}: ${error.message}`), - }; -}; +import { RuleResourceRetriever } from './util/rule_resource_retriever'; const ITERATION_BATCH_SIZE = 50 as const; const ITERATION_SLEEP_SECONDS = 10 as const; -export class RuleMigrationsTaskRunner { - private migrationsRunning: Map; - private taskLogger: TaskLogger; +type MigrationsRunning = Map; - constructor(private logger: Logger) { - this.migrationsRunning = new Map(); - this.taskLogger = getTaskLogger(logger); - } +export class RuleMigrationsTaskClient { + constructor( + private migrationsRunning: MigrationsRunning, + private logger: Logger, + private data: RuleMigrationsDataClient, + private currentUser: AuthenticatedUser + ) {} /** Starts a rule migration task */ async start(params: RuleMigrationTaskStartParams): Promise { - const { migrationId, dataClient } = params; + const { migrationId } = params; if (this.migrationsRunning.has(migrationId)) { return { exists: true, started: false }; } - // Just in case some previous execution was interrupted without releasing - await dataClient.releaseProcessable(migrationId); - - const { rules } = await dataClient.getStats(migrationId); + // Just in case some previous execution was interrupted without cleaning up + await this.data.rules.updateStatus( + migrationId, + SiemMigrationStatus.PROCESSING, + SiemMigrationStatus.PENDING, + { refresh: true } + ); + + const { rules } = await this.data.rules.getStats(migrationId); if (rules.total === 0) { return { exists: false, started: false }; } @@ -80,13 +74,14 @@ export class RuleMigrationsTaskRunner { // not awaiting the `run` promise to execute the task in the background this.run({ ...params, agent, abortController }).catch((err) => { // All errors in the `run` method are already catch, this should never happen, but just in case - this.taskLogger.error(`Unexpected error running the migration ID:${migrationId}`, err); + this.logger.error(`Unexpected error running the migration ID:${migrationId}`, err); }); return { exists: true, started: true }; } private async prepare({ + migrationId, connectorId, inferenceClient, actionsClient, @@ -95,6 +90,7 @@ export class RuleMigrationsTaskRunner { abortController, }: RuleMigrationTaskPrepareParams): Promise { const prebuiltRulesMap = await retrievePrebuiltRulesMap({ soClient, rulesClient }); + const resourceRetriever = new RuleResourceRetriever(migrationId, this.data); const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger); const model = await actionsClientChat.createModel({ @@ -107,6 +103,7 @@ export class RuleMigrationsTaskRunner { model, inferenceClient, prebuiltRulesMap, + resourceRetriever, logger: this.logger, }); return agent; @@ -115,8 +112,6 @@ export class RuleMigrationsTaskRunner { private async run({ migrationId, agent, - dataClient, - currentUser, invocationConfig, abortController, }: RuleMigrationTaskRunParams): Promise { @@ -124,9 +119,9 @@ export class RuleMigrationsTaskRunner { // This should never happen, but just in case throw new Error(`Task already running for migration ID:${migrationId} `); } - this.taskLogger.info(`Starting migration ID:${migrationId}`); + this.logger.info(`Starting migration ID:${migrationId}`); - this.migrationsRunning.set(migrationId, { user: currentUser.username, abortController }); + this.migrationsRunning.set(migrationId, { user: this.currentUser.username, abortController }); const config: RunnableConfig = { ...invocationConfig, // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319 @@ -136,7 +131,7 @@ export class RuleMigrationsTaskRunner { try { const sleep = async (seconds: number) => { - this.taskLogger.debug(`Sleeping ${seconds}s for migration ID:${migrationId}`); + this.logger.debug(`Sleeping ${seconds}s for migration ID:${migrationId}`); await Promise.race([ new Promise((resolve) => setTimeout(resolve, seconds * 1000)), abortPromise.promise, @@ -145,44 +140,42 @@ export class RuleMigrationsTaskRunner { let isDone: boolean = false; do { - const ruleMigrations = await dataClient.takePending(migrationId, ITERATION_BATCH_SIZE); - this.taskLogger.debug( + const ruleMigrations = await this.data.rules.takePending(migrationId, ITERATION_BATCH_SIZE); + this.logger.debug( `Processing ${ruleMigrations.length} rules for migration ID:${migrationId}` ); await Promise.all( ruleMigrations.map(async (ruleMigration) => { - this.taskLogger.debug( - `Starting migration of rule "${ruleMigration.original_rule.title}"` - ); + this.logger.debug(`Starting migration of rule "${ruleMigration.original_rule.title}"`); try { const start = Date.now(); - const ruleMigrationResult: MigrateRuleState = await Promise.race([ + const migrationResult: MigrateRuleState = await Promise.race([ agent.invoke({ original_rule: ruleMigration.original_rule }, config), abortPromise.promise, // workaround for the issue with the langGraph signal ]); const duration = (Date.now() - start) / 1000; - this.taskLogger.debug( + this.logger.debug( `Migration of rule "${ruleMigration.original_rule.title}" finished in ${duration}s` ); - await dataClient.saveFinished({ + await this.data.rules.saveCompleted({ ...ruleMigration, - elastic_rule: ruleMigrationResult.elastic_rule, - translation_result: ruleMigrationResult.translation_result, - comments: ruleMigrationResult.comments, + elastic_rule: migrationResult.elastic_rule, + translation_result: migrationResult.translation_result, + comments: migrationResult.comments, }); } catch (error) { if (error instanceof AbortError) { throw error; } - this.taskLogger.error( + this.logger.error( `Error migrating rule "${ruleMigration.original_rule.title}"`, error ); - await dataClient.saveError({ + await this.data.rules.saveError({ ...ruleMigration, comments: [`Error migrating rule: ${error.message}`], }); @@ -190,24 +183,24 @@ export class RuleMigrationsTaskRunner { }) ); - this.taskLogger.debug(`Batch processed successfully for migration ID:${migrationId}`); + this.logger.debug(`Batch processed successfully for migration ID:${migrationId}`); - const { rules } = await dataClient.getStats(migrationId); + const { rules } = await this.data.rules.getStats(migrationId); isDone = rules.pending === 0; if (!isDone) { await sleep(ITERATION_SLEEP_SECONDS); } } while (!isDone); - this.taskLogger.info(`Finished migration ID:${migrationId}`); + this.logger.info(`Finished migration ID:${migrationId}`); } catch (error) { - await dataClient.releaseProcessing(migrationId); + await this.data.rules.releaseProcessing(migrationId); if (error instanceof AbortError) { - this.taskLogger.info(`Abort signal received, stopping migration ID:${migrationId}`); + this.logger.info(`Abort signal received, stopping migration ID:${migrationId}`); return; } else { - this.taskLogger.error(`Error processing migration ID:${migrationId}`, error); + this.logger.error(`Error processing migration ID:${migrationId}`, error); } } finally { this.migrationsRunning.delete(migrationId); @@ -215,21 +208,28 @@ export class RuleMigrationsTaskRunner { } } + /** Updates all the rules in a migration to be re-executed */ + public async updateToRetry(migrationId: string): Promise<{ updated: boolean }> { + if (this.migrationsRunning.has(migrationId)) { + return { updated: false }; + } + // Update all the rules in the migration to pending + await this.data.rules.updateStatus(migrationId, undefined, SiemMigrationStatus.PENDING, { + refresh: true, + }); + return { updated: true }; + } + /** Returns the stats of a migration */ - async getStats({ - migrationId, - dataClient, - }: RuleMigrationTaskStatsParams): Promise { - const dataStats = await dataClient.getStats(migrationId); + public async getStats(migrationId: string): Promise { + const dataStats = await this.data.rules.getStats(migrationId); const status = this.getTaskStatus(migrationId, dataStats.rules); return { status, ...dataStats }; } /** Returns the stats of all migrations */ - async getAllStats({ - dataClient, - }: RuleMigrationAllTaskStatsParams): Promise { - const allDataStats = await dataClient.getAllStats(); + async getAllStats(): Promise { + const allDataStats = await this.data.rules.getAllStats(); return allDataStats.map((dataStats) => { const status = this.getTaskStatus(dataStats.migration_id, dataStats.rules); return { status, ...dataStats }; @@ -253,10 +253,7 @@ export class RuleMigrationsTaskRunner { } /** Stops one running migration */ - async stop({ - migrationId, - dataClient, - }: RuleMigrationTaskStopParams): Promise { + async stop(migrationId: string): Promise { try { const migrationRunning = this.migrationsRunning.get(migrationId); if (migrationRunning) { @@ -264,22 +261,14 @@ export class RuleMigrationsTaskRunner { return { exists: true, stopped: true }; } - const { rules } = await dataClient.getStats(migrationId); + const { rules } = await this.data.rules.getStats(migrationId); if (rules.total > 0) { return { exists: true, stopped: false }; } return { exists: false, stopped: false }; } catch (err) { - this.taskLogger.error(`Error stopping migration ID:${migrationId}`, err); + this.logger.error(`Error stopping migration ID:${migrationId}`, err); return { exists: true, stopped: false }; } } - - /** Stops all running migrations */ - stopAll() { - this.migrationsRunning.forEach((migrationRunning) => { - migrationRunning.abortController.abort(); - }); - this.migrationsRunning.clear(); - } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts new file mode 100644 index 000000000000..89147f296b32 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_service.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { RuleMigrationTaskCreateClientParams } from './types'; +import { RuleMigrationsTaskClient } from './rule_migrations_task_client'; + +export type MigrationRunning = Map; + +export class RuleMigrationsTaskService { + private migrationsRunning: MigrationRunning; + + constructor(private logger: Logger) { + this.migrationsRunning = new Map(); + } + + public createClient({ + currentUser, + dataClient, + }: RuleMigrationTaskCreateClientParams): RuleMigrationsTaskClient { + return new RuleMigrationsTaskClient( + this.migrationsRunning, + this.logger, + dataClient, + currentUser + ); + } + + /** Stops all running migrations */ + stopAll() { + this.migrationsRunning.forEach((migrationRunning) => { + migrationRunning.abortController.abort(); + }); + this.migrationsRunning.clear(); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts index e26a5b7216f4..7ac7e848ba80 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -10,24 +10,28 @@ import type { RunnableConfig } from '@langchain/core/runnables'; import type { InferenceClient } from '@kbn/inference-plugin/server'; import type { ActionsClient } from '@kbn/actions-plugin/server'; import type { RulesClient } from '@kbn/alerting-plugin/server'; -import type { RuleMigrationsDataClient } from '../data_stream/rule_migrations_data_client'; +import type { RuleMigrationsDataClient } from '../data/rule_migrations_data_client'; import type { getRuleMigrationAgent } from './agent'; export type MigrationAgent = ReturnType; +export interface RuleMigrationTaskCreateClientParams { + currentUser: AuthenticatedUser; + dataClient: RuleMigrationsDataClient; +} + export interface RuleMigrationTaskStartParams { migrationId: string; - currentUser: AuthenticatedUser; connectorId: string; invocationConfig: RunnableConfig; inferenceClient: InferenceClient; actionsClient: ActionsClient; rulesClient: RulesClient; soClient: SavedObjectsClientContract; - dataClient: RuleMigrationsDataClient; } export interface RuleMigrationTaskPrepareParams { + migrationId: string; connectorId: string; inferenceClient: InferenceClient; actionsClient: ActionsClient; @@ -38,27 +42,11 @@ export interface RuleMigrationTaskPrepareParams { export interface RuleMigrationTaskRunParams { migrationId: string; - currentUser: AuthenticatedUser; invocationConfig: RunnableConfig; agent: MigrationAgent; - dataClient: RuleMigrationsDataClient; abortController: AbortController; } -export interface RuleMigrationTaskStopParams { - migrationId: string; - dataClient: RuleMigrationsDataClient; -} - -export interface RuleMigrationTaskStatsParams { - migrationId: string; - dataClient: RuleMigrationsDataClient; -} - -export interface RuleMigrationAllTaskStatsParams { - dataClient: RuleMigrationsDataClient; -} - export interface RuleMigrationTaskStartResult { started: boolean; exists: boolean; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.test.ts new file mode 100644 index 000000000000..51618d5f3ca1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.test.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MAX_RECURSION_DEPTH, RuleResourceRetriever } from './rule_resource_retriever'; // Adjust path as needed +import type { OriginalRule } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import { MockRuleMigrationsDataClient } from '../../data/__mocks__/mocks'; + +const mockRuleResourceIdentifier = jest.fn(); +const mockGetRuleResourceIdentifier = jest.fn((_: unknown) => mockRuleResourceIdentifier); +jest.mock('../../../../../../common/siem_migrations/rules/resources', () => ({ + getRuleResourceIdentifier: (params: unknown) => mockGetRuleResourceIdentifier(params), +})); + +jest.mock('../../data/rule_migrations_data_service'); + +describe('RuleResourceRetriever', () => { + let retriever: RuleResourceRetriever; + const mockRuleMigrationsDataClient = new MockRuleMigrationsDataClient(); + const migrationId = 'test-migration-id'; + const ruleQuery = 'rule-query'; + const originalRule = { query: ruleQuery } as OriginalRule; + + beforeEach(() => { + retriever = new RuleResourceRetriever(migrationId, mockRuleMigrationsDataClient); + mockRuleResourceIdentifier.mockReturnValue({ list: [], macro: [] }); + + mockRuleMigrationsDataClient.resources.get.mockImplementation( + async (_: string, type: string, names: string[]) => + names.map((name) => ({ type, name, content: `${name}-content` })) + ); + + mockRuleResourceIdentifier.mockImplementation((query) => { + if (query === ruleQuery) { + return { list: ['list1', 'list2'], macro: ['macro1'] }; + } + return { list: [], macro: [] }; + }); + + jest.clearAllMocks(); + }); + + describe('getResources', () => { + it('should call resource identification', async () => { + await retriever.getResources(originalRule); + + expect(mockGetRuleResourceIdentifier).toHaveBeenCalledWith(originalRule); + expect(mockRuleResourceIdentifier).toHaveBeenCalledWith(ruleQuery); + expect(mockRuleResourceIdentifier).toHaveBeenCalledWith('macro1-content'); + }); + + it('should retrieve resources', async () => { + const resources = await retriever.getResources(originalRule); + + expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith(migrationId, 'list', [ + 'list1', + 'list2', + ]); + expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith( + migrationId, + 'macro', + ['macro1'] + ); + + expect(resources).toEqual({ + list: [ + { type: 'list', name: 'list1', content: 'list1-content' }, + { type: 'list', name: 'list2', content: 'list2-content' }, + ], + macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], + }); + }); + + it('should retrieve nested resources', async () => { + mockRuleResourceIdentifier.mockImplementation((query) => { + if (query === ruleQuery) { + return { list: ['list1', 'list2'], macro: ['macro1'] }; + } + if (query === 'macro1-content') { + return { list: ['list3'], macro: [] }; + } + return { list: [], macro: [] }; + }); + + const resources = await retriever.getResources(originalRule); + + expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith(migrationId, 'list', [ + 'list1', + 'list2', + ]); + expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith( + migrationId, + 'macro', + ['macro1'] + ); + expect(mockRuleMigrationsDataClient.resources.get).toHaveBeenCalledWith(migrationId, 'list', [ + 'list3', + ]); + + expect(resources).toEqual({ + list: [ + { type: 'list', name: 'list1', content: 'list1-content' }, + { type: 'list', name: 'list2', content: 'list2-content' }, + { type: 'list', name: 'list3', content: 'list3-content' }, + ], + macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], + }); + }); + + it('should handle missing macros', async () => { + mockRuleMigrationsDataClient.resources.get.mockImplementation( + async (_: string, type: string, names: string[]) => { + if (type === 'macro') { + return []; + } + return names.map((name) => ({ type, name, content: `${name}-content` })); + } + ); + + const resources = await retriever.getResources(originalRule); + + expect(resources).toEqual({ + list: [ + { type: 'list', name: 'list1', content: 'list1-content' }, + { type: 'list', name: 'list2', content: 'list2-content' }, + ], + }); + }); + + it('should handle missing lists', async () => { + mockRuleMigrationsDataClient.resources.get.mockImplementation( + async (_: string, type: string, names: string[]) => { + if (type === 'list') { + return []; + } + return names.map((name) => ({ type, name, content: `${name}-content` })); + } + ); + + const resources = await retriever.getResources(originalRule); + + expect(resources).toEqual({ + macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], + }); + }); + + it('should not include resources with missing content', async () => { + mockRuleMigrationsDataClient.resources.get.mockImplementation( + async (_: string, type: string, names: string[]) => { + return names.map((name) => { + if (name === 'list1') { + return { type, name, content: '' }; + } + return { type, name, content: `${name}-content` }; + }); + } + ); + + const resources = await retriever.getResources(originalRule); + + expect(resources).toEqual({ + list: [{ type: 'list', name: 'list2', content: 'list2-content' }], + macro: [{ type: 'macro', name: 'macro1', content: 'macro1-content' }], + }); + }); + + it('should stop recursion after reaching MAX_RECURSION_DEPTH', async () => { + mockRuleResourceIdentifier.mockImplementation(() => { + return { list: [], macro: ['infinite-macro'] }; + }); + + const resources = await retriever.getResources(originalRule); + + expect(resources.macro?.length).toEqual(MAX_RECURSION_DEPTH); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.ts new file mode 100644 index 000000000000..d80646dc27c4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/rule_resource_retriever.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash/fp'; +import type { QueryResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources/types'; +import { getRuleResourceIdentifier } from '../../../../../../common/siem_migrations/rules/resources'; +import type { + OriginalRule, + RuleMigrationResource, + RuleMigrationResourceType, +} from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationsDataClient } from '../../data/rule_migrations_data_client'; + +export type RuleMigrationResources = Partial< + Record +>; + +/* It's not a common practice to have more than 2-3 nested levels of resources. + * This limit is just to prevent infinite recursion in case something goes wrong. + */ +export const MAX_RECURSION_DEPTH = 30; + +export class RuleResourceRetriever { + constructor( + private readonly migrationId: string, + private readonly dataClient: RuleMigrationsDataClient + ) {} + + public async getResources(originalRule: OriginalRule): Promise { + const resourceIdentifier = getRuleResourceIdentifier(originalRule); + return this.recursiveRetriever(originalRule.query, resourceIdentifier); + } + + private recursiveRetriever = async ( + query: string, + resourceIdentifier: QueryResourceIdentifier, + it = 0 + ): Promise => { + if (it >= MAX_RECURSION_DEPTH) { + return {}; + } + + const identifiedResources = resourceIdentifier(query); + const resources: RuleMigrationResources = {}; + + const listNames = identifiedResources.list; + if (listNames.length > 0) { + const listsWithContent = await this.dataClient.resources + .get(this.migrationId, 'list', listNames) + .then(withContent); + + if (listsWithContent.length > 0) { + resources.list = listsWithContent; + } + } + + const macroNames = identifiedResources.macro; + if (macroNames.length > 0) { + const macrosWithContent = await this.dataClient.resources + .get(this.migrationId, 'macro', macroNames) + .then(withContent); + + if (macrosWithContent.length > 0) { + // retrieve nested resources inside macros + const macrosNestedResources = await Promise.all( + macrosWithContent.map(({ content }) => + this.recursiveRetriever(content, resourceIdentifier, it + 1) + ) + ); + + // Process lists inside macros + const macrosNestedLists = macrosNestedResources.flatMap( + (macroNestedResources) => macroNestedResources.list ?? [] + ); + if (macrosNestedLists.length > 0) { + resources.list = (resources.list ?? []).concat(macrosNestedLists); + } + + // Process macros inside macros + const macrosNestedMacros = macrosNestedResources.flatMap( + (macroNestedResources) => macroNestedResources.macro ?? [] + ); + + if (macrosNestedMacros.length > 0) { + macrosWithContent.push(...macrosNestedMacros); + } + resources.macro = macrosWithContent; + } + } + return resources; + }; +} + +const withContent = (resources: RuleMigrationResource[]) => { + return resources.filter((resource) => !isEmpty(resource.content)); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index 78ec2ef89c7a..34d008825628 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -5,58 +5,12 @@ * 2.0. */ -import type { - AuthenticatedUser, - IClusterClient, - KibanaRequest, - SavedObjectsClientContract, -} from '@kbn/core/server'; -import type { Subject } from 'rxjs'; -import type { InferenceClient } from '@kbn/inference-plugin/server'; -import type { RunnableConfig } from '@langchain/core/runnables'; -import type { ActionsClient } from '@kbn/actions-plugin/server'; -import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { RuleMigration, - RuleMigrationAllTaskStats, - RuleMigrationTaskStats, + RuleMigrationResource, } from '../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationsDataClient } from './data_stream/rule_migrations_data_client'; -import type { RuleMigrationTaskStopResult, RuleMigrationTaskStartResult } from './task/types'; - -export interface StoredRuleMigration extends RuleMigration { - _id: string; - _index: string; -} - -export interface SiemRulesMigrationsSetupParams { - esClusterClient: IClusterClient; - pluginStop$: Subject; - tasksTimeoutMs?: number; -} - -export interface SiemRuleMigrationsCreateClientParams { - request: KibanaRequest; - currentUser: AuthenticatedUser | null; - spaceId: string; -} -export interface SiemRuleMigrationsStartTaskParams { - migrationId: string; - connectorId: string; - invocationConfig: RunnableConfig; - inferenceClient: InferenceClient; - actionsClient: ActionsClient; - rulesClient: RulesClient; - soClient: SavedObjectsClientContract; -} +export type Stored = T & { _id: string }; -export interface SiemRuleMigrationsClient { - data: RuleMigrationsDataClient; - task: { - start: (params: SiemRuleMigrationsStartTaskParams) => Promise; - stop: (migrationId: string) => Promise; - getStats: (migrationId: string) => Promise; - getAllStats: () => Promise; - }; -} +export type StoredRuleMigration = Stored; +export type StoredRuleMigrationResource = Stored; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts index 7a85dd625fee..948ae89a39bb 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts @@ -5,18 +5,21 @@ * 2.0. */ -import type { Logger } from '@kbn/core/server'; +import type { LoggerFactory } from '@kbn/core/server'; import { ReplaySubject, type Subject } from 'rxjs'; import type { ConfigType } from '../../config'; -import { SiemRuleMigrationsService } from './rules/siem_rule_migrations_service'; -import type { SiemMigrationsSetupParams, SiemMigrationsCreateClientParams } from './types'; -import type { SiemRuleMigrationsClient } from './rules/types'; +import { + SiemRuleMigrationsService, + type SiemRuleMigrationsClient, + type SiemRuleMigrationsCreateClientParams, +} from './rules/siem_rule_migrations_service'; +import type { SiemMigrationsSetupParams } from './types'; export class SiemMigrationsService { private pluginStop$: Subject; private rules: SiemRuleMigrationsService; - constructor(private config: ConfigType, logger: Logger, kibanaVersion: string) { + constructor(private config: ConfigType, logger: LoggerFactory, kibanaVersion: string) { this.pluginStop$ = new ReplaySubject(1); this.rules = new SiemRuleMigrationsService(logger, kibanaVersion); } @@ -27,7 +30,7 @@ export class SiemMigrationsService { } } - createRulesClient(params: SiemMigrationsCreateClientParams): SiemRuleMigrationsClient { + createRulesClient(params: SiemRuleMigrationsCreateClientParams): SiemRuleMigrationsClient { return this.rules.createClient(params); } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts index d2af1e251872..62071c9e8bbb 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts @@ -6,11 +6,8 @@ */ import type { IClusterClient } from '@kbn/core/server'; -import type { SiemRuleMigrationsCreateClientParams } from './rules/types'; export interface SiemMigrationsSetupParams { esClusterClient: IClusterClient; tasksTimeoutMs?: number; } - -export type SiemMigrationsCreateClientParams = SiemRuleMigrationsCreateClientParams; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 087e3b8c3f05..e2ec9d0e1b53 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -165,7 +165,7 @@ export class Plugin implements ISecuritySolutionPlugin { ); this.siemMigrationsService = new SiemMigrationsService( this.config, - this.logger, + this.pluginContext.logger, this.pluginContext.env.packageInfo.version ); diff --git a/x-pack/plugins/security_solution/server/types.ts b/x-pack/plugins/security_solution/server/types.ts index 97b35133f824..72c8aa2a386e 100644 --- a/x-pack/plugins/security_solution/server/types.ts +++ b/x-pack/plugins/security_solution/server/types.ts @@ -37,7 +37,7 @@ import type { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk import type { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality'; import type { IDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface'; import type { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client'; -import type { SiemRuleMigrationsClient } from './lib/siem_migrations/rules/types'; +import type { SiemRuleMigrationsClient } from './lib/siem_migrations/rules/siem_rule_migrations_service'; export { AppClient }; export interface SecuritySolutionApiRequestHandlerContext { diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index cbd992ea78d8..43b4665b8d9d 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -229,6 +229,7 @@ "@kbn/core-lifecycle-server", "@kbn/core-user-profile-common", "@kbn/langchain", - "@kbn/react-hooks" + "@kbn/react-hooks", + "@kbn/index-adapter" ] } diff --git a/x-pack/test/api_integration/services/security_solution_api.gen.ts b/x-pack/test/api_integration/services/security_solution_api.gen.ts index 8f8845754738..3cffbef413fa 100644 --- a/x-pack/test/api_integration/services/security_solution_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_api.gen.ts @@ -32,7 +32,7 @@ import { CopyTimelineRequestBodyInput } from '@kbn/security-solution-plugin/comm import { CreateAlertsMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/signals_migration/create_signals_migration/create_signals_migration.gen'; import { CreateAssetCriticalityRecordRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/asset_criticality/create_asset_criticality.gen'; import { CreateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/create_rule/create_rule_route.gen'; -import { CreateRuleMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; +import { CreateRuleMigrationRequestBodyInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { CreateTimelinesRequestBodyInput } from '@kbn/security-solution-plugin/common/api/timeline/create_timelines/create_timelines_route.gen'; import { CreateUpdateProtectionUpdatesNoteRequestParamsInput, @@ -92,8 +92,12 @@ import { GetRuleExecutionResultsRequestQueryInput, GetRuleExecutionResultsRequestParamsInput, } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.gen'; -import { GetRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; -import { GetRuleMigrationStatsRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; +import { + GetRuleMigrationResourcesRequestQueryInput, + GetRuleMigrationResourcesRequestParamsInput, +} from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; +import { GetRuleMigrationStatsRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { GetTimelineRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timeline/get_timeline_route.gen'; import { GetTimelinesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timelines/get_timelines_route.gen'; import { ImportRulesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/import_rules/import_rules_route.gen'; @@ -129,12 +133,16 @@ import { StartEntityEngineRequestParamsInput } from '@kbn/security-solution-plug import { StartRuleMigrationRequestParamsInput, StartRuleMigrationRequestBodyInput, -} from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; +} from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { StopEntityEngineRequestParamsInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/engine/stop.gen'; -import { StopRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; +import { StopRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { SuggestUserProfilesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/users/suggest_user_profiles_route.gen'; import { TriggerRiskScoreCalculationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/risk_engine/entity_calculation_route.gen'; import { UpdateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/update_rule/update_rule_route.gen'; +import { + UpsertRuleMigrationResourcesRequestParamsInput, + UpsertRuleMigrationResourcesRequestBodyInput, +} from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rule_migration.gen'; import { routeWithNamespace } from '../../common/utils/security_solution'; import { FtrProviderContext } from '../ftr_provider_context'; @@ -928,6 +936,25 @@ finalize it. .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Retrieves resources for an existing SIEM rules migration + */ + getRuleMigrationResources( + props: GetRuleMigrationResourcesProps, + kibanaSpace: string = 'default' + ) { + return supertest + .get( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/resources', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .query(props.query); + }, /** * Retrieves the stats of a SIEM rules migration using the migration id provided */ @@ -1398,6 +1425,25 @@ detection engine rules. .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Creates or updates resources for an existing SIEM rules migration + */ + upsertRuleMigrationResources( + props: UpsertRuleMigrationResourcesProps, + kibanaSpace: string = 'default' + ) { + return supertest + .post( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/resources', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send(props.body as object); + }, }; } @@ -1564,6 +1610,10 @@ export interface GetRuleExecutionResultsProps { export interface GetRuleMigrationProps { params: GetRuleMigrationRequestParamsInput; } +export interface GetRuleMigrationResourcesProps { + query: GetRuleMigrationResourcesRequestQueryInput; + params: GetRuleMigrationResourcesRequestParamsInput; +} export interface GetRuleMigrationStatsProps { params: GetRuleMigrationStatsRequestParamsInput; } @@ -1658,3 +1708,7 @@ export interface TriggerRiskScoreCalculationProps { export interface UpdateRuleProps { body: UpdateRuleRequestBodyInput; } +export interface UpsertRuleMigrationResourcesProps { + params: UpsertRuleMigrationResourcesRequestParamsInput; + body: UpsertRuleMigrationResourcesRequestBodyInput; +}