From 7d02252d31803937876b34184096989d1854b69b Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Fri, 25 Oct 2024 17:50:10 +0200 Subject: [PATCH 01/14] task runner finished with dummy agent --- .../common/siem_migrations/constants.ts | 7 + .../model/api/rules/rules_migration.gen.ts | 51 ++++- .../api/rules/rules_migration.schema.yaml | 106 ++++++++- .../model/rule_migration.gen.ts | 39 ++++ .../model/rule_migration.schema.yaml | 42 ++++ .../routes/__mocks__/request_context.ts | 4 +- .../lib/siem_migrations/__mocks__/mocks.ts | 8 +- .../__mocks__/siem_migrations_service.ts | 9 + .../siem_migrations/rules/__mocks__/mocks.ts | 24 +- .../__mocks__/siem_rule_migrations_client.ts | 9 + .../lib/siem_migrations/rules/api/cancel.ts | 51 +++++ .../lib/siem_migrations/rules/api/create.ts | 20 +- .../lib/siem_migrations/rules/api/index.ts | 6 + .../lib/siem_migrations/rules/api/start.ts | 59 +++++ .../lib/siem_migrations/rules/api/stats.ts | 47 ++++ .../rule_migrations_data_client.ts | 192 ++++++++++++++++ .../rule_migrations_data_stream.ts | 41 +++- .../data_stream/rule_migrations_field_map.ts | 1 + .../siem_rule_migrations_service.test.ts | 24 +- .../rules/siem_rule_migrations_service.ts | 56 +++-- .../rules/task/agent/actions_client_chat.ts | 93 ++++++++ .../task/agent/esql_knowledge_base_caller.ts | 36 +++ .../siem_migrations/rules/task/agent/graph.ts | 49 +++++ .../rules/task/agent/prompts.ts | 33 +++ .../siem_migrations/rules/task/agent/state.ts | 27 +++ .../task/agent/tools/esql_translator_tool.ts | 115 ++++++++++ .../siem_migrations/rules/task/agent/types.ts | 22 ++ .../rules/task/rule_migrations_task_runner.ts | 205 ++++++++++++++++++ .../lib/siem_migrations/rules/task/types.ts | 36 +++ .../server/lib/siem_migrations/rules/types.ts | 29 ++- .../siem_migrations_service.ts | 14 +- .../server/lib/siem_migrations/types.ts | 8 +- .../server/request_context_factory.ts | 8 +- 33 files changed, 1370 insertions(+), 101 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/cancel.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/actions_client_chat.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/esql_knowledge_base_caller.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/prompts.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index 96ca75679f112..935f4868335c6 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -8,6 +8,13 @@ export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const; export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const; +export const SIEM_RULE_MIGRATIONS_START_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/start` as const; +export const SIEM_RULE_MIGRATIONS_STATS_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stats` as const; +export const SIEM_RULE_MIGRATIONS_CANCEL_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/cancel` as const; + export enum SiemMigrationsStatus { PENDING = 'pending', PROCESSING = 'processing', diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts index fa8a1cc8a6778..bf82b77ac5311 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts @@ -16,7 +16,24 @@ import { z } from '@kbn/zod'; -import { OriginalRule, RuleMigration } from '../../rule_migration.gen'; +import { OriginalRule, RuleMigration, RuleMigrationTaskStats } from '../../rule_migration.gen'; +import { ConnectorId, LangSmithOptions } from '../common.gen'; + +export type CancelRuleMigrationRequestParams = z.infer; +export const CancelRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type CancelRuleMigrationRequestParamsInput = z.input< + typeof CancelRuleMigrationRequestParams +>; + +export type CancelRuleMigrationResponse = z.infer; +export const CancelRuleMigrationResponse = z.object({ + /** + * Indicates the migration has been cancelled. + */ + cancelled: z.boolean(), +}); export type CreateRuleMigrationRequestBody = z.infer; export const CreateRuleMigrationRequestBody = z.array(OriginalRule); @@ -32,3 +49,35 @@ export const CreateRuleMigrationResponse = z.object({ export type GetRuleMigrationResponse = z.infer; export const GetRuleMigrationResponse = z.array(RuleMigration); + +export type GetRuleMigrationStatsRequestParams = z.infer; +export const GetRuleMigrationStatsRequestParams = z.object({ + migration_id: z.string(), +}); +export type GetRuleMigrationStatsRequestParamsInput = z.input< + typeof GetRuleMigrationStatsRequestParams +>; + +export type GetRuleMigrationStatsResponse = z.infer; +export const GetRuleMigrationStatsResponse = RuleMigrationTaskStats; + +export type StartRuleMigrationRequestParams = z.infer; +export const StartRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type StartRuleMigrationRequestParamsInput = z.input; + +export type StartRuleMigrationRequestBody = z.infer; +export const StartRuleMigrationRequestBody = z.object({ + connectorId: ConnectorId, + langSmithOptions: LangSmithOptions.optional(), +}); +export type StartRuleMigrationRequestBodyInput = z.input; + +export type StartRuleMigrationResponse = z.infer; +export const StartRuleMigrationResponse = z.object({ + /** + * Indicates the migration has been started. `false` means the migration does not need to be started. + */ + started: z.boolean(), +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml index 40596ba7e712d..1105736db29be 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml @@ -10,8 +10,7 @@ paths: x-codegen-enabled: true description: Creates a new SIEM rules migration using the original vendor rules provided tags: - - SIEM Migrations - - Rule Migrations + - SIEM Rule Migrations requestBody: required: true content: @@ -39,8 +38,7 @@ paths: x-codegen-enabled: true description: Retrieves the rule migrations stored in the system tags: - - SIEM Migrations - - Rule Migrations + - SIEM Rule Migrations responses: 200: description: Indicates rule migrations have been retrieved correctly. @@ -50,3 +48,103 @@ paths: type: array items: $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration' + + /internal/siem_migrations/rules/{migration_id}/start: + put: + summary: Starts a rule migration + operationId: StartRuleMigration + x-codegen-enabled: true + description: Starts a SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - connectorId + properties: + connectorId: + $ref: '../common.schema.yaml#/components/schemas/ConnectorId' + langSmithOptions: + $ref: '../common.schema.yaml#/components/schemas/LangSmithOptions' + responses: + 200: + description: Indicates the migration start request has been processed successfully. + content: + application/json: + schema: + type: object + required: + - started + properties: + started: + type: boolean + description: Indicates the migration has been started. `false` means the migration does not need to be started. + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/stats: + get: + summary: Gets a rule migration task stats + operationId: GetRuleMigrationStats + x-codegen-enabled: true + description: Retrieves the stats of a SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + responses: + 200: + description: Indicates the migration stats has been retrieved correctly. + content: + application/json: + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationTaskStats' + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/cancel: + put: + summary: Cancels an existing rule migration + operationId: CancelRuleMigration + x-codegen-enabled: true + description: Cancels a running SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to cancel + responses: + 200: + description: Indicates migration task cancellation has been processed successfully. + content: + application/json: + schema: + type: object + required: + - cancelled + properties: + cancelled: + type: boolean + description: Indicates the migration has been cancelled. + 204: + description: Indicates the migration id was not found running. diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 0e07ef2f208da..5bf3749088cf2 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -99,6 +99,10 @@ export const RuleMigration = z.object({ * The migration id. */ migration_id: z.string(), + /** + * The username of the user who created the migration. + */ + created_by: z.string(), original_rule: OriginalRule, elastic_rule: ElasticRule.optional(), /** @@ -122,3 +126,38 @@ export const RuleMigration = z.object({ */ updated_by: z.string().optional(), }); + +/** + * The rule migration task stats object. + */ +export type RuleMigrationTaskStats = z.infer; +export const RuleMigrationTaskStats = z.object({ + /** + * Indicates if the migration task status. + */ + status: z.enum(['not_started', 'processing', 'done', 'cancelled']), + /** + * The total number of rules to migrate. + */ + total: z.number().int().optional(), + /** + * The number of rules that have been migrated. + */ + finished: z.number().int(), + /** + * The number of rules that are being migrated. + */ + processing: z.number().int(), + /** + * The number of rules that are pending migration. + */ + pending: z.number().int(), + /** + * The number of rules that have failed migration. + */ + failed: z.number().int(), + /** + * The moment of the last execution. + */ + last_iteration_at: z.string().optional(), +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 9ec825389a52b..16795b6ff7bc9 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -84,6 +84,7 @@ components: - migration_id - original_rule - status + - created_by properties: "@timestamp": type: string @@ -91,6 +92,9 @@ components: migration_id: type: string description: The migration id. + created_by: + type: string + description: The username of the user who created the migration. original_rule: $ref: '#/components/schemas/OriginalRule' elastic_rule: @@ -122,3 +126,41 @@ components: updated_by: type: string description: The user who last updated the migration + + RuleMigrationTaskStats: + type: object + description: The rule migration task stats object. + required: + - status + - total_rules + - finished + - processing + - pending + - failed + properties: + status: + type: string + description: Indicates if the migration task status. + enum: + - not_started + - processing + - done + - cancelled + total: + type: integer + description: The total number of rules to migrate. + finished: + type: integer + description: The number of rules that have been migrated. + processing: + type: integer + description: The number of rules that are being migrated. + pending: + type: integer + description: The number of rules that are pending migration. + failed: + type: integer + description: The number of rules that have failed migration. + last_iteration_at: + type: string + description: The moment of the last execution. \ No newline at end of file diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts index ebc1706b309f8..bec7c4823ba01 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts @@ -79,7 +79,7 @@ export const createMockClients = () => { internalFleetServices: { packages: packageServiceMock.createClient(), }, - siemMigrationsClient: siemMigrationsServiceMock.createClient(), + siemRuleMigrationsClient: siemMigrationsServiceMock.createRulesClient(), }; }; @@ -165,7 +165,7 @@ const createSecuritySolutionRequestContextMock = ( getAssetCriticalityDataClient: jest.fn(() => clients.assetCriticalityDataClient), getAuditLogger: jest.fn(() => mockAuditLogger), getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient), - getSiemMigrationsClient: jest.fn(() => clients.siemMigrationsClient), + getSiemRuleMigrationsClient: jest.fn(() => clients.siemRuleMigrationsClient), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts index fcf119e19ece5..af961d48db5b1 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts @@ -7,18 +7,16 @@ import { createRuleMigrationClient } from '../rules/__mocks__/mocks'; -const createClient = () => ({ rules: createRuleMigrationClient() }); - export const mockSetup = jest.fn().mockResolvedValue(undefined); -export const mockCreateClient = jest.fn().mockReturnValue(createClient()); +export const mockCreateClient = jest.fn().mockReturnValue(createRuleMigrationClient()); export const mockStop = jest.fn(); export const siemMigrationsServiceMock = { create: () => jest.fn().mockImplementation(() => ({ setup: mockSetup, - createClient: mockCreateClient, + createRulesClient: mockCreateClient, stop: mockStop, })), - createClient: () => createClient(), + createRulesClient: () => createRuleMigrationClient(), }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts new file mode 100644 index 0000000000000..659929d47570f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { siemMigrationsServiceMock } from './mocks'; +export const SiemMigrationsService = siemMigrationsServiceMock.create(); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts index 8233151f513e4..75666bb48580a 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts @@ -5,17 +5,29 @@ * 2.0. */ -import type { SiemRuleMigrationsClient } from '../types'; - -export const createRuleMigrationClient = (): SiemRuleMigrationsClient => ({ +export const createRuleMigrationDataClient = () => ({ create: jest.fn().mockResolvedValue({ success: true }), - search: jest.fn().mockResolvedValue([]), + takePending: jest.fn().mockResolvedValue([]), + releaseProcessing: jest.fn(), + finishProcessing: jest.fn(), + finish: jest.fn(), +}); +export const createRuleMigrationTaskClient = () => ({ + run: jest.fn().mockResolvedValue({ processed: 0 }), + cancel: jest.fn(), }); +export const createRuleMigrationClient = () => ({ + data: createRuleMigrationDataClient(), + task: createRuleMigrationTaskClient(), +}); + +export const MockSiemRuleMigrationsClient = jest.fn().mockImplementation(createRuleMigrationClient); + export const mockSetup = jest.fn(); -export const mockGetClient = jest.fn().mockReturnValue(createRuleMigrationClient()); +export const mockCreateClient = jest.fn().mockReturnValue(createRuleMigrationClient()); export const MockSiemRuleMigrationsService = jest.fn().mockImplementation(() => ({ setup: mockSetup, - getClient: mockGetClient, + createClient: mockCreateClient, })); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts new file mode 100644 index 0000000000000..98032605ed233 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemRuleMigrationsClient } from './mocks'; +export const SiemRuleMigrationsClient = MockSiemRuleMigrationsClient; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/cancel.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/cancel.ts new file mode 100644 index 0000000000000..3993e0a3febea --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/cancel.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { CancelRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { CancelRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_CANCEL_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsCancelRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATIONS_CANCEL_PATH, + access: 'internal', + options: { tags: ['access:securitySolution'] }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(CancelRuleMigrationRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['core', 'actions', 'securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const { found, cancelled } = await ruleMigrationsClient.task.cancel(migrationId); + + if (!found) { + return res.noContent(); + } + return res.ok({ body: { cancelled } }); + } catch (err) { + logger.error(err); + return res.badRequest({ + body: err.message, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index e2cf97dd094a9..c8ae0a6c7c84f 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -8,14 +8,11 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { v4 as uuidV4 } from 'uuid'; -import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import type { CreateRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; import { CreateRuleMigrationRequestBody } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { - SIEM_RULE_MIGRATIONS_PATH, - SiemMigrationsStatus, -} from '../../../../../common/siem_migrations/constants'; +import { SIEM_RULE_MIGRATIONS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import type { CreateRuleMigrationInput } from '../data_stream/rule_migrations_data_client'; export const registerSiemRuleMigrationsCreateRoute = ( router: SecuritySolutionPluginRouter, @@ -37,20 +34,17 @@ export const registerSiemRuleMigrationsCreateRoute = ( async (context, req, res): Promise> => { const originalRules = req.body; try { - const ctx = await context.resolve(['core', 'actions', 'securitySolution']); - - const siemMigrationClient = ctx.securitySolution.getSiemMigrationsClient(); + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); const migrationId = uuidV4(); - const timestamp = new Date().toISOString(); - const ruleMigrations = originalRules.map((originalRule) => ({ - '@timestamp': timestamp, + const ruleMigrations = originalRules.map((originalRule) => ({ migration_id: migrationId, original_rule: originalRule, - status: SiemMigrationsStatus.PENDING, })); - await siemMigrationClient.rules.create(ruleMigrations); + + await ruleMigrationsClient.data.create(ruleMigrations); return res.ok({ body: { migration_id: migrationId } }); } catch (err) { diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index 0de49eb7df92b..81245597f7d0a 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -8,10 +8,16 @@ import type { Logger } from '@kbn/core/server'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { registerSiemRuleMigrationsCreateRoute } from './create'; +import { registerSiemRuleMigrationsStartRoute } from './start'; +import { registerSiemRuleMigrationsStatsRoute } from './stats'; +import { registerSiemRuleMigrationsCancelRoute } from './cancel'; export const registerSiemRuleMigrationsRoutes = ( router: SecuritySolutionPluginRouter, logger: Logger ) => { registerSiemRuleMigrationsCreateRoute(router, logger); + registerSiemRuleMigrationsStartRoute(router, logger); + registerSiemRuleMigrationsStatsRoute(router, logger); + registerSiemRuleMigrationsCancelRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts new file mode 100644 index 0000000000000..236512ea8f107 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { StartRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { + StartRuleMigrationRequestBody, + StartRuleMigrationRequestParams, +} from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_START_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStartRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATIONS_START_PATH, + access: 'internal', + options: { tags: ['access:securitySolution'] }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(StartRuleMigrationRequestParams), + body: buildRouteValidationWithZod(StartRuleMigrationRequestBody), + }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + // const { langSmithOptions, connectorId } = req.body; + try { + const ctx = await context.resolve(['core', 'actions', 'securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const { found, started } = await ruleMigrationsClient.task.start(migrationId); + + if (!found) { + return res.noContent(); + } + return res.ok({ body: { started } }); + } catch (err) { + logger.error(err); + return res.badRequest({ + body: err.message, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts new file mode 100644 index 0000000000000..7fa670d2519e3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { GetRuleMigrationStatsResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationStatsRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStatsRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_STATS_PATH, + access: 'internal', + options: { tags: ['access:securitySolution'] }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(GetRuleMigrationStatsRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const stats = await ruleMigrationsClient.task.stats(migrationId); + return res.ok({ body: stats }); + } catch (err) { + logger.error(err); + return res.badRequest({ + body: err.message, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts new file mode 100644 index 0000000000000..9460d380667ad --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; +import assert from 'assert'; +import type { + AggregationsFilterAggregate, + AggregationsMaxAggregate, + QueryDslQueryContainer, + SearchHit, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import type { StoredRuleMigration } from '../types'; +import { SiemMigrationsStatus } from '../../../../../common/siem_migrations/constants'; +import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; + +export type CreateRuleMigrationInput = Omit; +export interface RuleMigrationStats { + total: number; + pending: number; + processing: number; + finished: number; + failed: number; + lastUpdatedAt: string | undefined; +} + +export class RuleMigrationsDataClient { + constructor( + private dataStreamNamePromise: Promise, + private currentUser: AuthenticatedUser, + private esClient: ElasticsearchClient, + private logger: Logger + ) {} + + /** Indexes an array of rule migrations to be processed */ + async create(ruleMigrations: CreateRuleMigrationInput[]): Promise { + const index = await this.dataStreamNamePromise; + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: ruleMigrations.flatMap((ruleMigration) => [ + { create: { _index: index } }, + { + ...ruleMigration, + '@timestamp': new Date().toISOString(), + status: SiemMigrationsStatus.PENDING, + created_by: this.currentUser.username, + }, + ]), + }) + .catch((error) => { + this.logger.error(`Error creating rule migrations: ${error.message}`); + throw error; + }); + } + + /** Retrieves "pending" rule migrations with the provided id and updates their status to "processing" */ + async takePending(migrationId: string, size: number): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, SiemMigrationsStatus.PENDING); + + const storedRuleMigrations = await this.esClient + .search({ index, query, sort: '_doc', size }) + .catch((error) => { + this.logger.error(`Error searching for rule migrations: ${error.message}`); + throw error; + }) + .then((response) => + this.processHits(response.hits.hits, { status: SiemMigrationsStatus.PROCESSING }) + ); + + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: storedRuleMigrations.flatMap(({ _id, _index, status }) => [ + { update: { _id, _index } }, + { + doc: { + status, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }, + }, + ]), + }) + .catch((error) => { + this.logger.error( + `Error updating for rule migrations status to processing: ${error.message}` + ); + throw error; + }); + + return storedRuleMigrations; + } + + /** Updates one rule migration with the provided data and sets the status to "finished" */ + async saveFinished({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + const doc = { + ...ruleMigration, + status: SiemMigrationsStatus.FINISHED, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }; + await this.esClient + .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) + .catch((error) => { + this.logger.error(`Error updating rule migration status to finished: ${error.message}`); + throw error; + }); + } + + /** Updates all the rule migration with the provided id with status "processing" back to "pending" */ + async releaseProcessing(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, SiemMigrationsStatus.PROCESSING); + const script = { source: `ctx._source['status'] = '${SiemMigrationsStatus.PENDING}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh: false }).catch((error) => { + this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); + throw error; + }); + } + + /** Retrieves the stats for the rule migrations with the provided id */ + async getStats(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId); + const aggregations = { + pending: { filter: { term: { status: SiemMigrationsStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationsStatus.PROCESSING } } }, + finished: { filter: { term: { status: SiemMigrationsStatus.FINISHED } } }, + failed: { filter: { term: { status: SiemMigrationsStatus.ERROR } } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }; + const result = await this.esClient + .search< + unknown, + { + pending: AggregationsFilterAggregate; + processing: AggregationsFilterAggregate; + finished: AggregationsFilterAggregate; + failed: AggregationsFilterAggregate; + lastUpdatedAt: AggregationsMaxAggregate; + } + >({ index, query, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting rule migrations stats: ${error.message}`); + throw error; + }); + + const { pending, processing, finished, lastUpdatedAt, failed } = result.aggregations ?? {}; + return { + total: this.getTotalHits(result), + pending: pending?.doc_count ?? 0, + processing: processing?.doc_count ?? 0, + finished: finished?.doc_count ?? 0, + failed: failed?.doc_count ?? 0, + lastUpdatedAt: lastUpdatedAt?.value_as_string, + }; + } + + private getFilterQuery( + migrationId: string, + status?: SiemMigrationsStatus + ): QueryDslQueryContainer { + const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; + if (status) { + filter.push({ term: { status } }); + } + return { bool: { filter } }; + } + + private processHits( + hits: Array>, + override: Partial = {} + ): StoredRuleMigration[] { + return hits.map(({ _id, _index, _source }) => { + assert(_id, 'RuleMigration document should have _id'); + assert(_source, 'RuleMigration document should have _source'); + return { ..._source, ...override, _id, _index }; + }); + } + + private getTotalHits(response: SearchResponse) { + return typeof response.hits.total === 'number' + ? response.hits.total + : response.hits.total?.value ?? 0; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts index 83eb471e0cee3..41806183489cf 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts @@ -6,51 +6,70 @@ */ import { DataStreamSpacesAdapter, type InstallParams } from '@kbn/data-stream-adapter'; +import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; import { ruleMigrationsFieldMap } from './rule_migrations_field_map'; +import { RuleMigrationsDataClient } from './rule_migrations_data_client'; const TOTAL_FIELDS_LIMIT = 2500; const DATA_STREAM_NAME = '.kibana.siem-rule-migrations'; const ECS_COMPONENT_TEMPLATE_NAME = 'ecs'; +interface RuleMigrationsDataStreamCreateClientParams { + spaceId: string; + currentUser: AuthenticatedUser; + esClient: ElasticsearchClient; +} + export class RuleMigrationsDataStream { - private readonly dataStream: DataStreamSpacesAdapter; + private readonly dataStreamAdapter: DataStreamSpacesAdapter; private installPromise?: Promise; - constructor({ kibanaVersion }: { kibanaVersion: string }) { - this.dataStream = new DataStreamSpacesAdapter(DATA_STREAM_NAME, { + constructor(private logger: Logger, kibanaVersion: string) { + this.dataStreamAdapter = new DataStreamSpacesAdapter(DATA_STREAM_NAME, { kibanaVersion, totalFieldsLimit: TOTAL_FIELDS_LIMIT, }); - this.dataStream.setComponentTemplate({ + this.dataStreamAdapter.setComponentTemplate({ name: DATA_STREAM_NAME, fieldMap: ruleMigrationsFieldMap, }); - this.dataStream.setIndexTemplate({ + this.dataStreamAdapter.setIndexTemplate({ name: DATA_STREAM_NAME, componentTemplateRefs: [DATA_STREAM_NAME, ECS_COMPONENT_TEMPLATE_NAME], }); } - async install(params: InstallParams) { + async install(params: Omit) { try { - this.installPromise = this.dataStream.install(params); + this.installPromise = this.dataStreamAdapter.install({ ...params, logger: this.logger }); await this.installPromise; } catch (err) { - params.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err); + this.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err); } } - async installSpace(spaceId: string): Promise { + createClient({ + spaceId, + currentUser, + esClient, + }: RuleMigrationsDataStreamCreateClientParams): RuleMigrationsDataClient { + const dataStreamNamePromise = this.installSpace(spaceId); + return new RuleMigrationsDataClient(dataStreamNamePromise, currentUser, esClient, this.logger); + } + + // Installs the data stream for the specific space. it will only install if it hasn't been installed yet. + // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed. + private async installSpace(spaceId: string): Promise { if (!this.installPromise) { throw new Error('Siem rule migrations data stream not installed'); } // wait for install to complete, may reject if install failed, routes should handle this await this.installPromise; - let dataStreamName = await this.dataStream.getInstalledSpaceName(spaceId); + let dataStreamName = await this.dataStreamAdapter.getInstalledSpaceName(spaceId); if (!dataStreamName) { - dataStreamName = await this.dataStream.installSpace(spaceId); + dataStreamName = await this.dataStreamAdapter.installSpace(spaceId); } return dataStreamName; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts index ba9a706957bcb..c959efa3220fd 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts @@ -11,6 +11,7 @@ import type { RuleMigration } from '../../../../../common/siem_migrations/model/ export const ruleMigrationsFieldMap: FieldMap> = { '@timestamp': { type: 'date', required: false }, migration_id: { type: 'keyword', required: true }, + created_by: { type: 'keyword', required: true }, status: { type: 'keyword', required: true }, original_rule: { type: 'nested', required: true }, 'original_rule.vendor': { type: 'keyword', required: true }, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts index 390d302264cea..8103dc7c889a3 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts @@ -22,6 +22,8 @@ import type { KibanaRequest } from '@kbn/core/server'; jest.mock('./data_stream/rule_migrations_data_stream'); +const migrationId = 'dummy_migration_id'; + describe('SiemRuleMigrationsService', () => { let ruleMigrationsService: SiemRuleMigrationsService; const kibanaVersion = '8.16.0'; @@ -51,7 +53,7 @@ describe('SiemRuleMigrationsService', () => { }); }); - describe('when getClient is called', () => { + describe('when createClient is called', () => { let request: KibanaRequest; beforeEach(() => { request = httpServerMock.createKibanaRequest(); @@ -60,7 +62,7 @@ describe('SiemRuleMigrationsService', () => { describe('without setup', () => { it('should throw an error', () => { expect(() => { - ruleMigrationsService.getClient({ spaceId: 'default', request }); + ruleMigrationsService.createClient({ spaceId: 'default', request }); }).toThrowError('ES client not available, please call setup first'); }); }); @@ -71,42 +73,42 @@ describe('SiemRuleMigrationsService', () => { }); it('should call installSpace', () => { - ruleMigrationsService.getClient({ spaceId: 'default', request }); + ruleMigrationsService.createClient({ spaceId: 'default', request }); expect(mockInstallSpace).toHaveBeenCalledWith('default'); }); it('should return a client with create and search methods after setup', () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); + const client = ruleMigrationsService.createClient({ spaceId: 'default', request }); expect(client).toHaveProperty('create'); expect(client).toHaveProperty('search'); }); it('should call ES bulk create API with the correct parameters with create is called', async () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); + const client = ruleMigrationsService.createClient({ spaceId: 'default', request }); - const ruleMigrations = [{ migration_id: 'dummy_migration_id' } as RuleMigration]; + const ruleMigrations = [{ migration_id: migrationId } as RuleMigration]; await client.create(ruleMigrations); expect(esClusterClient.asScoped().asCurrentUser.bulk).toHaveBeenCalledWith( expect.objectContaining({ - body: [{ create: { _index: mockIndexName } }, { migration_id: 'dummy_migration_id' }], + body: [{ create: { _index: mockIndexName } }, { migration_id: migrationId }], refresh: 'wait_for', }) ); }); it('should call ES search API with the correct parameters with search is called', async () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); + const client = ruleMigrationsService.createClient({ spaceId: 'default', request }); - const term = { migration_id: 'dummy_migration_id' }; - await client.search(term); + const searchParams = { query: { term: { migration_id: migrationId } } }; + await client.takePending(migrationId); expect(esClusterClient.asScoped().asCurrentUser.search).toHaveBeenCalledWith( expect.objectContaining({ index: mockIndexName, - body: { query: { term } }, + ...searchParams, }) ); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index 5b20f957cb6fa..425d7c31b099d 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -5,52 +5,64 @@ * 2.0. */ +import assert from 'assert'; import type { IClusterClient, Logger } from '@kbn/core/server'; import { RuleMigrationsDataStream } from './data_stream/rule_migrations_data_stream'; import type { - SiemRuleMigrationsClient, SiemRulesMigrationsSetupParams, - SiemRuleMigrationsGetClientParams, + SiemRuleMigrationsCreateClientParams, + SiemRuleMigrationsClient, } from './types'; +import { RuleMigrationsTaskRunner } from './task/rule_migrations_task_runner'; export class SiemRuleMigrationsService { - private dataStreamAdapter: RuleMigrationsDataStream; + private rulesDataStream: RuleMigrationsDataStream; private esClusterClient?: IClusterClient; + private taskRunner: RuleMigrationsTaskRunner; constructor(private logger: Logger, kibanaVersion: string) { - this.dataStreamAdapter = new RuleMigrationsDataStream({ kibanaVersion }); + this.rulesDataStream = new RuleMigrationsDataStream(this.logger, kibanaVersion); + this.taskRunner = new RuleMigrationsTaskRunner(this.logger); } setup({ esClusterClient, ...params }: SiemRulesMigrationsSetupParams) { this.esClusterClient = esClusterClient; const esClient = esClusterClient.asInternalUser; - this.dataStreamAdapter.install({ ...params, esClient, logger: this.logger }).catch((err) => { + + this.rulesDataStream.install({ ...params, esClient }).catch((err) => { this.logger.error(`Error installing data stream for rule migrations: ${err.message}`); throw err; }); } - getClient({ spaceId, request }: SiemRuleMigrationsGetClientParams): SiemRuleMigrationsClient { - if (!this.esClusterClient) { - throw new Error('ES client not available, please call setup first'); - } - // Installs the data stream for the specific space. it will only install if it hasn't been installed yet. - // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed. - const dataStreamNamePromise = this.dataStreamAdapter.installSpace(spaceId); + createClient({ + spaceId, + currentUser, + request, + }: SiemRuleMigrationsCreateClientParams): SiemRuleMigrationsClient { + assert(currentUser, 'Current user must be authenticated'); + assert(this.esClusterClient, 'ES client not available, please call setup first'); const esClient = this.esClusterClient.asScoped(request).asCurrentUser; + const dataClient = this.rulesDataStream.createClient({ spaceId, currentUser, esClient }); + return { - create: async (ruleMigrations) => { - const _index = await dataStreamNamePromise; - return esClient.bulk({ - refresh: 'wait_for', - body: ruleMigrations.flatMap((ruleMigration) => [{ create: { _index } }, ruleMigration]), - }); - }, - search: async (term) => { - const index = await dataStreamNamePromise; - return esClient.search({ index, body: { query: { term } } }); + data: dataClient, + task: { + start: (migrationId: string) => { + return this.taskRunner.start({ migrationId, request, currentUser, dataClient }); + }, + stats: async (migrationId: string) => { + return this.taskRunner.stats({ migrationId, dataClient }); + }, + cancel: (migrationId: string) => { + return this.taskRunner.cancel({ migrationId, dataClient }); + }, }, }; } + + stop() { + this.taskRunner.stop(); + } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/actions_client_chat.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/actions_client_chat.ts new file mode 100644 index 0000000000000..204978c901df6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/actions_client_chat.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ActionsClientSimpleChatModel } from '@kbn/langchain/server'; +import { + ActionsClientBedrockChatModel, + ActionsClientChatOpenAI, + ActionsClientChatVertexAI, +} from '@kbn/langchain/server'; +import type { Logger } from '@kbn/core/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { ActionsClientChatOpenAIParams } from '@kbn/langchain/server/language_models/chat_openai'; +import type { CustomChatModelInput as ActionsClientBedrockChatModelParams } from '@kbn/langchain/server/language_models/bedrock_chat'; +import type { CustomChatModelInput as ActionsClientChatVertexAIParams } from '@kbn/langchain/server/language_models/gemini_chat'; +import type { CustomChatModelInput as ActionsClientSimpleChatModelParams } from '@kbn/langchain/server/language_models/simple_chat_model'; + +export type ChatModel = + | ActionsClientSimpleChatModel + | ActionsClientChatOpenAI + | ActionsClientBedrockChatModel + | ActionsClientChatVertexAI; + +export type ActionsClientChatModelClass = + | typeof ActionsClientSimpleChatModel + | typeof ActionsClientChatOpenAI + | typeof ActionsClientBedrockChatModel + | typeof ActionsClientChatVertexAI; + +export type ChatModelParams = Partial & + Partial & + Partial & + Partial & { + /** Enables the streaming mode of the response, disabled by default */ + streaming?: boolean; + }; + +const llmTypeDictionary: Record = { + [`.gen-ai`]: `openai`, + [`.bedrock`]: `bedrock`, + [`.gemini`]: `gemini`, +}; + +export class ActionsClientChat { + constructor( + private readonly connectorId: string, + private readonly actionsClient: ActionsClient, + private readonly logger: Logger + ) {} + + public async createModel(params?: ChatModelParams): Promise { + const connector = await this.actionsClient.get({ id: this.connectorId }); + if (!connector) { + throw new Error(`Connector not found: ${this.connectorId}`); + } + + const llmType = this.getLLMType(connector.actionTypeId); + const ChatModelClass = this.getLLMClass(llmType); + + const model = new ChatModelClass({ + actionsClient: this.actionsClient, + connectorId: this.connectorId, + logger: this.logger, + llmType, + model: connector.config?.defaultModel, + ...params, + streaming: params?.streaming ?? false, // disabling streaming by default, for some reason is enabled when omitted + }); + return model; + } + + private getLLMType(actionTypeId: string): string | undefined { + if (llmTypeDictionary[actionTypeId]) { + return llmTypeDictionary[actionTypeId]; + } + throw new Error(`Unknown LLM type for action type ID: ${actionTypeId}`); + } + + private getLLMClass(llmType?: string): ActionsClientChatModelClass { + switch (llmType) { + case 'bedrock': + return ActionsClientBedrockChatModel; + case 'gemini': + return ActionsClientChatVertexAI; + case 'openai': + default: + return ActionsClientChatOpenAI; + } + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/esql_knowledge_base_caller.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/esql_knowledge_base_caller.ts new file mode 100644 index 0000000000000..963eec6ace830 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/esql_knowledge_base_caller.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { naturalLanguageToEsql, type InferenceClient } from '@kbn/inference-plugin/server'; +import type { Logger } from '@kbn/core/server'; +import { lastValueFrom } from 'rxjs'; + +export type EsqlKnowledgeBaseCaller = (input: string) => Promise; + +type GetEsqlTranslatorToolParams = (params: { + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +}) => EsqlKnowledgeBaseCaller; + +export const getEsqlKnowledgeBase: GetEsqlTranslatorToolParams = + ({ inferenceClient: client, connectorId, logger }) => + async (input: string) => { + const { content } = await lastValueFrom( + naturalLanguageToEsql({ + client, + connectorId, + input, + logger: { + debug: (source) => { + logger.debug(typeof source === 'function' ? source() : source); + }, + }, + }) + ); + return content; + }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts new file mode 100644 index 0000000000000..bfa1fdb658abe --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { END, START, StateGraph } from '@langchain/langgraph'; +import { AIMessage } from '@langchain/core/messages'; +import { migrateRuleState } from './state'; +import type { MigrateRuleGraphParams, MigrateRuleState } from './types'; +import { getEsqlTranslationPrompt } from './prompts'; +import { getEsqlKnowledgeBase, type EsqlKnowledgeBaseCaller } from './esql_knowledge_base_caller'; + +type GraphNode = (state: MigrateRuleState) => Promise>; + +const createTranslationNode = (esqlKnowledgeBaseCaller: EsqlKnowledgeBaseCaller): GraphNode => { + return async (state) => { + const input = getEsqlTranslationPrompt(state); + const response = await esqlKnowledgeBaseCaller(input); + return { messages: [new AIMessage(response)] }; + }; +}; + +const responseNode: GraphNode = async (state) => { + const messages = state.messages; + const lastMessage = messages[messages.length - 1] as AIMessage; + return { response: lastMessage.content as string }; +}; + +export function getMigrateRuleGraph({ + inferenceClient, + connectorId, + logger, +}: MigrateRuleGraphParams) { + const esqlKnowledgeBaseCaller = getEsqlKnowledgeBase({ inferenceClient, connectorId, logger }); + const translationNode = createTranslationNode(esqlKnowledgeBaseCaller); + + const translateRuleGraph = new StateGraph(migrateRuleState) + // Nodes + .addNode('translation', translationNode) + .addNode('processResponse', responseNode) + // Edges + .addEdge(START, 'translation') + .addEdge('translation', 'processResponse') + .addEdge('processResponse', END); + + return translateRuleGraph.compile(); +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/prompts.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/prompts.ts new file mode 100644 index 0000000000000..75b4872044f71 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/prompts.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MigrateRuleState } from './types'; + +export const getEsqlTranslationPrompt = ( + state: MigrateRuleState +): string => `You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk to Elastic Security. +Below you will find Splunk rule information: the title, description and the SPL (Search Processing Language) query. +Your goal is to translate the SPL query into an equivalent Elastic Security Query Language (ES|QL) query. + +The output will be parsed and should contain: +- First, the ES|QL query inside an \`\`\`esql code block. +- At the end, the summary of the translation process followed in markdown, starting with "## Migration Summary". + +This is the Splunk rule information: + +<> +${state.original_rule.title} +<> + +<> +${state.original_rule.description} +<> + +<> +${state.original_rule.query} +<> +`; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts new file mode 100644 index 0000000000000..6c1ea925c0a0d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BaseMessage } from '@langchain/core/messages'; +import { Annotation, messagesStateReducer } from '@langchain/langgraph'; +import type { + ElasticRule, + OriginalRule, +} from '../../../../../../common/siem_migrations/model/rule_migration.gen'; + +export const migrateRuleState = Annotation.Root({ + messages: Annotation({ + reducer: messagesStateReducer, + default: () => [], + }), + original_rule: Annotation(), + elastic_rule: Annotation({ + reducer: (state, action) => ({ ...state, ...action }), + }), + translation_state: Annotation(), + prebuilt_rule_id: Annotation(), + response: Annotation(), +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts new file mode 100644 index 0000000000000..588437e7d2a4a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { naturalLanguageToEsql, type InferenceClient } from '@kbn/inference-plugin/server'; +import type { StructuredTool } from '@langchain/core/tools'; +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { lastValueFrom } from 'rxjs'; +import { z } from '@kbn/zod'; + +const TOOL_NAME = 'esql_translator'; +const schema = z.object({ + splQuery: z.string().describe(`The exact SPL query to translate to ES|QL`), + title: z.string().describe(`The title of the splunk rule`), + description: z.string().describe(`The description of the splunk rule`), +}); +type Schema = typeof schema; +type SchemaInput = z.output; + +const toolParams = { + name: TOOL_NAME, + description: `ALWAYS use the "${TOOL_NAME}" tool to convert the Splunk detection rule (SPL) to an Elastic ES|QL query.`, + schema, + tags: ['esql', 'query-translation', 'knowledge-base'], +}; + +const createInputPrompt = ({ + splQuery, + title, + description, +}: SchemaInput) => `Translate the following Splunk SPL (Search Processing Language) query rule to an ES|QL query, in order to be used as an Elastic Security detection rule: + +Splunk rule title: ${title} + +Splunk rule description: ${description} + +Splunk rule SPL query: +\`\`\`spl +${splQuery} +\`\`\` + +Along with the translated ES|QL query, you should also provide a summary of the translation process you followed, in markdown format. +The output should contain: +- First, the ES|QL query inside an \`\`\`esql code block. +- At the end, the summary of the translation process followed in markdown, starting with "## Translation Summary". +`; + +export type EsqlTranslatorTool = StructuredTool; + +interface GetEsqlTranslatorToolParams { + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +} +export const getEsqlTranslatorTool = ({ + inferenceClient: client, + connectorId, + logger, +}: GetEsqlTranslatorToolParams): EsqlTranslatorTool => { + const callNaturalLanguageToEsql = async (input: SchemaInput) => { + return lastValueFrom( + naturalLanguageToEsql({ + client, + connectorId, + input: createInputPrompt(input), + logger: { + debug: (source) => { + logger.debug(typeof source === 'function' ? source() : source); + }, + }, + }) + ); + }; + + const esqlTool = new DynamicStructuredTool({ + ...toolParams, + responseFormat: 'markdown', + func: async (input) => { + const generateEvent = await callNaturalLanguageToEsql(input); + const answer = generateEvent.content ?? 'An error occurred in the tool'; + + logger.debug(`Received response from NL to ESQL tool: ${answer}`); + return answer; + }, + }); + + return esqlTool; +}; + +export type EsqlKnowledgeBaseCaller = (input: string) => Promise; +export const getEsqlKnowledgeBase = + ({ + inferenceClient: client, + connectorId, + logger, + }: GetEsqlTranslatorToolParams): EsqlKnowledgeBaseCaller => + async (input: string) => { + const { content } = await lastValueFrom( + naturalLanguageToEsql({ + client, + connectorId, + input, + logger: { + debug: (source) => { + logger.debug(typeof source === 'function' ? source() : source); + }, + }, + }) + ); + return content; + }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts new file mode 100644 index 0000000000000..d5cdebb1d7bdc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { StructuredTool } from '@langchain/core/tools'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { ChatModel } from './actions_client_chat'; +import type { migrateRuleState } from './state'; + +export type MigrateRuleState = typeof migrateRuleState.State; + +export interface MigrateRuleGraphParams { + model: ChatModel; + tools: StructuredTool[]; + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts new file mode 100644 index 0000000000000..29a0506bb30cc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { AbortError, abortSignalToPromise } from '@kbn/kibana-utils-plugin/server'; +import type { RuleMigrationTaskStats } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationStats } from '../data_stream/rule_migrations_data_client'; +import type { StoredRuleMigration } from '../types'; +import type { + RuleMigrationTaskStartParams, + RuleMigrationTaskStartResult, + RuleMigrationTaskStatsParams, + RuleMigrationTaskCancelParams, + RuleMigrationTaskCancelResult, +} from './types'; + +interface MigrationProcessing { + abortController: AbortController; + user: string; +} + +interface RuleLogger { + info: (msg: string) => void; + debug: (msg: string) => void; + error: (msg: string, error: Error) => void; +} +const getRuleLogger = (logger: Logger): RuleLogger => { + const prefix = '[ruleMigrationsTask]: '; + return { + info: (msg) => logger.info(`${prefix}${msg}`), + debug: (msg) => logger.debug(`${prefix}${msg}`), + error: (msg, error) => logger.error(`${prefix}${msg}: ${error.message}`), + }; +}; + +const BATCH_SIZE = 2 as const; +const BATCH_SLEEP_MS = 10000 as const; + +export class RuleMigrationsTaskRunner { + private migrationsProcessing: Map; + private logger: RuleLogger; + + constructor(logger: Logger) { + this.migrationsProcessing = new Map(); + this.logger = getRuleLogger(logger); + } + + /** Starts a rule migration task */ + async start({ + migrationId, + currentUser, + request, + dataClient, + }: RuleMigrationTaskStartParams): Promise { + if (this.migrationsProcessing.has(migrationId)) { + return { found: true, started: false }; // already processing + } + // Just in case some previous execution was interrupted without releasing + await dataClient.releaseProcessing(migrationId); + + const stats = await dataClient.getStats(migrationId); + if (stats.total === 0) { + return { found: false, started: false }; + } + if (stats.pending === 0) { + return { found: true, started: false }; + } + + this.run({ migrationId, currentUser, request, dataClient }); + return { found: true, started: true }; + } + + private async run({ + migrationId, + currentUser, + request, + dataClient, + }: RuleMigrationTaskStartParams): Promise { + if (this.migrationsProcessing.has(migrationId)) { + throw new Error(`Migration ${migrationId} is already being processed`); + } + + const abortController = new AbortController(); + this.migrationsProcessing.set(migrationId, { abortController, user: currentUser.username }); + + const abortPromise = abortSignalToPromise(abortController.signal); + + try { + const sleep = async (ms: number) => { + this.logger.info(`Sleeping ${ms / 1000}s before next iteration`); + await Promise.race([ + new Promise((resolve) => setTimeout(resolve, ms)), + abortPromise.promise, + ]); + }; + + let isDone: boolean = false; + do { + const ruleMigrations = await dataClient.takePending(migrationId, BATCH_SIZE); + this.logger.info(`Processing ${ruleMigrations.length} rule migrations`); + + await Promise.all( + ruleMigrations.map(async (ruleMigration) => { + const ruleMigrationResult = await dummyRuleMigrationGraph.invoke(ruleMigration); + await dataClient.saveFinished(ruleMigrationResult); + }) + ); + + const { pending } = await dataClient.getStats(migrationId); + isDone = pending === 0; + if (!isDone) { + await sleep(BATCH_SLEEP_MS); + } + } while (!isDone); + } catch (error) { + await dataClient.releaseProcessing(migrationId); + + if (error instanceof AbortError) { + this.logger.info(`Abort signal received, migration ${migrationId} task stopped`); + return; + } else { + this.logger.error(`Error processing migration ${migrationId}`, error); + } + } finally { + this.migrationsProcessing.delete(migrationId); + abortPromise.cleanup(); + } + } + + /** Retries the status of a running migration */ + async stats({ + migrationId, + dataClient, + }: RuleMigrationTaskStatsParams): Promise { + const stats = await dataClient.getStats(migrationId); + const status = this.getTaskStatus(stats, migrationId); + return { + status, + total: stats.total, + finished: stats.finished, + pending: stats.pending, + processing: stats.processing, + failed: stats.failed, + last_iteration_at: stats.lastUpdatedAt, + }; + } + + private getTaskStatus( + stats: RuleMigrationStats, + migrationId: string + ): RuleMigrationTaskStats['status'] { + if (this.migrationsProcessing.has(migrationId)) { + return 'processing'; + } + if (stats.pending === stats.total) { + return 'not_started'; + } + if (stats.finished === stats.total) { + return 'done'; + } + return 'cancelled'; + } + + /** Aborts a running migration */ + async cancel({ + migrationId, + dataClient, + }: RuleMigrationTaskCancelParams): Promise { + const migrationProcessing = this.migrationsProcessing.get(migrationId); + if (migrationProcessing) { + migrationProcessing.abortController.abort(); + return { found: true, cancelled: true }; + } + + const stats = await dataClient.getStats(migrationId); + if (stats.total > 0) { + return { found: true, cancelled: false }; + } + return { found: false, cancelled: false }; + } + + /** Stops all running migrations */ + stop() { + this.migrationsProcessing.forEach((migrationProcessing) => { + migrationProcessing.abortController.abort(); + }); + this.migrationsProcessing.clear(); + } +} + +const dummyRuleMigrationGraph = { + invoke: async (ruleMigration: StoredRuleMigration): Promise => { + return new Promise((resolve) => { + console.log('dummyRuleMigrationGraph start'); + setTimeout(() => { + console.log('dummyRuleMigrationGraph resolved'); + resolve(ruleMigration); + }, 5000); + }); + }, +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts new file mode 100644 index 0000000000000..1f172bfb6ab78 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, KibanaRequest } from '@kbn/core/server'; +import type { RuleMigrationsDataClient } from '../data_stream/rule_migrations_data_client'; + +export interface RuleMigrationTaskStartParams { + migrationId: string; + currentUser: AuthenticatedUser; + request: KibanaRequest; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskCancelParams { + migrationId: string; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskStatsParams { + migrationId: string; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskStartResult { + started: boolean; + found: boolean; +} + +export interface RuleMigrationTaskCancelResult { + cancelled: boolean; + found: boolean; +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index 1892032a21723..fc1f1d7015231 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -5,10 +5,19 @@ * 2.0. */ -import type { BulkResponse, SearchResponse } from '@elastic/elasticsearch/lib/api/types'; -import type { IClusterClient, KibanaRequest } from '@kbn/core/server'; +import type { AuthenticatedUser, IClusterClient, KibanaRequest } from '@kbn/core/server'; import type { Subject } from 'rxjs'; -import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen'; +import type { + RuleMigration, + RuleMigrationTaskStats, +} from '../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationsDataClient } from './data_stream/rule_migrations_data_client'; +import type { RuleMigrationTaskCancelResult, RuleMigrationTaskStartResult } from './task/types'; + +export interface StoredRuleMigration extends RuleMigration { + _id: string; + _index: string; +} export interface SiemRulesMigrationsSetupParams { esClusterClient: IClusterClient; @@ -16,15 +25,17 @@ export interface SiemRulesMigrationsSetupParams { tasksTimeoutMs?: number; } -export interface SiemRuleMigrationsGetClientParams { +export interface SiemRuleMigrationsCreateClientParams { request: KibanaRequest; + currentUser: AuthenticatedUser | null; spaceId: string; } -export interface RuleMigrationSearchParams { - migration_id?: string; -} export interface SiemRuleMigrationsClient { - create: (body: RuleMigration[]) => Promise; - search: (params: RuleMigrationSearchParams) => Promise; + data: RuleMigrationsDataClient; + task: { + start: (migrationId: string) => Promise; + stats: (migrationId: string) => Promise; + cancel: (migrationId: string) => Promise; + }; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts index b84281eb13d9b..7a85dd625feec 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts @@ -9,11 +9,8 @@ import type { Logger } from '@kbn/core/server'; import { ReplaySubject, type Subject } from 'rxjs'; import type { ConfigType } from '../../config'; import { SiemRuleMigrationsService } from './rules/siem_rule_migrations_service'; -import type { - SiemMigrationsClient, - SiemMigrationsSetupParams, - SiemMigrationsGetClientParams, -} from './types'; +import type { SiemMigrationsSetupParams, SiemMigrationsCreateClientParams } from './types'; +import type { SiemRuleMigrationsClient } from './rules/types'; export class SiemMigrationsService { private pluginStop$: Subject; @@ -30,13 +27,12 @@ export class SiemMigrationsService { } } - createClient(params: SiemMigrationsGetClientParams): SiemMigrationsClient { - return { - rules: this.rules.getClient(params), - }; + createRulesClient(params: SiemMigrationsCreateClientParams): SiemRuleMigrationsClient { + return this.rules.createClient(params); } stop() { + this.rules.stop(); this.pluginStop$.next(); this.pluginStop$.complete(); } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts index b5647ff65e214..d2af1e2518722 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts @@ -6,15 +6,11 @@ */ import type { IClusterClient } from '@kbn/core/server'; -import type { SiemRuleMigrationsClient, SiemRuleMigrationsGetClientParams } from './rules/types'; +import type { SiemRuleMigrationsCreateClientParams } from './rules/types'; export interface SiemMigrationsSetupParams { esClusterClient: IClusterClient; tasksTimeoutMs?: number; } -export type SiemMigrationsGetClientParams = SiemRuleMigrationsGetClientParams; - -export interface SiemMigrationsClient { - rules: SiemRuleMigrationsClient; -} +export type SiemMigrationsCreateClientParams = SiemRuleMigrationsCreateClientParams; diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 8e3af9b9bce8a..9d6f49a26fe33 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -166,8 +166,12 @@ export class RequestContextFactory implements IRequestContextFactory { }) ), - getSiemMigrationsClient: memoize(() => - siemMigrationsService.createClient({ request, spaceId: getSpaceId() }) + getSiemRuleMigrationsClient: memoize(() => + siemMigrationsService.createRulesClient({ + request, + currentUser: coreContext.security.authc.getCurrentUser(), + spaceId: getSpaceId(), + }) ), getExceptionListClient: () => { From 2dd958e421b0959b28d5b52b5a958632936203cf Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Tue, 29 Oct 2024 16:05:57 +0100 Subject: [PATCH 02/14] integrate agent into the task --- .../common/siem_migrations/constants.ts | 6 +- .../model/api/rules/rules_migration.gen.ts | 30 ++- .../api/rules/rules_migration.schema.yaml | 18 +- .../model/rule_migration.gen.ts | 59 +++-- .../model/rule_migration.schema.yaml | 67 ++--- x-pack/plugins/security_solution/kibana.jsonc | 3 +- .../routes/__mocks__/request_context.ts | 2 + .../siem_migrations/rules/__mocks__/mocks.ts | 14 +- .../lib/siem_migrations/rules/api/index.ts | 4 +- .../lib/siem_migrations/rules/api/start.ts | 42 +++- .../rules/api/{cancel.ts => stop.ts} | 20 +- .../rule_migrations_data_client.ts | 51 +++- .../rules/siem_rule_migrations_service.ts | 12 +- .../siem_migrations/rules/task/agent/graph.ts | 43 ++-- .../siem_migrations/rules/task/agent/index.ts | 8 + .../agent/nodes/match_prebuilt_rule/index.ts | 8 + .../match_prebuilt_rule.ts | 59 +++++ .../nodes/match_prebuilt_rule/prompts.ts | 35 +++ .../task/agent/nodes/process_response.ts | 54 ++++ .../esql_knowledge_base_caller.ts | 2 +- .../task/agent/nodes/translate_query/index.ts | 7 + .../translate_query/prompt.ts} | 8 +- .../nodes/translate_query/translate_query.ts | 32 +++ .../siem_migrations/rules/task/agent/state.ts | 8 +- .../task/agent/tools/esql_translator_tool.ts | 115 --------- .../siem_migrations/rules/task/agent/types.ts | 9 +- .../rules/task/rule_migrations_task_runner.ts | 237 ++++++++++++------ .../lib/siem_migrations/rules/task/types.ts | 44 +++- .../{agent => util}/actions_client_chat.ts | 0 .../rules/task/util/prebuilt_rules.test.ts | 119 +++++++++ .../rules/task/util/prebuilt_rules.ts | 77 ++++++ .../server/lib/siem_migrations/rules/types.ts | 27 +- .../server/plugin_contract.ts | 2 + .../server/request_context_factory.ts | 2 + .../plugins/security_solution/server/types.ts | 6 +- 35 files changed, 871 insertions(+), 359 deletions(-) rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/{cancel.ts => stop.ts} (59%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/{ => nodes/translate_query}/esql_knowledge_base_caller.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/{prompts.ts => nodes/translate_query/prompt.ts} (66%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts rename x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/{agent => util}/actions_client_chat.ts (100%) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index 935f4868335c6..b085d854cd07d 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -12,12 +12,12 @@ export const SIEM_RULE_MIGRATIONS_START_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/start` as const; export const SIEM_RULE_MIGRATIONS_STATS_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stats` as const; -export const SIEM_RULE_MIGRATIONS_CANCEL_PATH = - `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/cancel` as const; +export const SIEM_RULE_MIGRATIONS_STOP_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stop` as const; export enum SiemMigrationsStatus { PENDING = 'pending', PROCESSING = 'processing', FINISHED = 'finished', - ERROR = 'error', + FAILED = 'failed', } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts index bf82b77ac5311..822219f9ec698 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts @@ -19,22 +19,6 @@ import { z } from '@kbn/zod'; import { OriginalRule, RuleMigration, RuleMigrationTaskStats } from '../../rule_migration.gen'; import { ConnectorId, LangSmithOptions } from '../common.gen'; -export type CancelRuleMigrationRequestParams = z.infer; -export const CancelRuleMigrationRequestParams = z.object({ - migration_id: z.string(), -}); -export type CancelRuleMigrationRequestParamsInput = z.input< - typeof CancelRuleMigrationRequestParams ->; - -export type CancelRuleMigrationResponse = z.infer; -export const CancelRuleMigrationResponse = z.object({ - /** - * Indicates the migration has been cancelled. - */ - cancelled: z.boolean(), -}); - export type CreateRuleMigrationRequestBody = z.infer; export const CreateRuleMigrationRequestBody = z.array(OriginalRule); export type CreateRuleMigrationRequestBodyInput = z.input; @@ -81,3 +65,17 @@ export const StartRuleMigrationResponse = z.object({ */ started: z.boolean(), }); + +export type StopRuleMigrationRequestParams = z.infer; +export const StopRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type StopRuleMigrationRequestParamsInput = z.input; + +export type StopRuleMigrationResponse = z.infer; +export const StopRuleMigrationResponse = z.object({ + /** + * Indicates the migration has been stopped. + */ + stopped: z.boolean(), +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml index 1105736db29be..16d7d6d45f1a4 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml @@ -118,12 +118,12 @@ paths: 204: description: Indicates the migration id was not found. - /internal/siem_migrations/rules/{migration_id}/cancel: + /internal/siem_migrations/rules/{migration_id}/stop: put: - summary: Cancels an existing rule migration - operationId: CancelRuleMigration + summary: Stops an existing rule migration + operationId: StopRuleMigration x-codegen-enabled: true - description: Cancels a running SIEM rules migration using the migration id provided + description: Stops a running SIEM rules migration using the migration id provided tags: - SIEM Rule Migrations parameters: @@ -132,19 +132,19 @@ paths: required: true schema: type: string - description: The migration id to cancel + description: The migration id to stop responses: 200: - description: Indicates migration task cancellation has been processed successfully. + description: Indicates migration task stop has been processed successfully. content: application/json: schema: type: object required: - - cancelled + - stopped properties: - cancelled: + stopped: type: boolean - description: Indicates the migration has been cancelled. + description: Indicates the migration has been stopped. 204: description: Indicates the migration id was not found running. diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 5bf3749088cf2..55dc16e464696 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -71,11 +71,11 @@ export const ElasticRule = z.object({ /** * The translated elastic query. */ - query: z.string(), + query: z.string().optional(), /** * The translated elastic query language. */ - query_language: z.literal('esql').default('esql'), + query_language: z.literal('esql').optional(), /** * The Elastic prebuilt rule id matched. */ @@ -135,29 +135,34 @@ export const RuleMigrationTaskStats = z.object({ /** * Indicates if the migration task status. */ - status: z.enum(['not_started', 'processing', 'done', 'cancelled']), - /** - * The total number of rules to migrate. - */ - total: z.number().int().optional(), - /** - * The number of rules that have been migrated. - */ - finished: z.number().int(), - /** - * The number of rules that are being migrated. - */ - processing: z.number().int(), - /** - * The number of rules that are pending migration. - */ - pending: z.number().int(), - /** - * The number of rules that have failed migration. - */ - failed: z.number().int(), - /** - * The moment of the last execution. - */ - last_iteration_at: z.string().optional(), + status: z.enum(['ready', 'running', 'stopped', 'done']), + /** + * The rules migration stats. + */ + rules: z.object({ + /** + * The total number of rules to migrate. + */ + total: z.number().int(), + /** + * The number of rules that have been migrated. + */ + finished: z.number().int(), + /** + * The number of rules that are being migrated. + */ + processing: z.number().int(), + /** + * The number of rules that are pending migration. + */ + pending: z.number().int(), + /** + * The number of rules that have failed migration. + */ + failed: z.number().int(), + }), + /** + * The moment of the last update. + */ + last_updated_at: z.string().optional(), }); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 16795b6ff7bc9..50605f9256b76 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -48,8 +48,6 @@ components: description: The migrated elastic rule. required: - title - - query - - query_language properties: title: type: string @@ -68,7 +66,6 @@ components: description: The translated elastic query language. enum: - esql - default: esql prebuilt_rule_id: type: string description: The Elastic prebuilt rule id matched. @@ -86,7 +83,7 @@ components: - status - created_by properties: - "@timestamp": + '@timestamp': type: string description: The moment of creation migration_id: @@ -113,7 +110,7 @@ components: - pending - processing - finished - - error + - failed default: pending comments: type: array @@ -132,35 +129,41 @@ components: description: The rule migration task stats object. required: - status - - total_rules - - finished - - processing - - pending - - failed + - rules properties: - status: + status: type: string description: Indicates if the migration task status. enum: - - not_started - - processing + - ready + - running + - stopped - done - - cancelled - total: - type: integer - description: The total number of rules to migrate. - finished: - type: integer - description: The number of rules that have been migrated. - processing: - type: integer - description: The number of rules that are being migrated. - pending: - type: integer - description: The number of rules that are pending migration. - failed: - type: integer - description: The number of rules that have failed migration. - last_iteration_at: - type: string - description: The moment of the last execution. \ No newline at end of file + rules: + type: object + description: The rules migration stats. + required: + - total + - finished + - processing + - pending + - failed + properties: + total: + type: integer + description: The total number of rules to migrate. + finished: + type: integer + description: The number of rules that have been migrated. + processing: + type: integer + description: The number of rules that are being migrated. + pending: + type: integer + description: The number of rules that are pending migration. + failed: + type: integer + description: The number of rules that have failed migration. + last_updated_at: + type: string + description: The moment of the last update. diff --git a/x-pack/plugins/security_solution/kibana.jsonc b/x-pack/plugins/security_solution/kibana.jsonc index e48a9794b7e5c..8c8b77d48bc9f 100644 --- a/x-pack/plugins/security_solution/kibana.jsonc +++ b/x-pack/plugins/security_solution/kibana.jsonc @@ -54,7 +54,8 @@ "savedSearch", "unifiedDocViewer", "charts", - "entityManager" + "entityManager", + "inference" ], "optionalPlugins": [ "encryptedSavedObjects", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts index bec7c4823ba01..d2aacbdeaeeaf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts @@ -80,6 +80,7 @@ export const createMockClients = () => { packages: packageServiceMock.createClient(), }, siemRuleMigrationsClient: siemMigrationsServiceMock.createRulesClient(), + getInferenceClient: jest.fn(), }; }; @@ -166,6 +167,7 @@ const createSecuritySolutionRequestContextMock = ( getAuditLogger: jest.fn(() => mockAuditLogger), getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient), getSiemRuleMigrationsClient: jest.fn(() => clients.siemRuleMigrationsClient), + getInferenceClient: jest.fn(() => clients.getInferenceClient()), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts index 75666bb48580a..a4d455ecd344a 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts @@ -13,8 +13,18 @@ export const createRuleMigrationDataClient = () => ({ finish: jest.fn(), }); export const createRuleMigrationTaskClient = () => ({ - run: jest.fn().mockResolvedValue({ processed: 0 }), - cancel: jest.fn(), + start: jest.fn().mockResolvedValue({ started: true }), + stop: jest.fn().mockResolvedValue({ stopped: true }), + stats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), }); export const createRuleMigrationClient = () => ({ diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index 81245597f7d0a..dbb721b49cdc8 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -10,7 +10,7 @@ import type { SecuritySolutionPluginRouter } from '../../../../types'; import { registerSiemRuleMigrationsCreateRoute } from './create'; import { registerSiemRuleMigrationsStartRoute } from './start'; import { registerSiemRuleMigrationsStatsRoute } from './stats'; -import { registerSiemRuleMigrationsCancelRoute } from './cancel'; +import { registerSiemRuleMigrationsStopRoute } from './stop'; export const registerSiemRuleMigrationsRoutes = ( router: SecuritySolutionPluginRouter, @@ -19,5 +19,5 @@ export const registerSiemRuleMigrationsRoutes = ( registerSiemRuleMigrationsCreateRoute(router, logger); registerSiemRuleMigrationsStartRoute(router, logger); registerSiemRuleMigrationsStatsRoute(router, logger); - registerSiemRuleMigrationsCancelRoute(router, logger); + registerSiemRuleMigrationsStopRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts index 236512ea8f107..2b2feb9f8c94e 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -7,6 +7,8 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { APMTracer } from '@kbn/langchain/server/tracers/apm'; +import { getLangSmithTracer } from '@kbn/langchain/server/tracers/langsmith'; import type { StartRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; import { StartRuleMigrationRequestBody, @@ -37,14 +39,46 @@ export const registerSiemRuleMigrationsStartRoute = ( }, async (context, req, res): Promise> => { const migrationId = req.params.migration_id; - // const { langSmithOptions, connectorId } = req.body; + const { langSmithOptions, connectorId } = req.body; + try { - const ctx = await context.resolve(['core', 'actions', 'securitySolution']); + const ctx = await context.resolve([ + 'core', + 'actions', + 'alerting', + 'securitySolution', + 'licensing', + ]); + if (!ctx.licensing.license.hasAtLeast('enterprise')) { + return res.forbidden({ + body: 'You must have a trial or enterprise license to use this feature', + }); + } + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const inferenceClient = ctx.securitySolution.getInferenceClient(); + const actionsClient = ctx.actions.getActionsClient(); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); - const { found, started } = await ruleMigrationsClient.task.start(migrationId); + const invocationConfig = { + callbacks: [ + new APMTracer({ projectName: langSmithOptions?.project_name ?? 'default' }, logger), + ...getLangSmithTracer({ ...langSmithOptions, logger }), + ], + }; + + const { exists, started } = await ruleMigrationsClient.task.start({ + migrationId, + connectorId, + invocationConfig, + inferenceClient, + actionsClient, + soClient, + rulesClient, + }); - if (!found) { + if (!exists) { return res.noContent(); } return res.ok({ body: { started } }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/cancel.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts similarity index 59% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/cancel.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts index 3993e0a3febea..d1097ad5c7cc8 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/cancel.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts @@ -7,18 +7,18 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; -import type { CancelRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { CancelRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { SIEM_RULE_MIGRATIONS_CANCEL_PATH } from '../../../../../common/siem_migrations/constants'; +import type { StopRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { StopRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_STOP_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -export const registerSiemRuleMigrationsCancelRoute = ( +export const registerSiemRuleMigrationsStopRoute = ( router: SecuritySolutionPluginRouter, logger: Logger ) => { router.versioned .put({ - path: SIEM_RULE_MIGRATIONS_CANCEL_PATH, + path: SIEM_RULE_MIGRATIONS_STOP_PATH, access: 'internal', options: { tags: ['access:securitySolution'] }, }) @@ -26,20 +26,20 @@ export const registerSiemRuleMigrationsCancelRoute = ( { version: '1', validate: { - request: { params: buildRouteValidationWithZod(CancelRuleMigrationRequestParams) }, + request: { params: buildRouteValidationWithZod(StopRuleMigrationRequestParams) }, }, }, - async (context, req, res): Promise> => { + async (context, req, res): Promise> => { const migrationId = req.params.migration_id; try { const ctx = await context.resolve(['core', 'actions', 'securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const { found, cancelled } = await ruleMigrationsClient.task.cancel(migrationId); + const { exists, stopped } = await ruleMigrationsClient.task.stop(migrationId); - if (!found) { + if (!exists) { return res.noContent(); } - return res.ok({ body: { cancelled } }); + return res.ok({ body: { stopped } }); } catch (err) { logger.error(err); return res.badRequest({ diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts index 9460d380667ad..ee80a4bfe9d1c 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts @@ -58,7 +58,12 @@ export class RuleMigrationsDataClient { }); } - /** Retrieves "pending" rule migrations with the provided id and updates their status to "processing" */ + /** + * Retrieves `pending` rule migrations with the provided id and updates their status to `processing`. + * This operation is not atomic at migration level: + * - Multiple tasks can process different migrations simultaneously. + * - Multiple tasks should not process the same migration simultaneously. + */ async takePending(migrationId: string, size: number): Promise { const index = await this.dataStreamNamePromise; const query = this.getFilterQuery(migrationId, SiemMigrationsStatus.PENDING); @@ -97,7 +102,7 @@ export class RuleMigrationsDataClient { return storedRuleMigrations; } - /** Updates one rule migration with the provided data and sets the status to "finished" */ + /** Updates one rule migration with the provided data and sets the status to `finished` */ async saveFinished({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { const doc = { ...ruleMigration, @@ -113,7 +118,23 @@ export class RuleMigrationsDataClient { }); } - /** Updates all the rule migration with the provided id with status "processing" back to "pending" */ + /** Updates one rule migration with the provided data and sets the status to `failed` */ + async saveError({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + const doc = { + ...ruleMigration, + status: SiemMigrationsStatus.FAILED, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }; + await this.esClient + .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) + .catch((error) => { + this.logger.error(`Error updating rule migration status to finished: ${error.message}`); + throw error; + }); + } + + /** Updates all the rule migration with the provided id with status `processing` back to `pending` */ async releaseProcessing(migrationId: string): Promise { const index = await this.dataStreamNamePromise; const query = this.getFilterQuery(migrationId, SiemMigrationsStatus.PROCESSING); @@ -124,6 +145,20 @@ export class RuleMigrationsDataClient { }); } + /** Updates all the rule migration with the provided id with status `processing` or `failed` back to `pending` */ + async releaseProcessable(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, [ + SiemMigrationsStatus.PROCESSING, + SiemMigrationsStatus.FAILED, + ]); + const script = { source: `ctx._source['status'] = '${SiemMigrationsStatus.PENDING}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh: true }).catch((error) => { + this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); + throw error; + }); + } + /** Retrieves the stats for the rule migrations with the provided id */ async getStats(migrationId: string): Promise { const index = await this.dataStreamNamePromise; @@ -132,7 +167,7 @@ export class RuleMigrationsDataClient { pending: { filter: { term: { status: SiemMigrationsStatus.PENDING } } }, processing: { filter: { term: { status: SiemMigrationsStatus.PROCESSING } } }, finished: { filter: { term: { status: SiemMigrationsStatus.FINISHED } } }, - failed: { filter: { term: { status: SiemMigrationsStatus.ERROR } } }, + failed: { filter: { term: { status: SiemMigrationsStatus.FAILED } } }, lastUpdatedAt: { max: { field: 'updated_at' } }, }; const result = await this.esClient @@ -164,11 +199,15 @@ export class RuleMigrationsDataClient { private getFilterQuery( migrationId: string, - status?: SiemMigrationsStatus + status?: SiemMigrationsStatus | SiemMigrationsStatus[] ): QueryDslQueryContainer { const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; if (status) { - filter.push({ term: { status } }); + if (Array.isArray(status)) { + filter.push({ terms: { status } }); + } else { + filter.push({ term: { status } }); + } } return { bool: { filter } }; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index 425d7c31b099d..c0e722315f21c 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -49,20 +49,20 @@ export class SiemRuleMigrationsService { return { data: dataClient, task: { - start: (migrationId: string) => { - return this.taskRunner.start({ migrationId, request, currentUser, dataClient }); + start: (params) => { + return this.taskRunner.start({ ...params, currentUser, dataClient }); }, - stats: async (migrationId: string) => { + stats: async (migrationId) => { return this.taskRunner.stats({ migrationId, dataClient }); }, - cancel: (migrationId: string) => { - return this.taskRunner.cancel({ migrationId, dataClient }); + stop: (migrationId) => { + return this.taskRunner.stop({ migrationId, dataClient }); }, }, }; } stop() { - this.taskRunner.stop(); + this.taskRunner.stopAll(); } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index bfa1fdb658abe..24e371484b17a 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -6,44 +6,39 @@ */ import { END, START, StateGraph } from '@langchain/langgraph'; -import { AIMessage } from '@langchain/core/messages'; import { migrateRuleState } from './state'; import type { MigrateRuleGraphParams, MigrateRuleState } from './types'; -import { getEsqlTranslationPrompt } from './prompts'; -import { getEsqlKnowledgeBase, type EsqlKnowledgeBaseCaller } from './esql_knowledge_base_caller'; +import { processResponseNode } from './nodes/process_response'; +import { getTranslateQueryNode } from './nodes/translate_query'; +import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule'; -type GraphNode = (state: MigrateRuleState) => Promise>; - -const createTranslationNode = (esqlKnowledgeBaseCaller: EsqlKnowledgeBaseCaller): GraphNode => { - return async (state) => { - const input = getEsqlTranslationPrompt(state); - const response = await esqlKnowledgeBaseCaller(input); - return { messages: [new AIMessage(response)] }; - }; -}; - -const responseNode: GraphNode = async (state) => { - const messages = state.messages; - const lastMessage = messages[messages.length - 1] as AIMessage; - return { response: lastMessage.content as string }; -}; - -export function getMigrateRuleGraph({ +export function getRuleMigrationAgent({ + model, inferenceClient, + prebuiltRulesMap, connectorId, logger, }: MigrateRuleGraphParams) { - const esqlKnowledgeBaseCaller = getEsqlKnowledgeBase({ inferenceClient, connectorId, logger }); - const translationNode = createTranslationNode(esqlKnowledgeBaseCaller); + const matchPrebuiltRuleNode = getMatchPrebuiltRuleNode({ model, prebuiltRulesMap, logger }); + const translationNode = getTranslateQueryNode({ inferenceClient, connectorId, logger }); const translateRuleGraph = new StateGraph(migrateRuleState) // Nodes + .addNode('matchPrebuiltRule', matchPrebuiltRuleNode) .addNode('translation', translationNode) - .addNode('processResponse', responseNode) + .addNode('processResponse', processResponseNode) // Edges - .addEdge(START, 'translation') + .addEdge(START, 'matchPrebuiltRule') + .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional) .addEdge('translation', 'processResponse') .addEdge('processResponse', END); return translateRuleGraph.compile(); } + +const matchedPrebuiltRuleConditional = (state: MigrateRuleState) => { + if (state.elastic_rule?.prebuilt_rule_id) { + return END; + } + return 'translation'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts new file mode 100644 index 0000000000000..febf5fc85f5a0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getRuleMigrationAgent } from './graph'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts new file mode 100644 index 0000000000000..2d8b81d00eafb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getMatchPrebuiltRuleNode } from './match_prebuilt_rule'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts new file mode 100644 index 0000000000000..4a0404acf653d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import type { ChatModel } from '../../../util/actions_client_chat'; +import type { GraphNode } from '../../types'; +import { filterPrebuiltRules, type PrebuiltRulesMapByName } from '../../../util/prebuilt_rules'; +import { MATCH_PREBUILT_RULE_PROMPT } from './prompts'; + +interface GetMatchPrebuiltRuleNodeParams { + model: ChatModel; + prebuiltRulesMap: PrebuiltRulesMapByName; + logger: Logger; +} + +export const getMatchPrebuiltRuleNode = + ({ model, prebuiltRulesMap }: GetMatchPrebuiltRuleNodeParams): GraphNode => + async (state) => { + const mitreAttackIds = state.original_rule.mitre_attack_ids; + if (!mitreAttackIds?.length) { + return {}; + } + const filteredPrebuiltRulesMap = filterPrebuiltRules(prebuiltRulesMap, mitreAttackIds); + if (filteredPrebuiltRulesMap.size === 0) { + return {}; + } + + const outputParser = new StringOutputParser(); + const matchPrebuiltRule = MATCH_PREBUILT_RULE_PROMPT.pipe(model).pipe(outputParser); + + const elasticSecurityRules = Array(filteredPrebuiltRulesMap.keys()).join('\n'); + const response = await matchPrebuiltRule.invoke({ + elasticSecurityRules, + ruleTitle: state.original_rule.title, + }); + const cleanResponse = response.trim(); + if (cleanResponse === 'no_match') { + return {}; + } + + const result = filteredPrebuiltRulesMap.get(cleanResponse); + if (result != null) { + return { + elastic_rule: { + title: result.rule.name, + description: result.rule.description, + prebuilt_rule_id: result.rule.rule_id, + id: result.installedRuleId, + }, + }; + } + + return {}; + }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts new file mode 100644 index 0000000000000..434636d0519b1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatPromptTemplate } from '@langchain/core/prompts'; +export const MATCH_PREBUILT_RULE_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are an expert assistant in Cybersecurity, your task is to help migrating a SIEM detection rule, from Splunk Security to Elastic Security. +You will be provided with a Splunk Detection Rule name by the user, your goal is to try find an Elastic Detection Rule that covers the same threat, if any. +The list of Elastic Detection Rules suggested is provided in the context below. + +Guidelines: +If there is no Elastic rule in the list that covers the same threat, answer only with the string: no_match +If there is one Elastic rule in the list that covers the same threat, answer only with its name without any further explanation. +If there are multiple rules in the list that cover the same threat, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". + + +{elasticSecurityRules} + +`, + ], + [ + 'human', + `The Splunk Detection Rule is: +<> +{ruleTitle} +<> +`, + ], + ['ai', 'Please find the answer below:'], +]); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts new file mode 100644 index 0000000000000..053a0c67456fe --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AIMessage } from '@langchain/core/messages'; +import type { GraphNode } from '../types'; + +export const processResponseNode: GraphNode = async (state) => { + const messages = state.messages; + const lastMessage = messages[messages.length - 1] as AIMessage; + const response = lastMessage.content as string; + + const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; + const summary = response.match(/## Migration Summary[\s\S]*$/)?.[0] ?? ''; + + // if (esqlQuery == null) { + // throw new Error('Could not find ESQL query in response'); + // } + // if (summary == null) { + // throw new Error('Could not find migration summary in response'); + // } + // const missingEntities = extractMissingEntities(esqlQuery); + const translationState = getTranslationState(esqlQuery); + + return { + response, + comments: [summary], + translation_state: translationState, + elastic_rule: { + title: state.original_rule.title, + description: state.original_rule.description, + severity: 'low', + query: esqlQuery, + query_language: 'esql', + // missing_entities: missingEntities, + }, + }; +}; + +// const extractMissingEntities = (esqlQuery: string) => { +// const result = Array.from(esqlQuery?.matchAll(/\[macro:[\s\S]+?\]/)); +// console.log(result); +// return result; +// }; + +const getTranslationState = (esqlQuery: string) => { + if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { + return 'partial'; + } + return 'complete'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/esql_knowledge_base_caller.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/esql_knowledge_base_caller.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts index 963eec6ace830..2277f2fae41a9 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/esql_knowledge_base_caller.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { naturalLanguageToEsql, type InferenceClient } from '@kbn/inference-plugin/server'; import type { Logger } from '@kbn/core/server'; +import { naturalLanguageToEsql, type InferenceClient } from '@kbn/inference-plugin/server'; import { lastValueFrom } from 'rxjs'; export type EsqlKnowledgeBaseCaller = (input: string) => Promise; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts new file mode 100644 index 0000000000000..7d247f755e9da --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getTranslateQueryNode } from './translate_query'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/prompts.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts similarity index 66% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/prompts.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts index 75b4872044f71..0b97faf7dc96f 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/prompts.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { MigrateRuleState } from './types'; +import type { MigrateRuleState } from '../../types'; export const getEsqlTranslationPrompt = ( state: MigrateRuleState @@ -13,6 +13,12 @@ export const getEsqlTranslationPrompt = ( Below you will find Splunk rule information: the title, description and the SPL (Search Processing Language) query. Your goal is to translate the SPL query into an equivalent Elastic Security Query Language (ES|QL) query. +Guidelines: +- Start the translation process by analyzing the SPL query and identifying the key components. +- Always use logs* index pattern for the ES|QL translated query. +- If, in the SPL query, you find a lookup list or macro that, based only on its name, you can not translate with confidence to ES|QL, mention it in the summary and +add a placeholder in the query with the format [macro:(parameters)] or [lookup:] including the [] keys, example: [macro:my_macro(first_param,second_param)] or [lookup:my_lookup]. + The output will be parsed and should contain: - First, the ES|QL query inside an \`\`\`esql code block. - At the end, the summary of the translation process followed in markdown, starting with "## Migration Summary". diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts new file mode 100644 index 0000000000000..1265507463bbc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import { AIMessage } from '@langchain/core/messages'; +import type { GraphNode } from '../../types'; +import { getEsqlKnowledgeBase } from './esql_knowledge_base_caller'; +import { getEsqlTranslationPrompt } from './prompt'; + +interface GetTranslateQueryNodeParams { + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +} + +export const getTranslateQueryNode = ({ + inferenceClient, + connectorId, + logger, +}: GetTranslateQueryNodeParams): GraphNode => { + const esqlKnowledgeBaseCaller = getEsqlKnowledgeBase({ inferenceClient, connectorId, logger }); + return async (state) => { + const input = getEsqlTranslationPrompt(state); + const response = await esqlKnowledgeBaseCaller(input); + return { messages: [new AIMessage(response)] }; + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts index 6c1ea925c0a0d..b3221c7766707 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -10,6 +10,7 @@ import { Annotation, messagesStateReducer } from '@langchain/langgraph'; import type { ElasticRule, OriginalRule, + RuleMigration, } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; export const migrateRuleState = Annotation.Root({ @@ -21,7 +22,10 @@ export const migrateRuleState = Annotation.Root({ elastic_rule: Annotation({ reducer: (state, action) => ({ ...state, ...action }), }), - translation_state: Annotation(), - prebuilt_rule_id: Annotation(), + translation_state: Annotation(), + comments: Annotation({ + reducer: (current, value) => (value ? (current ?? []).concat(value) : current), + default: () => [], + }), response: Annotation(), }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts deleted file mode 100644 index 588437e7d2a4a..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/tools/esql_translator_tool.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from '@kbn/core/server'; -import { naturalLanguageToEsql, type InferenceClient } from '@kbn/inference-plugin/server'; -import type { StructuredTool } from '@langchain/core/tools'; -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { lastValueFrom } from 'rxjs'; -import { z } from '@kbn/zod'; - -const TOOL_NAME = 'esql_translator'; -const schema = z.object({ - splQuery: z.string().describe(`The exact SPL query to translate to ES|QL`), - title: z.string().describe(`The title of the splunk rule`), - description: z.string().describe(`The description of the splunk rule`), -}); -type Schema = typeof schema; -type SchemaInput = z.output; - -const toolParams = { - name: TOOL_NAME, - description: `ALWAYS use the "${TOOL_NAME}" tool to convert the Splunk detection rule (SPL) to an Elastic ES|QL query.`, - schema, - tags: ['esql', 'query-translation', 'knowledge-base'], -}; - -const createInputPrompt = ({ - splQuery, - title, - description, -}: SchemaInput) => `Translate the following Splunk SPL (Search Processing Language) query rule to an ES|QL query, in order to be used as an Elastic Security detection rule: - -Splunk rule title: ${title} - -Splunk rule description: ${description} - -Splunk rule SPL query: -\`\`\`spl -${splQuery} -\`\`\` - -Along with the translated ES|QL query, you should also provide a summary of the translation process you followed, in markdown format. -The output should contain: -- First, the ES|QL query inside an \`\`\`esql code block. -- At the end, the summary of the translation process followed in markdown, starting with "## Translation Summary". -`; - -export type EsqlTranslatorTool = StructuredTool; - -interface GetEsqlTranslatorToolParams { - inferenceClient: InferenceClient; - connectorId: string; - logger: Logger; -} -export const getEsqlTranslatorTool = ({ - inferenceClient: client, - connectorId, - logger, -}: GetEsqlTranslatorToolParams): EsqlTranslatorTool => { - const callNaturalLanguageToEsql = async (input: SchemaInput) => { - return lastValueFrom( - naturalLanguageToEsql({ - client, - connectorId, - input: createInputPrompt(input), - logger: { - debug: (source) => { - logger.debug(typeof source === 'function' ? source() : source); - }, - }, - }) - ); - }; - - const esqlTool = new DynamicStructuredTool({ - ...toolParams, - responseFormat: 'markdown', - func: async (input) => { - const generateEvent = await callNaturalLanguageToEsql(input); - const answer = generateEvent.content ?? 'An error occurred in the tool'; - - logger.debug(`Received response from NL to ESQL tool: ${answer}`); - return answer; - }, - }); - - return esqlTool; -}; - -export type EsqlKnowledgeBaseCaller = (input: string) => Promise; -export const getEsqlKnowledgeBase = - ({ - inferenceClient: client, - connectorId, - logger, - }: GetEsqlTranslatorToolParams): EsqlKnowledgeBaseCaller => - async (input: string) => { - const { content } = await lastValueFrom( - naturalLanguageToEsql({ - client, - connectorId, - input, - logger: { - debug: (source) => { - logger.debug(typeof source === 'function' ? source() : source); - }, - }, - }) - ); - return content; - }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts index d5cdebb1d7bdc..643d200e4b0bf 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts @@ -6,17 +6,18 @@ */ import type { Logger } from '@kbn/core/server'; -import type { StructuredTool } from '@langchain/core/tools'; import type { InferenceClient } from '@kbn/inference-plugin/server'; -import type { ChatModel } from './actions_client_chat'; import type { migrateRuleState } from './state'; +import type { ChatModel } from '../util/actions_client_chat'; +import type { PrebuiltRulesMapByName } from '../util/prebuilt_rules'; export type MigrateRuleState = typeof migrateRuleState.State; +export type GraphNode = (state: MigrateRuleState) => Promise>; export interface MigrateRuleGraphParams { - model: ChatModel; - tools: StructuredTool[]; inferenceClient: InferenceClient; + model: ChatModel; connectorId: string; + prebuiltRulesMap: PrebuiltRulesMapByName; logger: Logger; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index 29a0506bb30cc..3b101df5239e8 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -7,16 +7,23 @@ import type { Logger } from '@kbn/core/server'; import { AbortError, abortSignalToPromise } from '@kbn/kibana-utils-plugin/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; import type { RuleMigrationTaskStats } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import type { RuleMigrationStats } from '../data_stream/rule_migrations_data_client'; -import type { StoredRuleMigration } from '../types'; import type { RuleMigrationTaskStartParams, RuleMigrationTaskStartResult, RuleMigrationTaskStatsParams, - RuleMigrationTaskCancelParams, - RuleMigrationTaskCancelResult, + RuleMigrationTaskStopParams, + RuleMigrationTaskStopResult, + RuleMigrationTaskPrepareParams, + RuleMigrationTaskRunParams, + MigrationAgent, } from './types'; +import { getRuleMigrationAgent } from './agent'; +import type { MigrateRuleState } from './agent/types'; +import { retrievePrebuiltRulesMap } from './util/prebuilt_rules'; +import { ActionsClientChat } from './util/actions_client_chat'; interface MigrationProcessing { abortController: AbortController; @@ -37,96 +44,169 @@ const getRuleLogger = (logger: Logger): RuleLogger => { }; }; -const BATCH_SIZE = 2 as const; -const BATCH_SLEEP_MS = 10000 as const; +const ITERATION_BATCH_SIZE = 1 as const; +const ITERATION_SLEEP_SECONDS = 5 as const; export class RuleMigrationsTaskRunner { - private migrationsProcessing: Map; - private logger: RuleLogger; + private migrationsExecuting: Map; + private taskLogger: RuleLogger; - constructor(logger: Logger) { - this.migrationsProcessing = new Map(); - this.logger = getRuleLogger(logger); + constructor(private logger: Logger) { + this.migrationsExecuting = new Map(); + this.taskLogger = getRuleLogger(logger); } /** Starts a rule migration task */ - async start({ - migrationId, - currentUser, - request, - dataClient, - }: RuleMigrationTaskStartParams): Promise { - if (this.migrationsProcessing.has(migrationId)) { - return { found: true, started: false }; // already processing + async start(params: RuleMigrationTaskStartParams): Promise { + const { migrationId, dataClient } = params; + if (this.migrationsExecuting.has(migrationId)) { + return { exists: true, started: false }; } // Just in case some previous execution was interrupted without releasing - await dataClient.releaseProcessing(migrationId); + await dataClient.releaseProcessable(migrationId); - const stats = await dataClient.getStats(migrationId); - if (stats.total === 0) { - return { found: false, started: false }; + const { total, pending } = await dataClient.getStats(migrationId); + if (total === 0) { + return { exists: false, started: false }; } - if (stats.pending === 0) { - return { found: true, started: false }; + if (pending === 0) { + return { exists: true, started: false }; } - this.run({ migrationId, currentUser, request, dataClient }); - return { found: true, started: true }; + const abortController = new AbortController(); + // Await the preparation to make sure the agent is created properly so the task can run + const agent = await this.prepare({ ...params, abortController }); + + // not awaiting the promise to execute the task in the background + this.run({ ...params, agent, abortController }); + + return { exists: true, started: true }; + } + + private async prepare({ + connectorId, + inferenceClient, + actionsClient, + rulesClient, + soClient, + abortController, + }: RuleMigrationTaskPrepareParams): Promise { + const prebuiltRulesMap = await retrievePrebuiltRulesMap({ soClient, rulesClient }); + + const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger); + const model = await actionsClientChat.createModel({ + signal: abortController.signal, + temperature: 0.05, + }); + + const agent = getRuleMigrationAgent({ + connectorId, + model, + inferenceClient, + prebuiltRulesMap, + logger: this.logger, + }); + return agent; } private async run({ migrationId, - currentUser, - request, + agent, dataClient, - }: RuleMigrationTaskStartParams): Promise { - if (this.migrationsProcessing.has(migrationId)) { - throw new Error(`Migration ${migrationId} is already being processed`); + currentUser, + invocationConfig, + abortController, + }: RuleMigrationTaskRunParams): Promise { + if (this.migrationsExecuting.has(migrationId)) { + // This should never happen, but just in case + throw new Error(`Task already running for migration ID:${migrationId} `); } + this.taskLogger.info(`Starting migration task run for ID:${migrationId}`); - const abortController = new AbortController(); - this.migrationsProcessing.set(migrationId, { abortController, user: currentUser.username }); + this.migrationsExecuting.set(migrationId, { abortController, user: currentUser.username }); + const config: RunnableConfig = { + ...invocationConfig, + // signal: abortController.signal, // not working properly + }; const abortPromise = abortSignalToPromise(abortController.signal); try { - const sleep = async (ms: number) => { - this.logger.info(`Sleeping ${ms / 1000}s before next iteration`); + const sleep = async (seconds: number) => { + this.taskLogger.info(`Sleeping ${seconds}s for migration ID:${migrationId}`); await Promise.race([ - new Promise((resolve) => setTimeout(resolve, ms)), + new Promise((resolve) => setTimeout(resolve, seconds * 1000)), abortPromise.promise, ]); }; let isDone: boolean = false; do { - const ruleMigrations = await dataClient.takePending(migrationId, BATCH_SIZE); - this.logger.info(`Processing ${ruleMigrations.length} rule migrations`); + const ruleMigrations = await dataClient.takePending(migrationId, ITERATION_BATCH_SIZE); + this.taskLogger.info( + `Processing ${ruleMigrations.length} rules for migration ID:${migrationId}` + ); await Promise.all( ruleMigrations.map(async (ruleMigration) => { - const ruleMigrationResult = await dummyRuleMigrationGraph.invoke(ruleMigration); - await dataClient.saveFinished(ruleMigrationResult); + this.taskLogger.info( + `Starting migration of rule "${ruleMigration.original_rule.title}"` + ); + try { + const start = Date.now(); + const ruleMigrationResult: MigrateRuleState = await agent.invoke( + { original_rule: ruleMigration.original_rule }, + config + ); + const duration = (Date.now() - start) / 1000; + this.taskLogger.info( + `Migration of rule "${ruleMigration.original_rule.title}" finished in ${duration}s` + ); + + await dataClient.saveFinished({ + ...ruleMigration, + elastic_rule: ruleMigrationResult.elastic_rule, + translation_state: ruleMigrationResult.translation_state, + comments: ruleMigrationResult.comments, + }); + } catch (error) { + if (error instanceof AbortError) { + throw error; + } + this.taskLogger.error( + `Error migrating rule "${ruleMigration.original_rule.title}"`, + error + ); + await dataClient.saveError({ + ...ruleMigration, + comments: [`Error migrating rule: ${error.message}`], + }); + } }) ); + this.taskLogger.info(`Batch processed successfully for migration ID:${migrationId}`); const { pending } = await dataClient.getStats(migrationId); isDone = pending === 0; if (!isDone) { - await sleep(BATCH_SLEEP_MS); + await sleep(ITERATION_SLEEP_SECONDS); } } while (!isDone); + + this.taskLogger.info(`Finished task for migration ID:${migrationId}`); } catch (error) { await dataClient.releaseProcessing(migrationId); if (error instanceof AbortError) { - this.logger.info(`Abort signal received, migration ${migrationId} task stopped`); + this.taskLogger.info( + `Abort signal received, stopping task for migration ID:${migrationId}` + ); return; } else { - this.logger.error(`Error processing migration ${migrationId}`, error); + this.taskLogger.error(`Error processing migration ID:${migrationId}`, error); } } finally { - this.migrationsProcessing.delete(migrationId); + this.migrationsExecuting.delete(migrationId); abortPromise.cleanup(); } } @@ -140,12 +220,14 @@ export class RuleMigrationsTaskRunner { const status = this.getTaskStatus(stats, migrationId); return { status, - total: stats.total, - finished: stats.finished, - pending: stats.pending, - processing: stats.processing, - failed: stats.failed, - last_iteration_at: stats.lastUpdatedAt, + rules: { + total: stats.total, + finished: stats.finished, + pending: stats.pending, + processing: stats.processing, + failed: stats.failed, + }, + last_updated_at: stats.lastUpdatedAt, }; } @@ -153,53 +235,46 @@ export class RuleMigrationsTaskRunner { stats: RuleMigrationStats, migrationId: string ): RuleMigrationTaskStats['status'] { - if (this.migrationsProcessing.has(migrationId)) { - return 'processing'; + if (this.migrationsExecuting.has(migrationId)) { + return 'running'; } if (stats.pending === stats.total) { - return 'not_started'; + return 'ready'; } - if (stats.finished === stats.total) { + if (stats.finished + stats.failed === stats.total) { return 'done'; } - return 'cancelled'; + return 'stopped'; } /** Aborts a running migration */ - async cancel({ + async stop({ migrationId, dataClient, - }: RuleMigrationTaskCancelParams): Promise { - const migrationProcessing = this.migrationsProcessing.get(migrationId); - if (migrationProcessing) { - migrationProcessing.abortController.abort(); - return { found: true, cancelled: true }; - } + }: RuleMigrationTaskStopParams): Promise { + try { + const migrationProcessing = this.migrationsExecuting.get(migrationId); + if (migrationProcessing) { + migrationProcessing.abortController.abort(); + return { exists: true, stopped: true }; + } - const stats = await dataClient.getStats(migrationId); - if (stats.total > 0) { - return { found: true, cancelled: false }; + const { total } = await dataClient.getStats(migrationId); + if (total > 0) { + return { exists: true, stopped: false }; + } + return { exists: false, stopped: false }; + } catch (err) { + this.taskLogger.error(`Error stopping migration ID:${migrationId}`, err); + return { exists: true, stopped: false }; } - return { found: false, cancelled: false }; } /** Stops all running migrations */ - stop() { - this.migrationsProcessing.forEach((migrationProcessing) => { + stopAll() { + this.migrationsExecuting.forEach((migrationProcessing) => { migrationProcessing.abortController.abort(); }); - this.migrationsProcessing.clear(); + this.migrationsExecuting.clear(); } } - -const dummyRuleMigrationGraph = { - invoke: async (ruleMigration: StoredRuleMigration): Promise => { - return new Promise((resolve) => { - console.log('dummyRuleMigrationGraph start'); - setTimeout(() => { - console.log('dummyRuleMigrationGraph resolved'); - resolve(ruleMigration); - }, 5000); - }); - }, -}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts index 1f172bfb6ab78..6431db0624901 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -5,17 +5,47 @@ * 2.0. */ -import type { AuthenticatedUser, KibanaRequest } from '@kbn/core/server'; +import type { AuthenticatedUser, SavedObjectsClientContract } from '@kbn/core/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { RuleMigrationsDataClient } from '../data_stream/rule_migrations_data_client'; +import type { getRuleMigrationAgent } from './agent'; + +export type MigrationAgent = ReturnType; export interface RuleMigrationTaskStartParams { migrationId: string; currentUser: AuthenticatedUser; - request: KibanaRequest; + connectorId: string; + invocationConfig: RunnableConfig; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskPrepareParams { + connectorId: string; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; + abortController: AbortController; +} + +export interface RuleMigrationTaskRunParams { + migrationId: string; + currentUser: AuthenticatedUser; + invocationConfig: RunnableConfig; + agent: MigrationAgent; dataClient: RuleMigrationsDataClient; + abortController: AbortController; } -export interface RuleMigrationTaskCancelParams { +export interface RuleMigrationTaskStopParams { migrationId: string; dataClient: RuleMigrationsDataClient; } @@ -27,10 +57,10 @@ export interface RuleMigrationTaskStatsParams { export interface RuleMigrationTaskStartResult { started: boolean; - found: boolean; + exists: boolean; } -export interface RuleMigrationTaskCancelResult { - cancelled: boolean; - found: boolean; +export interface RuleMigrationTaskStopResult { + stopped: boolean; + exists: boolean; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/actions_client_chat.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/actions_client_chat.ts rename to x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts new file mode 100644 index 0000000000000..dd3801dde0e33 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; +import { retrievePrebuiltRulesMap } from './prebuilt_rules'; +import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; +import type { SplunkRule } from '../../../../../../../../common/api/siem_migrations/splunk/rules/splunk_rule.gen'; + +jest.mock( + '../../../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client', + () => ({ createPrebuiltRuleObjectsClient: jest.fn() }) +); +jest.mock( + '../../../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client', + () => ({ createPrebuiltRuleAssetsClient: jest.fn() }) +); + +const rule1 = { + name: 'rule one', + id: 'rule1', + threat: [ + { + framework: 'MITRE ATT&CK', + technique: [{ id: 'T1234', name: 'tactic one' }], + }, + ], +}; +const rule2 = { + name: 'rule two', + id: 'rule2', +}; + +const defaultRuleVersionsTriad = new Map([ + ['rule1', { target: rule1 }], + ['rule2', { target: rule2, current: rule2 }], +]); +const mockFetchRuleVersionsTriad = jest.fn().mockResolvedValue(defaultRuleVersionsTriad); +jest.mock( + '../../../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad', + () => ({ + fetchRuleVersionsTriad: () => mockFetchRuleVersionsTriad(), + }) +); + +const splunkRule: SplunkRule = { + title: 'splunk rule', + description: 'splunk rule description', + search: 'index=*', + mitreAttackIds: ['T1234'], +}; + +const defaultParams = { + soClient: savedObjectsClientMock.create(), + rulesClient: rulesClientMock.create(), + splunkRule, +}; + +describe('retrievePrebuiltRulesMap', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('when prebuilt rule is installed', () => { + it('should return isInstalled flag', async () => { + const prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); + expect(prebuiltRulesMap.size).toBe(2); + expect(prebuiltRulesMap.get('rule one')).toEqual( + expect.objectContaining({ isInstalled: false }) + ); + expect(prebuiltRulesMap.get('rule two')).toEqual( + expect.objectContaining({ isInstalled: true }) + ); + }); + }); + + describe('when splunk rule does not contain mitreAttackIds', () => { + it('should return the full rules map', async () => { + const prebuiltRulesMap = await retrievePrebuiltRulesMap({ + ...defaultParams, + splunkRule: { ...splunkRule, mitreAttackIds: undefined }, + }); + expect(prebuiltRulesMap.size).toBe(2); + }); + }); + + describe('when splunk rule contains empty mitreAttackIds', () => { + it('should return the full rules map', async () => { + const prebuiltRulesMap = await retrievePrebuiltRulesMap({ + ...defaultParams, + splunkRule: { ...splunkRule, mitreAttackIds: [] }, + }); + expect(prebuiltRulesMap.size).toBe(2); + }); + }); + + describe('when splunk rule contains non matching mitreAttackIds', () => { + it('should return the full rules map', async () => { + const prebuiltRulesMap = await retrievePrebuiltRulesMap({ + ...defaultParams, + splunkRule: { ...splunkRule, mitreAttackIds: ['T2345'] }, + }); + expect(prebuiltRulesMap.size).toBe(1); + expect(prebuiltRulesMap.get('rule two')).toEqual(expect.objectContaining({ rule: rule2 })); + }); + }); + + describe('when splunk rule contains matching mitreAttackIds', () => { + it('should return the filtered rules map', async () => { + const prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); + expect(prebuiltRulesMap.size).toBe(2); + expect(prebuiltRulesMap.get('rule one')).toEqual(expect.objectContaining({ rule: rule1 })); + expect(prebuiltRulesMap.get('rule two')).toEqual(expect.objectContaining({ rule: rule2 })); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts new file mode 100644 index 0000000000000..ade6632aaa5b5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import type { PrebuiltRuleAsset } from '../../../../detection_engine/prebuilt_rules'; +import { fetchRuleVersionsTriad } from '../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad'; +import { createPrebuiltRuleObjectsClient } from '../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client'; +import { createPrebuiltRuleAssetsClient } from '../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; + +export interface PrebuiltRuleMapped { + rule: PrebuiltRuleAsset; + installedRuleId?: string; +} + +export type PrebuiltRulesMapByName = Map; + +interface RetrievePrebuiltRulesParams { + soClient: SavedObjectsClientContract; + rulesClient: RulesClient; +} + +export const retrievePrebuiltRulesMap = async ({ + soClient, + rulesClient, +}: RetrievePrebuiltRulesParams): Promise => { + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); + + const prebuiltRulesMap = await fetchRuleVersionsTriad({ + ruleAssetsClient, + ruleObjectsClient, + }); + const prebuiltRulesByName: PrebuiltRulesMapByName = new Map(); + prebuiltRulesMap.forEach((ruleVersions) => { + const rule = ruleVersions.target || ruleVersions.current; + if (rule) { + prebuiltRulesByName.set(rule.name, { + rule, + installedRuleId: ruleVersions.current?.id, + }); + } + }); + return prebuiltRulesByName; +}; + +export const filterPrebuiltRules = ( + prebuiltRulesByName: PrebuiltRulesMapByName, + mitreAttackIds: string[] +) => { + const filteredPrebuiltRulesByName = new Map(); + if (mitreAttackIds?.length) { + // If this rule has MITRE ATT&CK IDs, remove unrelated prebuilt rules + prebuiltRulesByName.forEach(({ rule }, ruleName) => { + const mitreAttackThreat = rule.threat?.filter( + ({ framework }) => framework === 'MITRE ATT&CK' + ); + if (!mitreAttackThreat) { + // If this rule has no MITRE ATT&CK reference we skip it + return; + } + + const sameTechnique = mitreAttackThreat.find((threat) => + threat.technique?.some(({ id }) => mitreAttackIds?.includes(id)) + ); + + if (sameTechnique) { + filteredPrebuiltRulesByName.set(ruleName, prebuiltRulesByName.get(ruleName)); + } + }); + } + return filteredPrebuiltRulesByName; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index fc1f1d7015231..cf1af4fdace69 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -5,14 +5,23 @@ * 2.0. */ -import type { AuthenticatedUser, IClusterClient, KibanaRequest } from '@kbn/core/server'; +import type { + AuthenticatedUser, + IClusterClient, + KibanaRequest, + SavedObjectsClientContract, +} from '@kbn/core/server'; import type { Subject } from 'rxjs'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { RuleMigration, RuleMigrationTaskStats, } from '../../../../common/siem_migrations/model/rule_migration.gen'; import type { RuleMigrationsDataClient } from './data_stream/rule_migrations_data_client'; -import type { RuleMigrationTaskCancelResult, RuleMigrationTaskStartResult } from './task/types'; +import type { RuleMigrationTaskStopResult, RuleMigrationTaskStartResult } from './task/types'; export interface StoredRuleMigration extends RuleMigration { _id: string; @@ -31,11 +40,21 @@ export interface SiemRuleMigrationsCreateClientParams { spaceId: string; } +export interface SiemRuleMigrationsStartTaskParams { + migrationId: string; + connectorId: string; + invocationConfig: RunnableConfig; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; +} + export interface SiemRuleMigrationsClient { data: RuleMigrationsDataClient; task: { - start: (migrationId: string) => Promise; + start: (params: SiemRuleMigrationsStartTaskParams) => Promise; stats: (migrationId: string) => Promise; - cancel: (migrationId: string) => Promise; + stop: (migrationId: string) => Promise; }; } diff --git a/x-pack/plugins/security_solution/server/plugin_contract.ts b/x-pack/plugins/security_solution/server/plugin_contract.ts index c7ec67c1b07fc..c178f0654d9bd 100644 --- a/x-pack/plugins/security_solution/server/plugin_contract.ts +++ b/x-pack/plugins/security_solution/server/plugin_contract.ts @@ -45,6 +45,7 @@ import type { SharePluginStart } from '@kbn/share-plugin/server'; import type { GuidedOnboardingPluginSetup } from '@kbn/guided-onboarding-plugin/server'; import type { PluginSetup as UnifiedSearchServerPluginSetup } from '@kbn/unified-search-plugin/server'; import type { ElasticAssistantPluginStart } from '@kbn/elastic-assistant-plugin/server'; +import type { InferenceServerStart } from '@kbn/inference-plugin/server'; import type { ProductFeaturesService } from './lib/product_features_service/product_features_service'; import type { ExperimentalFeatures } from '../common'; @@ -88,6 +89,7 @@ export interface SecuritySolutionPluginStartDependencies { telemetry?: TelemetryPluginStart; share: SharePluginStart; actions: ActionsPluginStartContract; + inference: InferenceServerStart; } export interface SecuritySolutionPluginSetup { diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 9d6f49a26fe33..85688e38968b9 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -174,6 +174,8 @@ export class RequestContextFactory implements IRequestContextFactory { }) ), + getInferenceClient: memoize(() => startPlugins.inference.getClient({ request })), + getExceptionListClient: () => { if (!lists) { return null; diff --git a/x-pack/plugins/security_solution/server/types.ts b/x-pack/plugins/security_solution/server/types.ts index 1355904dbe7f7..7afbf5dcff6d2 100644 --- a/x-pack/plugins/security_solution/server/types.ts +++ b/x-pack/plugins/security_solution/server/types.ts @@ -20,6 +20,7 @@ import type { AlertsClient, IRuleDataService } from '@kbn/rule-registry-plugin/s import type { Readable } from 'stream'; import type { AuditLogger } from '@kbn/security-plugin-types-server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; import type { Immutable } from '../common/endpoint/types'; import { AppClient } from './client'; import type { ConfigType } from './config'; @@ -35,7 +36,7 @@ import type { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk import type { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality'; import type { IDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface'; import type { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client'; -import type { SiemMigrationsClient } from './lib/siem_migrations/types'; +import type { SiemRuleMigrationsClient } from './lib/siem_migrations/rules/types'; export { AppClient }; export interface SecuritySolutionApiRequestHandlerContext { @@ -58,7 +59,8 @@ export interface SecuritySolutionApiRequestHandlerContext { getRiskScoreDataClient: () => RiskScoreDataClient; getAssetCriticalityDataClient: () => AssetCriticalityDataClient; getEntityStoreDataClient: () => EntityStoreDataClient; - getSiemMigrationsClient: () => SiemMigrationsClient; + getSiemRuleMigrationsClient: () => SiemRuleMigrationsClient; + getInferenceClient: () => InferenceClient; } export type SecuritySolutionRequestHandlerContext = CustomRequestHandlerContext<{ From 9f90900c503b1ddc257f0cf2b8729f5630e583f2 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Wed, 30 Oct 2024 10:16:25 +0100 Subject: [PATCH 03/14] add langgraph signal issue link --- .../siem_migrations/rules/task/rule_migrations_task_runner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index 3b101df5239e8..e7210eeeb313c 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -126,7 +126,7 @@ export class RuleMigrationsTaskRunner { this.migrationsExecuting.set(migrationId, { abortController, user: currentUser.username }); const config: RunnableConfig = { ...invocationConfig, - // signal: abortController.signal, // not working properly + // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319 }; const abortPromise = abortSignalToPromise(abortController.signal); From 7441f111ee785e08b945467ab9ca85e547afe5e1 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Mon, 4 Nov 2024 17:36:46 +0100 Subject: [PATCH 04/14] get and stats_all endpoints --- .../common/siem_migrations/constants.ts | 2 + .../model/api/rules/rules_migration.gen.ts | 16 +++- .../api/rules/rules_migration.schema.yaml | 33 ++++++- .../model/rule_migration.gen.ts | 14 ++- .../model/rule_migration.schema.yaml | 13 +++ .../lib/siem_migrations/rules/api/create.ts | 6 +- .../lib/siem_migrations/rules/api/get.ts | 47 +++++++++ .../lib/siem_migrations/rules/api/index.ts | 4 + .../lib/siem_migrations/rules/api/start.ts | 6 +- .../lib/siem_migrations/rules/api/stats.ts | 10 +- .../siem_migrations/rules/api/stats_all.ts | 39 ++++++++ .../lib/siem_migrations/rules/api/stop.ts | 7 +- .../rule_migrations_data_client.ts | 96 ++++++++++++++----- .../rules/siem_rule_migrations_service.ts | 9 +- .../siem_migrations/rules/task/agent/graph.ts | 4 +- .../rules/task/rule_migrations_task_runner.ts | 61 ++++++------ .../lib/siem_migrations/rules/task/types.ts | 4 + .../server/lib/siem_migrations/rules/types.ts | 4 +- 18 files changed, 294 insertions(+), 81 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index b085d854cd07d..4d0dd3252ae94 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -8,6 +8,8 @@ export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const; export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const; +export const SIEM_RULE_MIGRATIONS_ALL_STATS_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/stats` as const; +export const SIEM_RULE_MIGRATIONS_GET_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}` as const; export const SIEM_RULE_MIGRATIONS_START_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/start` as const; export const SIEM_RULE_MIGRATIONS_STATS_PATH = diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts index 822219f9ec698..a2a8edd15878f 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts @@ -16,7 +16,12 @@ import { z } from '@kbn/zod'; -import { OriginalRule, RuleMigration, RuleMigrationTaskStats } from '../../rule_migration.gen'; +import { + OriginalRule, + RuleMigrationAllTaskStats, + RuleMigration, + RuleMigrationTaskStats, +} from '../../rule_migration.gen'; import { ConnectorId, LangSmithOptions } from '../common.gen'; export type CreateRuleMigrationRequestBody = z.infer; @@ -31,6 +36,15 @@ export const CreateRuleMigrationResponse = z.object({ migration_id: z.string(), }); +export type GetAllStatsRuleMigrationResponse = z.infer; +export const GetAllStatsRuleMigrationResponse = RuleMigrationAllTaskStats; + +export type GetRuleMigrationRequestParams = z.infer; +export const GetRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type GetRuleMigrationRequestParamsInput = z.input; + export type GetRuleMigrationResponse = z.infer; export const GetRuleMigrationResponse = z.array(RuleMigration); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml index 16d7d6d45f1a4..4874997715b3e 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml @@ -32,22 +32,49 @@ paths: migration_id: type: string description: The migration id created. + + /internal/siem_migrations/rules/stats: get: - summary: Retrieves rule migrations - operationId: GetRuleMigration + summary: Retrieves the stats for all rule migrations + operationId: GetAllStatsRuleMigration x-codegen-enabled: true - description: Retrieves the rule migrations stored in the system + description: Retrieves the rule migrations stats for all migrations stored in the system tags: - SIEM Rule Migrations responses: 200: description: Indicates rule migrations have been retrieved correctly. + content: + application/json: + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationAllTaskStats' + + /internal/siem_migrations/rules/{migration_id}: + get: + summary: Retrieves all the rules of a migration + operationId: GetRuleMigration + x-codegen-enabled: true + description: Retrieves the rule documents stored in the system given the rule migration id + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + responses: + 200: + description: Indicates rule migration have been retrieved correctly. content: application/json: schema: type: array items: $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration' + 204: + description: Indicates the migration id was not found. /internal/siem_migrations/rules/{migration_id}/start: put: diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 55dc16e464696..0e0d846d74917 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -112,7 +112,7 @@ export const RuleMigration = z.object({ /** * The status of the rule migration. */ - status: z.enum(['pending', 'processing', 'finished', 'error']).default('pending'), + status: z.enum(['pending', 'processing', 'finished', 'failed']).default('pending'), /** * The comments for the migration including a summary from the LLM in markdown. */ @@ -166,3 +166,15 @@ export const RuleMigrationTaskStats = z.object({ */ last_updated_at: z.string().optional(), }); + +export type RuleMigrationAllTaskStats = z.infer; +export const RuleMigrationAllTaskStats = z.array( + RuleMigrationTaskStats.merge( + z.object({ + /** + * The migration id + */ + migration_id: z.string(), + }) + ) +); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 50605f9256b76..44bb8407dbd7d 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -167,3 +167,16 @@ components: last_updated_at: type: string description: The moment of the last update. + + RuleMigrationAllTaskStats: + type: array + items: + allOf: + - $ref: '#/components/schemas/RuleMigrationTaskStats' + - type: object + required: + - migration_id + properties: + migration_id: + type: string + description: The migration id diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index c8ae0a6c7c84f..e2505ca83beed 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -22,7 +22,7 @@ export const registerSiemRuleMigrationsCreateRoute = ( .post({ path: SIEM_RULE_MIGRATIONS_PATH, access: 'internal', - options: { tags: ['access:securitySolution'] }, + security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) .addVersion( { @@ -49,9 +49,7 @@ export const registerSiemRuleMigrationsCreateRoute = ( return res.ok({ body: { migration_id: migrationId } }); } catch (err) { logger.error(err); - return res.badRequest({ - body: err.message, - }); + return res.badRequest({ body: err.message }); } } ); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts new file mode 100644 index 0000000000000..0efb6706918f5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { GetRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_GET_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsGetRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_GET_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(GetRuleMigrationRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const migrationRules = await ruleMigrationsClient.data.getRules(migrationId); + + return res.ok({ body: migrationRules }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index dbb721b49cdc8..f37eb2108a8a4 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -8,15 +8,19 @@ import type { Logger } from '@kbn/core/server'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { registerSiemRuleMigrationsCreateRoute } from './create'; +import { registerSiemRuleMigrationsGetRoute } from './get'; import { registerSiemRuleMigrationsStartRoute } from './start'; import { registerSiemRuleMigrationsStatsRoute } from './stats'; import { registerSiemRuleMigrationsStopRoute } from './stop'; +import { registerSiemRuleMigrationsStatsAllRoute } from './stats_all'; export const registerSiemRuleMigrationsRoutes = ( router: SecuritySolutionPluginRouter, logger: Logger ) => { registerSiemRuleMigrationsCreateRoute(router, logger); + registerSiemRuleMigrationsStatsAllRoute(router, logger); + registerSiemRuleMigrationsGetRoute(router, logger); registerSiemRuleMigrationsStartRoute(router, logger); registerSiemRuleMigrationsStatsRoute(router, logger); registerSiemRuleMigrationsStopRoute(router, logger); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts index 2b2feb9f8c94e..7df94b20c0220 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -25,7 +25,7 @@ export const registerSiemRuleMigrationsStartRoute = ( .put({ path: SIEM_RULE_MIGRATIONS_START_PATH, access: 'internal', - options: { tags: ['access:securitySolution'] }, + security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) .addVersion( { @@ -84,9 +84,7 @@ export const registerSiemRuleMigrationsStartRoute = ( return res.ok({ body: { started } }); } catch (err) { logger.error(err); - return res.badRequest({ - body: err.message, - }); + return res.badRequest({ body: err.message }); } } ); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts index 7fa670d2519e3..8316e01fc6a9b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts @@ -20,7 +20,7 @@ export const registerSiemRuleMigrationsStatsRoute = ( .get({ path: SIEM_RULE_MIGRATIONS_STATS_PATH, access: 'internal', - options: { tags: ['access:securitySolution'] }, + security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) .addVersion( { @@ -34,13 +34,13 @@ export const registerSiemRuleMigrationsStatsRoute = ( try { const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); - const stats = await ruleMigrationsClient.task.stats(migrationId); + + const stats = await ruleMigrationsClient.task.getStats(migrationId); + return res.ok({ body: stats }); } catch (err) { logger.error(err); - return res.badRequest({ - body: err.message, - }); + return res.badRequest({ body: err.message }); } } ); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts new file mode 100644 index 0000000000000..dd2f2f503e19d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import type { GetAllStatsRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_ALL_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStatsAllRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_ALL_STATS_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { version: '1', validate: {} }, + async (context, req, res): Promise> => { + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const allStats = await ruleMigrationsClient.task.getAllStats(); + + return res.ok({ body: allStats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts index d1097ad5c7cc8..48bcb23ca94d8 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts @@ -20,7 +20,7 @@ export const registerSiemRuleMigrationsStopRoute = ( .put({ path: SIEM_RULE_MIGRATIONS_STOP_PATH, access: 'internal', - options: { tags: ['access:securitySolution'] }, + security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) .addVersion( { @@ -34,6 +34,7 @@ export const registerSiemRuleMigrationsStopRoute = ( try { const ctx = await context.resolve(['core', 'actions', 'securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const { exists, stopped } = await ruleMigrationsClient.task.stop(migrationId); if (!exists) { @@ -42,9 +43,7 @@ export const registerSiemRuleMigrationsStopRoute = ( return res.ok({ body: { stopped } }); } catch (err) { logger.error(err); - return res.badRequest({ - body: err.message, - }); + return res.badRequest({ body: err.message }); } } ); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts index ee80a4bfe9d1c..b96b281c3ecec 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts @@ -10,23 +10,22 @@ import assert from 'assert'; import type { AggregationsFilterAggregate, AggregationsMaxAggregate, + AggregationsStringTermsAggregate, + AggregationsStringTermsBucket, QueryDslQueryContainer, SearchHit, SearchResponse, } from '@elastic/elasticsearch/lib/api/types'; import type { StoredRuleMigration } from '../types'; import { SiemMigrationsStatus } from '../../../../../common/siem_migrations/constants'; -import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { + RuleMigration, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; export type CreateRuleMigrationInput = Omit; -export interface RuleMigrationStats { - total: number; - pending: number; - processing: number; - finished: number; - failed: number; - lastUpdatedAt: string | undefined; -} +export type RuleMigrationDataStats = Omit; +export type RuleMigrationAllDataStats = Array; export class RuleMigrationsDataClient { constructor( @@ -58,6 +57,21 @@ export class RuleMigrationsDataClient { }); } + /** Retrieves an array of rule documents of a specific migrations */ + async getRules(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId); + + const storedRuleMigrations = await this.esClient + .search({ index, query, sort: '_doc' }) + .catch((error) => { + this.logger.error(`Error searching getting rule migrations: ${error.message}`); + throw error; + }) + .then((response) => this.processHits(response.hits.hits)); + return storedRuleMigrations; + } + /** * Retrieves `pending` rule migrations with the provided id and updates their status to `processing`. * This operation is not atomic at migration level: @@ -160,7 +174,7 @@ export class RuleMigrationsDataClient { } /** Retrieves the stats for the rule migrations with the provided id */ - async getStats(migrationId: string): Promise { + async getStats(migrationId: string): Promise { const index = await this.dataStreamNamePromise; const query = this.getFilterQuery(migrationId); const aggregations = { @@ -171,16 +185,7 @@ export class RuleMigrationsDataClient { lastUpdatedAt: { max: { field: 'updated_at' } }, }; const result = await this.esClient - .search< - unknown, - { - pending: AggregationsFilterAggregate; - processing: AggregationsFilterAggregate; - finished: AggregationsFilterAggregate; - failed: AggregationsFilterAggregate; - lastUpdatedAt: AggregationsMaxAggregate; - } - >({ index, query, aggregations, _source: false }) + .search({ index, query, aggregations, _source: false }) .catch((error) => { this.logger.error(`Error getting rule migrations stats: ${error.message}`); throw error; @@ -188,13 +193,52 @@ export class RuleMigrationsDataClient { const { pending, processing, finished, lastUpdatedAt, failed } = result.aggregations ?? {}; return { - total: this.getTotalHits(result), - pending: pending?.doc_count ?? 0, - processing: processing?.doc_count ?? 0, - finished: finished?.doc_count ?? 0, - failed: failed?.doc_count ?? 0, - lastUpdatedAt: lastUpdatedAt?.value_as_string, + rules: { + total: this.getTotalHits(result), + pending: (pending as AggregationsFilterAggregate)?.doc_count ?? 0, + processing: (processing as AggregationsFilterAggregate)?.doc_count ?? 0, + finished: (finished as AggregationsFilterAggregate)?.doc_count ?? 0, + failed: (failed as AggregationsFilterAggregate)?.doc_count ?? 0, + }, + last_updated_at: (lastUpdatedAt as AggregationsMaxAggregate)?.value_as_string, + }; + } + + /** Retrieves the stats for all the rule migrations aggregated by migration id */ + async getAllStats(): Promise { + const index = await this.dataStreamNamePromise; + const aggregations = { + migrationIds: { + terms: { field: 'migration_id' }, + aggregations: { + pending: { filter: { term: { status: SiemMigrationsStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationsStatus.PROCESSING } } }, + finished: { filter: { term: { status: SiemMigrationsStatus.FINISHED } } }, + failed: { filter: { term: { status: SiemMigrationsStatus.FAILED } } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }, + }, }; + const result = await this.esClient + .search({ index, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting all rule migrations stats: ${error.message}`); + throw error; + }); + + const migrationsAgg = result.aggregations?.migrationIds as AggregationsStringTermsAggregate; + const buckets = (migrationsAgg?.buckets as AggregationsStringTermsBucket[]) ?? []; + return buckets.map((bucket) => ({ + migration_id: bucket.key, + rules: { + total: bucket.doc_count, + pending: bucket.pending?.doc_count ?? 0, + processing: bucket.processing?.doc_count ?? 0, + finished: bucket.finished?.doc_count ?? 0, + failed: bucket.failed?.doc_count ?? 0, + }, + last_updated_at: bucket.lastUpdatedAt?.value_as_string, + })); } private getFilterQuery( diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index c0e722315f21c..1bf9dcf11fd95 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -52,12 +52,15 @@ export class SiemRuleMigrationsService { start: (params) => { return this.taskRunner.start({ ...params, currentUser, dataClient }); }, - stats: async (migrationId) => { - return this.taskRunner.stats({ migrationId, dataClient }); - }, stop: (migrationId) => { return this.taskRunner.stop({ migrationId, dataClient }); }, + getStats: async (migrationId) => { + return this.taskRunner.getStats({ migrationId, dataClient }); + }, + getAllStats: async () => { + return this.taskRunner.getAllStats({ dataClient }); + }, }, }; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index 24e371484b17a..2e56277b174a1 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -33,7 +33,9 @@ export function getRuleMigrationAgent({ .addEdge('translation', 'processResponse') .addEdge('processResponse', END); - return translateRuleGraph.compile(); + const graph = translateRuleGraph.compile(); + graph.name = 'Rule Migration Graph'; // Customizes the name displayed in LangSmith + return graph; } const matchedPrebuiltRuleConditional = (state: MigrateRuleState) => { diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index e7210eeeb313c..34f42d6b34b5c 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -8,8 +8,11 @@ import type { Logger } from '@kbn/core/server'; import { AbortError, abortSignalToPromise } from '@kbn/kibana-utils-plugin/server'; import type { RunnableConfig } from '@langchain/core/runnables'; -import type { RuleMigrationTaskStats } from '../../../../../common/siem_migrations/model/rule_migration.gen'; -import type { RuleMigrationStats } from '../data_stream/rule_migrations_data_client'; +import type { + RuleMigrationAllTaskStats, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationDataStats } from '../data_stream/rule_migrations_data_client'; import type { RuleMigrationTaskStartParams, RuleMigrationTaskStartResult, @@ -19,6 +22,7 @@ import type { RuleMigrationTaskPrepareParams, RuleMigrationTaskRunParams, MigrationAgent, + RuleMigrationAllTaskStatsParams, } from './types'; import { getRuleMigrationAgent } from './agent'; import type { MigrateRuleState } from './agent/types'; @@ -65,11 +69,11 @@ export class RuleMigrationsTaskRunner { // Just in case some previous execution was interrupted without releasing await dataClient.releaseProcessable(migrationId); - const { total, pending } = await dataClient.getStats(migrationId); - if (total === 0) { + const { rules } = await dataClient.getStats(migrationId); + if (rules.total === 0) { return { exists: false, started: false }; } - if (pending === 0) { + if (rules.pending === 0) { return { exists: true, started: false }; } @@ -186,8 +190,8 @@ export class RuleMigrationsTaskRunner { ); this.taskLogger.info(`Batch processed successfully for migration ID:${migrationId}`); - const { pending } = await dataClient.getStats(migrationId); - isDone = pending === 0; + const { rules } = await dataClient.getStats(migrationId); + isDone = rules.pending === 0; if (!isDone) { await sleep(ITERATION_SLEEP_SECONDS); } @@ -211,37 +215,38 @@ export class RuleMigrationsTaskRunner { } } - /** Retries the status of a running migration */ - async stats({ + /** Returns the stats of a migration */ + async getStats({ migrationId, dataClient, }: RuleMigrationTaskStatsParams): Promise { - const stats = await dataClient.getStats(migrationId); - const status = this.getTaskStatus(stats, migrationId); - return { - status, - rules: { - total: stats.total, - finished: stats.finished, - pending: stats.pending, - processing: stats.processing, - failed: stats.failed, - }, - last_updated_at: stats.lastUpdatedAt, - }; + const dataStats = await dataClient.getStats(migrationId); + const status = this.getTaskStatus(migrationId, dataStats.rules); + return { status, ...dataStats }; + } + + /** Returns the stats of all migrations */ + async getAllStats({ + dataClient, + }: RuleMigrationAllTaskStatsParams): Promise { + const allDataStats = await dataClient.getAllStats(); + return allDataStats.map((dataStats) => { + const status = this.getTaskStatus(dataStats.migration_id, dataStats.rules); + return { status, ...dataStats }; + }); } private getTaskStatus( - stats: RuleMigrationStats, - migrationId: string + migrationId: string, + dataStats: RuleMigrationDataStats['rules'] ): RuleMigrationTaskStats['status'] { if (this.migrationsExecuting.has(migrationId)) { return 'running'; } - if (stats.pending === stats.total) { + if (dataStats.pending === dataStats.total) { return 'ready'; } - if (stats.finished + stats.failed === stats.total) { + if (dataStats.finished + dataStats.failed === dataStats.total) { return 'done'; } return 'stopped'; @@ -259,8 +264,8 @@ export class RuleMigrationsTaskRunner { return { exists: true, stopped: true }; } - const { total } = await dataClient.getStats(migrationId); - if (total > 0) { + const { rules } = await dataClient.getStats(migrationId); + if (rules.total > 0) { return { exists: true, stopped: false }; } return { exists: false, stopped: false }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts index 6431db0624901..e26a5b7216f48 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -55,6 +55,10 @@ export interface RuleMigrationTaskStatsParams { dataClient: RuleMigrationsDataClient; } +export interface RuleMigrationAllTaskStatsParams { + dataClient: RuleMigrationsDataClient; +} + export interface RuleMigrationTaskStartResult { started: boolean; exists: boolean; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index cf1af4fdace69..78ec2ef89c7a3 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -18,6 +18,7 @@ import type { ActionsClient } from '@kbn/actions-plugin/server'; import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { RuleMigration, + RuleMigrationAllTaskStats, RuleMigrationTaskStats, } from '../../../../common/siem_migrations/model/rule_migration.gen'; import type { RuleMigrationsDataClient } from './data_stream/rule_migrations_data_client'; @@ -54,7 +55,8 @@ export interface SiemRuleMigrationsClient { data: RuleMigrationsDataClient; task: { start: (params: SiemRuleMigrationsStartTaskParams) => Promise; - stats: (migrationId: string) => Promise; stop: (migrationId: string) => Promise; + getStats: (migrationId: string) => Promise; + getAllStats: () => Promise; }; } From 774de965cccdc03c60332181a552d3767f859ac9 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:51:02 +0000 Subject: [PATCH 05/14] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/security_solution/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index f8fdcfcd8f438..62b77b7c02f18 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -228,5 +228,6 @@ "@kbn/data-stream-adapter", "@kbn/core-lifecycle-server", "@kbn/core-user-profile-common", + "@kbn/langchain", ] } From 2669498064f09a6f4ea1e6c8589b5b233c5bea09 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Mon, 4 Nov 2024 19:54:10 +0100 Subject: [PATCH 06/14] fix tests --- .../siem_migrations/rules/__mocks__/mocks.ts | 2 + .../rules/data_stream/__mocks__/mocks.ts | 4 +- .../rule_migrations_data_stream.test.ts | 57 ++++++++++----- .../siem_rule_migrations_service.test.ts | 72 +++++++------------ .../rules/task/util/prebuilt_rules.test.ts | 70 ++++++++---------- .../siem_migrations_service.test.ts | 22 ++++-- 6 files changed, 114 insertions(+), 113 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts index a4d455ecd344a..3930c125ac06b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts @@ -36,8 +36,10 @@ export const MockSiemRuleMigrationsClient = jest.fn().mockImplementation(createR export const mockSetup = jest.fn(); export const mockCreateClient = jest.fn().mockReturnValue(createRuleMigrationClient()); +export const mockStop = jest.fn(); export const MockSiemRuleMigrationsService = jest.fn().mockImplementation(() => ({ setup: mockSetup, createClient: mockCreateClient, + stop: mockStop, })); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts index 103c0f9b0c952..1d9a181d2de5b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts @@ -7,9 +7,9 @@ export const mockIndexName = 'mocked_data_stream_name'; export const mockInstall = jest.fn().mockResolvedValue(undefined); -export const mockInstallSpace = jest.fn().mockResolvedValue(mockIndexName); +export const mockCreateClient = jest.fn().mockReturnValue({}); export const MockRuleMigrationsDataStream = jest.fn().mockImplementation(() => ({ install: mockInstall, - installSpace: mockInstallSpace, + createClient: mockCreateClient, })); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts index 56510da48f1bb..467d26a380945 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts @@ -11,9 +11,19 @@ import type { InstallParams } from '@kbn/data-stream-adapter'; import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { loggerMock } from '@kbn/logging-mocks'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { securityServiceMock } from '@kbn/core-security-server-mocks'; jest.mock('@kbn/data-stream-adapter'); +// This mock is required to have a way to await the data stream name promise +const mockDataStreamNamePromise = jest.fn(); +jest.mock('./rule_migrations_data_client', () => ({ + RuleMigrationsDataClient: jest.fn((dataStreamNamePromise: Promise) => { + mockDataStreamNamePromise.mockReturnValue(dataStreamNamePromise); + }), +})); + const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass< typeof DataStreamSpacesAdapter >; @@ -21,18 +31,21 @@ const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; describe('SiemRuleMigrationsDataStream', () => { + const kibanaVersion = '8.16.0'; + const logger = loggingSystemMock.createLogger(); + beforeEach(() => { jest.clearAllMocks(); }); describe('constructor', () => { it('should create DataStreamSpacesAdapter', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1); }); it('should create component templates', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) @@ -40,7 +53,7 @@ describe('SiemRuleMigrationsDataStream', () => { }); it('should create index templates', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) @@ -50,22 +63,20 @@ describe('SiemRuleMigrationsDataStream', () => { describe('install', () => { it('should install data stream', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - const params: InstallParams = { + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + const params: Omit = { esClient, - logger: loggerMock.create(), pluginStop$: new Subject(), }; await dataStream.install(params); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(params); + expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); }); it('should log error', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - const params: InstallParams = { + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + const params: Omit = { esClient, - logger: loggerMock.create(), pluginStop$: new Subject(), }; const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; @@ -73,13 +84,16 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); await dataStream.install(params); - expect(params.logger.error).toHaveBeenCalledWith(expect.any(String), error); + expect(logger.error).toHaveBeenCalledWith(expect.any(String), error); }); }); - describe('installSpace', () => { + describe('createClient', () => { + const currentUser = securityServiceMock.createMockAuthenticatedUser(); + const createClientParams = { spaceId: 'space1', currentUser, esClient }; + it('should install space data stream', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), @@ -89,19 +103,23 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); await dataStream.install(params); - await dataStream.installSpace('space1'); + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1'); expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1'); }); it('should not install space data stream if install not executed', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - await expect(dataStream.installSpace('space1')).rejects.toThrowError(); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + await expect(async () => { + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); + }).rejects.toThrowError(); }); it('should throw error if main install had error', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), @@ -112,7 +130,10 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); await dataStream.install(params); - await expect(dataStream.installSpace('space1')).rejects.toThrowError(error); + await expect(async () => { + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); + }).rejects.toThrowError(error); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts index 8103dc7c889a3..5c611d85e0464 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts @@ -8,27 +8,28 @@ import { loggingSystemMock, elasticsearchServiceMock, httpServerMock, + securityServiceMock, } from '@kbn/core/server/mocks'; import { SiemRuleMigrationsService } from './siem_rule_migrations_service'; import { Subject } from 'rxjs'; -import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen'; import { MockRuleMigrationsDataStream, mockInstall, - mockInstallSpace, - mockIndexName, + mockCreateClient, } from './data_stream/__mocks__/mocks'; -import type { KibanaRequest } from '@kbn/core/server'; +import type { SiemRuleMigrationsCreateClientParams } from './types'; jest.mock('./data_stream/rule_migrations_data_stream'); - -const migrationId = 'dummy_migration_id'; +jest.mock('./task/rule_migrations_task_runner', () => ({ + RuleMigrationsTaskRunner: jest.fn(), +})); describe('SiemRuleMigrationsService', () => { let ruleMigrationsService: SiemRuleMigrationsService; const kibanaVersion = '8.16.0'; const esClusterClient = elasticsearchServiceMock.createClusterClient(); + const currentUser = securityServiceMock.createMockAuthenticatedUser(); const logger = loggingSystemMock.createLogger(); const pluginStop$ = new Subject(); @@ -38,7 +39,7 @@ describe('SiemRuleMigrationsService', () => { }); it('should instantiate the rule migrations data stream adapter', () => { - expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith({ kibanaVersion }); + expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith(logger, kibanaVersion); }); describe('when setup is called', () => { @@ -47,22 +48,26 @@ describe('SiemRuleMigrationsService', () => { expect(mockInstall).toHaveBeenCalledWith({ esClient: esClusterClient.asInternalUser, - logger, pluginStop$, }); }); }); describe('when createClient is called', () => { - let request: KibanaRequest; + let createClientParams: SiemRuleMigrationsCreateClientParams; + beforeEach(() => { - request = httpServerMock.createKibanaRequest(); + createClientParams = { + spaceId: 'default', + currentUser, + request: httpServerMock.createKibanaRequest(), + }; }); describe('without setup', () => { it('should throw an error', () => { expect(() => { - ruleMigrationsService.createClient({ spaceId: 'default', request }); + ruleMigrationsService.createClient(createClientParams); }).toThrowError('ES client not available, please call setup first'); }); }); @@ -73,44 +78,19 @@ describe('SiemRuleMigrationsService', () => { }); it('should call installSpace', () => { - ruleMigrationsService.createClient({ spaceId: 'default', request }); - - expect(mockInstallSpace).toHaveBeenCalledWith('default'); - }); - - it('should return a client with create and search methods after setup', () => { - const client = ruleMigrationsService.createClient({ spaceId: 'default', request }); - - expect(client).toHaveProperty('create'); - expect(client).toHaveProperty('search'); + ruleMigrationsService.createClient(createClientParams); + expect(mockCreateClient).toHaveBeenCalledWith({ + spaceId: createClientParams.spaceId, + currentUser: createClientParams.currentUser, + esClient: esClusterClient.asScoped().asCurrentUser, + }); }); - it('should call ES bulk create API with the correct parameters with create is called', async () => { - const client = ruleMigrationsService.createClient({ spaceId: 'default', request }); - - const ruleMigrations = [{ migration_id: migrationId } as RuleMigration]; - await client.create(ruleMigrations); - - expect(esClusterClient.asScoped().asCurrentUser.bulk).toHaveBeenCalledWith( - expect.objectContaining({ - body: [{ create: { _index: mockIndexName } }, { migration_id: migrationId }], - refresh: 'wait_for', - }) - ); - }); - - it('should call ES search API with the correct parameters with search is called', async () => { - const client = ruleMigrationsService.createClient({ spaceId: 'default', request }); - - const searchParams = { query: { term: { migration_id: migrationId } } }; - await client.takePending(migrationId); + it('should return data and task clients', () => { + const client = ruleMigrationsService.createClient(createClientParams); - expect(esClusterClient.asScoped().asCurrentUser.search).toHaveBeenCalledWith( - expect.objectContaining({ - index: mockIndexName, - ...searchParams, - }) - ); + expect(client).toHaveProperty('data'); + expect(client).toHaveProperty('task'); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts index dd3801dde0e33..55256d0ad0fdd 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts @@ -6,26 +6,27 @@ */ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; -import { retrievePrebuiltRulesMap } from './prebuilt_rules'; +import type { PrebuiltRulesMapByName } from './prebuilt_rules'; +import { filterPrebuiltRules, retrievePrebuiltRulesMap } from './prebuilt_rules'; import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; -import type { SplunkRule } from '../../../../../../../../common/api/siem_migrations/splunk/rules/splunk_rule.gen'; jest.mock( - '../../../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client', + '../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client', () => ({ createPrebuiltRuleObjectsClient: jest.fn() }) ); jest.mock( - '../../../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client', + '../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client', () => ({ createPrebuiltRuleAssetsClient: jest.fn() }) ); +const mitreAttackIds = 'T1234'; const rule1 = { name: 'rule one', id: 'rule1', threat: [ { framework: 'MITRE ATT&CK', - technique: [{ id: 'T1234', name: 'tactic one' }], + technique: [{ id: mitreAttackIds, name: 'tactic one' }], }, ], }; @@ -40,23 +41,15 @@ const defaultRuleVersionsTriad = new Map([ ]); const mockFetchRuleVersionsTriad = jest.fn().mockResolvedValue(defaultRuleVersionsTriad); jest.mock( - '../../../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad', + '../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad', () => ({ fetchRuleVersionsTriad: () => mockFetchRuleVersionsTriad(), }) ); -const splunkRule: SplunkRule = { - title: 'splunk rule', - description: 'splunk rule description', - search: 'index=*', - mitreAttackIds: ['T1234'], -}; - const defaultParams = { soClient: savedObjectsClientMock.create(), rulesClient: rulesClientMock.create(), - splunkRule, }; describe('retrievePrebuiltRulesMap', () => { @@ -69,51 +62,44 @@ describe('retrievePrebuiltRulesMap', () => { const prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); expect(prebuiltRulesMap.size).toBe(2); expect(prebuiltRulesMap.get('rule one')).toEqual( - expect.objectContaining({ isInstalled: false }) + expect.objectContaining({ installedRuleId: undefined }) ); expect(prebuiltRulesMap.get('rule two')).toEqual( - expect.objectContaining({ isInstalled: true }) + expect.objectContaining({ installedRuleId: rule2.id }) ); }); }); +}); - describe('when splunk rule does not contain mitreAttackIds', () => { - it('should return the full rules map', async () => { - const prebuiltRulesMap = await retrievePrebuiltRulesMap({ - ...defaultParams, - splunkRule: { ...splunkRule, mitreAttackIds: undefined }, - }); - expect(prebuiltRulesMap.size).toBe(2); - }); +describe('filterPrebuiltRules', () => { + let prebuiltRulesMap: PrebuiltRulesMapByName; + + beforeEach(async () => { + prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); + jest.clearAllMocks(); }); describe('when splunk rule contains empty mitreAttackIds', () => { - it('should return the full rules map', async () => { - const prebuiltRulesMap = await retrievePrebuiltRulesMap({ - ...defaultParams, - splunkRule: { ...splunkRule, mitreAttackIds: [] }, - }); - expect(prebuiltRulesMap.size).toBe(2); + it('should return empty rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, []); + expect(filteredPrebuiltRules.size).toBe(0); }); }); - describe('when splunk rule contains non matching mitreAttackIds', () => { - it('should return the full rules map', async () => { - const prebuiltRulesMap = await retrievePrebuiltRulesMap({ - ...defaultParams, - splunkRule: { ...splunkRule, mitreAttackIds: ['T2345'] }, - }); - expect(prebuiltRulesMap.size).toBe(1); - expect(prebuiltRulesMap.get('rule two')).toEqual(expect.objectContaining({ rule: rule2 })); + describe('when splunk rule does not match mitreAttackIds', () => { + it('should return empty rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, [`${mitreAttackIds}_2`]); + expect(filteredPrebuiltRules.size).toBe(0); }); }); describe('when splunk rule contains matching mitreAttackIds', () => { it('should return the filtered rules map', async () => { - const prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); - expect(prebuiltRulesMap.size).toBe(2); - expect(prebuiltRulesMap.get('rule one')).toEqual(expect.objectContaining({ rule: rule1 })); - expect(prebuiltRulesMap.get('rule two')).toEqual(expect.objectContaining({ rule: rule2 })); + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, [mitreAttackIds]); + expect(filteredPrebuiltRules.size).toBe(1); + expect(filteredPrebuiltRules.get('rule one')).toEqual( + expect.objectContaining({ rule: rule1 }) + ); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts index 3d9e5b9fe179b..adf77756cce34 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts @@ -8,9 +8,15 @@ import { loggingSystemMock, elasticsearchServiceMock, httpServerMock, + securityServiceMock, } from '@kbn/core/server/mocks'; import { SiemMigrationsService } from './siem_migrations_service'; -import { MockSiemRuleMigrationsService, mockSetup, mockGetClient } from './rules/__mocks__/mocks'; +import { + MockSiemRuleMigrationsService, + mockSetup, + mockCreateClient, + mockStop, +} from './rules/__mocks__/mocks'; import type { ConfigType } from '../../config'; jest.mock('./rules/siem_rule_migrations_service'); @@ -25,6 +31,7 @@ describe('SiemMigrationsService', () => { let siemMigrationsService: SiemMigrationsService; const kibanaVersion = '8.16.0'; + const currentUser = securityServiceMock.createMockAuthenticatedUser(); const esClusterClient = elasticsearchServiceMock.createClusterClient(); const logger = loggingSystemMock.createLogger(); @@ -57,17 +64,22 @@ describe('SiemMigrationsService', () => { }); }); - describe('when createClient is called', () => { + describe('when createRulesClient is called', () => { it('should create rules client', async () => { - const request = httpServerMock.createKibanaRequest(); - siemMigrationsService.createClient({ spaceId: 'default', request }); - expect(mockGetClient).toHaveBeenCalledWith({ spaceId: 'default', request }); + const createRulesClientParams = { + spaceId: 'default', + request: httpServerMock.createKibanaRequest(), + currentUser, + }; + siemMigrationsService.createRulesClient(createRulesClientParams); + expect(mockCreateClient).toHaveBeenCalledWith(createRulesClientParams); }); }); describe('when stop is called', () => { it('should trigger the pluginStop subject', async () => { siemMigrationsService.stop(); + expect(mockStop).toHaveBeenCalled(); expect(mockReplaySubject$.next).toHaveBeenCalled(); expect(mockReplaySubject$.complete).toHaveBeenCalled(); }); From d3ac3440e5bbf815610ecab544076cbba1c06677 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Mon, 4 Nov 2024 20:05:16 +0100 Subject: [PATCH 07/14] fix mock lint error --- .../siem_migrations/rules/__mocks__/mocks.ts | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts index 3930c125ac06b..8811a54195e2b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts @@ -5,17 +5,31 @@ * 2.0. */ -export const createRuleMigrationDataClient = () => ({ +export const createRuleMigrationDataClient = jest.fn().mockImplementation(() => ({ create: jest.fn().mockResolvedValue({ success: true }), + getRules: jest.fn().mockResolvedValue([]), takePending: jest.fn().mockResolvedValue([]), - releaseProcessing: jest.fn(), - finishProcessing: jest.fn(), - finish: jest.fn(), -}); + saveFinished: jest.fn().mockResolvedValue({ success: true }), + saveError: jest.fn().mockResolvedValue({ success: true }), + releaseProcessing: jest.fn().mockResolvedValue({ success: true }), + releaseProcessable: jest.fn().mockResolvedValue({ success: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), +})); + export const createRuleMigrationTaskClient = () => ({ start: jest.fn().mockResolvedValue({ started: true }), stop: jest.fn().mockResolvedValue({ stopped: true }), - stats: jest.fn().mockResolvedValue({ + getStats: jest.fn().mockResolvedValue({ status: 'done', rules: { total: 1, @@ -25,6 +39,7 @@ export const createRuleMigrationTaskClient = () => ({ failed: 0, }, }), + getAllStats: jest.fn().mockResolvedValue([]), }); export const createRuleMigrationClient = () => ({ From 2b4dae921edacbb1d25b005a30a75899f2179796 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Tue, 5 Nov 2024 10:48:01 +0100 Subject: [PATCH 08/14] add promise catch --- .../rules/task/rule_migrations_task_runner.ts | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index 34f42d6b34b5c..a60fdc459e2c9 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -48,8 +48,8 @@ const getRuleLogger = (logger: Logger): RuleLogger => { }; }; -const ITERATION_BATCH_SIZE = 1 as const; -const ITERATION_SLEEP_SECONDS = 5 as const; +const ITERATION_BATCH_SIZE = 50 as const; +const ITERATION_SLEEP_SECONDS = 10 as const; export class RuleMigrationsTaskRunner { private migrationsExecuting: Map; @@ -78,11 +78,15 @@ export class RuleMigrationsTaskRunner { } const abortController = new AbortController(); + // Await the preparation to make sure the agent is created properly so the task can run const agent = await this.prepare({ ...params, abortController }); - // not awaiting the promise to execute the task in the background - this.run({ ...params, agent, abortController }); + // not awaiting the `run` promise to execute the task in the background + this.run({ ...params, agent, abortController }).catch((err) => { + // All errors in the `run` method are already catch, this should never happen, but just in case + this.taskLogger.error(`Unexpected error running the migration ID:${migrationId}`, err); + }); return { exists: true, started: true }; } @@ -125,7 +129,7 @@ export class RuleMigrationsTaskRunner { // This should never happen, but just in case throw new Error(`Task already running for migration ID:${migrationId} `); } - this.taskLogger.info(`Starting migration task run for ID:${migrationId}`); + this.taskLogger.info(`Starting migration task for ID:${migrationId}`); this.migrationsExecuting.set(migrationId, { abortController, user: currentUser.username }); const config: RunnableConfig = { @@ -137,7 +141,7 @@ export class RuleMigrationsTaskRunner { try { const sleep = async (seconds: number) => { - this.taskLogger.info(`Sleeping ${seconds}s for migration ID:${migrationId}`); + this.taskLogger.debug(`Sleeping ${seconds}s for migration ID:${migrationId}`); await Promise.race([ new Promise((resolve) => setTimeout(resolve, seconds * 1000)), abortPromise.promise, @@ -147,13 +151,13 @@ export class RuleMigrationsTaskRunner { let isDone: boolean = false; do { const ruleMigrations = await dataClient.takePending(migrationId, ITERATION_BATCH_SIZE); - this.taskLogger.info( + this.taskLogger.debug( `Processing ${ruleMigrations.length} rules for migration ID:${migrationId}` ); await Promise.all( ruleMigrations.map(async (ruleMigration) => { - this.taskLogger.info( + this.taskLogger.debug( `Starting migration of rule "${ruleMigration.original_rule.title}"` ); try { @@ -163,7 +167,7 @@ export class RuleMigrationsTaskRunner { config ); const duration = (Date.now() - start) / 1000; - this.taskLogger.info( + this.taskLogger.debug( `Migration of rule "${ruleMigration.original_rule.title}" finished in ${duration}s` ); @@ -188,7 +192,7 @@ export class RuleMigrationsTaskRunner { } }) ); - this.taskLogger.info(`Batch processed successfully for migration ID:${migrationId}`); + this.taskLogger.debug(`Batch processed successfully for migration ID:${migrationId}`); const { rules } = await dataClient.getStats(migrationId); isDone = rules.pending === 0; @@ -197,7 +201,7 @@ export class RuleMigrationsTaskRunner { } } while (!isDone); - this.taskLogger.info(`Finished task for migration ID:${migrationId}`); + this.taskLogger.info(`Finished migration task for ID:${migrationId}`); } catch (error) { await dataClient.releaseProcessing(migrationId); From aec01066224087e4658f5da1749b5393c4c03082 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:34:55 +0000 Subject: [PATCH 09/14] [CI] Auto-commit changed files from 'yarn openapi:generate' --- .../common/api/quickstart_client.gen.ts | 89 ++++++++++++++++++- .../services/security_solution_api.gen.ts | 87 +++++++++++++++++- 2 files changed, 170 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index 25560aeffdbbe..04650cbb1f54a 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -364,7 +364,16 @@ import type { import type { CreateRuleMigrationRequestBodyInput, CreateRuleMigrationResponse, + GetAllStatsRuleMigrationResponse, + GetRuleMigrationRequestParamsInput, GetRuleMigrationResponse, + GetRuleMigrationStatsRequestParamsInput, + GetRuleMigrationStatsResponse, + StartRuleMigrationRequestParamsInput, + StartRuleMigrationRequestBodyInput, + StartRuleMigrationResponse, + StopRuleMigrationRequestParamsInput, + StopRuleMigrationResponse, } from '../siem_migrations/model/api/rules/rules_migration.gen'; export interface ClientOptions { @@ -1225,6 +1234,21 @@ finalize it. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Retrieves the rule migrations stats for all migrations stored in the system + */ + async getAllStatsRuleMigration() { + this.log.info(`${new Date().toISOString()} Calling API GetAllStatsRuleMigration`); + return this.kbnClient + .request({ + path: '/internal/siem_migrations/rules/stats', + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Get the asset criticality record for a specific entity. */ @@ -1418,13 +1442,28 @@ finalize it. .catch(catchAxiosErrorFormatAndThrow); } /** - * Retrieves the rule migrations stored in the system + * Retrieves the rule documents stored in the system given the rule migration id */ - async getRuleMigration() { + async getRuleMigration(props: GetRuleMigrationProps) { this.log.info(`${new Date().toISOString()} Calling API GetRuleMigration`); return this.kbnClient .request({ - path: '/internal/siem_migrations/rules', + path: replaceParams('/internal/siem_migrations/rules/{migration_id}', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } + /** + * Retrieves the stats of a SIEM rules migration using the migration id provided + */ + async getRuleMigrationStats(props: GetRuleMigrationStatsProps) { + this.log.info(`${new Date().toISOString()} Calling API GetRuleMigrationStats`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/stats', props.params), headers: { [ELASTIC_HTTP_VERSION_HEADER]: '1', }, @@ -1960,6 +1999,22 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Starts a SIEM rules migration using the migration id provided + */ + async startRuleMigration(props: StartRuleMigrationProps) { + this.log.info(`${new Date().toISOString()} Calling API StartRuleMigration`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/start', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'PUT', + body: props.body, + }) + .catch(catchAxiosErrorFormatAndThrow); + } async stopEntityEngine(props: StopEntityEngineProps) { this.log.info(`${new Date().toISOString()} Calling API StopEntityEngine`); return this.kbnClient @@ -1972,6 +2027,21 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Stops a running SIEM rules migration using the migration id provided + */ + async stopRuleMigration(props: StopRuleMigrationProps) { + this.log.info(`${new Date().toISOString()} Calling API StopRuleMigration`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/stop', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'PUT', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Suggests user profiles. */ @@ -2208,6 +2278,12 @@ export interface GetRuleExecutionResultsProps { query: GetRuleExecutionResultsRequestQueryInput; params: GetRuleExecutionResultsRequestParamsInput; } +export interface GetRuleMigrationProps { + params: GetRuleMigrationRequestParamsInput; +} +export interface GetRuleMigrationStatsProps { + params: GetRuleMigrationStatsRequestParamsInput; +} export interface GetTimelineProps { query: GetTimelineRequestQueryInput; } @@ -2284,9 +2360,16 @@ export interface SetAlertTagsProps { export interface StartEntityEngineProps { params: StartEntityEngineRequestParamsInput; } +export interface StartRuleMigrationProps { + params: StartRuleMigrationRequestParamsInput; + body: StartRuleMigrationRequestBodyInput; +} export interface StopEntityEngineProps { params: StopEntityEngineRequestParamsInput; } +export interface StopRuleMigrationProps { + params: StopRuleMigrationRequestParamsInput; +} export interface SuggestUserProfilesProps { query: SuggestUserProfilesRequestQueryInput; } diff --git a/x-pack/test/api_integration/services/security_solution_api.gen.ts b/x-pack/test/api_integration/services/security_solution_api.gen.ts index 3503f07fec574..419ab4e76376b 100644 --- a/x-pack/test/api_integration/services/security_solution_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_api.gen.ts @@ -95,6 +95,8 @@ import { GetRuleExecutionResultsRequestQueryInput, GetRuleExecutionResultsRequestParamsInput, } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.gen'; +import { GetRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationStatsRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { GetTimelineRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timeline/get_timeline_route.gen'; import { GetTimelinesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timelines/get_timelines_route.gen'; import { ImportRulesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/import_rules/import_rules_route.gen'; @@ -127,7 +129,12 @@ import { SetAlertAssigneesRequestBodyInput } from '@kbn/security-solution-plugin import { SetAlertsStatusRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/signals/set_signal_status/set_signals_status_route.gen'; import { SetAlertTagsRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/alert_tags/set_alert_tags/set_alert_tags.gen'; import { StartEntityEngineRequestParamsInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/engine/start.gen'; +import { + StartRuleMigrationRequestParamsInput, + StartRuleMigrationRequestBodyInput, +} from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { StopEntityEngineRequestParamsInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/engine/stop.gen'; +import { StopRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { SuggestUserProfilesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/users/suggest_user_profiles_route.gen'; import { TriggerRiskScoreCalculationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/risk_engine/entity_calculation_route.gen'; import { UpdateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/update_rule/update_rule_route.gen'; @@ -775,6 +782,16 @@ finalize it. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Retrieves the rule migrations stats for all migrations stored in the system + */ + getAllStatsRuleMigration(kibanaSpace: string = 'default') { + return supertest + .get(routeWithNamespace('/internal/siem_migrations/rules/stats', kibanaSpace)) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, /** * Get the asset criticality record for a specific entity. */ @@ -932,11 +949,31 @@ finalize it. .query(props.query); }, /** - * Retrieves the rule migrations stored in the system + * Retrieves the rule documents stored in the system given the rule migration id */ - getRuleMigration(kibanaSpace: string = 'default') { + getRuleMigration(props: GetRuleMigrationProps, kibanaSpace: string = 'default') { return supertest - .get(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace)) + .get( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, + /** + * Retrieves the stats of a SIEM rules migration using the migration id provided + */ + getRuleMigrationStats(props: GetRuleMigrationStatsProps, kibanaSpace: string = 'default') { + return supertest + .get( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/stats', props.params), + kibanaSpace + ) + ) .set('kbn-xsrf', 'true') .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); @@ -1307,6 +1344,22 @@ detection engine rules. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Starts a SIEM rules migration using the migration id provided + */ + startRuleMigration(props: StartRuleMigrationProps, kibanaSpace: string = 'default') { + return supertest + .put( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/start', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send(props.body as object); + }, stopEntityEngine(props: StopEntityEngineProps, kibanaSpace: string = 'default') { return supertest .post( @@ -1319,6 +1372,21 @@ detection engine rules. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Stops a running SIEM rules migration using the migration id provided + */ + stopRuleMigration(props: StopRuleMigrationProps, kibanaSpace: string = 'default') { + return supertest + .put( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/stop', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, /** * Suggests user profiles. */ @@ -1537,6 +1605,12 @@ export interface GetRuleExecutionResultsProps { query: GetRuleExecutionResultsRequestQueryInput; params: GetRuleExecutionResultsRequestParamsInput; } +export interface GetRuleMigrationProps { + params: GetRuleMigrationRequestParamsInput; +} +export interface GetRuleMigrationStatsProps { + params: GetRuleMigrationStatsRequestParamsInput; +} export interface GetTimelineProps { query: GetTimelineRequestQueryInput; } @@ -1609,9 +1683,16 @@ export interface SetAlertTagsProps { export interface StartEntityEngineProps { params: StartEntityEngineRequestParamsInput; } +export interface StartRuleMigrationProps { + params: StartRuleMigrationRequestParamsInput; + body: StartRuleMigrationRequestBodyInput; +} export interface StopEntityEngineProps { params: StopEntityEngineRequestParamsInput; } +export interface StopRuleMigrationProps { + params: StopRuleMigrationRequestParamsInput; +} export interface SuggestUserProfilesProps { query: SuggestUserProfilesRequestQueryInput; } From 84e85852a3b316be895ba970bb4056524768ac47 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Tue, 5 Nov 2024 13:20:48 +0100 Subject: [PATCH 10/14] cleaning unnecessary code --- .../lib/siem_migrations/rules/api/stop.ts | 2 +- .../siem_migrations/rules/task/agent/graph.ts | 5 +- .../task/agent/nodes/process_response.ts | 54 ------------------- .../nodes/translate_query/translate_query.ts | 27 +++++++++- 4 files changed, 27 insertions(+), 61 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts index 48bcb23ca94d8..4767106910186 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts @@ -32,7 +32,7 @@ export const registerSiemRuleMigrationsStopRoute = ( async (context, req, res): Promise> => { const migrationId = req.params.migration_id; try { - const ctx = await context.resolve(['core', 'actions', 'securitySolution']); + const ctx = await context.resolve(['securitySolution']); const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); const { exists, stopped } = await ruleMigrationsClient.task.stop(migrationId); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts index 2e56277b174a1..a44197d82850f 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -8,7 +8,6 @@ import { END, START, StateGraph } from '@langchain/langgraph'; import { migrateRuleState } from './state'; import type { MigrateRuleGraphParams, MigrateRuleState } from './types'; -import { processResponseNode } from './nodes/process_response'; import { getTranslateQueryNode } from './nodes/translate_query'; import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule'; @@ -26,12 +25,10 @@ export function getRuleMigrationAgent({ // Nodes .addNode('matchPrebuiltRule', matchPrebuiltRuleNode) .addNode('translation', translationNode) - .addNode('processResponse', processResponseNode) // Edges .addEdge(START, 'matchPrebuiltRule') .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional) - .addEdge('translation', 'processResponse') - .addEdge('processResponse', END); + .addEdge('translation', END); const graph = translateRuleGraph.compile(); graph.name = 'Rule Migration Graph'; // Customizes the name displayed in LangSmith diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts deleted file mode 100644 index 053a0c67456fe..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/process_response.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { AIMessage } from '@langchain/core/messages'; -import type { GraphNode } from '../types'; - -export const processResponseNode: GraphNode = async (state) => { - const messages = state.messages; - const lastMessage = messages[messages.length - 1] as AIMessage; - const response = lastMessage.content as string; - - const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; - const summary = response.match(/## Migration Summary[\s\S]*$/)?.[0] ?? ''; - - // if (esqlQuery == null) { - // throw new Error('Could not find ESQL query in response'); - // } - // if (summary == null) { - // throw new Error('Could not find migration summary in response'); - // } - // const missingEntities = extractMissingEntities(esqlQuery); - const translationState = getTranslationState(esqlQuery); - - return { - response, - comments: [summary], - translation_state: translationState, - elastic_rule: { - title: state.original_rule.title, - description: state.original_rule.description, - severity: 'low', - query: esqlQuery, - query_language: 'esql', - // missing_entities: missingEntities, - }, - }; -}; - -// const extractMissingEntities = (esqlQuery: string) => { -// const result = Array.from(esqlQuery?.matchAll(/\[macro:[\s\S]+?\]/)); -// console.log(result); -// return result; -// }; - -const getTranslationState = (esqlQuery: string) => { - if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { - return 'partial'; - } - return 'complete'; -}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts index 1265507463bbc..84091c791c2a7 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts @@ -7,7 +7,6 @@ import type { Logger } from '@kbn/core/server'; import type { InferenceClient } from '@kbn/inference-plugin/server'; -import { AIMessage } from '@langchain/core/messages'; import type { GraphNode } from '../../types'; import { getEsqlKnowledgeBase } from './esql_knowledge_base_caller'; import { getEsqlTranslationPrompt } from './prompt'; @@ -27,6 +26,30 @@ export const getTranslateQueryNode = ({ return async (state) => { const input = getEsqlTranslationPrompt(state); const response = await esqlKnowledgeBaseCaller(input); - return { messages: [new AIMessage(response)] }; + + const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; + const summary = response.match(/## Migration Summary[\s\S]*$/)?.[0] ?? ''; + + const translationState = getTranslationState(esqlQuery); + + return { + response, + comments: [summary], + translation_state: translationState, + elastic_rule: { + title: state.original_rule.title, + description: state.original_rule.description, + severity: 'low', + query: esqlQuery, + query_language: 'esql', + }, + }; }; }; + +const getTranslationState = (esqlQuery: string) => { + if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { + return 'partial'; + } + return 'complete'; +}; From 435f15ee0fc588e68bf9651e2815cfe7055953d6 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Wed, 6 Nov 2024 10:38:29 +0100 Subject: [PATCH 11/14] add individual graph invocation abort logic --- .../rules/task/rule_migrations_task_runner.ts | 62 +++++++++---------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index a60fdc459e2c9..01f5023cb416d 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -29,17 +29,12 @@ import type { MigrateRuleState } from './agent/types'; import { retrievePrebuiltRulesMap } from './util/prebuilt_rules'; import { ActionsClientChat } from './util/actions_client_chat'; -interface MigrationProcessing { - abortController: AbortController; - user: string; -} - -interface RuleLogger { +interface TaskLogger { info: (msg: string) => void; debug: (msg: string) => void; error: (msg: string, error: Error) => void; } -const getRuleLogger = (logger: Logger): RuleLogger => { +const getTaskLogger = (logger: Logger): TaskLogger => { const prefix = '[ruleMigrationsTask]: '; return { info: (msg) => logger.info(`${prefix}${msg}`), @@ -52,18 +47,18 @@ const ITERATION_BATCH_SIZE = 50 as const; const ITERATION_SLEEP_SECONDS = 10 as const; export class RuleMigrationsTaskRunner { - private migrationsExecuting: Map; - private taskLogger: RuleLogger; + private migrationsRunning: Map; + private taskLogger: TaskLogger; constructor(private logger: Logger) { - this.migrationsExecuting = new Map(); - this.taskLogger = getRuleLogger(logger); + this.migrationsRunning = new Map(); + this.taskLogger = getTaskLogger(logger); } /** Starts a rule migration task */ async start(params: RuleMigrationTaskStartParams): Promise { const { migrationId, dataClient } = params; - if (this.migrationsExecuting.has(migrationId)) { + if (this.migrationsRunning.has(migrationId)) { return { exists: true, started: false }; } // Just in case some previous execution was interrupted without releasing @@ -79,7 +74,7 @@ export class RuleMigrationsTaskRunner { const abortController = new AbortController(); - // Await the preparation to make sure the agent is created properly so the task can run + // Await the preparation for the request to make sure the agent is created properly so the task can run const agent = await this.prepare({ ...params, abortController }); // not awaiting the `run` promise to execute the task in the background @@ -125,13 +120,13 @@ export class RuleMigrationsTaskRunner { invocationConfig, abortController, }: RuleMigrationTaskRunParams): Promise { - if (this.migrationsExecuting.has(migrationId)) { + if (this.migrationsRunning.has(migrationId)) { // This should never happen, but just in case throw new Error(`Task already running for migration ID:${migrationId} `); } - this.taskLogger.info(`Starting migration task for ID:${migrationId}`); + this.taskLogger.info(`Starting migration ID:${migrationId}`); - this.migrationsExecuting.set(migrationId, { abortController, user: currentUser.username }); + this.migrationsRunning.set(migrationId, { user: currentUser.username, abortController }); const config: RunnableConfig = { ...invocationConfig, // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319 @@ -162,10 +157,12 @@ export class RuleMigrationsTaskRunner { ); try { const start = Date.now(); - const ruleMigrationResult: MigrateRuleState = await agent.invoke( - { original_rule: ruleMigration.original_rule }, - config - ); + + const ruleMigrationResult: MigrateRuleState = await Promise.race([ + agent.invoke({ original_rule: ruleMigration.original_rule }, config), + abortPromise.promise, // workaround for the issue with the langGraph signal + ]); + const duration = (Date.now() - start) / 1000; this.taskLogger.debug( `Migration of rule "${ruleMigration.original_rule.title}" finished in ${duration}s` @@ -192,6 +189,7 @@ export class RuleMigrationsTaskRunner { } }) ); + this.taskLogger.debug(`Batch processed successfully for migration ID:${migrationId}`); const { rules } = await dataClient.getStats(migrationId); @@ -201,20 +199,18 @@ export class RuleMigrationsTaskRunner { } } while (!isDone); - this.taskLogger.info(`Finished migration task for ID:${migrationId}`); + this.taskLogger.info(`Finished migration ID:${migrationId}`); } catch (error) { await dataClient.releaseProcessing(migrationId); if (error instanceof AbortError) { - this.taskLogger.info( - `Abort signal received, stopping task for migration ID:${migrationId}` - ); + this.taskLogger.info(`Abort signal received, stopping migration ID:${migrationId}`); return; } else { this.taskLogger.error(`Error processing migration ID:${migrationId}`, error); } } finally { - this.migrationsExecuting.delete(migrationId); + this.migrationsRunning.delete(migrationId); abortPromise.cleanup(); } } @@ -244,7 +240,7 @@ export class RuleMigrationsTaskRunner { migrationId: string, dataStats: RuleMigrationDataStats['rules'] ): RuleMigrationTaskStats['status'] { - if (this.migrationsExecuting.has(migrationId)) { + if (this.migrationsRunning.has(migrationId)) { return 'running'; } if (dataStats.pending === dataStats.total) { @@ -256,15 +252,15 @@ export class RuleMigrationsTaskRunner { return 'stopped'; } - /** Aborts a running migration */ + /** Stops one running migration */ async stop({ migrationId, dataClient, }: RuleMigrationTaskStopParams): Promise { try { - const migrationProcessing = this.migrationsExecuting.get(migrationId); - if (migrationProcessing) { - migrationProcessing.abortController.abort(); + const migrationRunning = this.migrationsRunning.get(migrationId); + if (migrationRunning) { + migrationRunning.abortController.abort(); return { exists: true, stopped: true }; } @@ -281,9 +277,9 @@ export class RuleMigrationsTaskRunner { /** Stops all running migrations */ stopAll() { - this.migrationsExecuting.forEach((migrationProcessing) => { - migrationProcessing.abortController.abort(); + this.migrationsRunning.forEach((migrationRunning) => { + migrationRunning.abortController.abort(); }); - this.migrationsExecuting.clear(); + this.migrationsRunning.clear(); } } From 0c119edefc4f571c5d0a60e006851fa4756c47bb Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Wed, 6 Nov 2024 10:38:57 +0100 Subject: [PATCH 12/14] fix comment --- .../siem_migrations/rules/task/rule_migrations_task_runner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index 01f5023cb416d..0feb286122a95 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -74,7 +74,7 @@ export class RuleMigrationsTaskRunner { const abortController = new AbortController(); - // Await the preparation for the request to make sure the agent is created properly so the task can run + // Await the preparation to make sure the agent is created properly so the task can run const agent = await this.prepare({ ...params, abortController }); // not awaiting the `run` promise to execute the task in the background From 3fd76b5ac671debba98516126a3b4e232d3b8021 Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Wed, 6 Nov 2024 15:28:24 +0100 Subject: [PATCH 13/14] remove ecs component template --- .../rules/data_stream/rule_migrations_data_stream.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts index 41806183489cf..a5855cefb1324 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts @@ -13,7 +13,6 @@ import { RuleMigrationsDataClient } from './rule_migrations_data_client'; const TOTAL_FIELDS_LIMIT = 2500; const DATA_STREAM_NAME = '.kibana.siem-rule-migrations'; -const ECS_COMPONENT_TEMPLATE_NAME = 'ecs'; interface RuleMigrationsDataStreamCreateClientParams { spaceId: string; @@ -37,7 +36,7 @@ export class RuleMigrationsDataStream { this.dataStreamAdapter.setIndexTemplate({ name: DATA_STREAM_NAME, - componentTemplateRefs: [DATA_STREAM_NAME, ECS_COMPONENT_TEMPLATE_NAME], + componentTemplateRefs: [DATA_STREAM_NAME], }); } From 0854e37911d25f28e60e8ac9c1d1f217d8eb2fee Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Wed, 6 Nov 2024 16:26:17 +0100 Subject: [PATCH 14/14] small status names refactor --- .../common/siem_migrations/constants.ts | 10 +++- .../model/api/rules/rules_migration.gen.ts | 4 +- .../api/rules/rules_migration.schema.yaml | 6 +-- .../model/rule_migration.gen.ts | 18 +++---- .../model/rule_migration.schema.yaml | 26 +++++----- .../lib/siem_migrations/rules/api/start.ts | 6 +-- .../rule_migrations_data_client.ts | 52 +++++++++---------- .../data_stream/rule_migrations_field_map.ts | 2 +- .../nodes/translate_query/translate_query.ts | 11 ++-- .../siem_migrations/rules/task/agent/state.ts | 3 +- .../rules/task/rule_migrations_task_runner.ts | 6 +-- 11 files changed, 76 insertions(+), 68 deletions(-) diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index 4d0dd3252ae94..f2efc646a8101 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -17,9 +17,15 @@ export const SIEM_RULE_MIGRATIONS_STATS_PATH = export const SIEM_RULE_MIGRATIONS_STOP_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stop` as const; -export enum SiemMigrationsStatus { +export enum SiemMigrationStatus { PENDING = 'pending', PROCESSING = 'processing', - FINISHED = 'finished', + COMPLETED = 'completed', FAILED = 'failed', } + +export enum SiemMigrationRuleTranslationResult { + FULL = 'full', + PARTIAL = 'partial', + UNTRANSLATABLE = 'untranslatable', +} diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts index a2a8edd15878f..120505ec43cb7 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts @@ -67,8 +67,8 @@ export type StartRuleMigrationRequestParamsInput = z.input; export const StartRuleMigrationRequestBody = z.object({ - connectorId: ConnectorId, - langSmithOptions: LangSmithOptions.optional(), + connector_id: ConnectorId, + langsmith_options: LangSmithOptions.optional(), }); export type StartRuleMigrationRequestBodyInput = z.input; diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml index 4874997715b3e..7b06c3d6a22ac 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml @@ -98,11 +98,11 @@ paths: schema: type: object required: - - connectorId + - connector_id properties: - connectorId: + connector_id: $ref: '../common.schema.yaml#/components/schemas/ConnectorId' - langSmithOptions: + langsmith_options: $ref: '../common.schema.yaml#/components/schemas/LangSmithOptions' responses: 200: diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 0e0d846d74917..fe00c4b4df1c6 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -106,13 +106,13 @@ export const RuleMigration = z.object({ original_rule: OriginalRule, elastic_rule: ElasticRule.optional(), /** - * The translation state. + * The rule translation result. */ - translation_state: z.enum(['complete', 'partial', 'untranslatable']).optional(), + translation_result: z.enum(['full', 'partial', 'untranslatable']).optional(), /** - * The status of the rule migration. + * The status of the rule migration process. */ - status: z.enum(['pending', 'processing', 'finished', 'failed']).default('pending'), + status: z.enum(['pending', 'processing', 'completed', 'failed']).default('pending'), /** * The comments for the migration including a summary from the LLM in markdown. */ @@ -135,7 +135,7 @@ export const RuleMigrationTaskStats = z.object({ /** * Indicates if the migration task status. */ - status: z.enum(['ready', 'running', 'stopped', 'done']), + status: z.enum(['ready', 'running', 'stopped', 'finished']), /** * The rules migration stats. */ @@ -145,17 +145,17 @@ export const RuleMigrationTaskStats = z.object({ */ total: z.number().int(), /** - * The number of rules that have been migrated. + * The number of rules that are pending migration. */ - finished: z.number().int(), + pending: z.number().int(), /** * The number of rules that are being migrated. */ processing: z.number().int(), /** - * The number of rules that are pending migration. + * The number of rules that have been migrated successfully. */ - pending: z.number().int(), + completed: z.number().int(), /** * The number of rules that have failed migration. */ diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 44bb8407dbd7d..c9841856a6914 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -96,20 +96,20 @@ components: $ref: '#/components/schemas/OriginalRule' elastic_rule: $ref: '#/components/schemas/ElasticRule' - translation_state: + translation_result: type: string - description: The translation state. - enum: - - complete + description: The rule translation result. + enum: # should match SiemMigrationRuleTranslationResult enum at ../constants.ts + - full - partial - untranslatable status: type: string - description: The status of the rule migration. + description: The status of the rule migration process. enum: # should match SiemMigrationsStatus enum at ../constants.ts - pending - processing - - finished + - completed - failed default: pending comments: @@ -138,29 +138,29 @@ components: - ready - running - stopped - - done + - finished rules: type: object description: The rules migration stats. required: - total - - finished - - processing - pending + - processing + - completed - failed properties: total: type: integer description: The total number of rules to migrate. - finished: + pending: type: integer - description: The number of rules that have been migrated. + description: The number of rules that are pending migration. processing: type: integer description: The number of rules that are being migrated. - pending: + completed: type: integer - description: The number of rules that are pending migration. + description: The number of rules that have been migrated successfully. failed: type: integer description: The number of rules that have failed migration. diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts index 7df94b20c0220..f97a4f2ce2398 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -39,7 +39,7 @@ export const registerSiemRuleMigrationsStartRoute = ( }, async (context, req, res): Promise> => { const migrationId = req.params.migration_id; - const { langSmithOptions, connectorId } = req.body; + const { langsmith_options: langsmithOptions, connector_id: connectorId } = req.body; try { const ctx = await context.resolve([ @@ -63,8 +63,8 @@ export const registerSiemRuleMigrationsStartRoute = ( const invocationConfig = { callbacks: [ - new APMTracer({ projectName: langSmithOptions?.project_name ?? 'default' }, logger), - ...getLangSmithTracer({ ...langSmithOptions, logger }), + new APMTracer({ projectName: langsmithOptions?.project_name ?? 'default' }, logger), + ...getLangSmithTracer({ ...langsmithOptions, logger }), ], }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts index b96b281c3ecec..83808901a0bd1 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts @@ -17,7 +17,7 @@ import type { SearchResponse, } from '@elastic/elasticsearch/lib/api/types'; import type { StoredRuleMigration } from '../types'; -import { SiemMigrationsStatus } from '../../../../../common/siem_migrations/constants'; +import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; import type { RuleMigration, RuleMigrationTaskStats, @@ -46,7 +46,7 @@ export class RuleMigrationsDataClient { { ...ruleMigration, '@timestamp': new Date().toISOString(), - status: SiemMigrationsStatus.PENDING, + status: SiemMigrationStatus.PENDING, created_by: this.currentUser.username, }, ]), @@ -80,7 +80,7 @@ export class RuleMigrationsDataClient { */ async takePending(migrationId: string, size: number): Promise { const index = await this.dataStreamNamePromise; - const query = this.getFilterQuery(migrationId, SiemMigrationsStatus.PENDING); + const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PENDING); const storedRuleMigrations = await this.esClient .search({ index, query, sort: '_doc', size }) @@ -89,7 +89,7 @@ export class RuleMigrationsDataClient { throw error; }) .then((response) => - this.processHits(response.hits.hits, { status: SiemMigrationsStatus.PROCESSING }) + this.processHits(response.hits.hits, { status: SiemMigrationStatus.PROCESSING }) ); await this.esClient @@ -116,18 +116,18 @@ export class RuleMigrationsDataClient { return storedRuleMigrations; } - /** Updates one rule migration with the provided data and sets the status to `finished` */ + /** Updates one rule migration with the provided data and sets the status to `completed` */ async saveFinished({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { const doc = { ...ruleMigration, - status: SiemMigrationsStatus.FINISHED, + status: SiemMigrationStatus.COMPLETED, updated_by: this.currentUser.username, updated_at: new Date().toISOString(), }; await this.esClient .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) .catch((error) => { - this.logger.error(`Error updating rule migration status to finished: ${error.message}`); + this.logger.error(`Error updating rule migration status to completed: ${error.message}`); throw error; }); } @@ -136,14 +136,14 @@ export class RuleMigrationsDataClient { async saveError({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { const doc = { ...ruleMigration, - status: SiemMigrationsStatus.FAILED, + status: SiemMigrationStatus.FAILED, updated_by: this.currentUser.username, updated_at: new Date().toISOString(), }; await this.esClient .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) .catch((error) => { - this.logger.error(`Error updating rule migration status to finished: ${error.message}`); + this.logger.error(`Error updating rule migration status to completed: ${error.message}`); throw error; }); } @@ -151,8 +151,8 @@ export class RuleMigrationsDataClient { /** Updates all the rule migration with the provided id with status `processing` back to `pending` */ async releaseProcessing(migrationId: string): Promise { const index = await this.dataStreamNamePromise; - const query = this.getFilterQuery(migrationId, SiemMigrationsStatus.PROCESSING); - const script = { source: `ctx._source['status'] = '${SiemMigrationsStatus.PENDING}'` }; + const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PROCESSING); + const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; await this.esClient.updateByQuery({ index, query, script, refresh: false }).catch((error) => { this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); throw error; @@ -163,10 +163,10 @@ export class RuleMigrationsDataClient { async releaseProcessable(migrationId: string): Promise { const index = await this.dataStreamNamePromise; const query = this.getFilterQuery(migrationId, [ - SiemMigrationsStatus.PROCESSING, - SiemMigrationsStatus.FAILED, + SiemMigrationStatus.PROCESSING, + SiemMigrationStatus.FAILED, ]); - const script = { source: `ctx._source['status'] = '${SiemMigrationsStatus.PENDING}'` }; + const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; await this.esClient.updateByQuery({ index, query, script, refresh: true }).catch((error) => { this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); throw error; @@ -178,10 +178,10 @@ export class RuleMigrationsDataClient { const index = await this.dataStreamNamePromise; const query = this.getFilterQuery(migrationId); const aggregations = { - pending: { filter: { term: { status: SiemMigrationsStatus.PENDING } } }, - processing: { filter: { term: { status: SiemMigrationsStatus.PROCESSING } } }, - finished: { filter: { term: { status: SiemMigrationsStatus.FINISHED } } }, - failed: { filter: { term: { status: SiemMigrationsStatus.FAILED } } }, + pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationStatus.PROCESSING } } }, + completed: { filter: { term: { status: SiemMigrationStatus.COMPLETED } } }, + failed: { filter: { term: { status: SiemMigrationStatus.FAILED } } }, lastUpdatedAt: { max: { field: 'updated_at' } }, }; const result = await this.esClient @@ -191,13 +191,13 @@ export class RuleMigrationsDataClient { throw error; }); - const { pending, processing, finished, lastUpdatedAt, failed } = result.aggregations ?? {}; + const { pending, processing, completed, lastUpdatedAt, failed } = result.aggregations ?? {}; return { rules: { total: this.getTotalHits(result), pending: (pending as AggregationsFilterAggregate)?.doc_count ?? 0, processing: (processing as AggregationsFilterAggregate)?.doc_count ?? 0, - finished: (finished as AggregationsFilterAggregate)?.doc_count ?? 0, + completed: (completed as AggregationsFilterAggregate)?.doc_count ?? 0, failed: (failed as AggregationsFilterAggregate)?.doc_count ?? 0, }, last_updated_at: (lastUpdatedAt as AggregationsMaxAggregate)?.value_as_string, @@ -211,10 +211,10 @@ export class RuleMigrationsDataClient { migrationIds: { terms: { field: 'migration_id' }, aggregations: { - pending: { filter: { term: { status: SiemMigrationsStatus.PENDING } } }, - processing: { filter: { term: { status: SiemMigrationsStatus.PROCESSING } } }, - finished: { filter: { term: { status: SiemMigrationsStatus.FINISHED } } }, - failed: { filter: { term: { status: SiemMigrationsStatus.FAILED } } }, + pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationStatus.PROCESSING } } }, + completed: { filter: { term: { status: SiemMigrationStatus.COMPLETED } } }, + failed: { filter: { term: { status: SiemMigrationStatus.FAILED } } }, lastUpdatedAt: { max: { field: 'updated_at' } }, }, }, @@ -234,7 +234,7 @@ export class RuleMigrationsDataClient { total: bucket.doc_count, pending: bucket.pending?.doc_count ?? 0, processing: bucket.processing?.doc_count ?? 0, - finished: bucket.finished?.doc_count ?? 0, + completed: bucket.completed?.doc_count ?? 0, failed: bucket.failed?.doc_count ?? 0, }, last_updated_at: bucket.lastUpdatedAt?.value_as_string, @@ -243,7 +243,7 @@ export class RuleMigrationsDataClient { private getFilterQuery( migrationId: string, - status?: SiemMigrationsStatus | SiemMigrationsStatus[] + status?: SiemMigrationStatus | SiemMigrationStatus[] ): QueryDslQueryContainer { const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; if (status) { diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts index c959efa3220fd..a65cd45b832e9 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts @@ -29,7 +29,7 @@ export const ruleMigrationsFieldMap: FieldMap> 'elastic_rule.severity': { type: 'keyword', required: false }, 'elastic_rule.prebuilt_rule_id': { type: 'keyword', required: false }, 'elastic_rule.id': { type: 'keyword', required: false }, - translation_state: { type: 'keyword', required: false }, + translation_result: { type: 'keyword', required: false }, comments: { type: 'text', array: true, required: false }, updated_at: { type: 'date', required: false }, updated_by: { type: 'keyword', required: false }, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts index 84091c791c2a7..00e1e60c7b5f3 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts @@ -10,6 +10,7 @@ import type { InferenceClient } from '@kbn/inference-plugin/server'; import type { GraphNode } from '../../types'; import { getEsqlKnowledgeBase } from './esql_knowledge_base_caller'; import { getEsqlTranslationPrompt } from './prompt'; +import { SiemMigrationRuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; interface GetTranslateQueryNodeParams { inferenceClient: InferenceClient; @@ -30,12 +31,12 @@ export const getTranslateQueryNode = ({ const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; const summary = response.match(/## Migration Summary[\s\S]*$/)?.[0] ?? ''; - const translationState = getTranslationState(esqlQuery); + const translationResult = getTranslationResult(esqlQuery); return { response, comments: [summary], - translation_state: translationState, + translation_result: translationResult, elastic_rule: { title: state.original_rule.title, description: state.original_rule.description, @@ -47,9 +48,9 @@ export const getTranslateQueryNode = ({ }; }; -const getTranslationState = (esqlQuery: string) => { +const getTranslationResult = (esqlQuery: string): SiemMigrationRuleTranslationResult => { if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { - return 'partial'; + return SiemMigrationRuleTranslationResult.PARTIAL; } - return 'complete'; + return SiemMigrationRuleTranslationResult.FULL; }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts index b3221c7766707..c1e510bdc052d 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -12,6 +12,7 @@ import type { OriginalRule, RuleMigration, } from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { SiemMigrationRuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; export const migrateRuleState = Annotation.Root({ messages: Annotation({ @@ -22,7 +23,7 @@ export const migrateRuleState = Annotation.Root({ elastic_rule: Annotation({ reducer: (state, action) => ({ ...state, ...action }), }), - translation_state: Annotation(), + translation_result: Annotation(), comments: Annotation({ reducer: (current, value) => (value ? (current ?? []).concat(value) : current), default: () => [], diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts index 0feb286122a95..6ae7294fb5257 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -171,7 +171,7 @@ export class RuleMigrationsTaskRunner { await dataClient.saveFinished({ ...ruleMigration, elastic_rule: ruleMigrationResult.elastic_rule, - translation_state: ruleMigrationResult.translation_state, + translation_result: ruleMigrationResult.translation_result, comments: ruleMigrationResult.comments, }); } catch (error) { @@ -246,8 +246,8 @@ export class RuleMigrationsTaskRunner { if (dataStats.pending === dataStats.total) { return 'ready'; } - if (dataStats.finished + dataStats.failed === dataStats.total) { - return 'done'; + if (dataStats.completed + dataStats.failed === dataStats.total) { + return 'finished'; } return 'stopped'; }