From 2431a08d2ba8af32399905ef0087668f66197d4c Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Mon, 1 Nov 2021 14:40:14 +0100 Subject: [PATCH] [Security Solution][Detections] Reading last 5 failures from Event Log v1 - raw implementation (#115574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Ticket:** https://github.com/elastic/kibana/issues/106469, https://github.com/elastic/kibana/issues/101013 ## Summary TL;DR: New internal endpoint for reading data from Event Log (raw version), legacy status SO under the hood. With this PR we now read the Failure History (last 5 failures) on the Rule Details page from Event Log. We continue getting the Current Status from the legacy `siem-detection-engine-rule-status` saved objects. Rule Management page also gets data from the legacy saved objects. - [x] Deprecate existing methods for reading data in `IRuleExecutionLogClient`: `.find()` and `.findBulk()` - [x] Introduce new methods for reading data in IRuleExecutionLogClient: - for reading last N execution events for 1 rule from event log - for reading current status and metrics for 1 rule from legacy status SOs - for reading current statuses and metrics for N rules from legacy status SOs - [x] New methods should return data in the legacy status SO format. - [x] Update all the existing endpoints that depend on `IRuleExecutionLogClient` to use the new methods. - [x] Implement a new internal endpoint for fetching current status of the rule execution and execution events from Event Log for a given rule. - [x] The API of the new endpoint should be the same as `rules/_find_statuses` to minimise changes in the app. - [x] Use the new endpoint on the Rule Details page. ## Near-term plan for technical implementation of the Rule Execution Log (https://github.com/elastic/kibana/issues/101013) **Stage 1. Reading last 5 failures from Event Log v1 - raw implementation** - :heavy_check_mark: done in this PR TL;DR: New internal endpoint for reading data from Event Log (raw version), legacy status SO under the hood. - Deprecate existing methods for reading data in `IRuleExecutionLogClient`: `.find()` and `.findBulk()` - Introduce new methods for reading data in IRuleExecutionLogClient: - for reading last N execution events for 1 rule from event log - for reading current status and metrics for 1 rule from legacy status SOs - for reading current statuses and metrics for N rules from legacy status SOs - New methods should return data in the legacy status SO format. - Update all the existing endpoints that depend on `IRuleExecutionLogClient` to use the new methods. - Implement a new internal endpoint for fetching current status of the rule execution and execution events from Event Log for a given rule. - The API of the new endpoint should be the same as `rules/_find_statuses` to minimise changes in the app. - Use the new endpoint on the Rule Details page. **Stage 2: Reading last 5 failures from Event Log v2 - clean implementation** TL;DR: Clean HTTP API, legacy Rule Status SO under the hood. 🚨🚨🚨 Possible breaking changes in Detections API 🚨🚨🚨 - Design a new data model for the Current Rule Execution Info (the TO-BE new SO type and later the TO-BE data in the rule object itself). - Design a new data model for the Rule Execution Event (read model to be used on the Rule Details page) - Think over changes in `IRuleExecutionLogClient` to support the new data model. - Think over changes in all the endpoints that return any data related to rule monitoring (statuses, metrics, etc). Make sure to check our docs to identify what's documented there regarding rule monitoring. - Update `IRuleExecutionLogClient` to return data in the new format. - Update all the endpoints (including the raw new one) to return data in the new format. - Update Rule Details page to consume data in the new format. - Update Rule Management page to consume data in the new format. **Stage 3: Reading last 5 failures from Event Log v3 - new SO** TL;DR: Clean HTTP API, new Rule Execution Info SO under the hood. - Implement a new SO type for storing the current rule execution info. Relation type: 1 rule - 1 current execution info. - Swap the legacy SO with the new SO in the implementation of `IRuleExecutionLogClient`. **Stage 4: Cleanup and misc** - Revisit the problem of deterministic ordering ([comment](https://github.com/elastic/kibana/pull/115574#discussion_r735803087)) - Remove rule execution log's glue code: adapters, feature switch. - Remove the legacy rule status SO. - Mark the legacy rule status SO as deleted in Kibana Core. - Encapsulate the current space id in the instance of IRuleExecutionLogClient. Remove it from parameters of its methods. - Introduce a Rule Execution Logger scoped to a rule instance. For use in rule executors. - Add test coverage. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../security_solution/common/constants.ts | 7 + .../request/find_rule_statuses_schema.ts | 10 ++ .../detection_engine/rules/api.test.ts | 4 +- .../containers/detection_engine/rules/api.ts | 5 +- .../routes/__mocks__/request_responses.ts | 154 ++++++++---------- .../routes/rules/create_rules_route.test.ts | 6 +- .../routes/rules/create_rules_route.ts | 5 +- .../routes/rules/delete_rules_bulk_route.ts | 8 +- .../routes/rules/delete_rules_route.test.ts | 6 +- .../routes/rules/delete_rules_route.ts | 8 +- .../find_rule_status_internal_route.test.ts | 115 +++++++++++++ .../rules/find_rule_status_internal_route.ts | 85 ++++++++++ .../routes/rules/find_rules_route.test.ts | 4 +- .../routes/rules/find_rules_route.ts | 7 +- .../rules/find_rules_status_route.test.ts | 6 +- .../routes/rules/find_rules_status_route.ts | 34 ++-- .../routes/rules/patch_rules_bulk_route.ts | 5 +- .../routes/rules/patch_rules_route.test.ts | 13 +- .../routes/rules/patch_rules_route.ts | 9 +- .../routes/rules/perform_bulk_action_route.ts | 8 +- .../routes/rules/read_rules_route.test.ts | 5 +- .../routes/rules/read_rules_route.ts | 4 +- .../routes/rules/update_rules_bulk_route.ts | 5 +- .../routes/rules/update_rules_route.test.ts | 6 +- .../routes/rules/update_rules_route.ts | 9 +- .../detection_engine/routes/rules/utils.ts | 17 +- .../routes/rules/validate.test.ts | 6 +- .../detection_engine/routes/rules/validate.ts | 15 +- .../__mocks__/rule_execution_log_client.ts | 9 +- .../event_log_adapter/event_log_adapter.ts | 53 +++--- .../event_log_adapter/event_log_client.ts | 90 ++++++++-- .../rule_execution_log_client.ts | 50 +++--- .../saved_objects_adapter.ts | 60 +++++-- .../rule_execution_log/types.ts | 88 ++++++---- .../create_security_rule_type_wrapper.ts | 2 +- .../rules/delete_rules.test.ts | 35 +--- .../detection_engine/rules/delete_rules.ts | 14 +- .../lib/detection_engine/rules/enable_rule.ts | 22 +-- .../lib/detection_engine/rules/types.ts | 16 +- .../preview_rule_execution_log_client.ts | 57 +++++-- .../signals/signal_rule_alert_type.ts | 5 +- .../server/request_context_factory.ts | 17 +- .../security_solution/server/routes/index.ts | 2 + 43 files changed, 696 insertions(+), 390 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.ts diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index c746ed1006e56..2772c3de51065 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -250,6 +250,13 @@ export const DETECTION_ENGINE_RULES_PREVIEW = `${DETECTION_ENGINE_RULES_URL}/pre export const DETECTION_ENGINE_RULES_PREVIEW_INDEX_URL = `${DETECTION_ENGINE_RULES_PREVIEW}/index` as const; +/** + * Internal detection engine routes + */ +export const INTERNAL_DETECTION_ENGINE_URL = '/internal/detection_engine' as const; +export const INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL = + `${INTERNAL_DETECTION_ENGINE_URL}/rules/_find_status` as const; + export const TIMELINE_RESOLVE_URL = '/api/timeline/resolve' as const; export const TIMELINE_URL = '/api/timeline' as const; export const TIMELINES_URL = '/api/timelines' as const; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/find_rule_statuses_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/find_rule_statuses_schema.ts index 1437a8230b67a..d489ad562f304 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/find_rule_statuses_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/find_rule_statuses_schema.ts @@ -16,3 +16,13 @@ export const findRulesStatusesSchema = t.exact( export type FindRulesStatusesSchema = t.TypeOf; export type FindRulesStatusesSchemaDecoded = FindRulesStatusesSchema; + +export const findRuleStatusSchema = t.exact( + t.type({ + ruleId: t.string, + }) +); + +export type FindRuleStatusSchema = t.TypeOf; + +export type FindRuleStatusSchemaDecoded = FindRuleStatusSchema; diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts index e5b9808bf1e46..0045f69968b2a 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts @@ -669,8 +669,8 @@ describe('Detections Rules API', () => { test('check parameter url, query', async () => { await getRuleStatusById({ id: 'mySuperRuleId', signal: abortCtrl.signal }); - expect(fetchMock).toHaveBeenCalledWith('/api/detection_engine/rules/_find_statuses', { - body: '{"ids":["mySuperRuleId"]}', + expect(fetchMock).toHaveBeenCalledWith('/internal/detection_engine/rules/_find_status', { + body: '{"ruleId":"mySuperRuleId"}', method: 'POST', signal: abortCtrl.signal, }); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts index f5f7fbd846623..5f9ad7fdd2bf5 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts @@ -19,6 +19,7 @@ import { DETECTION_ENGINE_TAGS_URL, DETECTION_ENGINE_RULES_BULK_ACTION, DETECTION_ENGINE_RULES_PREVIEW, + INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL, } from '../../../../../common/constants'; import { UpdateRulesProps, @@ -372,9 +373,9 @@ export const getRuleStatusById = async ({ id: string; signal: AbortSignal; }): Promise => - KibanaServices.get().http.fetch(DETECTION_ENGINE_RULES_STATUS_URL, { + KibanaServices.get().http.fetch(INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL, { method: 'POST', - body: JSON.stringify({ ids: [id] }), + body: JSON.stringify({ ruleId: id }), signal, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index a890b12d3b7aa..3c1a49c640863 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ALERT_WORKFLOW_STATUS } from '@kbn/rule-data-utils'; import { ruleTypeMappings } from '@kbn/securitysolution-rules'; -import { SavedObjectsFindResponse, SavedObjectsFindResult } from 'kibana/server'; +import { SavedObjectsFindResponse } from 'src/core/server'; import { ActionResult } from '../../../../../../actions/server'; import { @@ -23,6 +23,7 @@ import { DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL, DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL, DETECTION_ENGINE_RULES_BULK_ACTION, + INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL, } from '../../../../../common/constants'; import { RuleAlertType, @@ -42,7 +43,7 @@ import { SanitizedAlert, ResolvedSanitizedRule } from '../../../../../../alertin import { getQueryRuleParams } from '../../schemas/rule_schemas.mock'; import { getPerformBulkActionSchemaMock } from '../../../../../common/detection_engine/schemas/request/perform_bulk_action_schema.mock'; import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; -import { FindBulkExecutionLogResponse } from '../../rule_execution_log/types'; +import { GetCurrentStatusBulkResult } from '../../rule_execution_log/types'; // eslint-disable-next-line no-restricted-imports import type { LegacyRuleNotificationAlertType } from '../../notifications/legacy_types'; @@ -232,6 +233,13 @@ export const ruleStatusRequest = () => body: { ids: ['04128c15-0d1b-4716-a4c5-46997ac7f3bd'] }, }); +export const internalRuleStatusRequest = () => + requestMock.create({ + method: 'post', + path: INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL, + body: { ruleId: '04128c15-0d1b-4716-a4c5-46997ac7f3bd' }, + }); + export const getImportRulesRequest = (hapiStream?: HapiReadableStream) => requestMock.create({ method: 'post', @@ -475,94 +483,64 @@ export const getEmptySavedObjectsResponse = saved_objects: [], }); -export const getRuleExecutionStatuses = (): Array< - SavedObjectsFindResult -> => [ - { - type: 'my-type', - id: 'e0b86950-4e9f-11ea-bdbd-07b56aa159b3', - attributes: { - statusDate: '2020-02-18T15:26:49.783Z', - status: RuleExecutionStatus.succeeded, - lastFailureAt: undefined, - lastSuccessAt: '2020-02-18T15:26:49.783Z', - lastFailureMessage: undefined, - lastSuccessMessage: 'succeeded', - lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), - gap: '500.32', - searchAfterTimeDurations: ['200.00'], - bulkCreateTimeDurations: ['800.43'], - }, - score: 1, - references: [ - { - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bc', - type: 'alert', - name: 'alert_0', - }, - ], - updated_at: '2020-02-18T15:26:51.333Z', - version: 'WzQ2LDFd', - }, - { - type: 'my-type', - id: '91246bd0-5261-11ea-9650-33b954270f67', - attributes: { - statusDate: '2020-02-18T15:15:58.806Z', - status: RuleExecutionStatus.failed, - lastFailureAt: '2020-02-18T15:15:58.806Z', - lastSuccessAt: '2020-02-13T20:31:59.855Z', - lastFailureMessage: - 'Signal rule name: "Query with a rule id Number 1", id: "1ea5a820-4da1-4e82-92a1-2b43a7bece08", rule_id: "query-rule-id-1" has a time gap of 5 days (412682928ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.', - lastSuccessMessage: 'succeeded', - lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), - gap: '500.32', - searchAfterTimeDurations: ['200.00'], - bulkCreateTimeDurations: ['800.43'], - }, - score: 1, - references: [ - { - id: '1ea5a820-4da1-4e82-92a1-2b43a7bece08', - type: 'alert', - name: 'alert_0', - }, - ], - updated_at: '2020-02-18T15:15:58.860Z', - version: 'WzMyLDFd', - }, +export const getRuleExecutionStatusSucceeded = (): IRuleStatusSOAttributes => ({ + statusDate: '2020-02-18T15:26:49.783Z', + status: RuleExecutionStatus.succeeded, + lastFailureAt: undefined, + lastSuccessAt: '2020-02-18T15:26:49.783Z', + lastFailureMessage: undefined, + lastSuccessMessage: 'succeeded', + lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), + gap: '500.32', + searchAfterTimeDurations: ['200.00'], + bulkCreateTimeDurations: ['800.43'], +}); + +export const getRuleExecutionStatusFailed = (): IRuleStatusSOAttributes => ({ + statusDate: '2020-02-18T15:15:58.806Z', + status: RuleExecutionStatus.failed, + lastFailureAt: '2020-02-18T15:15:58.806Z', + lastSuccessAt: '2020-02-13T20:31:59.855Z', + lastFailureMessage: + 'Signal rule name: "Query with a rule id Number 1", id: "1ea5a820-4da1-4e82-92a1-2b43a7bece08", rule_id: "query-rule-id-1" has a time gap of 5 days (412682928ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.', + lastSuccessMessage: 'succeeded', + lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), + gap: '500.32', + searchAfterTimeDurations: ['200.00'], + bulkCreateTimeDurations: ['800.43'], +}); + +export const getRuleExecutionStatuses = (): IRuleStatusSOAttributes[] => [ + getRuleExecutionStatusSucceeded(), + getRuleExecutionStatusFailed(), ]; -export const getFindBulkResultStatus = (): FindBulkExecutionLogResponse => ({ - '04128c15-0d1b-4716-a4c5-46997ac7f3bd': [ - { - statusDate: '2020-02-18T15:26:49.783Z', - status: RuleExecutionStatus.succeeded, - lastFailureAt: undefined, - lastSuccessAt: '2020-02-18T15:26:49.783Z', - lastFailureMessage: undefined, - lastSuccessMessage: 'succeeded', - lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), - gap: '500.32', - searchAfterTimeDurations: ['200.00'], - bulkCreateTimeDurations: ['800.43'], - }, - ], - '1ea5a820-4da1-4e82-92a1-2b43a7bece08': [ - { - statusDate: '2020-02-18T15:15:58.806Z', - status: RuleExecutionStatus.failed, - lastFailureAt: '2020-02-18T15:15:58.806Z', - lastSuccessAt: '2020-02-13T20:31:59.855Z', - lastFailureMessage: - 'Signal rule name: "Query with a rule id Number 1", id: "1ea5a820-4da1-4e82-92a1-2b43a7bece08", rule_id: "query-rule-id-1" has a time gap of 5 days (412682928ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.', - lastSuccessMessage: 'succeeded', - lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), - gap: '500.32', - searchAfterTimeDurations: ['200.00'], - bulkCreateTimeDurations: ['800.43'], - }, - ], +export const getFindBulkResultStatus = (): GetCurrentStatusBulkResult => ({ + '04128c15-0d1b-4716-a4c5-46997ac7f3bd': { + statusDate: '2020-02-18T15:26:49.783Z', + status: RuleExecutionStatus.succeeded, + lastFailureAt: undefined, + lastSuccessAt: '2020-02-18T15:26:49.783Z', + lastFailureMessage: undefined, + lastSuccessMessage: 'succeeded', + lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), + gap: '500.32', + searchAfterTimeDurations: ['200.00'], + bulkCreateTimeDurations: ['800.43'], + }, + '1ea5a820-4da1-4e82-92a1-2b43a7bece08': { + statusDate: '2020-02-18T15:15:58.806Z', + status: RuleExecutionStatus.failed, + lastFailureAt: '2020-02-18T15:15:58.806Z', + lastSuccessAt: '2020-02-13T20:31:59.855Z', + lastFailureMessage: + 'Signal rule name: "Query with a rule id Number 1", id: "1ea5a820-4da1-4e82-92a1-2b43a7bece08", rule_id: "query-rule-id-1" has a time gap of 5 days (412682928ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.', + lastSuccessMessage: 'succeeded', + lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), + gap: '500.32', + searchAfterTimeDurations: ['200.00'], + bulkCreateTimeDurations: ['800.43'], + }, }); export const getBasicEmptySearchResponse = (): estypes.SearchResponse => ({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.test.ts index 010c4b27507bb..a9f5938abb921 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.test.ts @@ -10,7 +10,7 @@ import { getEmptyFindResult, getAlertMock, getCreateRequest, - getRuleExecutionStatuses, + getRuleExecutionStatusSucceeded, getFindResultWithSingleHit, createMlRuleRequest, getBasicEmptySearchResponse, @@ -43,7 +43,9 @@ describe.each([ clients.rulesClient.create.mockResolvedValue( getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()) ); // creation succeeds - clients.ruleExecutionLogClient.find.mockResolvedValue(getRuleExecutionStatuses()); // needed to transform: ; + clients.ruleExecutionLogClient.getCurrentStatus.mockResolvedValue( + getRuleExecutionStatusSucceeded() + ); context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise(getBasicEmptySearchResponse()) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 9e03e5f8f2143..71d453809d0fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -106,14 +106,13 @@ export const createRulesRoute = ( await rulesClient.muteAll({ id: createdRule.id }); } - const ruleStatuses = await context.securitySolution.getExecutionLogClient().find({ - logsCount: 1, + const ruleStatus = await context.securitySolution.getExecutionLogClient().getCurrentStatus({ ruleId: createdRule.id, spaceId: context.securitySolution.getSpaceId(), }); const [validated, errors] = newTransformValidate( createdRule, - ruleStatuses[0], + ruleStatus, isRuleRegistryEnabled ); if (errors != null) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_bulk_route.ts index 6aecfff1178bc..054238cf6fa45 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_bulk_route.ts @@ -80,21 +80,19 @@ export const deleteRulesBulkRoute = ( return getIdBulkError({ id, ruleId }); } - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 6, + const ruleStatus = await ruleStatusClient.getCurrentStatus({ ruleId: rule.id, spaceId: context.securitySolution.getSpaceId(), }); await deleteRules({ + ruleId: rule.id, rulesClient, ruleStatusClient, - ruleStatuses, - id: rule.id, }); return transformValidateBulkError( idOrRuleIdOrUnknown, rule, - ruleStatuses, + ruleStatus, isRuleRegistryEnabled ); } catch (err) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.test.ts index 466012a045eb3..9c126a177eeb5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.test.ts @@ -12,7 +12,7 @@ import { getDeleteRequest, getFindResultWithSingleHit, getDeleteRequestById, - getRuleExecutionStatuses, + getRuleExecutionStatusSucceeded, getEmptySavedObjectsResponse, } from '../__mocks__/request_responses'; import { requestContextMock, serverMock, requestMock } from '../__mocks__'; @@ -32,7 +32,9 @@ describe.each([ clients.rulesClient.find.mockResolvedValue(getFindResultWithSingleHit(isRuleRegistryEnabled)); clients.savedObjectsClient.find.mockResolvedValue(getEmptySavedObjectsResponse()); - clients.ruleExecutionLogClient.find.mockResolvedValue(getRuleExecutionStatuses()); + clients.ruleExecutionLogClient.getCurrentStatus.mockResolvedValue( + getRuleExecutionStatusSucceeded() + ); deleteRulesRoute(server.router, isRuleRegistryEnabled); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts index 77b8dd6fc5b54..abcf0d07a33b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts @@ -62,18 +62,16 @@ export const deleteRulesRoute = ( }); } - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 6, + const currentStatus = await ruleStatusClient.getCurrentStatus({ ruleId: rule.id, spaceId: context.securitySolution.getSpaceId(), }); await deleteRules({ + ruleId: rule.id, rulesClient, ruleStatusClient, - ruleStatuses, - id: rule.id, }); - const transformed = transform(rule, ruleStatuses[0], isRuleRegistryEnabled); + const transformed = transform(rule, currentStatus, isRuleRegistryEnabled); if (transformed == null) { return siemResponse.error({ statusCode: 500, body: 'failed to transform alert' }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.test.ts new file mode 100644 index 0000000000000..285b839cacb9f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL } from '../../../../../common/constants'; +import { + internalRuleStatusRequest, + getAlertMock, + getRuleExecutionStatusSucceeded, + getRuleExecutionStatusFailed, +} from '../__mocks__/request_responses'; +import { serverMock, requestContextMock, requestMock } from '../__mocks__'; +import { findRuleStatusInternalRoute } from './find_rule_status_internal_route'; +import { RuleStatusResponse } from '../../rules/types'; +import { AlertExecutionStatusErrorReasons } from '../../../../../../alerting/common'; +import { getQueryRuleParams } from '../../schemas/rule_schemas.mock'; + +describe.each([ + ['Legacy', false], + ['RAC', true], +])(`${INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL} - %s`, (_, isRuleRegistryEnabled) => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + + beforeEach(async () => { + server = serverMock.create(); + ({ clients, context } = requestContextMock.createTools()); + + clients.ruleExecutionLogClient.getCurrentStatus.mockResolvedValue( + getRuleExecutionStatusSucceeded() + ); + clients.ruleExecutionLogClient.getLastFailures.mockResolvedValue([ + getRuleExecutionStatusFailed(), + ]); + clients.rulesClient.get.mockResolvedValue( + getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()) + ); + + findRuleStatusInternalRoute(server.router); + }); + + describe('status codes with actionClient and alertClient', () => { + test('returns 200 when finding a single rule status with a valid rulesClient', async () => { + const response = await server.inject(internalRuleStatusRequest(), context); + expect(response.status).toEqual(200); + }); + + test('returns 404 if alertClient is not available on the route', async () => { + context.alerting.getRulesClient = jest.fn(); + const response = await server.inject(internalRuleStatusRequest(), context); + expect(response.status).toEqual(404); + expect(response.body).toEqual({ message: 'Not Found', status_code: 404 }); + }); + + test('catch error when status search throws error', async () => { + clients.ruleExecutionLogClient.getCurrentStatus.mockImplementation(async () => { + throw new Error('Test error'); + }); + const response = await server.inject(internalRuleStatusRequest(), context); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: 'Test error', + status_code: 500, + }); + }); + + test('returns success if rule status client writes an error status', async () => { + // 0. task manager tried to run the rule but couldn't, so the alerting framework + // wrote an error to the executionStatus. + const failingExecutionRule = getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()); + failingExecutionRule.executionStatus = { + status: 'error', + lastExecutionDate: failingExecutionRule.executionStatus.lastExecutionDate, + error: { + reason: AlertExecutionStatusErrorReasons.Read, + message: 'oops', + }, + }; + + // 1. getFailingRules api found a rule where the executionStatus was 'error' + clients.rulesClient.get.mockResolvedValue({ + ...failingExecutionRule, + }); + + const request = internalRuleStatusRequest(); + const { ruleId } = request.body; + + const response = await server.inject(request, context); + const responseBody: RuleStatusResponse = response.body; + const ruleStatus = responseBody[ruleId].current_status; + + expect(response.status).toEqual(200); + expect(ruleStatus?.status).toEqual('failed'); + expect(ruleStatus?.last_failure_message).toEqual('Reason: read Message: oops'); + }); + }); + + describe('request validation', () => { + test('disallows singular id query param', async () => { + const request = requestMock.create({ + method: 'post', + path: INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL, + body: { id: ['someId'] }, + }); + const result = server.validate(request); + + expect(result.badRequest).toHaveBeenCalledWith( + 'Invalid value "undefined" supplied to "ruleId"' + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.ts new file mode 100644 index 0000000000000..6d9b371a9370c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rule_status_internal_route.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; +import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL } from '../../../../../common/constants'; +import { buildSiemResponse, mergeStatuses, getFailingRules } from '../utils'; +import { + findRuleStatusSchema, + FindRuleStatusSchemaDecoded, +} from '../../../../../common/detection_engine/schemas/request/find_rule_statuses_schema'; +import { mergeAlertWithSidecarStatus } from '../../schemas/rule_converters'; + +/** + * Returns the current execution status and metrics + last five failed statuses of a given rule. + * Accepts a rule id. + * + * NOTE: This endpoint is a raw implementation of an endpoint for reading rule execution + * status and logs for a given rule (e.g. for use on the Rule Details page). It will be reworked. + * See the plan in https://github.com/elastic/kibana/pull/115574 + * + * @param router + * @returns RuleStatusResponse containing data only for the given rule (normally it contains data for N rules). + */ +export const findRuleStatusInternalRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: INTERNAL_DETECTION_ENGINE_RULE_STATUS_URL, + validate: { + body: buildRouteValidation( + findRuleStatusSchema + ), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const { ruleId } = request.body; + + const siemResponse = buildSiemResponse(response); + const rulesClient = context.alerting?.getRulesClient(); + + if (!rulesClient) { + return siemResponse.error({ statusCode: 404 }); + } + + try { + const ruleStatusClient = context.securitySolution.getExecutionLogClient(); + const spaceId = context.securitySolution.getSpaceId(); + + const [currentStatus, lastFailures, failingRules] = await Promise.all([ + ruleStatusClient.getCurrentStatus({ ruleId, spaceId }), + ruleStatusClient.getLastFailures({ ruleId, spaceId }), + getFailingRules([ruleId], rulesClient), + ]); + + const failingRule = failingRules[ruleId]; + let statuses = {}; + + if (currentStatus != null) { + const finalCurrentStatus = + failingRule != null + ? mergeAlertWithSidecarStatus(failingRule, currentStatus) + : currentStatus; + + statuses = mergeStatuses(ruleId, [finalCurrentStatus, ...lastFailures], statuses); + } + + return response.ok({ body: statuses }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts index 0b0650d48872f..9f151d1db9292 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts @@ -36,7 +36,9 @@ describe.each([ getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()) ); clients.savedObjectsClient.find.mockResolvedValue(getEmptySavedObjectsResponse()); - clients.ruleExecutionLogClient.findBulk.mockResolvedValue(getFindBulkResultStatus()); + clients.ruleExecutionLogClient.getCurrentStatusBulk.mockResolvedValue( + getFindBulkResultStatus() + ); findRulesRoute(server.router, logger, isRuleRegistryEnabled); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts index a55a525806b17..199ef75e22f25 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts @@ -68,15 +68,14 @@ export const findRulesRoute = ( }); const alertIds = rules.data.map((rule) => rule.id); - const [ruleStatuses, ruleActions] = await Promise.all([ - execLogClient.findBulk({ + const [currentStatusesByRuleId, ruleActions] = await Promise.all([ + execLogClient.getCurrentStatusBulk({ ruleIds: alertIds, - logsCount: 1, spaceId: context.securitySolution.getSpaceId(), }), legacyGetBulkRuleActionsSavedObject({ alertIds, savedObjectsClient, logger }), ]); - const transformed = transformFindAlerts(rules, ruleStatuses, ruleActions); + const transformed = transformFindAlerts(rules, currentStatusesByRuleId, ruleActions); if (transformed == null) { return siemResponse.error({ statusCode: 500, body: 'Internal error transforming' }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts index 5d6b9810a2cda..2286c010a0a5a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts @@ -27,7 +27,9 @@ describe.each([ beforeEach(async () => { server = serverMock.create(); ({ clients, context } = requestContextMock.createTools()); - clients.ruleExecutionLogClient.findBulk.mockResolvedValue(getFindBulkResultStatus()); // successful status search + clients.ruleExecutionLogClient.getCurrentStatusBulk.mockResolvedValue( + getFindBulkResultStatus() + ); // successful status search clients.rulesClient.get.mockResolvedValue( getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()) ); @@ -48,7 +50,7 @@ describe.each([ }); test('catch error when status search throws error', async () => { - clients.ruleExecutionLogClient.findBulk.mockImplementation(async () => { + clients.ruleExecutionLogClient.getCurrentStatusBulk.mockImplementation(async () => { throw new Error('Test error'); }); const response = await server.inject(ruleStatusRequest(), context); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts index 71ebe23f124d2..af4f8ddbb9ec8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts @@ -17,11 +17,16 @@ import { import { mergeAlertWithSidecarStatus } from '../../schemas/rule_converters'; /** - * Given a list of rule ids, return the current status and - * last five errors for each associated rule. + * Returns the current execution status and metrics for N rules. + * Accepts an array of rule ids. + * + * NOTE: This endpoint is used on the Rule Management page and will be reworked. + * See the plan in https://github.com/elastic/kibana/pull/115574 * * @param router - * @returns RuleStatusResponse + * @returns RuleStatusResponse containing data for N requested rules. + * RuleStatusResponse[ruleId].failures is always an empty array, because + * we don't need failure history of every rule when we render tables with rules. */ export const findRulesStatusesRoute = (router: SecuritySolutionPluginRouter) => { router.post( @@ -48,31 +53,30 @@ export const findRulesStatusesRoute = (router: SecuritySolutionPluginRouter) => const ids = body.ids; try { const ruleStatusClient = context.securitySolution.getExecutionLogClient(); - const [statusesById, failingRules] = await Promise.all([ - ruleStatusClient.findBulk({ + const [currentStatusesByRuleId, failingRules] = await Promise.all([ + ruleStatusClient.getCurrentStatusBulk({ ruleIds: ids, - logsCount: 6, spaceId: context.securitySolution.getSpaceId(), }), getFailingRules(ids, rulesClient), ]); const statuses = ids.reduce((acc, id) => { - const lastFiveErrorsForId = statusesById[id]; + const currentStatus = currentStatusesByRuleId[id]; + const failingRule = failingRules[id]; - if (lastFiveErrorsForId == null || lastFiveErrorsForId.length === 0) { + if (currentStatus == null) { return acc; } - const failingRule = failingRules[id]; + const finalCurrentStatus = + failingRule != null + ? mergeAlertWithSidecarStatus(failingRule, currentStatus) + : currentStatus; - if (failingRule != null) { - const currentStatus = mergeAlertWithSidecarStatus(failingRule, lastFiveErrorsForId[0]); - const updatedLastFiveErrorsSO = [currentStatus, ...lastFiveErrorsForId.slice(1)]; - return mergeStatuses(id, updatedLastFiveErrorsSO, acc); - } - return mergeStatuses(id, [...lastFiveErrorsForId], acc); + return mergeStatuses(id, [finalCurrentStatus], acc); }, {}); + return response.ok({ body: statuses }); } catch (err) { const error = transformError(err); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index 2b514ba911091..838bfe63782c8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -194,12 +194,11 @@ export const patchRulesBulkRoute = ( exceptionsList, }); if (rule != null && rule.enabled != null && rule.name != null) { - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 1, + const ruleStatus = await ruleStatusClient.getCurrentStatus({ ruleId: rule.id, spaceId: context.securitySolution.getSpaceId(), }); - return transformValidateBulkError(rule.id, rule, ruleStatuses, isRuleRegistryEnabled); + return transformValidateBulkError(rule.id, rule, ruleStatus, isRuleRegistryEnabled); } else { return getIdBulkError({ id, ruleId }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts index 00d7180dfc9be..fe8e4470a61cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts @@ -10,7 +10,7 @@ import { mlServicesMock, mlAuthzMock as mockMlAuthzFactory } from '../../../mach import { buildMlAuthz } from '../../../machine_learning/authz'; import { getEmptyFindResult, - getRuleExecutionStatuses, + getRuleExecutionStatusSucceeded, getAlertMock, getPatchRequest, getFindResultWithSingleHit, @@ -46,8 +46,15 @@ describe.each([ getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()) ); // successful update clients.savedObjectsClient.find.mockResolvedValue(getEmptySavedObjectsResponse()); // successful transform - clients.savedObjectsClient.create.mockResolvedValue(getRuleExecutionStatuses()[0]); // successful transform - clients.ruleExecutionLogClient.find.mockResolvedValue(getRuleExecutionStatuses()); + clients.savedObjectsClient.create.mockResolvedValue({ + type: 'my-type', + id: 'e0b86950-4e9f-11ea-bdbd-07b56aa159b3', + attributes: getRuleExecutionStatusSucceeded(), + references: [], + }); // successful transform + clients.ruleExecutionLogClient.getCurrentStatus.mockResolvedValue( + getRuleExecutionStatusSucceeded() + ); patchRulesRoute(server.router, ml, isRuleRegistryEnabled); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index 0096cd2e38180..bb9f7e1475247 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -195,17 +195,12 @@ export const patchRulesRoute = ( exceptionsList, }); if (rule != null && rule.enabled != null && rule.name != null) { - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 1, + const ruleStatus = await ruleStatusClient.getCurrentStatus({ ruleId: rule.id, spaceId: context.securitySolution.getSpaceId(), }); - const [validated, errors] = transformValidate( - rule, - ruleStatuses[0], - isRuleRegistryEnabled - ); + const [validated, errors] = transformValidate(rule, ruleStatus, isRuleRegistryEnabled); if (errors != null) { return siemResponse.error({ statusCode: 500, body: errors }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts index 44f2577e032b5..251ff1e6e5f38 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts @@ -109,16 +109,10 @@ export const performBulkActionRoute = ( case BulkAction.delete: await Promise.all( rules.data.map(async (rule) => { - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 6, - ruleId: rule.id, - spaceId: context.securitySolution.getSpaceId(), - }); await deleteRules({ + ruleId: rule.id, rulesClient, ruleStatusClient, - ruleStatuses, - id: rule.id, }); }) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.test.ts index bc9fa43b56ae7..4264ca9961bd4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.test.ts @@ -16,6 +16,7 @@ import { getFindResultWithSingleHit, nonRuleFindResult, getEmptySavedObjectsResponse, + getRuleExecutionStatusSucceeded, resolveAlertMock, } from '../__mocks__/request_responses'; import { requestMock, requestContextMock, serverMock } from '../__mocks__'; @@ -37,7 +38,9 @@ describe.each([ clients.rulesClient.find.mockResolvedValue(getFindResultWithSingleHit(isRuleRegistryEnabled)); // rule exists clients.savedObjectsClient.find.mockResolvedValue(getEmptySavedObjectsResponse()); // successful transform - clients.ruleExecutionLogClient.find.mockResolvedValue([]); + clients.ruleExecutionLogClient.getCurrentStatus.mockResolvedValue( + getRuleExecutionStatusSucceeded() + ); clients.rulesClient.resolve.mockResolvedValue({ ...resolveAlertMock(isRuleRegistryEnabled, { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts index c3d6f09c306f0..06d0b9d8c327a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts @@ -70,12 +70,10 @@ export const readRulesRoute = ( ruleAlertId: rule.id, logger, }); - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 1, + const currentStatus = await ruleStatusClient.getCurrentStatus({ ruleId: rule.id, spaceId: context.securitySolution.getSpaceId(), }); - const [currentStatus] = ruleStatuses; if (currentStatus != null && rule.executionStatus.status === 'error') { currentStatus.attributes.lastFailureMessage = `Reason: ${rule.executionStatus.error?.reason} Message: ${rule.executionStatus.error?.message}`; currentStatus.attributes.lastFailureAt = diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index 067f7b80dfca1..80b77722e79b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -95,12 +95,11 @@ export const updateRulesBulkRoute = ( isRuleRegistryEnabled, }); if (rule != null) { - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 1, + const ruleStatus = await ruleStatusClient.getCurrentStatus({ ruleId: rule.id, spaceId: context.securitySolution.getSpaceId(), }); - return transformValidateBulkError(rule.id, rule, ruleStatuses, isRuleRegistryEnabled); + return transformValidateBulkError(rule.id, rule, ruleStatus, isRuleRegistryEnabled); } else { return getIdBulkError({ id: payloadRule.id, ruleId: payloadRule.rule_id }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.test.ts index 37df792b421b0..131015880053c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.test.ts @@ -12,6 +12,7 @@ import { getAlertMock, getUpdateRequest, getFindResultWithSingleHit, + getRuleExecutionStatusSucceeded, nonRuleFindResult, typicalMlRulePayload, } from '../__mocks__/request_responses'; @@ -43,8 +44,11 @@ describe.each([ clients.rulesClient.update.mockResolvedValue( getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()) ); // successful update - clients.ruleExecutionLogClient.find.mockResolvedValue([]); // successful transform: ; + clients.ruleExecutionLogClient.getCurrentStatus.mockResolvedValue( + getRuleExecutionStatusSucceeded() + ); clients.appClient.getSignalsIndex.mockReturnValue('.siem-signals-test-index'); + updateRulesRoute(server.router, ml, isRuleRegistryEnabled); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index 543591c415a6b..1aad28d110bd9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -86,16 +86,11 @@ export const updateRulesRoute = ( }); if (rule != null) { - const ruleStatuses = await ruleStatusClient.find({ - logsCount: 1, + const ruleStatus = await ruleStatusClient.getCurrentStatus({ ruleId: rule.id, spaceId: context.securitySolution.getSpaceId(), }); - const [validated, errors] = transformValidate( - rule, - ruleStatuses[0], - isRuleRegistryEnabled - ); + const [validated, errors] = transformValidate(rule, ruleStatus, isRuleRegistryEnabled); if (errors != null) { return siemResponse.error({ statusCode: 500, body: errors }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index 7472d41b9ab77..e706a3c914974 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -6,7 +6,6 @@ */ import { countBy } from 'lodash/fp'; -import { SavedObject } from 'kibana/server'; import uuid from 'uuid'; import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema'; @@ -18,8 +17,7 @@ import { INTERNAL_IDENTIFIER } from '../../../../../common/constants'; import { RuleAlertType, isAlertType, - IRuleSavedAttributesSavedObjectAttributes, - isRuleStatusSavedObjectType, + isRuleStatusSavedObjectAttributes, IRuleStatusSOAttributes, } from '../../rules/types'; import { createBulkErrorObject, BulkError, OutputError } from '../utils'; @@ -98,10 +96,10 @@ export const transformTags = (tags: string[]): string[] => { // those on the export export const transformAlertToRule = ( alert: SanitizedAlert, - ruleStatus?: SavedObject, + ruleStatus?: IRuleStatusSOAttributes, legacyRuleActions?: LegacyRulesActionsSavedObject | null ): Partial => { - return internalRuleToAPIResponse(alert, ruleStatus?.attributes, legacyRuleActions); + return internalRuleToAPIResponse(alert, ruleStatus, legacyRuleActions); }; export const transformAlertsToRules = ( @@ -113,7 +111,7 @@ export const transformAlertsToRules = ( export const transformFindAlerts = ( findResults: FindResult, - ruleStatuses: { [key: string]: IRuleStatusSOAttributes[] | undefined }, + currentStatusesByRuleId: { [key: string]: IRuleStatusSOAttributes | undefined }, legacyRuleActions: Record ): { page: number; @@ -126,8 +124,7 @@ export const transformFindAlerts = ( perPage: findResults.perPage, total: findResults.total, data: findResults.data.map((alert) => { - const statuses = ruleStatuses[alert.id]; - const status = statuses ? statuses[0] : undefined; + const status = currentStatusesByRuleId[alert.id]; return internalRuleToAPIResponse(alert, status, legacyRuleActions[alert.id]); }), }; @@ -135,14 +132,14 @@ export const transformFindAlerts = ( export const transform = ( alert: PartialAlert, - ruleStatus?: SavedObject, + ruleStatus?: IRuleStatusSOAttributes, isRuleRegistryEnabled?: boolean, legacyRuleActions?: LegacyRulesActionsSavedObject | null ): Partial | null => { if (isAlertType(isRuleRegistryEnabled ?? false, alert)) { return transformAlertToRule( alert, - isRuleStatusSavedObjectType(ruleStatus) ? ruleStatus : undefined, + isRuleStatusSavedObjectAttributes(ruleStatus) ? ruleStatus : undefined, legacyRuleActions ); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts index a7ba1ac77b7bf..032988bcca8be 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts @@ -8,7 +8,7 @@ import { transformValidate, transformValidateBulkError } from './validate'; import { BulkError } from '../utils'; import { RulesSchema } from '../../../../../common/detection_engine/schemas/response'; -import { getAlertMock, getRuleExecutionStatuses } from '../__mocks__/request_responses'; +import { getAlertMock, getRuleExecutionStatusSucceeded } from '../__mocks__/request_responses'; import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { getThreatMock } from '../../../../../common/detection_engine/schemas/types/threat.mock'; import { getQueryRuleParams } from '../../schemas/rule_schemas.mock'; @@ -121,12 +121,12 @@ describe.each([ }); test('it should do a validation correctly of a rule id with ruleStatus passed in', () => { - const ruleStatuses = getRuleExecutionStatuses(); + const ruleStatus = getRuleExecutionStatusSucceeded(); const ruleAlert = getAlertMock(isRuleRegistryEnabled, getQueryRuleParams()); const validatedOrError = transformValidateBulkError( 'rule-1', ruleAlert, - ruleStatuses, + ruleStatus, isRuleRegistryEnabled ); const expected: RulesSchema = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.ts index 307b6c96da3e5..d4bb020cfb672 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { SavedObject, SavedObjectsFindResult } from 'kibana/server'; - import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; import { FullResponseSchema, @@ -19,9 +17,8 @@ import { import { PartialAlert } from '../../../../../../alerting/server'; import { isAlertType, - IRuleSavedAttributesSavedObjectAttributes, IRuleStatusSOAttributes, - isRuleStatusSavedObjectType, + isRuleStatusSavedObjectAttributes, } from '../../rules/types'; import { createBulkErrorObject, BulkError } from '../utils'; import { transform, transformAlertToRule } from './utils'; @@ -31,7 +28,7 @@ import { LegacyRulesActionsSavedObject } from '../../rule_actions/legacy_get_rul export const transformValidate = ( alert: PartialAlert, - ruleStatus?: SavedObject, + ruleStatus?: IRuleStatusSOAttributes, isRuleRegistryEnabled?: boolean, legacyRuleActions?: LegacyRulesActionsSavedObject | null ): [RulesSchema | null, string | null] => { @@ -45,7 +42,7 @@ export const transformValidate = ( export const newTransformValidate = ( alert: PartialAlert, - ruleStatus?: SavedObject, + ruleStatus?: IRuleStatusSOAttributes, isRuleRegistryEnabled?: boolean, legacyRuleActions?: LegacyRulesActionsSavedObject | null ): [FullResponseSchema | null, string | null] => { @@ -60,12 +57,12 @@ export const newTransformValidate = ( export const transformValidateBulkError = ( ruleId: string, alert: PartialAlert, - ruleStatus?: Array>, + ruleStatus?: IRuleStatusSOAttributes, isRuleRegistryEnabled?: boolean ): RulesSchema | BulkError => { if (isAlertType(isRuleRegistryEnabled ?? false, alert)) { - if (ruleStatus && ruleStatus?.length > 0 && isRuleStatusSavedObjectType(ruleStatus[0])) { - const transformed = transformAlertToRule(alert, ruleStatus[0]); + if (ruleStatus && isRuleStatusSavedObjectAttributes(ruleStatus)) { + const transformed = transformAlertToRule(alert, ruleStatus); const [validated, errors] = validateNonExact(transformed, rulesSchema); if (errors != null || validated == null) { return createBulkErrorObject({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts index 910e1ecaa508f..518e4aa903d95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts @@ -11,8 +11,13 @@ export const ruleExecutionLogClientMock = { create: (): jest.Mocked => ({ find: jest.fn(), findBulk: jest.fn(), - update: jest.fn(), - delete: jest.fn(), + + getLastFailures: jest.fn(), + getCurrentStatus: jest.fn(), + getCurrentStatusBulk: jest.fn(), + + deleteCurrentStatus: jest.fn(), + logStatusChange: jest.fn(), logExecutionMetrics: jest.fn(), }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts index a3fb50f1f6b0b..e5660da8d4cf4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts @@ -7,18 +7,25 @@ import { sum } from 'lodash'; import { SavedObjectsClientContract } from '../../../../../../../../src/core/server'; -import { IEventLogService } from '../../../../../../event_log/server'; +import { IEventLogClient, IEventLogService } from '../../../../../../event_log/server'; +import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; +import { IRuleStatusSOAttributes } from '../../rules/types'; import { SavedObjectsAdapter } from '../saved_objects_adapter/saved_objects_adapter'; import { FindBulkExecutionLogArgs, FindExecutionLogArgs, + GetCurrentStatusArgs, + GetCurrentStatusBulkArgs, + GetCurrentStatusBulkResult, + GetLastFailuresArgs, IRuleExecutionLogClient, LogExecutionMetricsArgs, LogStatusChangeArgs, - UpdateExecutionLogArgs, } from '../types'; import { EventLogClient } from './event_log_client'; +const MAX_LAST_FAILURES = 5; + export class EventLogAdapter implements IRuleExecutionLogClient { private eventLogClient: EventLogClient; /** @@ -28,38 +35,46 @@ export class EventLogAdapter implements IRuleExecutionLogClient { */ private savedObjectsAdapter: IRuleExecutionLogClient; - constructor(eventLogService: IEventLogService, savedObjectsClient: SavedObjectsClientContract) { - this.eventLogClient = new EventLogClient(eventLogService); + constructor( + eventLogService: IEventLogService, + eventLogClient: IEventLogClient | undefined, + savedObjectsClient: SavedObjectsClientContract + ) { + this.eventLogClient = new EventLogClient(eventLogService, eventLogClient); this.savedObjectsAdapter = new SavedObjectsAdapter(savedObjectsClient); } + /** @deprecated */ public async find(args: FindExecutionLogArgs) { return this.savedObjectsAdapter.find(args); } + /** @deprecated */ public async findBulk(args: FindBulkExecutionLogArgs) { return this.savedObjectsAdapter.findBulk(args); } - public async update(args: UpdateExecutionLogArgs) { - const { attributes, spaceId, ruleId, ruleName, ruleType } = args; + public getLastFailures(args: GetLastFailuresArgs): Promise { + const { ruleId } = args; + return this.eventLogClient.getLastStatusChanges({ + ruleId, + count: MAX_LAST_FAILURES, + includeStatuses: [RuleExecutionStatus.failed], + }); + } - await this.savedObjectsAdapter.update(args); + public getCurrentStatus( + args: GetCurrentStatusArgs + ): Promise { + return this.savedObjectsAdapter.getCurrentStatus(args); + } - // EventLog execution events are immutable, so we just log a status change istead of updating previous - if (attributes.status) { - this.eventLogClient.logStatusChange({ - ruleName, - ruleType, - ruleId, - newStatus: attributes.status, - spaceId, - }); - } + public getCurrentStatusBulk(args: GetCurrentStatusBulkArgs): Promise { + return this.savedObjectsAdapter.getCurrentStatusBulk(args); } - public async delete(id: string) { - await this.savedObjectsAdapter.delete(id); + public async deleteCurrentStatus(ruleId: string): Promise { + await this.savedObjectsAdapter.deleteCurrentStatus(ruleId); // EventLog execution events are immutable, nothing to do here } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts index d85c67e422035..6ce9d3d1c26ee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts @@ -7,11 +7,14 @@ import { SavedObjectsUtils } from '../../../../../../../../src/core/server'; import { + IEventLogClient, IEventLogger, IEventLogService, SAVED_OBJECT_REL_PRIMARY, } from '../../../../../../event_log/server'; import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; +import { invariant } from '../../../../../common/utils/invariant'; +import { IRuleStatusSOAttributes } from '../../rules/types'; import { LogStatusChangeArgs } from '../types'; import { RuleExecutionLogAction, @@ -21,6 +24,8 @@ import { const spaceIdToNamespace = SavedObjectsUtils.namespaceStringToId; +const now = () => new Date().toISOString(); + const statusSeverityDict: Record = { [RuleExecutionStatus.succeeded]: 0, [RuleExecutionStatus['going to run']]: 10, @@ -29,13 +34,6 @@ const statusSeverityDict: Record = { [RuleExecutionStatus.failed]: 30, }; -interface FindExecutionLogArgs { - ruleIds: string[]; - spaceId: string; - logsCount?: number; - statuses?: RuleExecutionStatus[]; -} - interface LogExecutionMetricsArgs { ruleId: string; ruleName: string; @@ -50,24 +48,88 @@ interface EventLogExecutionMetrics { executionGapDuration?: number; } +interface GetLastStatusChangesArgs { + ruleId: string; + count: number; + includeStatuses?: RuleExecutionStatus[]; +} + interface IExecLogEventLogClient { - find: (args: FindExecutionLogArgs) => Promise<{}>; + getLastStatusChanges(args: GetLastStatusChangesArgs): Promise; logStatusChange: (args: LogStatusChangeArgs) => void; logExecutionMetrics: (args: LogExecutionMetricsArgs) => void; } export class EventLogClient implements IExecLogEventLogClient { + private readonly eventLogClient: IEventLogClient | undefined; + private readonly eventLogger: IEventLogger; private sequence = 0; - private eventLogger: IEventLogger; - constructor(eventLogService: IEventLogService) { + constructor(eventLogService: IEventLogService, eventLogClient: IEventLogClient | undefined) { + this.eventLogClient = eventLogClient; this.eventLogger = eventLogService.getLogger({ event: { provider: RULE_EXECUTION_LOG_PROVIDER }, }); } - public async find({ ruleIds, spaceId, statuses, logsCount = 1 }: FindExecutionLogArgs) { - return {}; // TODO implement + public async getLastStatusChanges( + args: GetLastStatusChangesArgs + ): Promise { + if (!this.eventLogClient) { + throw new Error('Querying Event Log from a rule executor is not supported at this moment'); + } + + const soType = ALERT_SAVED_OBJECT_TYPE; + const soIds = [args.ruleId]; + const count = args.count; + const includeStatuses = (args.includeStatuses ?? []).map((status) => `"${status}"`); + + const filterBy: string[] = [ + `event.provider: ${RULE_EXECUTION_LOG_PROVIDER}`, + 'event.kind: event', + `event.action: ${RuleExecutionLogAction['status-change']}`, + includeStatuses.length > 0 + ? `kibana.alert.rule.execution.status:${includeStatuses.join(' ')}` + : '', + ]; + + const kqlFilter = filterBy + .filter(Boolean) + .map((item) => `(${item})`) + .join(' and '); + + const findResult = await this.eventLogClient.findEventsBySavedObjectIds(soType, soIds, { + page: 1, + per_page: count, + sort_field: '@timestamp', + sort_order: 'desc', + filter: kqlFilter, + }); + + return findResult.data.map((event) => { + invariant(event, 'Event not found'); + invariant(event['@timestamp'], 'Required "@timestamp" field is not found'); + + const statusDate = event['@timestamp']; + const status = event.kibana?.alert?.rule?.execution?.status as + | RuleExecutionStatus + | undefined; + const isStatusFailed = status === RuleExecutionStatus.failed; + const message = event.message ?? ''; + + return { + statusDate, + status, + lastFailureAt: isStatusFailed ? statusDate : undefined, + lastFailureMessage: isStatusFailed ? message : undefined, + lastSuccessAt: !isStatusFailed ? statusDate : undefined, + lastSuccessMessage: !isStatusFailed ? message : undefined, + lastLookBackDate: undefined, + gap: undefined, + bulkCreateTimeDurations: undefined, + searchAfterTimeDurations: undefined, + }; + }); } public logExecutionMetrics({ @@ -78,6 +140,7 @@ export class EventLogClient implements IExecLogEventLogClient { spaceId, }: LogExecutionMetricsArgs) { this.eventLogger.logEvent({ + '@timestamp': now(), rule: { id: ruleId, name: ruleName, @@ -122,6 +185,8 @@ export class EventLogClient implements IExecLogEventLogClient { spaceId, }: LogStatusChangeArgs) { this.eventLogger.logEvent({ + '@timestamp': now(), + message, rule: { id: ruleId, name: ruleName, @@ -132,7 +197,6 @@ export class EventLogClient implements IExecLogEventLogClient { action: RuleExecutionLogAction['status-change'], sequence: this.sequence++, }, - message, kibana: { alert: { rule: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts index 7ae2f179f9692..005097ac3fd82 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts @@ -6,7 +6,8 @@ */ import { SavedObjectsClientContract } from '../../../../../../../src/core/server'; -import { IEventLogService } from '../../../../../event_log/server'; +import { IEventLogClient, IEventLogService } from '../../../../../event_log/server'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { EventLogAdapter } from './event_log_adapter/event_log_adapter'; import { SavedObjectsAdapter } from './saved_objects_adapter/saved_objects_adapter'; import { @@ -15,58 +16,63 @@ import { FindExecutionLogArgs, IRuleExecutionLogClient, LogStatusChangeArgs, - UpdateExecutionLogArgs, UnderlyingLogClient, + GetLastFailuresArgs, + GetCurrentStatusArgs, + GetCurrentStatusBulkArgs, + GetCurrentStatusBulkResult, } from './types'; import { truncateMessage } from './utils/normalization'; -export interface RuleExecutionLogClientArgs { +interface ConstructorParams { + underlyingClient: UnderlyingLogClient; savedObjectsClient: SavedObjectsClientContract; eventLogService: IEventLogService; - underlyingClient: UnderlyingLogClient; + eventLogClient?: IEventLogClient; } export class RuleExecutionLogClient implements IRuleExecutionLogClient { private client: IRuleExecutionLogClient; - constructor({ - savedObjectsClient, - eventLogService, - underlyingClient, - }: RuleExecutionLogClientArgs) { + constructor(params: ConstructorParams) { + const { underlyingClient, eventLogService, eventLogClient, savedObjectsClient } = params; + switch (underlyingClient) { case UnderlyingLogClient.savedObjects: this.client = new SavedObjectsAdapter(savedObjectsClient); break; case UnderlyingLogClient.eventLog: - this.client = new EventLogAdapter(eventLogService, savedObjectsClient); + this.client = new EventLogAdapter(eventLogService, eventLogClient, savedObjectsClient); break; } } + /** @deprecated */ public find(args: FindExecutionLogArgs) { return this.client.find(args); } + /** @deprecated */ public findBulk(args: FindBulkExecutionLogArgs) { return this.client.findBulk(args); } - public async update(args: UpdateExecutionLogArgs) { - const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = args.attributes; + public getLastFailures(args: GetLastFailuresArgs): Promise { + return this.client.getLastFailures(args); + } + + public getCurrentStatus( + args: GetCurrentStatusArgs + ): Promise { + return this.client.getCurrentStatus(args); + } - return this.client.update({ - ...args, - attributes: { - lastFailureMessage: truncateMessage(lastFailureMessage), - lastSuccessMessage: truncateMessage(lastSuccessMessage), - ...restAttributes, - }, - }); + public getCurrentStatusBulk(args: GetCurrentStatusBulkArgs): Promise { + return this.client.getCurrentStatusBulk(args); } - public async delete(id: string) { - return this.client.delete(id); + public deleteCurrentStatus(ruleId: string): Promise { + return this.client.deleteCurrentStatus(ruleId); } public async logExecutionMetrics(args: LogExecutionMetricsArgs) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts index 70db3a768fdb1..53b50bb8fe638 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { mapValues } from 'lodash'; import { SavedObject, SavedObjectReference } from 'src/core/server'; import { SavedObjectsClientContract } from '../../../../../../../../src/core/server'; import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; @@ -23,7 +24,10 @@ import { IRuleExecutionLogClient, ExecutionMetrics, LogStatusChangeArgs, - UpdateExecutionLogArgs, + GetLastFailuresArgs, + GetCurrentStatusArgs, + GetCurrentStatusBulkArgs, + GetCurrentStatusBulkResult, } from '../types'; import { assertUnreachable } from '../../../../../common'; @@ -48,26 +52,52 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { this.ruleStatusClient = ruleStatusSavedObjectsClientFactory(savedObjectsClient); } - public find({ ruleId, logsCount = 1 }: FindExecutionLogArgs) { + private findRuleStatusSavedObjects(ruleId: string, count: number) { return this.ruleStatusClient.find({ - perPage: logsCount, + perPage: count, sortField: 'statusDate', sortOrder: 'desc', ruleId, }); } + /** @deprecated */ + public find({ ruleId, logsCount = 1 }: FindExecutionLogArgs) { + return this.findRuleStatusSavedObjects(ruleId, logsCount); + } + + /** @deprecated */ public findBulk({ ruleIds, logsCount = 1 }: FindBulkExecutionLogArgs) { return this.ruleStatusClient.findBulk(ruleIds, logsCount); } - public async update({ id, attributes, ruleId }: UpdateExecutionLogArgs) { - const references: SavedObjectReference[] = [legacyGetRuleReference(ruleId)]; - await this.ruleStatusClient.update(id, attributes, { references }); + public async getLastFailures(args: GetLastFailuresArgs): Promise { + const result = await this.findRuleStatusSavedObjects(args.ruleId, MAX_RULE_STATUSES); + + // The first status is always the current one followed by 5 last failures. + // We skip the current status and return only the failures. + return result.map((so) => so.attributes).slice(1); + } + + public async getCurrentStatus( + args: GetCurrentStatusArgs + ): Promise { + const result = await this.findRuleStatusSavedObjects(args.ruleId, 1); + const currentStatusSavedObject = result[0]; + return currentStatusSavedObject?.attributes; + } + + public async getCurrentStatusBulk( + args: GetCurrentStatusBulkArgs + ): Promise { + const { ruleIds } = args; + const result = await this.ruleStatusClient.findBulk(ruleIds, 1); + return mapValues(result, (attributes = []) => attributes[0]); } - public async delete(id: string) { - await this.ruleStatusClient.delete(id); + public async deleteCurrentStatus(ruleId: string): Promise { + const statusSavedObjects = await this.findRuleStatusSavedObjects(ruleId, MAX_RULE_STATUSES); + await Promise.all(statusSavedObjects.map((so) => this.ruleStatusClient.delete(so.id))); } public async logExecutionMetrics({ ruleId, metrics }: LogExecutionMetricsArgs) { @@ -109,16 +139,12 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { private getOrCreateRuleStatuses = async ( ruleId: string ): Promise>> => { - const ruleStatuses = await this.find({ - spaceId: '', // spaceId is a required argument but it's not used by savedObjectsClient, any string would work here - ruleId, - logsCount: MAX_RULE_STATUSES, - }); - if (ruleStatuses.length > 0) { - return ruleStatuses; + const existingStatuses = await this.findRuleStatusSavedObjects(ruleId, MAX_RULE_STATUSES); + if (existingStatuses.length > 0) { + return existingStatuses; } - const newStatus = await this.createNewRuleStatus(ruleId); + const newStatus = await this.createNewRuleStatus(ruleId); return [newStatus]; }; @@ -159,7 +185,7 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { // drop oldest failures const oldStatuses = [lastStatus, ...ruleStatuses].slice(MAX_RULE_STATUSES); - await Promise.all(oldStatuses.map((status) => this.delete(status.id))); + await Promise.all(oldStatuses.map((status) => this.ruleStatusClient.delete(status.id))); return; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts index 564145cfc5d1f..88802f9f28822 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts @@ -15,74 +15,92 @@ export enum UnderlyingLogClient { 'eventLog' = 'eventLog', } +export interface IRuleExecutionLogClient { + /** @deprecated */ + find(args: FindExecutionLogArgs): Promise>>; + /** @deprecated */ + findBulk(args: FindBulkExecutionLogArgs): Promise; + + getLastFailures(args: GetLastFailuresArgs): Promise; + getCurrentStatus(args: GetCurrentStatusArgs): Promise; + getCurrentStatusBulk(args: GetCurrentStatusBulkArgs): Promise; + + deleteCurrentStatus(ruleId: string): Promise; + + logStatusChange(args: LogStatusChangeArgs): Promise; + logExecutionMetrics(args: LogExecutionMetricsArgs): Promise; +} + +/** @deprecated */ export interface FindExecutionLogArgs { ruleId: string; spaceId: string; logsCount?: number; } +/** @deprecated */ export interface FindBulkExecutionLogArgs { ruleIds: string[]; spaceId: string; logsCount?: number; } -export interface ExecutionMetrics { - searchDurations?: string[]; - indexingDurations?: string[]; - /** - * @deprecated lastLookBackDate is logged only by SavedObjectsAdapter and should be removed in the future - */ - lastLookBackDate?: string; - executionGap?: Duration; +/** @deprecated */ +export interface FindBulkExecutionLogResponse { + [ruleId: string]: IRuleStatusSOAttributes[] | undefined; } -export interface LogStatusChangeArgs { +export interface GetLastFailuresArgs { ruleId: string; - ruleName: string; - ruleType: string; spaceId: string; - newStatus: RuleExecutionStatus; - message?: string; - /** - * @deprecated Use RuleExecutionLogClient.logExecutionMetrics to write metrics instead - */ - metrics?: ExecutionMetrics; } -export interface UpdateExecutionLogArgs { - id: string; - attributes: IRuleStatusSOAttributes; +export interface GetCurrentStatusArgs { ruleId: string; - ruleName: string; - ruleType: string; spaceId: string; } +export interface GetCurrentStatusBulkArgs { + ruleIds: string[]; + spaceId: string; +} + +export interface GetCurrentStatusBulkResult { + [ruleId: string]: IRuleStatusSOAttributes; +} + export interface CreateExecutionLogArgs { attributes: IRuleStatusSOAttributes; spaceId: string; } -export interface LogExecutionMetricsArgs { +export interface LogStatusChangeArgs { ruleId: string; ruleName: string; ruleType: string; spaceId: string; - metrics: ExecutionMetrics; + newStatus: RuleExecutionStatus; + message?: string; + /** + * @deprecated Use RuleExecutionLogClient.logExecutionMetrics to write metrics instead + */ + metrics?: ExecutionMetrics; } -export interface FindBulkExecutionLogResponse { - [ruleId: string]: IRuleStatusSOAttributes[] | undefined; +export interface LogExecutionMetricsArgs { + ruleId: string; + ruleName: string; + ruleType: string; + spaceId: string; + metrics: ExecutionMetrics; } -export interface IRuleExecutionLogClient { - find: ( - args: FindExecutionLogArgs - ) => Promise>>; - findBulk: (args: FindBulkExecutionLogArgs) => Promise; - update: (args: UpdateExecutionLogArgs) => Promise; - delete: (id: string) => Promise; - logStatusChange: (args: LogStatusChangeArgs) => Promise; - logExecutionMetrics: (args: LogExecutionMetricsArgs) => Promise; +export interface ExecutionMetrics { + searchDurations?: string[]; + indexingDurations?: string[]; + /** + * @deprecated lastLookBackDate is logged only by SavedObjectsAdapter and should be removed in the future + */ + lastLookBackDate?: string; + executionGap?: Duration; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index c472494138b7f..bc13a12e01ca4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -67,9 +67,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = const esClient = scopedClusterClient.asCurrentUser; const ruleStatusClient = new RuleExecutionLogClient({ + underlyingClient: config.ruleExecutionLog.underlyingClient, savedObjectsClient, eventLogService, - underlyingClient: config.ruleExecutionLog.underlyingClient, }); const completeRule = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts index 2d82cd7f8732a..42d7f960beb22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.test.ts @@ -6,10 +6,9 @@ */ import { rulesClientMock } from '../../../../../alerting/server/mocks'; -import { deleteRules } from './delete_rules'; -import { SavedObjectsFindResult } from '../../../../../../../src/core/server'; -import { DeleteRuleOptions, IRuleStatusSOAttributes } from './types'; import { ruleExecutionLogClientMock } from '../rule_execution_log/__mocks__/rule_execution_log_client'; +import { deleteRules } from './delete_rules'; +import { DeleteRuleOptions } from './types'; describe('deleteRules', () => { let rulesClient: ReturnType; @@ -21,35 +20,15 @@ describe('deleteRules', () => { }); it('should delete the rule along with its actions, and statuses', async () => { - const ruleStatus: SavedObjectsFindResult = { - id: 'statusId', - type: '', - references: [], - attributes: { - statusDate: '', - lastFailureAt: null, - lastFailureMessage: null, - lastSuccessAt: null, - lastSuccessMessage: null, - status: null, - lastLookBackDate: null, - gap: null, - bulkCreateTimeDurations: null, - searchAfterTimeDurations: null, - }, - score: 0, - }; - - const rule: DeleteRuleOptions = { + const options: DeleteRuleOptions = { + ruleId: 'ruleId', rulesClient, ruleStatusClient, - id: 'ruleId', - ruleStatuses: [ruleStatus], }; - await deleteRules(rule); + await deleteRules(options); - expect(rulesClient.delete).toHaveBeenCalledWith({ id: rule.id }); - expect(ruleStatusClient.delete).toHaveBeenCalledWith(ruleStatus.id); + expect(rulesClient.delete).toHaveBeenCalledWith({ id: options.ruleId }); + expect(ruleStatusClient.deleteCurrentStatus).toHaveBeenCalledWith(options.ruleId); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.ts index 5003dbf0279e4..880132434f7cc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/delete_rules.ts @@ -5,17 +5,9 @@ * 2.0. */ -import { asyncForEach } from '@kbn/std'; import { DeleteRuleOptions } from './types'; -export const deleteRules = async ({ - rulesClient, - ruleStatusClient, - ruleStatuses, - id, -}: DeleteRuleOptions) => { - await rulesClient.delete({ id }); - await asyncForEach(ruleStatuses, async (obj) => { - await ruleStatusClient.delete(obj.id); - }); +export const deleteRules = async ({ ruleId, rulesClient, ruleStatusClient }: DeleteRuleOptions) => { + await rulesClient.delete({ id: ruleId }); + await ruleStatusClient.deleteCurrentStatus(ruleId); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts index b75a1b0d80e9a..e24da8a2ba0d4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts @@ -33,25 +33,11 @@ export const enableRule = async ({ }: EnableRuleArgs) => { await rulesClient.enable({ id: rule.id }); - const ruleCurrentStatus = await ruleStatusClient.find({ - logsCount: 1, + await ruleStatusClient.logStatusChange({ ruleId: rule.id, + ruleName: rule.name, + ruleType: rule.alertTypeId, spaceId, + newStatus: RuleExecutionStatus['going to run'], }); - - // set current status for this rule to be 'going to run' - if (ruleCurrentStatus && ruleCurrentStatus.length > 0) { - const currentStatusToDisable = ruleCurrentStatus[0]; - await ruleStatusClient.update({ - id: currentStatusToDisable.id, - ruleId: rule.id, - ruleName: rule.name, - ruleType: rule.alertTypeId, - attributes: { - ...currentStatusToDisable.attributes, - status: RuleExecutionStatus['going to run'], - }, - spaceId, - }); - } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 037f85091bfcc..ed0f0447ad3b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -8,12 +8,7 @@ import { get } from 'lodash/fp'; import { Readable } from 'stream'; -import { - SavedObject, - SavedObjectAttributes, - SavedObjectsClientContract, - SavedObjectsFindResult, -} from 'kibana/server'; +import { SavedObject, SavedObjectAttributes, SavedObjectsClientContract } from 'kibana/server'; import type { MachineLearningJobIdOrUndefined, From, @@ -207,10 +202,8 @@ export const isAlertType = ( : partialAlert.alertTypeId === SIGNALS_ID; }; -export const isRuleStatusSavedObjectType = ( - obj: unknown -): obj is SavedObject => { - return get('attributes', obj) != null; +export const isRuleStatusSavedObjectAttributes = (obj: unknown): obj is IRuleStatusSOAttributes => { + return get('status', obj) != null; }; export interface CreateRulesOptions { @@ -342,10 +335,9 @@ export interface ReadRuleOptions { } export interface DeleteRuleOptions { + ruleId: Id; rulesClient: RulesClient; ruleStatusClient: IRuleExecutionLogClient; - ruleStatuses: Array>; - id: Id; } export interface FindRuleOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/preview/preview_rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/preview/preview_rule_execution_log_client.ts index d3ccafddab6e4..c2c1b5d7615c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/preview/preview_rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/preview/preview_rule_execution_log_client.ts @@ -7,13 +7,16 @@ import { SavedObjectsFindResult } from 'kibana/server'; import { - LogExecutionMetricsArgs, IRuleExecutionLogClient, + LogStatusChangeArgs, + LogExecutionMetricsArgs, FindBulkExecutionLogArgs, FindBulkExecutionLogResponse, FindExecutionLogArgs, - LogStatusChangeArgs, - UpdateExecutionLogArgs, + GetLastFailuresArgs, + GetCurrentStatusArgs, + GetCurrentStatusBulkArgs, + GetCurrentStatusBulkResult, } from '../../rule_execution_log'; import { IRuleStatusSOAttributes } from '../../rules/types'; @@ -21,26 +24,50 @@ export const createWarningsAndErrors = () => { const warningsAndErrorsStore: LogStatusChangeArgs[] = []; const previewRuleExecutionLogClient: IRuleExecutionLogClient = { - async delete(id: string): Promise { - return Promise.resolve(undefined); - }, - async find( + find( args: FindExecutionLogArgs ): Promise>> { return Promise.resolve([]); }, - async findBulk(args: FindBulkExecutionLogArgs): Promise { + + findBulk(args: FindBulkExecutionLogArgs): Promise { return Promise.resolve({}); }, - async logStatusChange(args: LogStatusChangeArgs): Promise { - warningsAndErrorsStore.push(args); - return Promise.resolve(undefined); + + getLastFailures(args: GetLastFailuresArgs): Promise { + return Promise.resolve([]); }, - async update(args: UpdateExecutionLogArgs): Promise { - return Promise.resolve(undefined); + + getCurrentStatus(args: GetCurrentStatusArgs): Promise { + return Promise.resolve({ + statusDate: new Date().toISOString(), + status: null, + lastFailureAt: null, + lastFailureMessage: null, + lastSuccessAt: null, + lastSuccessMessage: null, + lastLookBackDate: null, + gap: null, + bulkCreateTimeDurations: null, + searchAfterTimeDurations: null, + }); }, - async logExecutionMetrics(args: LogExecutionMetricsArgs): Promise { - return Promise.resolve(undefined); + + getCurrentStatusBulk(args: GetCurrentStatusBulkArgs): Promise { + return Promise.resolve({}); + }, + + deleteCurrentStatus(ruleId: string): Promise { + return Promise.resolve(); + }, + + logStatusChange(args: LogStatusChangeArgs): Promise { + warningsAndErrorsStore.push(args); + return Promise.resolve(); + }, + + logExecutionMetrics(args: LogExecutionMetricsArgs): Promise { + return Promise.resolve(); }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 6de039f083ba3..85285eed2817a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -142,12 +142,13 @@ export const signalRulesAlertType = ({ const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); let hasError: boolean = false; let result = createSearchAfterReturnType(); + const ruleStatusClient = ruleExecutionLogClientOverride ? ruleExecutionLogClientOverride : new RuleExecutionLogClient({ - eventLogService, - savedObjectsClient: services.savedObjectsClient, underlyingClient: config.ruleExecutionLog.underlyingClient, + savedObjectsClient: services.savedObjectsClient, + eventLogService, }); const completeRule: CompleteRule = { diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index c2e622bc495c9..0028d624c2955 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -36,7 +36,13 @@ export class RequestContextFactory implements IRequestContextFactory { private readonly appClientFactory: AppClientFactory; constructor(private readonly options: ConstructorOptions) { + const { config, plugins } = options; + this.appClientFactory = new AppClientFactory(); + this.appClientFactory.setup({ + getSpaceId: plugins.spaces?.spacesService?.getSpaceId, + config, + }); } public async create( @@ -44,14 +50,10 @@ export class RequestContextFactory implements IRequestContextFactory { request: KibanaRequest ): Promise { const { options, appClientFactory } = this; - const { config, plugins } = options; + const { config, core, plugins } = options; const { lists, ruleRegistry, security, spaces } = plugins; - appClientFactory.setup({ - getSpaceId: plugins.spaces?.spacesService?.getSpaceId, - config, - }); - + const [, startPlugins] = await core.getStartServices(); const frameworkRequest = await buildFrameworkRequest(context, security, request); return { @@ -69,9 +71,10 @@ export class RequestContextFactory implements IRequestContextFactory { getExecutionLogClient: () => new RuleExecutionLogClient({ + underlyingClient: config.ruleExecutionLog.underlyingClient, savedObjectsClient: context.core.savedObjects.client, eventLogService: plugins.eventLog, - underlyingClient: config.ruleExecutionLog.underlyingClient, + eventLogClient: startPlugins.eventLog.getClient(request), }), getExceptionListClient: () => { diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index f3e8cc1dee4b1..20fbf44e77a48 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -36,6 +36,7 @@ import { performBulkActionRoute } from '../lib/detection_engine/routes/rules/per import { importRulesRoute } from '../lib/detection_engine/routes/rules/import_rules_route'; import { exportRulesRoute } from '../lib/detection_engine/routes/rules/export_rules_route'; import { findRulesStatusesRoute } from '../lib/detection_engine/routes/rules/find_rules_status_route'; +import { findRuleStatusInternalRoute } from '../lib/detection_engine/routes/rules/find_rule_status_internal_route'; import { getPrepackagedRulesStatusRoute } from '../lib/detection_engine/routes/rules/get_prepackaged_rules_status_route'; import { createTimelinesRoute, @@ -122,6 +123,7 @@ export const initRoutes = ( persistPinnedEventRoute(router, config, security); findRulesStatusesRoute(router); + findRuleStatusInternalRoute(router); // Detection Engine Signals routes that have the REST endpoints of /api/detection_engine/signals // POST /api/detection_engine/signals/status