From 3d466a72a8ab181aadf562ab6c27a5affa32dc96 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 14 Oct 2024 16:29:12 +0200 Subject: [PATCH] Execution type field (#195884) ## Added new field - execution type for alerts Added new field only for security type alerts: `kibana.alert.rule.execution.type` - can be `manual` or `scheduled` Also, move intended timestamp settings from `create_persistence_rule_type_wrapper` to `build_alert` Also added those new field to Alert schema and types. https://github.com/user-attachments/assets/c5b021a6-4763-47ae-b46c-814a138be65a For tests: - tests all rule types with and without suppression: `kibana.alert.rule.execution.type` - should be `scheduled`, `kibana.alert.intended_timestamp` - should equal alert timestamp - tests all rules with and without suppression with manual run - `kibana.alert.rule.execution.type` - should be `manual`, `kibana.alert.intended_timestamp` - should equal date inside you manual rule run date range --------- Co-authored-by: Elastic Machine --- .../src/field_maps/alert_field_map.ts | 6 + .../src/schemas/generated/alert_schema.ts | 1 + .../src/schemas/generated/security_schema.ts | 1 + .../src/default_alerts_as_data.ts | 5 + .../field_maps/mapping_from_field_map.test.ts | 3 + .../alert_as_data_fields.test.ts.snap | 40 ++++ .../technical_rule_field_map.test.ts | 5 + .../create_persistence_rule_type_wrapper.ts | 18 -- .../model/alerts/8.16.0/index.ts | 57 +++++ .../detection_engine/model/alerts/index.ts | 35 +-- .../create_security_rule_type_wrapper.ts | 5 + .../eql/build_alert_group_from_sequence.ts | 11 +- .../rule_types/eql/create_eql_alert_type.ts | 2 + .../rule_types/eql/wrap_sequences_factory.ts | 5 +- .../detection_engine/rule_types/esql/esql.ts | 3 + .../rule_types/esql/wrap_esql_alerts.test.ts | 2 + .../rule_types/esql/wrap_esql_alerts.ts | 3 + .../esql/wrap_suppressed_esql_alerts.test.ts | 3 + .../esql/wrap_suppressed_esql_alerts.ts | 3 + .../__snapshots__/build_alert.test.ts.snap | 226 ++++++++++++++++++ .../factories/utils/build_alert.test.ts | 18 ++ .../rule_types/factories/utils/build_alert.ts | 13 + .../utils/transform_hit_to_alert.test.ts | 48 ++++ .../factories/utils/transform_hit_to_alert.ts | 3 + .../rule_types/factories/wrap_hits_factory.ts | 3 + .../create_indicator_match_alert_type.ts | 2 + .../rule_types/ml/create_ml_alert_type.ts | 2 + .../new_terms/create_new_terms_alert_type.ts | 3 + .../new_terms/wrap_new_terms_alerts.test.ts | 4 + .../new_terms/wrap_new_terms_alerts.ts | 3 + .../wrap_suppressed_new_terms_alerts.test.ts | 4 + .../wrap_suppressed_new_terms_alerts.ts | 3 + .../group_and_bulk_create.ts | 1 + .../wrap_suppressed_alerts.ts | 3 + ...bulk_create_suppressed_threshold_alerts.ts | 1 + .../wrap_suppressed_threshold_alerts.ts | 3 + .../lib/detection_engine/rule_types/types.ts | 1 + .../utils/enrichments/__mocks__/alerts.ts | 4 + .../utils/search_after_bulk_create.test.ts | 1 + .../utils/wrap_suppressed_alerts.test.ts | 1 + .../utils/wrap_suppressed_alerts.ts | 3 + .../alerts_compatibility.ts | 2 + .../execution_logic/custom_query.ts | 17 +- .../execution_logic/eql.ts | 4 +- .../execution_logic/esql.ts | 6 +- .../execution_logic/indicator_match.ts | 3 + .../execution_logic/new_terms.ts | 5 +- .../execution_logic/threshold.ts | 3 + 48 files changed, 551 insertions(+), 47 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.16.0/index.ts diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts index ed6a7211c7a90..68208951eea48 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts @@ -23,6 +23,7 @@ import { ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_RULE_EXECUTION_TYPE, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_NAME, ALERT_RULE_PARAMETERS, @@ -134,6 +135,11 @@ export const alertFieldMap = { array: false, required: false, }, + [ALERT_RULE_EXECUTION_TYPE]: { + type: 'keyword', + array: false, + required: false, + }, [ALERT_INTENDED_TIMESTAMP]: { type: 'date', array: false, diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts index 4a4117a8f2197..e377039566dd9 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts @@ -99,6 +99,7 @@ const AlertOptional = rt.partial({ 'kibana.alert.previous_action_group': schemaString, 'kibana.alert.reason': schemaString, 'kibana.alert.rule.execution.timestamp': schemaDate, + 'kibana.alert.rule.execution.type': schemaString, 'kibana.alert.rule.execution.uuid': schemaString, 'kibana.alert.rule.parameters': schemaUnknown, 'kibana.alert.rule.tags': schemaStringArray, diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts index 3573efa5535e4..58e1d8ebd1f46 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts @@ -164,6 +164,7 @@ const SecurityAlertOptional = rt.partial({ 'kibana.alert.rule.description': schemaString, 'kibana.alert.rule.enabled': schemaString, 'kibana.alert.rule.execution.timestamp': schemaDate, + 'kibana.alert.rule.execution.type': schemaString, 'kibana.alert.rule.execution.uuid': schemaString, 'kibana.alert.rule.from': schemaString, 'kibana.alert.rule.immutable': schemaStringArray, diff --git a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts index d56510e51465d..3430680906be4 100644 --- a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts +++ b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts @@ -122,6 +122,9 @@ const ALERT_URL = `${ALERT_NAMESPACE}.url` as const; // kibana.alert.rule.uuid - rule ID for rule that generated this alert const ALERT_RULE_UUID = `${ALERT_RULE_NAMESPACE}.uuid` as const; +// kibana.alert.rule.execution.type - rule execution type for rule that generated this alert (manual /scheduled) +const ALERT_RULE_EXECUTION_TYPE = `${ALERT_RULE_NAMESPACE}.execution.type` as const; + const namespaces = { KIBANA_NAMESPACE, ALERT_NAMESPACE, @@ -144,6 +147,7 @@ const fields = { ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_RULE_EXECUTION_TYPE, ALERT_INTENDED_TIMESTAMP, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_NAME, @@ -189,6 +193,7 @@ export { ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_RULE_EXECUTION_TYPE, ALERT_INTENDED_TIMESTAMP, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_NAME, diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts index e450cdd1e6f94..739e2d48bffd4 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts @@ -276,6 +276,9 @@ describe('mappingFromFieldMap', () => { timestamp: { type: 'date', }, + type: { + type: 'keyword', + }, uuid: { type: 'keyword', }, diff --git a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap index 072d8c59a53ff..8c65843f2d844 100644 --- a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap +++ b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap @@ -1075,6 +1075,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, @@ -2173,6 +2178,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, @@ -3271,6 +3281,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, @@ -4369,6 +4384,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, @@ -5467,6 +5487,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, @@ -6571,6 +6596,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, @@ -7669,6 +7699,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, @@ -8767,6 +8802,11 @@ Object { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts index c2963de50419c..125019674e9ee 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts @@ -157,6 +157,11 @@ it('matches snapshot', () => { "required": false, "type": "date", }, + "kibana.alert.rule.execution.type": Object { + "array": false, + "required": false, + "type": "keyword", + }, "kibana.alert.rule.execution.uuid": Object { "array": false, "required": false, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 78c6238ce7886..afdc63712ba84 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -25,7 +25,6 @@ import { TIMESTAMP, VERSION, ALERT_RULE_EXECUTION_TIMESTAMP, - ALERT_INTENDED_TIMESTAMP, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; import type { IRuleDataClient } from '..'; @@ -56,13 +55,11 @@ const augmentAlerts = async ({ options, kibanaVersion, currentTimeOverride, - intendedTimestamp, }: { alerts: Array<{ _id: string; _source: T }>; options: RuleExecutorOptions; kibanaVersion: string; currentTimeOverride: Date | undefined; - intendedTimestamp: Date | undefined; }) => { const commonRuleFields = getCommonAlertFields(options); const maintenanceWindowIds: string[] = @@ -77,9 +74,6 @@ const augmentAlerts = async ({ [ALERT_RULE_EXECUTION_TIMESTAMP]: currentDate, [ALERT_START]: timestampOverrideOrCurrent, [ALERT_LAST_DETECTED]: timestampOverrideOrCurrent, - [ALERT_INTENDED_TIMESTAMP]: intendedTimestamp - ? intendedTimestamp - : timestampOverrideOrCurrent, [VERSION]: kibanaVersion, ...(maintenanceWindowIds.length ? { [ALERT_MAINTENANCE_WINDOW_IDS]: maintenanceWindowIds } @@ -309,17 +303,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alertsWereTruncated = true; } - let intendedTimestamp; - if (options.startedAtOverridden) { - intendedTimestamp = options.startedAt; - } - const augmentedAlerts = await augmentAlerts({ alerts: enrichedAlerts, options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride: undefined, - intendedTimestamp, }); const response = await ruleDataClientWriter.bulk({ @@ -399,11 +387,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper let alertsWereTruncated = false; - let intendedTimestamp; - if (options.startedAtOverridden) { - intendedTimestamp = options.startedAt; - } - if (writeAlerts && alerts.length > 0) { const suppressionWindowStart = dateMath.parse(suppressionWindow, { forceNow: currentTimeOverride, @@ -583,7 +566,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride, - intendedTimestamp, }); const bulkResponse = await ruleDataClientWriter.bulk({ diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.16.0/index.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.16.0/index.ts new file mode 100644 index 0000000000000..3e5b8843e893a --- /dev/null +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.16.0/index.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ALERT_RULE_EXECUTION_TYPE, ALERT_INTENDED_TIMESTAMP } from '@kbn/rule-data-utils'; +import type { AlertWithCommonFields800 } from '@kbn/rule-registry-plugin/common/schemas/8.0.0'; +import type { + Ancestor8130, + BaseFields8130, + EqlBuildingBlockFields8130, + EqlShellFields8130, + NewTermsFields8130, +} from '../8.13.0'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.12.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.12.0. +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export type { Ancestor8130 as Ancestor8160 }; + +export interface BaseFields8160 extends BaseFields8130 { + [ALERT_RULE_EXECUTION_TYPE]: string; + [ALERT_INTENDED_TIMESTAMP]: string; +} + +export interface WrappedFields8160 { + _id: string; + _index: string; + _source: T; +} + +export type GenericAlert8160 = AlertWithCommonFields800; + +export type EqlShellFields8160 = EqlShellFields8130 & BaseFields8160; + +export type EqlBuildingBlockFields8160 = EqlBuildingBlockFields8130 & BaseFields8160; + +export type NewTermsFields8160 = NewTermsFields8130 & BaseFields8160; + +export type NewTermsAlert8160 = NewTermsFields8130 & BaseFields8160; + +export type EqlBuildingBlockAlert8160 = AlertWithCommonFields800; + +export type EqlShellAlert8160 = AlertWithCommonFields800; + +export type DetectionAlert8160 = + | GenericAlert8160 + | EqlShellAlert8160 + | EqlBuildingBlockAlert8160 + | NewTermsAlert8160; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/index.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/index.ts index 6bf7b1d5dfd7e..816c9a4c81897 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/index.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/index.ts @@ -13,15 +13,17 @@ import type { DetectionAlert870 } from './8.7.0'; import type { DetectionAlert880 } from './8.8.0'; import type { DetectionAlert890 } from './8.9.0'; import type { DetectionAlert8120 } from './8.12.0'; +import type { DetectionAlert8130 } from './8.13.0'; + import type { - Ancestor8130, - BaseFields8130, - DetectionAlert8130, - EqlBuildingBlockFields8130, - EqlShellFields8130, - NewTermsFields8130, - WrappedFields8130, -} from './8.13.0'; + Ancestor8160, + BaseFields8160, + DetectionAlert8160, + EqlBuildingBlockFields8160, + EqlShellFields8160, + NewTermsFields8160, + WrappedFields8160, +} from './8.16.0'; // When new Alert schemas are created for new Kibana versions, add the DetectionAlert type from the new version // here, e.g. `export type DetectionAlert = DetectionAlert800 | DetectionAlert820` if a new schema is created in 8.2.0 @@ -33,14 +35,15 @@ export type DetectionAlert = | DetectionAlert880 | DetectionAlert890 | DetectionAlert8120 - | DetectionAlert8130; + | DetectionAlert8130 + | DetectionAlert8160; export type { - Ancestor8130 as AncestorLatest, - BaseFields8130 as BaseFieldsLatest, - DetectionAlert8130 as DetectionAlertLatest, - WrappedFields8130 as WrappedFieldsLatest, - EqlBuildingBlockFields8130 as EqlBuildingBlockFieldsLatest, - EqlShellFields8130 as EqlShellFieldsLatest, - NewTermsFields8130 as NewTermsFieldsLatest, + Ancestor8160 as AncestorLatest, + BaseFields8160 as BaseFieldsLatest, + DetectionAlert8160 as DetectionAlertLatest, + WrappedFields8160 as WrappedFieldsLatest, + EqlBuildingBlockFields8160 as EqlBuildingBlockFieldsLatest, + EqlShellFields8160 as EqlShellFieldsLatest, + NewTermsFields8160 as NewTermsFieldsLatest, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index ac56d2c41a8fe..aa63359b6ccb1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -128,6 +128,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = params, previousStartedAt, startedAt, + startedAtOverridden, services, spaceId, state, @@ -363,6 +364,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = ignoreFieldsStandard.forEach((field) => { ignoreFieldsObject[field] = true; }); + const intendedTimestamp = startedAtOverridden ? startedAt : undefined; const wrapHits = wrapHitsFactory({ ignoreFields: ignoreFieldsObject, ignoreFieldsRegexes, @@ -373,6 +375,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = alertTimestampOverride, publicBaseUrl, ruleExecutionLogger, + intendedTimestamp, }); const wrapSequences = wrapSequencesFactory({ @@ -384,6 +387,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = publicBaseUrl, indicesToQuery: inputIndex, alertTimestampOverride, + intendedTimestamp, }); const { filter: exceptionFilter, unprocessedExceptions } = await buildExceptionFilter({ @@ -427,6 +431,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = refreshOnIndexingAlerts: refresh, publicBaseUrl, experimentalFeatures, + intendedTimestamp, }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index d865ae6232005..185aa1236a234 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -47,7 +47,8 @@ export const buildAlertGroupFromSequence = ( buildReasonMessage: BuildReasonMessage, indicesToQuery: string[], alertTimestampOverride: Date | undefined, - publicBaseUrl?: string + publicBaseUrl?: string, + intendedTimestamp?: Date ): Array> => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { @@ -73,6 +74,7 @@ export const buildAlertGroupFromSequence = ( ruleExecutionLogger, alertUuid: 'placeholder-alert-uuid', // This is overriden below publicBaseUrl, + intendedTimestamp, }) ); } catch (error) { @@ -104,7 +106,8 @@ export const buildAlertGroupFromSequence = ( buildReasonMessage, indicesToQuery, alertTimestampOverride, - publicBaseUrl + publicBaseUrl, + intendedTimestamp ); const sequenceAlert: WrappedFieldsLatest = { _id: shellAlert[ALERT_UUID], @@ -146,7 +149,8 @@ export const buildAlertRoot = ( buildReasonMessage: BuildReasonMessage, indicesToQuery: string[], alertTimestampOverride: Date | undefined, - publicBaseUrl?: string + publicBaseUrl?: string, + intendedTimestamp?: Date ): EqlShellFieldsLatest => { const mergedAlerts = objectArrayIntersection(wrappedBuildingBlocks.map((alert) => alert._source)); const reason = buildReasonMessage({ @@ -163,6 +167,7 @@ export const buildAlertRoot = ( alertUuid: 'placeholder-uuid', // These will be overriden below publicBaseUrl, // Not necessary now, but when the ID is created ahead of time this can be passed alertTimestampOverride, + intendedTimestamp, }); const alertId = generateAlertId(doc); const alertUrl = getAlertDetailsUrl({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index 12af1966b7dce..2bdf0d913d156 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -79,6 +79,7 @@ export const createEqlAlertType = ( alertTimestampOverride, publicBaseUrl, alertWithSuppression, + intendedTimestamp, }, services, state, @@ -101,6 +102,7 @@ export const createEqlAlertType = ( publicBaseUrl, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }); const isNonSeqAlertSuppressionActive = await getIsAlertSuppressionActive({ alertSuppression: completeRule.ruleParams.alertSuppression, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts index e21f09438e8b8..39f9df366627e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts @@ -25,6 +25,7 @@ export const wrapSequencesFactory = spaceId, indicesToQuery, alertTimestampOverride, + intendedTimestamp, }: { ruleExecutionLogger: IRuleExecutionLogForExecutors; completeRule: CompleteRule; @@ -34,6 +35,7 @@ export const wrapSequencesFactory = indicesToQuery: string[]; alertTimestampOverride: Date | undefined; publicBaseUrl: string | undefined; + intendedTimestamp: Date | undefined; }): WrapSequences => (sequences, buildReasonMessage) => sequences.reduce( @@ -48,7 +50,8 @@ export const wrapSequencesFactory = buildReasonMessage, indicesToQuery, alertTimestampOverride, - publicBaseUrl + publicBaseUrl, + intendedTimestamp ), ], [] diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts index a076ea0c62635..173d722d782a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts @@ -59,6 +59,7 @@ export const esqlExecutor = async ({ alertTimestampOverride, publicBaseUrl, alertWithSuppression, + intendedTimestamp, }, services, state, @@ -167,6 +168,7 @@ export const esqlExecutor = async ({ ruleExecutionLogger, publicBaseUrl, tuple, + intendedTimestamp, }); const syntheticHits: Array> = results.map((document) => { @@ -194,6 +196,7 @@ export const esqlExecutor = async ({ primaryTimestamp, secondaryTimestamp, tuple, + intendedTimestamp, }); const bulkCreateResult = await bulkCreateSuppressedAlertsInMemory({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts index d54f91c088958..4021ed0eae620 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts @@ -44,6 +44,7 @@ describe('wrapSuppressedEsqlAlerts', () => { from: moment('2010-10-20 04:43:12'), maxSignals: 100, }, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('ed7fbf575371c898e0f0aea48cdf0bf1865939a9'); @@ -71,6 +72,7 @@ describe('wrapSuppressedEsqlAlerts', () => { from: moment('2010-10-20 04:43:12'), maxSignals: 100, }, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('mocked-alert-id'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts index 762f3cff4ce45..917635c26b6ac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts @@ -30,6 +30,7 @@ export const wrapEsqlAlerts = ({ publicBaseUrl, tuple, isRuleAggregating, + intendedTimestamp, }: { isRuleAggregating: boolean; events: Array>; @@ -44,6 +45,7 @@ export const wrapEsqlAlerts = ({ from: Moment; maxSignals: number; }; + intendedTimestamp: Date | undefined; }): Array> => { const wrapped = events.map>((event, i) => { const id = generateAlertId({ @@ -69,6 +71,7 @@ export const wrapEsqlAlerts = ({ ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts index c52dd25689e1b..e8602a1993ea2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts @@ -53,6 +53,7 @@ describe('wrapSuppressedEsqlAlerts', () => { from: moment('2010-10-20 04:43:12'), maxSignals: 100, }, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('ed7fbf575371c898e0f0aea48cdf0bf1865939a9'); @@ -92,6 +93,7 @@ describe('wrapSuppressedEsqlAlerts', () => { from: moment('2010-10-20 04:43:12'), maxSignals: 100, }, + intendedTimestamp: undefined, }); expect(alerts[0]._source[ALERT_URL]).toContain( @@ -128,6 +130,7 @@ describe('wrapSuppressedEsqlAlerts', () => { from: moment('2010-10-20 04:43:12'), maxSignals: 100, }, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('mocked-alert-id'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts index 2a6e9a42acf28..b2d01e4d5ee7a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts @@ -36,6 +36,7 @@ export const wrapSuppressedEsqlAlerts = ({ isRuleAggregating, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }: { isRuleAggregating: boolean; events: Array>; @@ -52,6 +53,7 @@ export const wrapSuppressedEsqlAlerts = ({ }; primaryTimestamp: string; secondaryTimestamp?: string; + intendedTimestamp: Date | undefined; }): Array> => { const wrapped = events.map>( (event, i) => { @@ -87,6 +89,7 @@ export const wrapSuppressedEsqlAlerts = ({ ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap index 17d8ceb0c50b6..25d08db218f20 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap @@ -19,6 +19,7 @@ Object { "kibana.alert.building_block_type": "default", "kibana.alert.depth": 1, "kibana.alert.host.criticality_level": undefined, + "kibana.alert.intended_timestamp": "2020-01-01T00:00:00.000Z", "kibana.alert.original_time": "2020-04-20T21:27:45.000Z", "kibana.alert.reason": "test reason", "kibana.alert.risk_score": 50, @@ -46,6 +47,231 @@ Object { "type": "endpoint", }, ], + "kibana.alert.rule.execution.type": "scheduled", + "kibana.alert.rule.false_positives": Array [], + "kibana.alert.rule.from": "now-6m", + "kibana.alert.rule.immutable": false, + "kibana.alert.rule.indices": Array [], + "kibana.alert.rule.interval": "5m", + "kibana.alert.rule.license": "Elastic License", + "kibana.alert.rule.max_signals": 10000, + "kibana.alert.rule.meta.someMeta": "someField", + "kibana.alert.rule.name": "rule-name", + "kibana.alert.rule.namespace": undefined, + "kibana.alert.rule.note": "# Investigative notes", + "kibana.alert.rule.parameters": Object { + "alert_suppression": undefined, + "author": Array [ + "Elastic", + ], + "building_block_type": "default", + "data_view_id": undefined, + "description": "Detecting root and admin users", + "exceptions_list": Array [ + Object { + "id": "some_uuid", + "list_id": "list_id_single", + "namespace_type": "single", + "type": "detection", + }, + Object { + "id": "endpoint_list", + "list_id": "endpoint_list", + "namespace_type": "agnostic", + "type": "endpoint", + }, + ], + "false_positives": Array [], + "filters": Array [ + Object { + "query": Object { + "match_phrase": Object { + "host.name": "some-host", + }, + }, + }, + ], + "from": "now-6m", + "immutable": false, + "index": Array [ + "auditbeat-*", + "filebeat-*", + "packetbeat-*", + "winlogbeat-*", + ], + "investigation_fields": undefined, + "language": "kuery", + "license": "Elastic License", + "max_signals": 10000, + "meta": Object { + "someMeta": "someField", + }, + "namespace": undefined, + "note": "# Investigative notes", + "query": "user.name: root or user.name: admin", + "references": Array [ + "http://example.com", + "https://example.com", + ], + "related_integrations": Array [], + "required_fields": Array [], + "response_actions": undefined, + "risk_score": 50, + "risk_score_mapping": Array [], + "rule_id": "rule-1", + "rule_name_override": undefined, + "rule_source": Object { + "type": "internal", + }, + "saved_id": undefined, + "setup": "", + "severity": "high", + "severity_mapping": Array [], + "threat": Array [ + Object { + "framework": "MITRE ATT&CK", + "tactic": Object { + "id": "TA0000", + "name": "test tactic", + "reference": "https://attack.mitre.org/tactics/TA0000/", + }, + "technique": Array [ + Object { + "id": "T0000", + "name": "test technique", + "reference": "https://attack.mitre.org/techniques/T0000/", + "subtechnique": Array [ + Object { + "id": "T0000.000", + "name": "test subtechnique", + "reference": "https://attack.mitre.org/techniques/T0000/000/", + }, + ], + }, + ], + }, + ], + "timeline_id": "some-timeline-id", + "timeline_title": "some-timeline-title", + "timestamp_override": undefined, + "timestamp_override_fallback_disabled": undefined, + "to": "now", + "type": "query", + "version": 1, + }, + "kibana.alert.rule.references": Array [ + "http://example.com", + "https://example.com", + ], + "kibana.alert.rule.risk_score": 50, + "kibana.alert.rule.risk_score_mapping": Array [], + "kibana.alert.rule.rule_id": "rule-1", + "kibana.alert.rule.rule_name_override": undefined, + "kibana.alert.rule.severity": "high", + "kibana.alert.rule.severity_mapping": Array [], + "kibana.alert.rule.tags": Array [ + "some fake tag 1", + "some fake tag 2", + ], + "kibana.alert.rule.threat": Array [ + Object { + "framework": "MITRE ATT&CK", + "tactic": Object { + "id": "TA0000", + "name": "test tactic", + "reference": "https://attack.mitre.org/tactics/TA0000/", + }, + "technique": Array [ + Object { + "id": "T0000", + "name": "test technique", + "reference": "https://attack.mitre.org/techniques/T0000/", + "subtechnique": Array [ + Object { + "id": "T0000.000", + "name": "test subtechnique", + "reference": "https://attack.mitre.org/techniques/T0000/000/", + }, + ], + }, + ], + }, + ], + "kibana.alert.rule.throttle": "no_actions", + "kibana.alert.rule.timeline_id": "some-timeline-id", + "kibana.alert.rule.timeline_title": "some-timeline-title", + "kibana.alert.rule.timestamp_override": undefined, + "kibana.alert.rule.to": "now", + "kibana.alert.rule.type": "query", + "kibana.alert.rule.updated_at": "2020-03-27T22:55:59.577Z", + "kibana.alert.rule.updated_by": "sample user", + "kibana.alert.rule.uuid": "04128c15-0d1b-4716-a4c5-46997ac7f3bd", + "kibana.alert.rule.version": 1, + "kibana.alert.severity": "high", + "kibana.alert.status": "active", + "kibana.alert.url": "test/url/app/security/alerts/redirect/test-uuid?index=.alerts-security.alerts-default×tamp=2020-01-01T00:00:00.000Z", + "kibana.alert.user.criticality_level": undefined, + "kibana.alert.uuid": "test-uuid", + "kibana.alert.workflow_assignee_ids": Array [], + "kibana.alert.workflow_status": "open", + "kibana.alert.workflow_tags": Array [], + "kibana.space_ids": Array [ + "default", + ], + "user.asset.criticality": undefined, + "user.risk.calculated_level": undefined, + "user.risk.calculated_score_norm": undefined, +} +`; + +exports[`buildAlertFields it creates the expected alert fields for manual run 1`] = ` +Object { + "@timestamp": "2020-01-01T00:00:00.000Z", + "event.kind": "signal", + "host.asset.criticality": undefined, + "host.risk.calculated_level": undefined, + "host.risk.calculated_score_norm": undefined, + "kibana.alert.ancestors": Array [ + Object { + "depth": 0, + "id": "d5e8eb51-a6a0-456d-8a15-4b79bfec3d71", + "index": "myFakeSignalIndex", + "rule": undefined, + "type": "event", + }, + ], + "kibana.alert.building_block_type": "default", + "kibana.alert.depth": 1, + "kibana.alert.host.criticality_level": undefined, + "kibana.alert.intended_timestamp": "2019-01-01T00:00:00.000Z", + "kibana.alert.original_time": "2020-04-20T21:27:45.000Z", + "kibana.alert.reason": "test reason", + "kibana.alert.risk_score": 50, + "kibana.alert.rule.actions": Array [], + "kibana.alert.rule.author": Array [ + "Elastic", + ], + "kibana.alert.rule.building_block_type": "default", + "kibana.alert.rule.consumer": "siem", + "kibana.alert.rule.created_at": "2020-03-27T22:55:59.577Z", + "kibana.alert.rule.created_by": "sample user", + "kibana.alert.rule.description": "Detecting root and admin users", + "kibana.alert.rule.enabled": true, + "kibana.alert.rule.exceptions_list": Array [ + Object { + "id": "some_uuid", + "list_id": "list_id_single", + "namespace_type": "single", + "type": "detection", + }, + Object { + "id": "endpoint_list", + "list_id": "endpoint_list", + "namespace_type": "agnostic", + "type": "endpoint", + }, + ], + "kibana.alert.rule.execution.type": "manual", "kibana.alert.rule.false_positives": Array [], "kibana.alert.rule.from": "now-6m", "kibana.alert.rule.immutable": false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts index b7f83106ea0b9..6c4a77e7f90d7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts @@ -46,6 +46,24 @@ describe('buildAlertFields', () => { alertUuid: 'test-uuid', publicBaseUrl: 'test/url', alertTimestampOverride: new Date('2020-01-01T00:00:00.000Z'), + intendedTimestamp: undefined, + }); + expect(alertFields).toMatchSnapshot(); + }); + + test('it creates the expected alert fields for manual run', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const completeRule = getCompleteRuleMock(getQueryRuleParams()); + const alertFields = buildAlertFields({ + docs: [sampleDoc], + completeRule, + spaceId: 'default', + reason: 'test reason', + indicesToQuery: [], + alertUuid: 'test-uuid', + publicBaseUrl: 'test/url', + alertTimestampOverride: new Date('2020-01-01T00:00:00.000Z'), + intendedTimestamp: new Date('2019-01-01T00:00:00.000Z'), }); expect(alertFields).toMatchSnapshot(); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index 036ba8c9a644a..e49070f483f97 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -42,6 +42,8 @@ import { EVENT_KIND, SPACE_IDS, TIMESTAMP, + ALERT_INTENDED_TIMESTAMP, + ALERT_RULE_EXECUTION_TYPE, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { requiredOptional } from '@kbn/zod-helpers'; @@ -109,6 +111,7 @@ export interface BuildAlertFieldsProps { severityOverride: string; riskScoreOverride: number; }; + intendedTimestamp: Date | undefined; } export const generateAlertId = (alert: BaseFieldsLatest) => { @@ -151,6 +154,11 @@ export const buildAncestors = (doc: SimpleHit): AncestorLatest[] => { return [...existingAncestors, newAncestor]; }; +enum RULE_EXECUTION_TYPE { + MANUAL = 'manual', + SCHEDULED = 'scheduled', +} + /** * Builds the `kibana.alert.*` fields that are common across all alerts. * @param docs The parent alerts/events of the new alert to be built. @@ -169,6 +177,7 @@ export const buildAlertFields = ({ publicBaseUrl, alertTimestampOverride, overrides, + intendedTimestamp, }: BuildAlertFieldsProps): BaseFieldsLatest => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; @@ -283,6 +292,10 @@ export const buildAlertFields = ({ [ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, [ALERT_USER_RISK_SCORE_CALCULATED_LEVEL]: undefined, [ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, + [ALERT_INTENDED_TIMESTAMP]: intendedTimestamp ? intendedTimestamp.toISOString() : timestamp, + [ALERT_RULE_EXECUTION_TYPE]: intendedTimestamp + ? RULE_EXECUTION_TYPE.MANUAL + : RULE_EXECUTION_TYPE.SCHEDULED, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts index 92d02e1b3ac4a..2dfffebfdd6eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts @@ -5,9 +5,11 @@ * 2.0. */ import { + ALERT_INTENDED_TIMESTAMP, ALERT_REASON, ALERT_RISK_SCORE, ALERT_RULE_CONSUMER, + ALERT_RULE_EXECUTION_TYPE, ALERT_RULE_NAMESPACE, ALERT_RULE_PARAMETERS, ALERT_SEVERITY, @@ -80,6 +82,7 @@ describe('transformHitToAlert', () => { ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp: undefined, }); expect(alert['kibana.alert.original_event.action']).toEqual('process'); @@ -112,6 +115,7 @@ describe('transformHitToAlert', () => { ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp: undefined, }); expect(get(alert.event, 'kind')).toEqual(undefined); @@ -142,6 +146,7 @@ describe('transformHitToAlert', () => { ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp: undefined, }); expect(get(alert.event, 'kind')).toEqual(undefined); @@ -172,6 +177,7 @@ describe('transformHitToAlert', () => { ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp: undefined, }); expect(alert.event).toEqual(undefined); @@ -202,6 +208,7 @@ describe('transformHitToAlert', () => { ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp: undefined, }); expect(alert['kibana.alert.original_event.action']).toEqual(['process']); @@ -224,6 +231,7 @@ describe('transformHitToAlert', () => { ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp: undefined, }); delete doc._source.event; @@ -392,6 +400,8 @@ describe('transformHitToAlert', () => { source: { ip: '127.0.0.1', }, + [ALERT_RULE_EXECUTION_TYPE]: 'scheduled', + [ALERT_INTENDED_TIMESTAMP]: timestamp, }; expect(alert).toEqual(expected); }); @@ -423,6 +433,7 @@ describe('transformHitToAlert', () => { ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp: undefined, }); const expected = { ...alert, @@ -435,4 +446,41 @@ describe('transformHitToAlert', () => { }; expect(alert).toEqual(expected); }); + + it('build alert with manual execution type if intendedTimestamp is provided', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + [EVENT_ACTION]: 'socket_opened', + [EVENT_DATASET]: 'socket', + [EVENT_KIND]: 'event', + [EVENT_MODULE]: 'system', + }, + }; + const completeRule = getCompleteRuleMock(getQueryRuleParams()); + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + intendedTimestamp: new Date('2019-01-01T00:00:00.000Z'), + }); + const expected = { + ...alert, + [ALERT_RULE_EXECUTION_TYPE]: 'manual', + [ALERT_INTENDED_TIMESTAMP]: new Date('2019-01-01T00:00:00.000Z').toISOString(), + }; + expect(alert).toEqual(expected); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts index c5112006ae251..b95d335642b2f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts @@ -44,6 +44,7 @@ export interface TransformHitToAlertProps { ruleExecutionLogger: IRuleExecutionLogForExecutors; alertUuid: string; publicBaseUrl?: string; + intendedTimestamp: Date | undefined; } /** @@ -69,6 +70,7 @@ export const transformHitToAlert = ({ ruleExecutionLogger, alertUuid, publicBaseUrl, + intendedTimestamp, }: TransformHitToAlertProps): BaseFieldsLatest => { const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields, ignoreFieldsRegexes }); const thresholdResult = mergedDoc._source?.threshold_result; @@ -110,6 +112,7 @@ export const transformHitToAlert = ({ publicBaseUrl, alertTimestampOverride, overrides, + intendedTimestamp, }); const { result: validatedSource, removed: removedSourceFields } = traverseAndMutateDoc( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 2495e48c1cc81..cdcf9e0716ec5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -29,6 +29,7 @@ export const wrapHitsFactory = alertTimestampOverride, publicBaseUrl, ruleExecutionLogger, + intendedTimestamp, }: { completeRule: CompleteRule; ignoreFields: Record; @@ -39,6 +40,7 @@ export const wrapHitsFactory = alertTimestampOverride: Date | undefined; publicBaseUrl: string | undefined; ruleExecutionLogger: IRuleExecutionLogForExecutors; + intendedTimestamp: Date | undefined; }) => ( events: Array>, @@ -67,6 +69,7 @@ export const wrapHitsFactory = ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts index 9c51d22d31ee1..0de5f580b2c98 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts @@ -82,6 +82,7 @@ export const createIndicatorMatchAlertType = ( secondaryTimestamp, exceptionFilter, unprocessedExceptions, + intendedTimestamp, }, services, spaceId, @@ -105,6 +106,7 @@ export const createIndicatorMatchAlertType = ( publicBaseUrl: runOpts.publicBaseUrl, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }); const result = await indicatorMatchExecutor({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts index 4d896c4efdaa4..c5d918becefdf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts @@ -65,6 +65,7 @@ export const createMlAlertType = ( alertWithSuppression, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }, services, spaceId, @@ -89,6 +90,7 @@ export const createMlAlertType = ( publicBaseUrl, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }); const result = await mlExecutor({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 6b50f0fe0505e..99a7bfc0c0b4d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -111,6 +111,7 @@ export const createNewTermsAlertType = ( alertTimestampOverride, publicBaseUrl, alertWithSuppression, + intendedTimestamp, }, services, params, @@ -212,6 +213,7 @@ export const createNewTermsAlertType = ( alertTimestampOverride, ruleExecutionLogger, publicBaseUrl, + intendedTimestamp, }); const wrapSuppressedHits = (eventsAndTerms: EventsAndTerms[]) => @@ -226,6 +228,7 @@ export const createNewTermsAlertType = ( publicBaseUrl, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }); const eventsAndTerms: EventsAndTerms[] = ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.test.ts index 67a3c69af9850..eb4115bbf1281 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.test.ts @@ -29,6 +29,7 @@ describe('wrapNewTermsAlerts', () => { alertTimestampOverride: undefined, ruleExecutionLogger, publicBaseUrl, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); @@ -51,6 +52,7 @@ describe('wrapNewTermsAlerts', () => { alertTimestampOverride: undefined, ruleExecutionLogger, publicBaseUrl, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545'); @@ -73,6 +75,7 @@ describe('wrapNewTermsAlerts', () => { alertTimestampOverride: undefined, ruleExecutionLogger, publicBaseUrl, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea'); @@ -95,6 +98,7 @@ describe('wrapNewTermsAlerts', () => { alertTimestampOverride: undefined, ruleExecutionLogger, publicBaseUrl, + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('86a216cfa4884767d9bb26d2b8db911cb4aa85ce'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts index efd33c5442bf3..666062592237c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts @@ -34,6 +34,7 @@ export const wrapNewTermsAlerts = ({ alertTimestampOverride, ruleExecutionLogger, publicBaseUrl, + intendedTimestamp, }: { eventsAndTerms: EventsAndTerms[]; spaceId: string | null | undefined; @@ -43,6 +44,7 @@ export const wrapNewTermsAlerts = ({ alertTimestampOverride: Date | undefined; ruleExecutionLogger: IRuleExecutionLogForExecutors; publicBaseUrl: string | undefined; + intendedTimestamp: Date | undefined; }): Array> => { return eventsAndTerms.map((eventAndTerms) => { const id = objectHash([ @@ -66,6 +68,7 @@ export const wrapNewTermsAlerts = ({ ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.test.ts index d5cf50fbefcdd..1299b5f905f42 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.test.ts @@ -46,6 +46,7 @@ describe('wrapSuppressedNewTermsAlerts', () => { ruleExecutionLogger, publicBaseUrl, primaryTimestamp: '@timestamp', + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); @@ -81,6 +82,7 @@ describe('wrapSuppressedNewTermsAlerts', () => { ruleExecutionLogger, publicBaseUrl, primaryTimestamp: '@timestamp', + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('a36d9fe6fe4b2f65058fb1a487733275f811af58'); @@ -109,6 +111,7 @@ describe('wrapSuppressedNewTermsAlerts', () => { ruleExecutionLogger, publicBaseUrl, primaryTimestamp: '@timestamp', + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('f7877a31b1cc83373dbc9ba5939ebfab1db66545'); @@ -130,6 +133,7 @@ describe('wrapSuppressedNewTermsAlerts', () => { ruleExecutionLogger, publicBaseUrl, primaryTimestamp: '@timestamp', + intendedTimestamp: undefined, }); expect(alerts[0]._id).toEqual('75e5a507a4bc48bcd983820c7fd2d9621ff4e2ea'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts index ad34feb81eab1..fa3781b2a2e63 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts @@ -39,6 +39,7 @@ export const wrapSuppressedNewTermsAlerts = ({ publicBaseUrl, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }: { eventsAndTerms: EventsAndTerms[]; spaceId: string | null | undefined; @@ -50,6 +51,7 @@ export const wrapSuppressedNewTermsAlerts = ({ publicBaseUrl: string | undefined; primaryTimestamp: string; secondaryTimestamp?: string; + intendedTimestamp: Date | undefined; }): Array> => { return eventsAndTerms.map((eventAndTerms) => { const event = eventAndTerms.event; @@ -83,6 +85,7 @@ export const wrapSuppressedNewTermsAlerts = ({ ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index 38018060e2252..97640be9bbe0a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -261,6 +261,7 @@ export const groupAndBulkCreate = async ({ buildReasonMessage, alertTimestampOverride: runOpts.alertTimestampOverride, ruleExecutionLogger: runOpts.ruleExecutionLogger, + intendedTimestamp: runOpts.intendedTimestamp, }); const suppressionDuration = runOpts.completeRule.ruleParams.alertSuppression?.duration; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts index abff900a5dcb4..6961c3bcef15e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts @@ -56,6 +56,7 @@ export const wrapSuppressedAlerts = ({ alertTimestampOverride, ruleExecutionLogger, publicBaseUrl, + intendedTimestamp, }: { suppressionBuckets: SuppressionBucket[]; spaceId: string; @@ -66,6 +67,7 @@ export const wrapSuppressedAlerts = ({ alertTimestampOverride: Date | undefined; ruleExecutionLogger: IRuleExecutionLogForExecutors; publicBaseUrl: string | undefined; + intendedTimestamp: Date | undefined; }): Array> => { return suppressionBuckets.map((bucket) => { const id = objectHash([ @@ -96,6 +98,7 @@ export const wrapSuppressedAlerts = ({ ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts index b5e288909b412..d3b5f3fb88fa8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/bulk_create_suppressed_threshold_alerts.ts @@ -84,6 +84,7 @@ export const bulkCreateSuppressedThresholdAlerts = async ({ to, suppressionWindow, threshold: ruleParams.threshold, + intendedTimestamp: runOpts.intendedTimestamp, }); const bulkCreateResult = await bulkCreateWithSuppression({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts index e9acb8284b740..a0a4f8148f482 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts @@ -53,6 +53,7 @@ export const wrapSuppressedThresholdALerts = ({ from, to, threshold, + intendedTimestamp, }: { buckets: ThresholdBucket[]; spaceId: string; @@ -69,6 +70,7 @@ export const wrapSuppressedThresholdALerts = ({ to: Date; suppressionWindow: string; threshold: ThresholdNormalized; + intendedTimestamp: Date | undefined; }): Array> => { return buckets.map((bucket) => { const hit = transformBucketIntoHit( @@ -100,6 +102,7 @@ export const wrapSuppressedThresholdALerts = ({ ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 34307ea495268..c1e8165fe3aed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -105,6 +105,7 @@ export interface RunOpts { refreshOnIndexingAlerts: RefreshTypes; publicBaseUrl: string | undefined; experimentalFeatures?: ExperimentalFeatures; + intendedTimestamp: Date | undefined; } export type SecurityAlertType< diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts index 218ba29bfdc45..8e43453dd086d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/enrichments/__mocks__/alerts.ts @@ -6,6 +6,7 @@ */ import { ALERT_BUILDING_BLOCK_TYPE, + ALERT_INTENDED_TIMESTAMP, ALERT_REASON, ALERT_RISK_SCORE, ALERT_RULE_AUTHOR, @@ -15,6 +16,7 @@ import { ALERT_RULE_CREATED_BY, ALERT_RULE_DESCRIPTION, ALERT_RULE_ENABLED, + ALERT_RULE_EXECUTION_TYPE, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_FROM, ALERT_RULE_INTERVAL, @@ -210,6 +212,8 @@ export const createAlert = ( [ALERT_HOST_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, [ALERT_USER_RISK_SCORE_CALCULATED_LEVEL]: undefined, [ALERT_USER_RISK_SCORE_CALCULATED_SCORE_NORM]: undefined, + [ALERT_INTENDED_TIMESTAMP]: '2020-04-20T21:27:45+0000', + [ALERT_RULE_EXECUTION_TYPE]: 'scheduled', ...data, }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts index 60b7e3edacedf..d2e51e03908f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts @@ -120,6 +120,7 @@ describe('searchAfterAndBulkCreate', () => { alertTimestampOverride: undefined, ruleExecutionLogger, publicBaseUrl: 'http://testkibanabaseurl.com', + intendedTimestamp: undefined, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts index 4d47c279f1495..0d54fd91ba336 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts @@ -115,6 +115,7 @@ const wrappedParams = { publicBaseUrl: 'public-url-mock', primaryTimestamp: '@timestamp', secondaryTimestamp: 'event.ingested', + intendedTimestamp: undefined, }; describe('wrapSuppressedAlerts', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 87d1e64d8ece9..2e40026c38e4a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -48,6 +48,7 @@ export const wrapSuppressedAlerts = ({ publicBaseUrl, primaryTimestamp, secondaryTimestamp, + intendedTimestamp, }: { events: SignalSourceHit[]; spaceId: string; @@ -60,6 +61,7 @@ export const wrapSuppressedAlerts = ({ publicBaseUrl: string | undefined; primaryTimestamp: string; secondaryTimestamp?: string; + intendedTimestamp: Date | undefined; }): Array> => { return events.map((event) => { const suppressionTerms = getSuppressionTerms({ @@ -91,6 +93,7 @@ export const wrapSuppressedAlerts = ({ ruleExecutionLogger, alertUuid: id, publicBaseUrl, + intendedTimestamp, }); return { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts index 9c8d50631cc53..d98afe99addc7 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts @@ -387,6 +387,7 @@ export default ({ getService }: FtrProviderContext) => { 'kibana.alert.original_event.ingested': '2022-03-23T16:50:28.994Z', 'kibana.alert.original_event.dataset': 'elastic_agent.filebeat', 'kibana.alert.original_event.kind': 'signal', + 'kibana.alert.rule.execution.type': 'scheduled', }); }); @@ -555,6 +556,7 @@ export default ({ getService }: FtrProviderContext) => { 'kibana.alert.original_event.ingested': '2022-03-23T16:50:28.994Z', 'kibana.alert.original_event.dataset': 'elastic_agent.filebeat', 'kibana.alert.original_event.kind': 'signal', + 'kibana.alert.rule.execution.type': 'scheduled', }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts index 0b39a7287bacb..8c1462a84a971 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts @@ -19,7 +19,7 @@ import { TIMESTAMP, ALERT_LAST_DETECTED, ALERT_INTENDED_TIMESTAMP, - ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_RULE_EXECUTION_TYPE, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { Rule } from '@kbn/alerting-plugin/common'; @@ -2462,8 +2462,7 @@ export default ({ getService }: FtrProviderContext) => { ); }); - // Flakey test - https://github.com/elastic/kibana/issues/192935 - it.skip('alerts has intended_timestamp set to the time of the manual run', async () => { + it('alerts has intended_timestamp set to the time of the manual run', async () => { const id = uuidv4(); const firstTimestamp = moment(new Date()).subtract(3, 'h').toISOString(); const secondTimestamp = new Date().toISOString(); @@ -2496,9 +2495,11 @@ export default ({ getService }: FtrProviderContext) => { expect(alerts.hits.hits).toHaveLength(1); expect(alerts.hits.hits[0]?._source?.[ALERT_INTENDED_TIMESTAMP]).toEqual( - alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TIMESTAMP] + alerts.hits.hits[0]?._source?.[TIMESTAMP] ); + expect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('scheduled'); + const backfillStartDate = moment(firstTimestamp).startOf('hour'); const backfillEndDate = moment(backfillStartDate).add(1, 'h'); const backfill = await scheduleRuleRun(supertest, [createdRule.id], { @@ -2508,9 +2509,11 @@ export default ({ getService }: FtrProviderContext) => { await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log }); const allNewAlerts = await getAlerts(supertest, log, es, createdRule); - expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_INTENDED_TIMESTAMP]).toEqual( - backfillEndDate.toISOString() + expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_INTENDED_TIMESTAMP]).not.toEqual( + allNewAlerts.hits.hits[1]?._source?.[TIMESTAMP] ); + + expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('manual'); }); it('alerts when run on a time range that the rule has not previously seen, and deduplicates if run there more than once', async () => { @@ -2747,6 +2750,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log }); const allNewAlerts = await getAlerts(supertest, log, es, createdRule); expect(allNewAlerts.hits.hits).toHaveLength(1); + expect(allNewAlerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('manual'); }); it('supression with time window should work for manual rule runs and update alert', async () => { @@ -2817,6 +2821,7 @@ export default ({ getService }: FtrProviderContext) => { expect(updatedAlerts.hits.hits[0]._source).toEqual({ ...updatedAlerts.hits.hits[0]._source, [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + [ALERT_RULE_EXECUTION_TYPE]: 'manual', }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts index 9077873274fa5..e2d39bce4b024 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts @@ -18,6 +18,7 @@ import { ALERT_WORKFLOW_ASSIGNEE_IDS, ALERT_SUPPRESSION_DOCS_COUNT, EVENT_KIND, + ALERT_RULE_EXECUTION_TYPE, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; @@ -997,8 +998,8 @@ export default ({ getService }: FtrProviderContext) => { const createdRule = await createRule(supertest, log, rule); const alerts = await getAlerts(supertest, log, es, createdRule); - expect(alerts.hits.hits.length).equal(3); + expect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('scheduled'); const backfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), @@ -1008,6 +1009,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log }); const allNewAlerts = await getAlerts(supertest, log, es, createdRule); expect(allNewAlerts.hits.hits.length).equal(6); + expect(allNewAlerts.hits.hits[5]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('manual'); const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts index ee976de14186d..d44896115fae3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts @@ -8,7 +8,7 @@ import expect from 'expect'; import { v4 as uuidv4 } from 'uuid'; import moment from 'moment'; -import { ALERT_SUPPRESSION_DOCS_COUNT } from '@kbn/rule-data-utils'; +import { ALERT_RULE_EXECUTION_TYPE, ALERT_SUPPRESSION_DOCS_COUNT } from '@kbn/rule-data-utils'; import { EsqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks'; import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; @@ -167,6 +167,7 @@ export default ({ getService }: FtrProviderContext) => { 'kibana.alert.workflow_assignee_ids': [], 'kibana.alert.rule.risk_score': 55, 'kibana.alert.rule.severity': 'high', + 'kibana.alert.rule.execution.type': 'scheduled', }); }); @@ -1189,6 +1190,7 @@ export default ({ getService }: FtrProviderContext) => { const alerts = await getAlerts(supertest, log, es, createdRule); expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('scheduled'); const backfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), @@ -1199,6 +1201,8 @@ export default ({ getService }: FtrProviderContext) => { const allNewAlerts = await getAlerts(supertest, log, es, createdRule); expect(allNewAlerts.hits.hits).toHaveLength(2); + expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('manual'); + const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), endDate: moment(firstTimestamp).add(5, 'm'), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts index 0dd5a93bb9e60..663da2aef5784 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts @@ -22,6 +22,7 @@ import { ALERT_WORKFLOW_TAGS, ALERT_WORKFLOW_ASSIGNEE_IDS, ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_RULE_EXECUTION_TYPE, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types'; @@ -1786,6 +1787,7 @@ export default ({ getService }: FtrProviderContext) => { const alerts = await getAlerts(supertest, log, es, createdRule); expect(alerts.hits.hits.length).equal(1); + expect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('scheduled'); const backfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), @@ -1795,6 +1797,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log }); const allNewAlerts = await getAlerts(supertest, log, es, createdRule); expect(allNewAlerts.hits.hits.length).equal(2); + expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('manual'); const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts index e274366e54aa7..a1695ec04021c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts @@ -8,7 +8,7 @@ import expect from 'expect'; import moment from 'moment'; import { v4 as uuidv4 } from 'uuid'; -import { ALERT_SUPPRESSION_DOCS_COUNT } from '@kbn/rule-data-utils'; +import { ALERT_RULE_EXECUTION_TYPE, ALERT_SUPPRESSION_DOCS_COUNT } from '@kbn/rule-data-utils'; import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; import { orderBy } from 'lodash'; import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks'; @@ -252,6 +252,7 @@ export default ({ getService }: FtrProviderContext) => { 'kibana.alert.original_event.origin': '/var/log/wtmp', 'kibana.alert.original_event.outcome': 'success', 'kibana.alert.original_event.type': 'authentication_success', + 'kibana.alert.rule.execution.type': 'scheduled', }); }); @@ -1179,6 +1180,7 @@ export default ({ getService }: FtrProviderContext) => { const alerts = await getAlerts(supertest, log, es, createdRule); expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('scheduled'); const backfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), @@ -1188,6 +1190,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log }); const allNewAlerts = await getAlerts(supertest, log, es, createdRule); expect(allNewAlerts.hits.hits).toHaveLength(2); + expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('manual'); const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts index 46c45a4c94de5..e0e93ba8ed300 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/threshold.ts @@ -14,6 +14,7 @@ import { ALERT_WORKFLOW_STATUS, EVENT_KIND, ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_RULE_EXECUTION_TYPE, } from '@kbn/rule-data-utils'; import { ThresholdRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; @@ -540,6 +541,7 @@ export default ({ getService }: FtrProviderContext) => { const alerts = await getAlerts(supertest, log, es, createdRule); expect(alerts.hits.hits).toHaveLength(1); + expect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('scheduled'); const backfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'), @@ -549,6 +551,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log }); const allNewAlerts = await getAlerts(supertest, log, es, createdRule); expect(allNewAlerts.hits.hits).toHaveLength(2); + expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_RULE_EXECUTION_TYPE]).toEqual('manual'); const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], { startDate: moment(firstTimestamp).subtract(5, 'm'),