From d48454d1461e75f22c6553974b7c856bb58e795d Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 15 Dec 2022 15:19:02 -0800 Subject: [PATCH 01/22] WIP --- .../src/technical_field_names.ts | 3 + .../common/schemas/8.7.0/index.ts | 32 +++ .../server/rule_data_client/types.ts | 9 +- .../create_persistence_rule_type_wrapper.ts | 201 +++++++++++++++++- .../server/utils/persistence_types.ts | 17 ++ .../specific_attributes/query_attributes.ts | 31 ++- .../schemas/alerts/8.7.0/index.ts | 28 +++ .../detection_engine/schemas/alerts/index.ts | 11 +- .../pages/rule_creation/helpers.ts | 8 +- .../rules/description_step/helpers.tsx | 35 +++ .../rules/description_step/index.tsx | 5 + .../components/rules/duration_input/index.tsx | 169 +++++++++++++++ .../rules/duration_input/translations.ts | 36 ++++ .../rules/step_define_rule/index.tsx | 36 +++- .../rules/step_define_rule/schema.tsx | 42 ++++ .../pages/detection_engine/rules/helpers.tsx | 1 + .../pages/detection_engine/rules/types.ts | 6 + .../pages/detection_engine/rules/utils.ts | 4 + .../rule_management/utils/utils.ts | 2 + .../rule_schema/model/rule_schemas.ts | 19 +- .../create_security_rule_type_wrapper.ts | 1 + .../factories/utils/wrap_suppressed_alerts.ts | 9 + .../signals/single_search_after.ts | 9 +- 23 files changed, 686 insertions(+), 28 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/duration_input/translations.ts diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts index 6b51906cca1ef..f41a40e13038a 100644 --- a/packages/kbn-rule-data-utils/src/technical_field_names.ts +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -50,6 +50,7 @@ const ALERT_SUPPRESSION_VALUE = `${ALERT_SUPPRESSION_TERMS}.value` as const; const ALERT_SUPPRESSION_START = `${ALERT_SUPPRESSION_META}.start` as const; const ALERT_SUPPRESSION_END = `${ALERT_SUPPRESSION_META}.end` as const; const ALERT_SUPPRESSION_DOCS_COUNT = `${ALERT_SUPPRESSION_META}.docs_count` as const; +const ALERT_SUPPRESSION_GROUP_ID = `${ALERT_SUPPRESSION_META}.group_id` as const; // Fields pertaining to the rule associated with the alert const ALERT_RULE_AUTHOR = `${ALERT_RULE_NAMESPACE}.author` as const; @@ -180,6 +181,7 @@ const fields = { ALERT_SUPPRESSION_START, ALERT_SUPPRESSION_END, ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_GROUP_ID, SPACE_IDS, VERSION, }; @@ -255,6 +257,7 @@ export { ALERT_SUPPRESSION_START, ALERT_SUPPRESSION_END, ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_GROUP_ID, TAGS, TIMESTAMP, SPACE_IDS, diff --git a/x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts new file mode 100644 index 0000000000000..fb91c91829b89 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, +} from '@kbn/rule-data-utils'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.0.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.0.0. + +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. + +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export interface SuppressionFields { + [ALERT_SUPPRESSION_TERMS]: Array<{ field: string; value: string | number | null }>; + [ALERT_SUPPRESSION_START]: Date; + [ALERT_SUPPRESSION_END]: Date; + [ALERT_SUPPRESSION_DOCS_COUNT]: number; + [ALERT_INSTANCE_ID]: string; +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index 7a3a504cc9b6d..dc8199c1e2963 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -23,11 +23,12 @@ export interface IRuleDataClient { } export interface IRuleDataReader { - search( + search< + TSearchRequest extends ESSearchRequest, + TAlertDoc = Partial + >( request: TSearchRequest - ): Promise< - ESSearchResponse, TSearchRequest> - >; + ): Promise>; getDynamicIndexPattern(target?: string): Promise<{ title: string; diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 8c5265dfedfea..aa22eabfcdf8d 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -5,13 +5,22 @@ * 2.0. */ +import dateMath from '@elastic/datemath'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { chunk } from 'lodash'; -import { ALERT_UUID, VERSION } from '@kbn/rule-data-utils'; +import { chunk, partition } from 'lodash'; +import { + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_UUID, + TIMESTAMP, + VERSION, +} from '@kbn/rule-data-utils'; import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; import { errorAggregator } from './utils'; import { createGetSummarizedAlertsFn } from './create_get_summarized_alerts_fn'; +import { SuppressionFields } from '../../common/schemas/8.7.0'; export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper = ({ logger, ruleDataClient }) => @@ -58,6 +67,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper aggs: { uuids: { terms: { + // TODO: we can probably use `include` here instead of the query above on `ids` field: ALERT_UUID, size: CHUNK_SIZE, }, @@ -94,7 +104,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper spaceId: options.spaceId, }); } catch (e) { - logger.debug('Enrichemnts failed'); + logger.debug('Enrichments failed'); } } @@ -146,6 +156,191 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper return { createdAlerts: [], errors: {}, alertsWereTruncated: false }; } }, + alertWithSuppression: async ( + alerts, + refresh, + suppressionWindow, + enrichAlerts, + currentTimeOverride + ) => { + const ruleDataClientWriter = await ruleDataClient.getWriter({ + namespace: options.spaceId, + }); + + // Only write alerts if: + // - writing is enabled + // AND + // - rule execution has not been cancelled due to timeout + // OR + // - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway + const writeAlerts = + ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); + + if (writeAlerts && alerts.length > 0) { + const commonRuleFields = getCommonAlertFields(options); + + // TODO: proper error handling if suppressionWindowStart is undefined + const suppressionWindowStart = dateMath.parse(suppressionWindow, { + forceNow: currentTimeOverride, + }); + if (!suppressionWindowStart) { + throw new Error('Failed to parse suppression window'); + } + + const suppressionAlertSearchRequest = { + body: { + size: alerts.length, + query: { + bool: { + filter: [ + { + range: { + [TIMESTAMP]: { + gte: suppressionWindowStart.toISOString(), + }, + }, + }, + { + term: { + [ALERT_INSTANCE_ID]: alerts.map((alert) => alert.instanceId), + }, + }, + ], + }, + }, + collapse: { + field: ALERT_INSTANCE_ID, + }, + sort: [ + { + '@timestamp': { + order: 'desc' as const, + }, + }, + ], + /*aggs: { + suppressionAlerts: { + terms: { + field: ALERT_INSTANCE_ID, + size: alerts.length, + include: alerts.map((alert) => alert.instanceId), + }, + aggs: { + docs: { + top_hits: { + sort: [ + { + [TIMESTAMP]: { + order: 'desc' as const, + }, + }, + ], + }, + }, + }, + }, + },*/ + }, + }; + + const response = await ruleDataClient + .getReader({ namespace: options.spaceId }) + .search( + suppressionAlertSearchRequest + ); + + if (!response.aggregations) { + throw new Error( + 'Expected to find aggregations on suppression alert search response' + ); + } + + const existingAlertsByInstanceId = response.hits.hits.reduce< + Record> + >((acc, hit) => { + acc[hit._source['kibana.alert.instance.id']] = hit; + return acc; + }, {}); + + const [duplicateAlerts, newAlerts] = partition( + alerts, + (alert) => existingAlertsByInstanceId[alert.instanceId] != null + ); + + const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { + const existingAlert = existingAlertsByInstanceId[alert.instanceId]; + const existingDocsCount = + existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; + return [ + { update: { _id: existingAlert._id } }, + { + doc: { + // TODO: add last detected at based on now or currentTimeOverride + [ALERT_SUPPRESSION_END]: alert._source[ALERT_SUPPRESSION_END], + [ALERT_SUPPRESSION_DOCS_COUNT]: + existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1, + }, + }, + ]; + }); + + let enrichedAlerts = newAlerts; + + if (enrichAlerts) { + try { + enrichedAlerts = await enrichAlerts(enrichedAlerts, { + spaceId: options.spaceId, + }); + } catch (e) { + logger.debug('Enrichments failed'); + } + } + + const augmentedAlerts = enrichedAlerts.map((alert) => { + return { + ...alert, + _source: { + [VERSION]: ruleDataClient.kibanaVersion, + ...commonRuleFields, + ...alert._source, + }, + }; + }); + + const newAlertCreates = augmentedAlerts.flatMap((alert) => [ + { create: { _id: alert._id } }, + alert._source, + ]); + + const bulkResponse = await ruleDataClientWriter.bulk({ + body: [...duplicateAlertUpdates, ...newAlertCreates], + refresh, + }); + + if (bulkResponse == null) { + return { createdAlerts: [], errors: {} }; + } + + return { + createdAlerts: augmentedAlerts + .map((alert, idx) => { + const responseItem = + bulkResponse.body.items[idx + duplicateAlertUpdates.length].create; + return { + _id: responseItem?._id ?? '', + _index: responseItem?._index ?? '', + ...alert._source, + }; + }) + .filter((_, idx) => bulkResponse.body.items[idx].create?.status === 201), + // updatedAlerts: TODO: add updated alerts in response and to context + errors: errorAggregator(bulkResponse.body, [409]), + }; + } else { + logger.debug('Writing is disabled.'); + return { createdAlerts: [], errors: {}, alertsWereTruncated: false }; + } + }, }, }); diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 63d9cfdd55aea..2ed587f8c95f5 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -18,6 +18,7 @@ import { WithoutReservedActionGroups } from '@kbn/alerting-plugin/common'; import { IRuleDataClient } from '../rule_data_client'; import { BulkResponseErrorAggregation } from './utils'; import { AlertWithCommonFieldsLatest } from '../../common/schemas'; +import { SuppressionFields } from '../../common/schemas/8.7.0'; export type PersistenceAlertService = ( alerts: Array<{ @@ -40,6 +41,21 @@ export type PersistenceAlertService = ( > ) => Promise>; +export type SuppressedAlertService = ( + alerts: Array<{ + _id: string; + _source: T; + instanceId: string; + }>, + refresh: boolean | 'wait_for', + suppressionWindow: string, + enrichAlerts?: ( + alerts: TAlert[], + params: { spaceId: string } + ) => Promise, + currentTimeOverride?: Date +) => Promise, 'alertsWereTruncated'>>; + export interface PersistenceAlertServiceResult { createdAlerts: Array & { _id: string; _index: string }>; errors: BulkResponseErrorAggregation; @@ -48,6 +64,7 @@ export interface PersistenceAlertServiceResult { export interface PersistenceServices { alertWithPersistence: PersistenceAlertService; + alertWithSuppression: SuppressedAlertService; } export type PersistenceAlertType< diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts index 4edb5c6885af1..4e4c8a8a2a486 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts @@ -6,7 +6,10 @@ */ import * as t from 'io-ts'; -import { LimitedSizeArray } from '@kbn/securitysolution-io-ts-types'; +import { + LimitedSizeArray, + PositiveIntegerGreaterThanZero, +} from '@kbn/securitysolution-io-ts-types'; export const AlertSuppressionGroupBy = LimitedSizeArray({ codec: t.string, @@ -14,16 +17,32 @@ export const AlertSuppressionGroupBy = LimitedSizeArray({ maxSize: 3, }); +export const AlertSuppressionDuration = t.type({ + value: PositiveIntegerGreaterThanZero, + unit: t.keyof({ + s: null, + m: null, + h: null, + }), +}); + /** * Schema for fields relating to alert suppression, which enables limiting the number of alerts per entity. * e.g. group_by: ['host.name'] would create only one alert per value of host.name. The created alert * contains metadata about how many other candidate alerts with the same host.name value were suppressed. */ export type AlertSuppression = t.TypeOf; -export const AlertSuppression = t.exact( - t.type({ - group_by: AlertSuppressionGroupBy, - }) -); +export const AlertSuppression = t.intersection([ + t.exact( + t.type({ + group_by: AlertSuppressionGroupBy, + }) + ), + t.exact( + t.partial({ + duration: AlertSuppressionDuration, + }) + ), +]); export const minimumLicenseForSuppression = 'platinum'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts new file mode 100644 index 0000000000000..160589a6b4270 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AlertWithCommonFields800 } from '@kbn/rule-registry-plugin/common/schemas/8.0.0'; +import type { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; + +import type { DetectionAlert840 } from '../8.4.0'; +import type { SuppressionFields860 } from '../8.6.0'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.6.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.6.0. +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export interface SuppressionFields870 extends SuppressionFields860 { + [ALERT_INSTANCE_ID]: string; +} + +export type SuppressionAlert870 = AlertWithCommonFields800; + +export type DetectionAlert870 = DetectionAlert840 | SuppressionAlert870; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts index 93436ffa52d6b..128675b4620c3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts @@ -17,11 +17,16 @@ import type { NewTermsFields840, } from './8.4.0'; -import type { DetectionAlert860, SuppressionFields860 } from './8.6.0'; +import type { DetectionAlert860 } from './8.6.0'; +import type { DetectionAlert870, SuppressionFields870 } from './8.7.0'; // When new Alert schemas are created for new Kibana versions, add the DetectionAlert type from the new version // here, e.g. `export type DetectionAlert = DetectionAlert800 | DetectionAlert820` if a new schema is created in 8.2.0 -export type DetectionAlert = DetectionAlert800 | DetectionAlert840 | DetectionAlert860; +export type DetectionAlert = + | DetectionAlert800 + | DetectionAlert840 + | DetectionAlert860 + | DetectionAlert870; export type { Ancestor840 as AncestorLatest, @@ -31,5 +36,5 @@ export type { EqlBuildingBlockFields840 as EqlBuildingBlockFieldsLatest, EqlShellFields840 as EqlShellFieldsLatest, NewTermsFields840 as NewTermsFieldsLatest, - SuppressionFields860 as SuppressionFieldsLatest, + SuppressionFields870 as SuppressionFieldsLatest, }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index d01bc59c0bbe3..e13c978d4f323 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -436,7 +436,13 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep } : { ...(ruleFields.groupByFields.length > 0 - ? { alert_suppression: { group_by: ruleFields.groupByFields } } + ? { + alert_suppression: { + group_by: ruleFields.groupByFields, + duration: + ruleFields.groupByDuration.value != null ? ruleFields.groupByDuration : undefined, + }, + } : {}), index: ruleFields.index, filters: ruleFields.queryBar?.filters, diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index a10fa66099ef0..352e23c802c5a 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -50,6 +50,7 @@ import type { import { defaultToEmptyTag } from '../../../../common/components/empty_value'; import { ThreatEuiFlexGroup } from './threat_description'; import type { LicenseService } from '../../../../../common/license'; +import type { Duration } from '../../../pages/detection_engine/rules/types'; const NoteDescriptionContainer = styled(EuiFlexItem)` height: 105px; @@ -560,3 +561,37 @@ export const buildAlertSuppressionDescription = ( }, ]; }; + +export const buildAlertSuppressionWindowDescription = ( + label: string, + value: Duration, + license: LicenseService +): ListItems[] => { + console.log(typeof value.value); + if (value.value == null) { + return []; + } + const description = `${value.value}${value.unit}`; + + const title = ( + <> + {label} + + {!license.isAtLeast(minimumLicenseForSuppression) && ( + + + + )} + + ); + return [ + { + title, + description, + }, + ]; +}; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index b9b2abffdc96d..6bf0ecf272aee 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -25,6 +25,7 @@ import { useKibana } from '../../../../common/lib/kibana'; import type { AboutStepRiskScore, AboutStepSeverity, + Duration, } from '../../../pages/detection_engine/rules/types'; import type { FieldValueTimeline } from '../pick_timeline'; import type { FormSchema } from '../../../../shared_imports'; @@ -44,6 +45,7 @@ import { buildEqlOptionsDescription, buildRequiredFieldsDescription, buildAlertSuppressionDescription, + buildAlertSuppressionWindowDescription, } from './helpers'; import { buildMlJobsDescription } from './build_ml_jobs_description'; import { buildActionsDescription } from './actions_description'; @@ -199,6 +201,9 @@ export const getDescriptionItem = ( } else if (field === 'groupByFields') { const values: string[] = get(field, data); return buildAlertSuppressionDescription(label, values, license); + } else if (field === 'groupByDuration') { + const value: Duration = get(field, data); + return buildAlertSuppressionWindowDescription(label, value, license); } else if (field === 'eqlOptions') { const eqlOptions: EqlOptionsSelected = get(field, data); return buildEqlOptionsDescription(eqlOptions); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx new file mode 100644 index 0000000000000..0d3fe66330797 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiFlexGroup, + EuiFlexItem, + EuiFieldNumber, + EuiFormRow, + EuiSelect, + EuiFormControlLayout, + transparentize, +} from '@elastic/eui'; +import React, { useCallback, useMemo } from 'react'; +import styled from 'styled-components'; + +import type { FieldHook } from '../../../../shared_imports'; +import { getFieldValidityAndErrorMessage } from '../../../../shared_imports'; +import * as I18n from './translations'; + +interface DurationInputProps { + durationValueField: FieldHook; + durationUnitField: FieldHook; + minimumValue?: number; + isDisabled: boolean; + durationUnitOptions?: Array<{ value: 's' | 'm' | 'h' | 'd'; text: string }>; +} + +// TODO: share with ScheduleItem impl +const getNumberFromUserInput = (input: string, minimumValue = 0): number | undefined => { + if (input == null || input.length === 0) { + return undefined; + } + const number = parseInt(input, 10); + if (Number.isNaN(number)) { + return minimumValue; + } else { + return Math.max(minimumValue, Math.min(number, Number.MAX_SAFE_INTEGER)); + } +}; + +// move optional label to the end of input +const StyledLabelAppend = styled(EuiFlexItem)` + &.euiFlexItem { + margin-left: 31px; + } +`; + +const StyledEuiFormRow = styled(EuiFormRow)` + max-width: none; + + .euiFormControlLayout { + max-width: 235px; + width: auto; + } + + .euiFormControlLayout__childrenWrapper > *:first-child { + box-shadow: none; + height: 38px; + width: 100%; + } + + .euiFormControlLayout__childrenWrapper > select { + background-color: ${({ theme }) => transparentize(theme.eui.euiColorPrimary, 0.1)}; + color: ${({ theme }) => theme.eui.euiColorPrimary}; + } + + .euiFormControlLayout--group .euiFormControlLayout { + min-width: 100px; + } + + .euiFormControlLayoutIcons { + color: ${({ theme }) => theme.eui.euiColorPrimary}; + } + + .euiFormControlLayout:not(:first-child) { + border-left: 1px solid ${({ theme }) => theme.eui.euiColorLightShade}; + } +`; + +const MyEuiSelect = styled(EuiSelect)` + width: auto; +`; + +const DurationInputComponent: React.FC = ({ + durationValueField, + durationUnitField, + minimumValue = 0, + isDisabled, + durationUnitOptions = [ + { value: 's', text: I18n.SECONDS }, + { value: 'm', text: I18n.MINUTES }, + { value: 'h', text: I18n.HOURS }, + ], +}: DurationInputProps) => { + const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(durationValueField); + const { value: durationValue, setValue: setDurationValue } = durationValueField; + const { value: durationUnit, setValue: setDurationUnit } = durationUnitField; + + const onChangeTimeType: React.ChangeEventHandler = useCallback( + (e) => { + setDurationUnit(e.target.value); + }, + [setDurationUnit] + ); + + const onChangeTimeVal: React.ChangeEventHandler = useCallback( + (e) => { + const sanitizedValue = getNumberFromUserInput(e.target.value, minimumValue); + setDurationValue(sanitizedValue); + }, + [minimumValue, setDurationValue] + ); + + // EUI missing some props + const rest = { disabled: isDisabled }; + const label = useMemo( + () => ( + + + {durationValueField.label} + + + {durationValueField.labelAppend} + + + ), + [durationValueField.label, durationValueField.labelAppend] + ); + + return ( + + + } + > + + + + ); +}; + +export const DurationInput = React.memo(DurationInputComponent); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/translations.ts b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/translations.ts new file mode 100644 index 0000000000000..c460d2f7198b3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/translations.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const SECONDS = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepScheduleRuleForm.secondsOptionDescription', + { + defaultMessage: 'Seconds', + } +); + +export const MINUTES = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepScheduleRuleForm.minutesOptionDescription', + { + defaultMessage: 'Minutes', + } +); + +export const HOURS = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepScheduleRuleForm.hoursOptionDescription', + { + defaultMessage: 'Hours', + } +); + +export const DAYS = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepScheduleRuleForm.daysOptionDescription', + { + defaultMessage: 'Days', + } +); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 554eb0df47774..e1b61b4fb5b54 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -83,6 +83,7 @@ import { getIsRulePreviewDisabled } from '../rule_preview/helpers'; import { GroupByFields } from '../group_by_fields'; import { useLicense } from '../../../../common/hooks/use_license'; import { minimumLicenseForSuppression } from '../../../../../common/detection_engine/rule_schema'; +import { DurationInput } from '../duration_input'; const CommonUseField = getUseField({ component: Field }); @@ -143,7 +144,7 @@ const StepDefineRuleComponent: FC = ({ const { form } = useForm({ defaultValue: initialState, - options: { stripEmptyFields: false }, + options: { stripEmptyFields: true }, schema, }); @@ -169,6 +170,8 @@ const StepDefineRuleComponent: FC = ({ 'historyWindowSize', 'shouldLoadQueryDynamically', 'groupByFields', + 'groupByDuration.value', + 'groupByDuration.unit', ], onChange: (data: DefineStepRule) => { if (onRuleDataChange) { @@ -201,6 +204,7 @@ const StepDefineRuleComponent: FC = ({ dataSourceType: formDataSourceType, newTermsFields, shouldLoadQueryDynamically: formShouldLoadQueryDynamically, + groupByFields, } = formData; const [isQueryBarValid, setIsQueryBarValid] = useState(false); @@ -489,6 +493,22 @@ const StepDefineRuleComponent: FC = ({ ] ); + const GroupByChildren = useCallback( + ({ groupByDurationUnit, groupByDurationValue }) => ( + + ), + [license, groupByFields] + ); + const dataViewIndexPatternToggleButtonOptions: EuiButtonGroupOptionProps[] = useMemo( () => [ { @@ -808,6 +828,20 @@ const StepDefineRuleComponent: FC = ({ }} /> + + + {GroupByChildren} + + <> diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index 1f47db53a1087..478e63bab53ea 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -605,6 +605,48 @@ export const schema: FormSchema = { }, ], }, + groupByDuration: { + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.groupByDurationValueLabel', + { + defaultMessage: 'Suppress alerts for', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldGroupByDurationValueHelpText', + { + defaultMessage: 'Suppress alerts for', + } + ), + value: { + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.groupByDurationValueLabel', + { + defaultMessage: 'Suppress alerts for', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldGroupByDurationValueHelpText', + { + defaultMessage: 'Suppress alerts for', + } + ), + }, + unit: { + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.groupByDurationUnitLabel', + { + defaultMessage: 'Suppress alerts for', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldGroupByDurationUnitHelpText', + { + defaultMessage: 'Suppress alerts for', + } + ), + }, + }, newTermsFields: { type: FIELD_TYPES.COMBO_BOX, label: i18n.translate( diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 0452644b12d00..b2fce1b119e54 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -136,6 +136,7 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ : '7d', shouldLoadQueryDynamically: Boolean(rule.type === 'saved_query' && rule.saved_id), groupByFields: rule.alert_suppression?.group_by ?? [], + groupByDuration: rule.alert_suppression?.duration ?? { unit: 'm' }, }); const convertHistoryStartToSize = (relativeTime: string) => { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 680249fb8f89f..cf238cdd434da 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -169,6 +169,12 @@ export interface DefineStepRule { historyWindowSize: string; shouldLoadQueryDynamically: boolean; groupByFields: string[]; + groupByDuration: Duration; +} + +export interface Duration { + value?: number; + unit: string; } export interface ScheduleStepRule { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts index e7cc2967ba62d..ff32debb4b3bd 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts @@ -144,6 +144,10 @@ export const stepDefineDefaultValue: DefineStepRule = { historyWindowSize: '7d', shouldLoadQueryDynamically: false, groupByFields: [], + groupByDuration: { + value: undefined, + unit: 'm', + }, }; export const stepAboutDefaultValue: AboutStepRule = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts index 173e1a5f5b906..275fa0afec201 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts @@ -365,6 +365,7 @@ export const convertAlertSuppressionToCamel = ( input ? { groupBy: input.group_by, + duration: input.duration, } : undefined; @@ -374,5 +375,6 @@ export const convertAlertSuppressionToSnake = ( input ? { group_by: input.groupBy, + duration: input.duration, } : undefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index 1be28852dadeb..65f739ba7a5d5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -24,6 +24,7 @@ import { threat_query, threatIndicatorPathOrUndefined, } from '@kbn/securitysolution-io-ts-alerting-types'; +import { NonEmptyString } from '@kbn/securitysolution-io-ts-types'; import { SIGNALS_ID, EQL_RULE_TYPE_ID, @@ -39,6 +40,7 @@ import type { SanitizedRuleConfig } from '@kbn/alerting-plugin/common'; import { AlertsIndex, AlertsIndexNamespace, + AlertSuppressionDuration, AlertSuppressionGroupBy, BuildingBlockType, DataViewId, @@ -87,11 +89,18 @@ import { ResponseActionRuleParamsOrUndefined } from '../../../../../common/detec const nonEqlLanguages = t.keyof({ kuery: null, lucene: null }); export type AlertSuppressionCamel = t.TypeOf; -const AlertSuppressionCamel = t.exact( - t.type({ - groupBy: AlertSuppressionGroupBy, - }) -); +const AlertSuppressionCamel = t.intersection([ + t.exact( + t.type({ + groupBy: AlertSuppressionGroupBy, + }) + ), + t.exact( + t.partial({ + duration: AlertSuppressionDuration, + }) + ), +]); export const baseRuleParams = t.exact( t.type({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 8ec879c1e814c..512732b6f3ff2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -85,6 +85,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = } = params; const { alertWithPersistence, + alertWithSuppression, savedObjectsClient, scopedClusterClient, uiSettingsClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts index e708e3d906efc..b3876c0dfd231 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts @@ -13,6 +13,7 @@ import { ALERT_SUPPRESSION_DOCS_COUNT, ALERT_SUPPRESSION_END, ALERT_SUPPRESSION_START, + ALERT_INSTANCE_ID, } from '@kbn/rule-data-utils'; import type { BaseFieldsLatest, @@ -33,6 +34,12 @@ export interface SuppressionBuckets { terms: Array<{ field: string; value: string | number | null }>; } +export const createSuppressedAlertInstanceId = ( + terms: Array<{ field: string; value: string | number | null }> +): string => { + return objectHash(terms); +}; + export const wrapSuppressedAlerts = ({ suppressionBuckets, spaceId, @@ -60,6 +67,7 @@ export const wrapSuppressedAlerts = ({ bucket.start, bucket.end, ]); + const instanceId = createSuppressedAlertInstanceId(bucket.terms); const baseAlert: BaseFieldsLatest = buildBulkBody( spaceId, completeRule, @@ -81,6 +89,7 @@ export const wrapSuppressedAlerts = ({ [ALERT_SUPPRESSION_END]: bucket.end, [ALERT_SUPPRESSION_DOCS_COUNT]: bucket.count - 1, [ALERT_UUID]: id, + [ALERT_INSTANCE_ID]: instanceId, }, }; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index e1ef6e5867859..859e0c34574eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -79,11 +79,10 @@ export const singleSearchAfter = async < }); const start = performance.now(); - const { body: nextSearchAfterResult } = - await services.scopedClusterClient.asCurrentUser.search( - searchAfterQuery as estypes.SearchRequest, - { meta: true } - ); + const nextSearchAfterResult = await services.scopedClusterClient.asCurrentUser.search< + SignalSource, + TAggregations + >(searchAfterQuery as estypes.SearchRequest); const end = performance.now(); const searchErrors = createErrorsFromShard({ From 382f0c3b7056df42d302dfc6024339a2eb2480ee Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 11 Jan 2023 09:41:18 -0800 Subject: [PATCH 02/22] Add radio button to rule create, implement alert update logic --- .../src/technical_field_names.ts | 3 + .../create_persistence_rule_type_wrapper.ts | 48 ++----- .../server/utils/persistence_types.ts | 7 +- .../pages/rule_creation/helpers.ts | 15 ++- .../rules/description_step/index.tsx | 17 ++- .../components/rules/duration_input/index.tsx | 33 +---- .../rules/step_define_rule/index.tsx | 44 ++++++- .../rules/step_define_rule/schema.tsx | 31 +---- .../pages/detection_engine/rules/helpers.tsx | 7 +- .../pages/detection_engine/rules/types.ts | 8 +- .../pages/detection_engine/rules/utils.ts | 5 +- .../create_security_rule_type_wrapper.ts | 2 + .../factories/bulk_create_with_suppression.ts | 123 ++++++++++++++++++ .../factories/utils/wrap_suppressed_alerts.ts | 22 +++- .../lib/detection_engine/rule_types/types.ts | 4 + .../group_and_bulk_create.ts | 26 +++- .../signals/enrichments/types.ts | 2 +- .../lib/detection_engine/signals/utils.ts | 2 +- 18 files changed, 270 insertions(+), 129 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts index f67ceac08ab59..458406ae40910 100644 --- a/packages/kbn-rule-data-utils/src/technical_field_names.ts +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -32,6 +32,7 @@ const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const; const ALERT_FLAPPING = `${ALERT_NAMESPACE}.flapping` as const; const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const; +const ALERT_LAST_DETECTED = `${ALERT_NAMESPACE}.last_detected_at` as const; const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const; const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const; const ALERT_SEVERITY = `${ALERT_NAMESPACE}.severity` as const; @@ -129,6 +130,7 @@ const fields = { ALERT_EVALUATION_VALUE, ALERT_FLAPPING, ALERT_INSTANCE_ID, + ALERT_LAST_DETECTED, ALERT_RULE_CONSUMER, ALERT_RULE_PRODUCER, ALERT_REASON, @@ -199,6 +201,7 @@ export { ALERT_EVALUATION_VALUE, ALERT_FLAPPING, ALERT_INSTANCE_ID, + ALERT_LAST_DETECTED, ALERT_NAMESPACE, ALERT_RULE_NAMESPACE, ALERT_RULE_CONSUMER, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index aa22eabfcdf8d..ac0d5e80634e8 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -10,6 +10,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { chunk, partition } from 'lodash'; import { ALERT_INSTANCE_ID, + ALERT_LAST_DETECTED, ALERT_SUPPRESSION_DOCS_COUNT, ALERT_SUPPRESSION_END, ALERT_UUID, @@ -201,8 +202,10 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, }, { - term: { - [ALERT_INSTANCE_ID]: alerts.map((alert) => alert.instanceId), + terms: { + [ALERT_INSTANCE_ID]: alerts.map( + (alert) => alert._source['kibana.alert.instance.id'] + ), }, }, ], @@ -218,28 +221,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, }, ], - /*aggs: { - suppressionAlerts: { - terms: { - field: ALERT_INSTANCE_ID, - size: alerts.length, - include: alerts.map((alert) => alert.instanceId), - }, - aggs: { - docs: { - top_hits: { - sort: [ - { - [TIMESTAMP]: { - order: 'desc' as const, - }, - }, - ], - }, - }, - }, - }, - },*/ }, }; @@ -249,12 +230,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper suppressionAlertSearchRequest ); - if (!response.aggregations) { - throw new Error( - 'Expected to find aggregations on suppression alert search response' - ); - } - const existingAlertsByInstanceId = response.hits.hits.reduce< Record> >((acc, hit) => { @@ -264,18 +239,20 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper const [duplicateAlerts, newAlerts] = partition( alerts, - (alert) => existingAlertsByInstanceId[alert.instanceId] != null + (alert) => + existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']] != null ); const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { - const existingAlert = existingAlertsByInstanceId[alert.instanceId]; + const existingAlert = + existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']]; const existingDocsCount = existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; return [ { update: { _id: existingAlert._id } }, { doc: { - // TODO: add last detected at based on now or currentTimeOverride + [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), [ALERT_SUPPRESSION_END]: alert._source[ALERT_SUPPRESSION_END], [ALERT_SUPPRESSION_DOCS_COUNT]: existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1, @@ -301,6 +278,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ...alert, _source: { [VERSION]: ruleDataClient.kibanaVersion, + [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), ...commonRuleFields, ...alert._source, }, @@ -314,7 +292,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper const bulkResponse = await ruleDataClientWriter.bulk({ body: [...duplicateAlertUpdates, ...newAlertCreates], - refresh, + refresh: 'wait_for', }); if (bulkResponse == null) { @@ -338,7 +316,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }; } else { logger.debug('Writing is disabled.'); - return { createdAlerts: [], errors: {}, alertsWereTruncated: false }; + return { createdAlerts: [], errors: {} }; } }, }, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 2ed587f8c95f5..8f5832955d76a 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -45,14 +45,13 @@ export type SuppressedAlertService = ( alerts: Array<{ _id: string; _source: T; - instanceId: string; }>, refresh: boolean | 'wait_for', suppressionWindow: string, - enrichAlerts?: ( - alerts: TAlert[], + enrichAlerts?: ( + alerts: Array<{ _id: string; _source: T }>, params: { spaceId: string } - ) => Promise, + ) => Promise>, currentTimeOverride?: Date ) => Promise, 'alertsWereTruncated'>>; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index e13c978d4f323..5442727561ce1 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -5,6 +5,8 @@ * 2.0. */ +/* eslint-disable complexity */ + import { has, isEmpty } from 'lodash/fp'; import type { Unit } from '@kbn/datemath'; import moment from 'moment'; @@ -42,7 +44,10 @@ import type { RuleStepsFormData, RuleStep, } from '../../../../detections/pages/detection_engine/rules/types'; -import { DataSourceType } from '../../../../detections/pages/detection_engine/rules/types'; +import { + DataSourceType, + GroupByOptions, +} from '../../../../detections/pages/detection_engine/rules/types'; import type { RuleCreateProps } from '../../../../../common/detection_engine/rule_schema'; import { stepActionsDefaultValue } from '../../../../detections/components/rules/step_rule_actions'; @@ -440,7 +445,9 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep alert_suppression: { group_by: ruleFields.groupByFields, duration: - ruleFields.groupByDuration.value != null ? ruleFields.groupByDuration : undefined, + ruleFields.groupByRadioSelection === GroupByOptions.PerTimePeriod + ? ruleFields.groupByDuration + : undefined, }, } : {}), @@ -449,12 +456,12 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep language: ruleFields.queryBar?.query?.language, query: ruleFields.queryBar?.query?.query as string, saved_id: undefined, - type: 'query' as Type, + type: 'query' as const, // rule only be updated as saved_query type if it has saved_id and shouldLoadQueryDynamically checkbox checked ...(['query', 'saved_query'].includes(ruleType) && ruleFields.queryBar?.saved_id && ruleFields.shouldLoadQueryDynamically && { - type: 'saved_query' as Type, + type: 'saved_query' as const, query: undefined, filters: undefined, saved_id: ruleFields.queryBar.saved_id, diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index 6bf0ecf272aee..59c4e74fe3e92 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -27,6 +27,7 @@ import type { AboutStepSeverity, Duration, } from '../../../pages/detection_engine/rules/types'; +import { GroupByOptions } from '../../../pages/detection_engine/rules/types'; import type { FieldValueTimeline } from '../pick_timeline'; import type { FormSchema } from '../../../../shared_imports'; import type { ListItems } from './types'; @@ -201,9 +202,21 @@ export const getDescriptionItem = ( } else if (field === 'groupByFields') { const values: string[] = get(field, data); return buildAlertSuppressionDescription(label, values, license); + } else if (field === 'groupByRadioSelection') { + return []; } else if (field === 'groupByDuration') { - const value: Duration = get(field, data); - return buildAlertSuppressionWindowDescription(label, value, license); + if (get('groupByRadioSelection', data) === GroupByOptions.PerTimePeriod) { + const value: Duration = get(field, data); + return buildAlertSuppressionWindowDescription(label, value, license); + } else if (get('groupByRadioSelection', data) === GroupByOptions.PerRuleExecution) { + return [ + { + title: label, + // TODO: copywriting and i18n + description: 'One rule execution', + }, + ]; + } } else if (field === 'eqlOptions') { const eqlOptions: EqlOptionsSelected = get(field, data); return buildEqlOptionsDescription(eqlOptions); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx index 0d3fe66330797..1ca0b9e606139 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx @@ -6,15 +6,13 @@ */ import { - EuiFlexGroup, - EuiFlexItem, EuiFieldNumber, EuiFormRow, EuiSelect, EuiFormControlLayout, transparentize, } from '@elastic/eui'; -import React, { useCallback, useMemo } from 'react'; +import React, { useCallback } from 'react'; import styled from 'styled-components'; import type { FieldHook } from '../../../../shared_imports'; @@ -42,13 +40,6 @@ const getNumberFromUserInput = (input: string, minimumValue = 0): number | undef } }; -// move optional label to the end of input -const StyledLabelAppend = styled(EuiFlexItem)` - &.euiFlexItem { - margin-left: 31px; - } -`; - const StyledEuiFormRow = styled(EuiFormRow)` max-width: none; @@ -117,29 +108,9 @@ const DurationInputComponent: React.FC = ({ // EUI missing some props const rest = { disabled: isDisabled }; - const label = useMemo( - () => ( - - - {durationValueField.label} - - - {durationValueField.labelAppend} - - - ), - [durationValueField.label, durationValueField.labelAppend] - ); return ( - + = ({ 'historyWindowSize', 'shouldLoadQueryDynamically', 'groupByFields', + 'groupByRadioSelection', 'groupByDuration.value', 'groupByDuration.unit', ], @@ -494,16 +500,37 @@ const StepDefineRuleComponent: FC = ({ ); const GroupByChildren = useCallback( - ({ groupByDurationUnit, groupByDurationValue }) => ( - ( + + {`Per time period`} + + + ), + }, + ]} + onChange={(id: string) => { + groupByRadioSelection.setValue(id); + }} /> ), [license, groupByFields] @@ -831,6 +858,9 @@ const StepDefineRuleComponent: FC = ({ = { }, ], }, + groupByRadioSelection: {}, groupByDuration: { label: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.groupByDurationValueLabel', @@ -618,34 +619,8 @@ export const schema: FormSchema = { defaultMessage: 'Suppress alerts for', } ), - value: { - label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.groupByDurationValueLabel', - { - defaultMessage: 'Suppress alerts for', - } - ), - helpText: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldGroupByDurationValueHelpText', - { - defaultMessage: 'Suppress alerts for', - } - ), - }, - unit: { - label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.groupByDurationUnitLabel', - { - defaultMessage: 'Suppress alerts for', - } - ), - helpText: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldGroupByDurationUnitHelpText', - { - defaultMessage: 'Suppress alerts for', - } - ), - }, + value: {}, + unit: {}, }, newTermsFields: { type: FIELD_TYPES.COMBO_BOX, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index b2fce1b119e54..a290ac92f6a66 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -37,7 +37,7 @@ import type { ScheduleStepRule, ActionsStepRule, } from './types'; -import { DataSourceType } from './types'; +import { DataSourceType, GroupByOptions } from './types'; import { severityOptions } from '../../../components/rules/step_about_rule/data'; export interface GetStepsData { @@ -136,7 +136,10 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ : '7d', shouldLoadQueryDynamically: Boolean(rule.type === 'saved_query' && rule.saved_id), groupByFields: rule.alert_suppression?.group_by ?? [], - groupByDuration: rule.alert_suppression?.duration ?? { unit: 'm' }, + groupByRadioSelection: rule.alert_suppression?.duration + ? GroupByOptions.PerTimePeriod + : GroupByOptions.PerRuleExecution, + groupByDuration: rule.alert_suppression?.duration ?? { value: 5, unit: 'm' }, }); const convertHistoryStartToSize = (relativeTime: string) => { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index cf238cdd434da..edbb3b4ecbf97 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -143,6 +143,11 @@ export enum DataSourceType { DataView = 'dataView', } +export enum GroupByOptions { + PerRuleExecution = 'per-rule-execution', + PerTimePeriod = 'per-time-period', +} + /** * add / update data source types to show XOR relationship between 'index' and 'dataViewId' fields * Maybe something with io-ts? @@ -169,11 +174,12 @@ export interface DefineStepRule { historyWindowSize: string; shouldLoadQueryDynamically: boolean; groupByFields: string[]; + groupByRadioSelection: GroupByOptions; groupByDuration: Duration; } export interface Duration { - value?: number; + value: number; unit: string; } diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts index ff32debb4b3bd..f48c35cecb425 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts @@ -18,7 +18,7 @@ import type { RouteSpyState } from '../../../../common/utils/route/types'; import { SecurityPageName } from '../../../../app/types'; import { DEFAULT_THREAT_MATCH_QUERY, RULES_PATH } from '../../../../../common/constants'; import type { AboutStepRule, DefineStepRule, RuleStepsOrder, ScheduleStepRule } from './types'; -import { DataSourceType, RuleStep } from './types'; +import { DataSourceType, GroupByOptions, RuleStep } from './types'; import type { GetSecuritySolutionUrl } from '../../../../common/components/link_to'; import { RuleDetailTabs, @@ -144,8 +144,9 @@ export const stepDefineDefaultValue: DefineStepRule = { historyWindowSize: '7d', shouldLoadQueryDynamically: false, groupByFields: [], + groupByRadioSelection: GroupByOptions.PerRuleExecution, groupByDuration: { - value: undefined, + value: 5, unit: 'm', }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 512732b6f3ff2..968ea96a42174 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -343,6 +343,8 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = ruleExecutionLogger, aggregatableTimestampField, alertTimestampOverride, + alertWithSuppression, + refreshOnIndexingAlerts: refresh, }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts new file mode 100644 index 0000000000000..cd4a156611f0e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { performance } from 'perf_hooks'; +import { isEmpty } from 'lodash'; + +import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; +import type { AlertWithCommonFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import { makeFloatString } from '../../signals/utils'; +import type { RefreshTypes } from '../../types'; +import type { + BaseFieldsLatest, + SuppressionFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/detection_engine/schemas/alerts'; +import type { RuleServices } from '../../signals/types'; +import { createEnrichEventsFunction } from '../../signals/enrichments'; + +export interface GenericBulkCreateResponse { + success: boolean; + bulkCreateDuration: string; + enrichmentDuration: string; + createdItemsCount: number; + createdItems: Array & { _id: string; _index: string }>; + errors: string[]; +} + +export const bulkCreateWithSuppression = async ({ + alertWithSuppression, + refreshForBulkCreate, + ruleExecutionLogger, + wrappedDocs, + services, + suppressionWindow, + alertTimestampOverride, +}: { + alertWithSuppression: SuppressedAlertService; + refreshForBulkCreate: RefreshTypes; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + wrappedDocs: Array>; + services: RuleServices; + suppressionWindow: string; + alertTimestampOverride: Date | undefined; +}): Promise> => { + if (wrappedDocs.length === 0) { + return { + errors: [], + success: true, + enrichmentDuration: '0', + bulkCreateDuration: '0', + createdItemsCount: 0, + createdItems: [], + }; + } + + const start = performance.now(); + + const enrichAlerts = createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }); + + let enrichmentsTimeStart = 0; + let enrichmentsTimeFinish = 0; + const enrichAlertsWrapper: typeof enrichAlerts = async (alerts, params) => { + enrichmentsTimeStart = performance.now(); + try { + const enrichedAlerts = await enrichAlerts(alerts, params); + return enrichedAlerts; + } catch (error) { + ruleExecutionLogger.error(`Enrichments failed ${error}`); + throw error; + } finally { + enrichmentsTimeFinish = performance.now(); + } + }; + + const { createdAlerts, errors } = await alertWithSuppression( + wrappedDocs.map((doc) => ({ + _id: doc._id, + // `fields` should have already been merged into `doc._source` + _source: doc._source, + })), + refreshForBulkCreate, + suppressionWindow, + enrichAlertsWrapper, + alertTimestampOverride + ); + + const end = performance.now(); + + ruleExecutionLogger.debug( + `individual bulk process time took: ${makeFloatString(end - start)} milliseconds` + ); + + if (!isEmpty(errors)) { + ruleExecutionLogger.debug( + `[-] bulkResponse had errors with responses of: ${JSON.stringify(errors)}` + ); + return { + errors: Object.keys(errors), + success: false, + enrichmentDuration: makeFloatString(enrichmentsTimeFinish - enrichmentsTimeStart), + bulkCreateDuration: makeFloatString(end - start), + createdItemsCount: createdAlerts.length, + createdItems: createdAlerts, + }; + } else { + return { + errors: [], + success: true, + bulkCreateDuration: makeFloatString(end - start), + enrichmentDuration: makeFloatString(enrichmentsTimeFinish - enrichmentsTimeStart), + createdItemsCount: createdAlerts.length, + createdItems: createdAlerts, + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts index b3876c0dfd231..cdcff1bcfbcf9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts @@ -34,10 +34,16 @@ export interface SuppressionBuckets { terms: Array<{ field: string; value: string | number | null }>; } -export const createSuppressedAlertInstanceId = ( - terms: Array<{ field: string; value: string | number | null }> -): string => { - return objectHash(terms); +export const createSuppressedAlertInstanceId = ({ + terms, + ruleId, + spaceId, +}: { + terms: Array<{ field: string; value: string | number | null }>; + ruleId: string; + spaceId: string; +}): string => { + return objectHash([terms, ruleId, spaceId]); }; export const wrapSuppressedAlerts = ({ @@ -50,7 +56,7 @@ export const wrapSuppressedAlerts = ({ alertTimestampOverride, }: { suppressionBuckets: SuppressionBuckets[]; - spaceId: string | null | undefined; + spaceId: string; completeRule: CompleteRule; mergeStrategy: ConfigType['alertMergeStrategy']; indicesToQuery: string[]; @@ -67,7 +73,11 @@ export const wrapSuppressedAlerts = ({ bucket.start, bucket.end, ]); - const instanceId = createSuppressedAlertInstanceId(bucket.terms); + const instanceId = createSuppressedAlertInstanceId({ + terms: bucket.terms, + ruleId: completeRule.alertId, + spaceId, + }); const baseAlert: BaseFieldsLatest = buildBulkBody( spaceId, completeRule, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 4bbc32b371108..614236391d041 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -25,6 +25,7 @@ import type { PersistenceServices, IRuleDataClient, IRuleDataReader, + SuppressedAlertService, } from '@kbn/rule-registry-plugin/server'; import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; @@ -41,6 +42,7 @@ import type { import type { ExperimentalFeatures } from '../../../../common/experimental_features'; import type { ITelemetryEventsSender } from '../../telemetry/sender'; import type { IRuleExecutionLogForExecutors, IRuleExecutionLogService } from '../rule_monitoring'; +import type { RefreshTypes } from '../types'; export interface SecurityAlertTypeReturnValue { bulkCreateTimes: string[]; @@ -79,6 +81,8 @@ export interface RunOpts { unprocessedExceptions: ExceptionListItemSchema[]; exceptionFilter: Filter | undefined; alertTimestampOverride: Date | undefined; + alertWithSuppression: SuppressedAlertService; + refreshOnIndexingAlerts: RefreshTypes; } export type SecurityAlertType< diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts index 690b0f698f306..ceec94e6c0a54 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts @@ -21,6 +21,7 @@ import type { SuppressionBuckets } from '../../rule_types/factories/utils/wrap_s import { wrapSuppressedAlerts } from '../../rule_types/factories/utils/wrap_suppressed_alerts'; import { buildGroupByFieldAggregation } from './build_group_by_field_aggregation'; import { singleSearchAfter } from '../single_search_after'; +import { bulkCreateWithSuppression } from '../../rule_types/factories/bulk_create_with_suppression'; export interface BucketHistory { key: Record; @@ -187,11 +188,25 @@ export const groupAndBulkCreate = async ({ alertTimestampOverride: runOpts.alertTimestampOverride, }); - const bulkCreateResult = await runOpts.bulkCreate(wrappedAlerts); - - addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); - - runOpts.ruleExecutionLogger.debug(`created ${bulkCreateResult.createdItemsCount} signals`); + const suppressionDuration = runOpts.completeRule.ruleParams.alertSuppression?.duration; + + if (suppressionDuration) { + const suppressionWindow = `now-${suppressionDuration.value}${suppressionDuration.unit}`; + const bulkCreateResult = await bulkCreateWithSuppression({ + alertWithSuppression: runOpts.alertWithSuppression, + refreshForBulkCreate: runOpts.refreshOnIndexingAlerts, + ruleExecutionLogger: runOpts.ruleExecutionLogger, + wrappedDocs: wrappedAlerts, + services, + suppressionWindow, + alertTimestampOverride: runOpts.alertTimestampOverride, + }); + addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); + } else { + const bulkCreateResult = await runOpts.bulkCreate(wrappedAlerts); + addToSearchAfterReturn({ current: toReturn, next: bulkCreateResult }); + runOpts.ruleExecutionLogger.debug(`created ${bulkCreateResult.createdItemsCount} signals`); + } const newBucketHistory: BucketHistory[] = buckets .filter((bucket) => { @@ -207,6 +222,7 @@ export const groupAndBulkCreate = async ({ }; }); + // TODO: combine history buckets with identical keys toReturn.state.suppressionGroupHistory.push(...newBucketHistory); } catch (exc) { toReturn.success = false; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts index ef0864c32cbd1..244311fe67df5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/types.ts @@ -99,7 +99,7 @@ export type EnrichEventsFunction = ( } ) => Promise>>; -export type CreateEnrichEventsFunction = (params: { +export type CreateEnrichEventsFunction = (params: { services: RuleServices; logger: IRuleExecutionLogForExecutors; }) => EnrichEvents; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index fbf538af284d9..379bc59318ac2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -711,7 +711,7 @@ export const addToSearchAfterReturn = ({ next, }: { current: SearchAfterAndBulkCreateReturnType; - next: GenericBulkCreateResponse; + next: Omit, 'alertsWereTruncated'>; }) => { current.success = current.success && next.success; current.createdSignalsCount += next.createdItemsCount; From bf6ec62e972b3fea900bd21a365ed74be1f39263 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 12 Jan 2023 11:20:49 -0800 Subject: [PATCH 03/22] Add explicit index to bulk updates --- .../server/utils/create_persistence_rule_type_wrapper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index ac0d5e80634e8..78f7aa40049f7 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -249,7 +249,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper const existingDocsCount = existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; return [ - { update: { _id: existingAlert._id } }, + { update: { _id: existingAlert._id, _index: existingAlert._index } }, { doc: { [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), From b210b3b6c2b95cdfab3c5fa57d5feb5fe208c62f Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 12 Jan 2023 16:45:01 -0800 Subject: [PATCH 04/22] Fix deduplication bug with null values, add tests --- .../create_persistence_rule_type_wrapper.ts | 13 +- .../server/utils/persistence_types.ts | 1 - .../factories/bulk_create_with_suppression.ts | 4 - .../utils/wrap_suppressed_alerts.test.ts | 83 ++++++ .../group_and_bulk_create.ts | 45 +-- .../rule_execution_logic/query.ts | 259 ++++++++++++++++++ 6 files changed, 374 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.test.ts diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 78f7aa40049f7..5aed686ee8bf4 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -159,7 +159,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, alertWithSuppression: async ( alerts, - refresh, suppressionWindow, enrichAlerts, currentTimeOverride @@ -180,7 +179,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper if (writeAlerts && alerts.length > 0) { const commonRuleFields = getCommonAlertFields(options); - // TODO: proper error handling if suppressionWindowStart is undefined const suppressionWindowStart = dateMath.parse(suppressionWindow, { forceNow: currentTimeOverride, }); @@ -249,7 +247,13 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper const existingDocsCount = existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; return [ - { update: { _id: existingAlert._id, _index: existingAlert._index } }, + { + update: { + _id: existingAlert._id, + _index: existingAlert._index, + require_alias: false, + }, + }, { doc: { [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), @@ -292,7 +296,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper const bulkResponse = await ruleDataClientWriter.bulk({ body: [...duplicateAlertUpdates, ...newAlertCreates], - refresh: 'wait_for', + refresh: true, }); if (bulkResponse == null) { @@ -311,7 +315,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }; }) .filter((_, idx) => bulkResponse.body.items[idx].create?.status === 201), - // updatedAlerts: TODO: add updated alerts in response and to context errors: errorAggregator(bulkResponse.body, [409]), }; } else { diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 8f5832955d76a..5dcb63c40d899 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -46,7 +46,6 @@ export type SuppressedAlertService = ( _id: string; _source: T; }>, - refresh: boolean | 'wait_for', suppressionWindow: string, enrichAlerts?: ( alerts: Array<{ _id: string; _source: T }>, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts index cd4a156611f0e..36ae3051547f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts @@ -12,7 +12,6 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { AlertWithCommonFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { makeFloatString } from '../../signals/utils'; -import type { RefreshTypes } from '../../types'; import type { BaseFieldsLatest, SuppressionFieldsLatest, @@ -32,7 +31,6 @@ export interface GenericBulkCreateResponse { export const bulkCreateWithSuppression = async ({ alertWithSuppression, - refreshForBulkCreate, ruleExecutionLogger, wrappedDocs, services, @@ -40,7 +38,6 @@ export const bulkCreateWithSuppression = async >; services: RuleServices; @@ -86,7 +83,6 @@ export const bulkCreateWithSuppression = async { + test('should generate unique instance IDs', () => { + expect( + createSuppressedAlertInstanceId({ + terms: [ + { field: 'host.name', value: 'host-1' }, + { field: 'user.id', value: 2 }, + ], + ruleId: 'abc', + spaceId: 'default', + }) + ).toEqual('89a50ea19a73a06ceb2f1882de27be64ef6bb4c0'); + + expect( + createSuppressedAlertInstanceId({ + terms: [ + { field: 'host.name', value: 'host-1' }, + { field: 'user.id', value: 3 }, + ], + ruleId: 'abc', + spaceId: 'default', + }) + ).toEqual('b40340d217f64a6b97aaa1d3d3120d3b6a1a6955'); + + expect( + createSuppressedAlertInstanceId({ + terms: [ + { field: 'host.name', value: 'host-1' }, + { field: 'user.id', value: 2 }, + ], + ruleId: 'abc', + spaceId: 'other-space', + }) + ).toEqual('8dace33aaeea3e38cbe608b8ac21439b3a9fac05'); + + expect( + createSuppressedAlertInstanceId({ + terms: [ + { field: 'host.name', value: 'host-1' }, + { field: 'user.id', value: 2 }, + ], + ruleId: 'other-id', + spaceId: 'default', + }) + ).toEqual('2c7b540ae3a3310271f7cbc44a3ba49fd840636b'); + + expect( + createSuppressedAlertInstanceId({ + terms: [ + { field: 'host.name', value: 'host-2' }, + { field: 'user.id', value: 2 }, + ], + ruleId: 'abc', + spaceId: 'default', + }) + ).toEqual('d8505a6fe8fd4c9b74367b89a7fd19a72ed1077b'); + + expect( + createSuppressedAlertInstanceId({ + terms: [{ field: 'host.name', value: 'host-1' }], + ruleId: 'abc', + spaceId: 'default', + }) + ).toEqual('657c4202086c305bb4406217099cd586e27417f7'); + + expect( + createSuppressedAlertInstanceId({ + terms: [{ field: 'host.name', value: null }], + ruleId: 'abc', + spaceId: 'default', + }) + ).toEqual('879f4db2774775831acd1b1477109a3757540b8c'); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts index ceec94e6c0a54..a5b0a30b670fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/alert_suppression/group_and_bulk_create.ts @@ -24,7 +24,7 @@ import { singleSearchAfter } from '../single_search_after'; import { bulkCreateWithSuppression } from '../../rule_types/factories/bulk_create_with_suppression'; export interface BucketHistory { - key: Record; + key: Record; endDate: string; } @@ -51,11 +51,21 @@ export const buildBucketHistoryFilter = ({ must_not: bucketHistory.map((bucket) => ({ bool: { filter: [ - ...Object.entries(bucket.key).map(([field, value]) => ({ - term: { - [field]: value, - }, - })), + ...Object.entries(bucket.key).map(([field, value]) => + value != null + ? { + term: { + [field]: value, + }, + } + : { + must_not: { + exists: { + field, + }, + }, + } + ), buildTimeRangeFilter({ to: bucket.endDate, from: from.toISOString(), @@ -194,7 +204,6 @@ export const groupAndBulkCreate = async ({ const suppressionWindow = `now-${suppressionDuration.value}${suppressionDuration.unit}`; const bulkCreateResult = await bulkCreateWithSuppression({ alertWithSuppression: runOpts.alertWithSuppression, - refreshForBulkCreate: runOpts.refreshOnIndexingAlerts, ruleExecutionLogger: runOpts.ruleExecutionLogger, wrappedDocs: wrappedAlerts, services, @@ -208,21 +217,15 @@ export const groupAndBulkCreate = async ({ runOpts.ruleExecutionLogger.debug(`created ${bulkCreateResult.createdItemsCount} signals`); } - const newBucketHistory: BucketHistory[] = buckets - .filter((bucket) => { - return !Object.values(bucket.key).includes(null); - }) - .map((bucket) => { - return { - // This cast should be safe as we just filtered out buckets where any key has a null value. - key: bucket.key as Record, - endDate: bucket.max_timestamp.value_as_string - ? bucket.max_timestamp.value_as_string - : tuple.to.toISOString(), - }; - }); + const newBucketHistory: BucketHistory[] = buckets.map((bucket) => { + return { + key: bucket.key, + endDate: bucket.max_timestamp.value_as_string + ? bucket.max_timestamp.value_as_string + : tuple.to.toISOString(), + }; + }); - // TODO: combine history buckets with identical keys toReturn.state.suppressionGroupHistory.push(...newBucketHistory); } catch (exc) { toReturn.success = false; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts index 074989e424cfe..ce8f400c9a21e 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -16,6 +16,7 @@ import { ALERT_SUPPRESSION_END, ALERT_SUPPRESSION_DOCS_COUNT, ALERT_SUPPRESSION_TERMS, + TIMESTAMP, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; @@ -673,6 +674,264 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_DOCS_COUNT]: 16, }); }); + + it('should correctly deduplicate null values across rule runs', async () => { + // We set a long lookback then run the rule twice so both runs cover all documents from the test data. + // The last alert, with null for destination.ip, should be found by the first rule run but not duplicated + // by the second run. + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['suppression-data']), + query: `*:*`, + alert_suppression: { + group_by: ['destination.ip'], + }, + from: 'now-2h', + interval: '1h', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 1000, + sort: ['destination.ip'], + }); + expect(previewAlerts.length).to.eql(3); + + // We also expect to have a separate group for documents that don't populate the groupBy field + expect(previewAlerts[2]._source).to.eql({ + ...previewAlerts[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'destination.ip', + value: null, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 34, + }); + }); + + describe('with a suppression time window', async () => { + it('should generate an alert per rule run when duration is less than rule interval', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['suppression-data']), + query: `host.name: "host-0"`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 30, + unit: 'm', + }, + }, + from: 'now-1h', + interval: '1h', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).to.eql(2); + expect(previewAlerts[0]._source).to.eql({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T05:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 5, + }); + expect(previewAlerts[1]._source).to.eql({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 5, + }); + }); + + it('should update an existing alert in the time window', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['suppression-data']), + query: `host.name: "host-0"`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 2, + unit: 'h', + }, + }, + from: 'now-1h', + interval: '1h', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).to.eql(1); + expect(previewAlerts[0]._source).to.eql({ + ...previewAlerts[0]._source, + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 11, + }); + }); + + it('should update the correct alerts based on group_by field-value pair', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['suppression-data']), + query: `host.name: *`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 2, + unit: 'h', + }, + }, + from: 'now-1h', + interval: '1h', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).to.eql(3); + expect(previewAlerts[0]._source).to.eql({ + ...previewAlerts[0]._source, + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 11, + }); + expect(previewAlerts[1]._source).to.eql({ + ...previewAlerts[1]._source, + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-1', + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 11, + }); + expect(previewAlerts[2]._source).to.eql({ + ...previewAlerts[2]._source, + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-2', + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 11, + }); + }); + + it('should update the correct alerts based on group_by field-value pair even when value is null', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['suppression-data']), + query: `host.name: *`, + alert_suppression: { + group_by: ['destination.ip'], // Only 1 document populates destination.ip + duration: { + value: 2, + unit: 'h', + }, + }, + from: 'now-1h', + interval: '1h', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['destination.ip', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).to.eql(3); + expect(previewAlerts[2]._source).to.eql({ + ...previewAlerts[2]._source, + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'destination.ip', + value: null, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 34, + }); + }); + }); }); }); }; From 5d52a88c1c8c143eb9d7f50d1a59c30d273a9780 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 12 Jan 2023 18:30:51 -0800 Subject: [PATCH 05/22] Cleanup --- .../create_persistence_rule_type_wrapper.ts | 1 - .../rules/description_step/helpers.tsx | 15 ++-- .../rules/description_step/index.tsx | 20 +++-- .../rules/description_step/translations.ts | 7 ++ .../components/rules/duration_input/index.tsx | 7 +- .../rule_execution_logic/query.ts | 76 +++++++++++++++++-- .../es_archives/entity/host_risk/data.json | 20 +++++ 7 files changed, 118 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 5aed686ee8bf4..3ffc323549353 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -68,7 +68,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper aggs: { uuids: { terms: { - // TODO: we can probably use `include` here instead of the query above on `ids` field: ALERT_UUID, size: CHUNK_SIZE, }, diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index 826c7e4340b84..046ae474517f2 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -45,11 +45,12 @@ import { SeverityBadge } from '../severity_badge'; import type { AboutStepRiskScore, AboutStepSeverity, + Duration, } from '../../../pages/detection_engine/rules/types'; +import { GroupByOptions } from '../../../pages/detection_engine/rules/types'; import { defaultToEmptyTag } from '../../../../common/components/empty_value'; import { ThreatEuiFlexGroup } from './threat_description'; import type { LicenseService } from '../../../../../common/license'; -import type { Duration } from '../../../pages/detection_engine/rules/types'; const NoteDescriptionContainer = styled(EuiFlexItem)` height: 105px; @@ -560,13 +561,13 @@ export const buildAlertSuppressionDescription = ( export const buildAlertSuppressionWindowDescription = ( label: string, value: Duration, - license: LicenseService + license: LicenseService, + groupByRadioSelection: GroupByOptions ): ListItems[] => { - console.log(typeof value.value); - if (value.value == null) { - return []; - } - const description = `${value.value}${value.unit}`; + const description = + groupByRadioSelection === GroupByOptions.PerTimePeriod + ? `${value.value}${value.unit}` + : i18n.ALERT_SUPPRESSION_PER_RULE_EXECUTION; const title = ( <> diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index 59c4e74fe3e92..8c47d40f49383 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -27,7 +27,6 @@ import type { AboutStepSeverity, Duration, } from '../../../pages/detection_engine/rules/types'; -import { GroupByOptions } from '../../../pages/detection_engine/rules/types'; import type { FieldValueTimeline } from '../pick_timeline'; import type { FormSchema } from '../../../../shared_imports'; import type { ListItems } from './types'; @@ -205,17 +204,16 @@ export const getDescriptionItem = ( } else if (field === 'groupByRadioSelection') { return []; } else if (field === 'groupByDuration') { - if (get('groupByRadioSelection', data) === GroupByOptions.PerTimePeriod) { + if (get('groupByFields', data).length > 0) { const value: Duration = get(field, data); - return buildAlertSuppressionWindowDescription(label, value, license); - } else if (get('groupByRadioSelection', data) === GroupByOptions.PerRuleExecution) { - return [ - { - title: label, - // TODO: copywriting and i18n - description: 'One rule execution', - }, - ]; + return buildAlertSuppressionWindowDescription( + label, + value, + license, + get('groupByRadioSelection', data) + ); + } else { + return []; } } else if (field === 'eqlOptions') { const eqlOptions: EqlOptionsSelected = get(field, data); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.ts b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.ts index b4b2df5515342..626a0648d7bac 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/translations.ts @@ -140,3 +140,10 @@ export const ALERT_SUPPRESSION_TECHNICAL_PREVIEW = i18n.translate( defaultMessage: 'Technical Preview', } ); + +export const ALERT_SUPPRESSION_PER_RULE_EXECUTION = i18n.translate( + 'xpack.securitySolution.detectionEngine.ruleDescription.alertSuppressionPerRuleExecution', + { + defaultMessage: 'One rule execution', + } +); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx index 1ca0b9e606139..2133a10c38b8b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/duration_input/index.tsx @@ -27,11 +27,7 @@ interface DurationInputProps { durationUnitOptions?: Array<{ value: 's' | 'm' | 'h' | 'd'; text: string }>; } -// TODO: share with ScheduleItem impl const getNumberFromUserInput = (input: string, minimumValue = 0): number | undefined => { - if (input == null || input.length === 0) { - return undefined; - } const number = parseInt(input, 10); if (Number.isNaN(number)) { return minimumValue; @@ -76,6 +72,9 @@ const MyEuiSelect = styled(EuiSelect)` width: auto; `; +// This component is similar to the ScheduleItem component, but instead of combining the value +// and unit into a single string it keeps them separate. This makes the component simpler and +// allows for easier validation of values and units in APIs as well. const DurationInputComponent: React.FC = ({ durationValueField, durationUnitField, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts index ce8f400c9a21e..bd68a3c84c966 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -17,6 +17,7 @@ import { ALERT_SUPPRESSION_DOCS_COUNT, ALERT_SUPPRESSION_TERMS, TIMESTAMP, + ALERT_LAST_DETECTED, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; @@ -755,6 +756,8 @@ export default ({ getService }: FtrProviderContext) => { value: 'host-0', }, ], + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T05:30:00.000Z', [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_END]: '2020-10-28T05:00:02.000Z', @@ -768,6 +771,8 @@ export default ({ getService }: FtrProviderContext) => { value: 'host-0', }, ], + [TIMESTAMP]: '2020-10-28T06:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', [ALERT_ORIGINAL_TIME]: '2020-10-28T06:00:00.000Z', [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', @@ -804,13 +809,14 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).to.eql(1); expect(previewAlerts[0]._source).to.eql({ ...previewAlerts[0]._source, - [TIMESTAMP]: '2020-10-28T05:30:00.000Z', [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', value: 'host-0', }, ], + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', @@ -847,13 +853,14 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).to.eql(3); expect(previewAlerts[0]._source).to.eql({ ...previewAlerts[0]._source, - [TIMESTAMP]: '2020-10-28T05:30:00.000Z', [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', value: 'host-0', }, ], + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', @@ -861,13 +868,14 @@ export default ({ getService }: FtrProviderContext) => { }); expect(previewAlerts[1]._source).to.eql({ ...previewAlerts[1]._source, - [TIMESTAMP]: '2020-10-28T05:30:00.000Z', [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', value: 'host-1', }, ], + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', @@ -875,13 +883,14 @@ export default ({ getService }: FtrProviderContext) => { }); expect(previewAlerts[2]._source).to.eql({ ...previewAlerts[2]._source, - [TIMESTAMP]: '2020-10-28T05:30:00.000Z', [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', value: 'host-2', }, ], + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', @@ -918,19 +927,76 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).to.eql(3); expect(previewAlerts[2]._source).to.eql({ ...previewAlerts[2]._source, - [TIMESTAMP]: '2020-10-28T05:30:00.000Z', [ALERT_SUPPRESSION_TERMS]: [ { field: 'destination.ip', value: null, }, ], + [TIMESTAMP]: '2020-10-28T05:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', [ALERT_ORIGINAL_TIME]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_START]: '2020-10-28T05:00:00.000Z', [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', [ALERT_SUPPRESSION_DOCS_COUNT]: 34, }); }); + + describe('with host risk index', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['suppression-data']), + query: `host.name: "host-0"`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 30, + unit: 'm', + }, + }, + from: 'now-1h', + interval: '1h', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).to.eql(1); + expect(previewAlerts[0]._source).to.eql({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [TIMESTAMP]: '2020-10-28T06:30:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:30:00.000Z', + [ALERT_ORIGINAL_TIME]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_END]: '2020-10-28T06:00:02.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 5, + }); + expect(previewAlerts[0]._source?.host?.risk?.calculated_level).to.eql('Low'); + expect(previewAlerts[0]._source?.host?.risk?.calculated_score_norm).to.eql(1); + }); + }); }); }); }); diff --git a/x-pack/test/functional/es_archives/entity/host_risk/data.json b/x-pack/test/functional/es_archives/entity/host_risk/data.json index c6e609cbcaf7b..d17ae4cfe8353 100644 --- a/x-pack/test/functional/es_archives/entity/host_risk/data.json +++ b/x-pack/test/functional/es_archives/entity/host_risk/data.json @@ -98,3 +98,23 @@ } } +{ + "type": "doc", + "value": { + "id": "5", + "index": "ml_host_risk_score_latest_default", + "source": { + "host": { + "name": "host-0", + "risk": { + "calculated_score_norm": 1, + "calculated_level": "Low" + } + }, + "ingest_timestamp": "2022-08-15T16:32:16.142561766Z", + "@timestamp": "2022-08-12T14:45:36.171Z" + }, + "type": "_doc" + } +} + From 6743a5dc0b162be2f56678387ab490fc90b748f2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 13 Jan 2023 03:06:41 +0000 Subject: [PATCH 06/22] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../lib/detection_engine/rule_schema/model/rule_schemas.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index 65f739ba7a5d5..7e0c7568a3d67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -24,7 +24,6 @@ import { threat_query, threatIndicatorPathOrUndefined, } from '@kbn/securitysolution-io-ts-alerting-types'; -import { NonEmptyString } from '@kbn/securitysolution-io-ts-types'; import { SIGNALS_ID, EQL_RULE_TYPE_ID, From 899d160381f91534b22548b250a576a2b1c58cc5 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 12 Jan 2023 19:29:12 -0800 Subject: [PATCH 07/22] Fix types --- .../server/rule_data_client/rule_data_client.ts | 14 ++++++-------- .../create_persistence_rule_type_wrapper.mock.ts | 1 + 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts index 11ccdbed97bf9..92ae633cf16de 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts @@ -106,20 +106,18 @@ export class RuleDataClient implements IRuleDataClient { }; return { - search: async ( + search: async < + TSearchRequest extends ESSearchRequest, + TAlertDoc = Partial + >( request: TSearchRequest - ): Promise< - ESSearchResponse, TSearchRequest> - > => { + ): Promise> => { try { const clusterClient = await waitUntilReady(); return (await clusterClient.search({ ...request, index: indexPattern, - })) as unknown as ESSearchResponse< - Partial, - TSearchRequest - >; + })) as unknown as ESSearchResponse; } catch (err) { this.options.logger.error(`Error performing search in RuleDataClient - ${err.message}`); throw err; diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts index 1aec64a408060..4a95fd2df5b84 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts @@ -11,6 +11,7 @@ import { PersistenceServices } from './persistence_types'; export const createPersistenceServicesMock = (): jest.Mocked => { return { alertWithPersistence: jest.fn(), + alertWithSuppression: jest.fn(), }; }; From f39021f5bb08ce1d64b9e0e853d37a69666bb304 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 12 Jan 2023 20:23:52 -0800 Subject: [PATCH 08/22] Fix more types --- .../components/rules_table/__mocks__/mock.ts | 10 +++++++++- .../components/rules/step_about_rule/index.test.tsx | 7 ++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts index f28d6b53821e2..40707a4307f27 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts @@ -13,7 +13,10 @@ import type { DefineStepRule, ScheduleStepRule, } from '../../../../../detections/pages/detection_engine/rules/types'; -import { DataSourceType } from '../../../../../detections/pages/detection_engine/rules/types'; +import { + DataSourceType, + GroupByOptions, +} from '../../../../../detections/pages/detection_engine/rules/types'; import type { FieldValueQueryBar } from '../../../../../detections/components/rules/query_bar'; import { fillEmptySeverityMappings } from '../../../../../detections/pages/detection_engine/rules/helpers'; import { getThreatMock } from '../../../../../../common/detection_engine/schemas/types/threat.mock'; @@ -230,6 +233,11 @@ export const mockDefineStepRule = (): DefineStepRule => ({ historyWindowSize: '7d', shouldLoadQueryDynamically: false, groupByFields: [], + groupByRadioSelection: GroupByOptions.PerRuleExecution, + groupByDuration: { + unit: 'm', + value: 5, + }, }); export const mockScheduleStepRule = (): ScheduleStepRule => ({ diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx index 1da78d8d97a29..429aebefa5347 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.test.tsx @@ -22,7 +22,7 @@ import type { RuleStep, DefineStepRule, } from '../../../pages/detection_engine/rules/types'; -import { DataSourceType } from '../../../pages/detection_engine/rules/types'; +import { DataSourceType, GroupByOptions } from '../../../pages/detection_engine/rules/types'; import { fillEmptySeverityMappings } from '../../../pages/detection_engine/rules/helpers'; import { TestProviders } from '../../../../common/mock'; @@ -57,6 +57,11 @@ export const stepDefineStepMLRule: DefineStepRule = { eqlOptions: {}, dataSourceType: DataSourceType.IndexPatterns, groupByFields: ['host.name'], + groupByRadioSelection: GroupByOptions.PerRuleExecution, + groupByDuration: { + unit: 'm', + value: 5, + }, newTermsFields: ['host.ip'], historyWindowSize: '7d', shouldLoadQueryDynamically: false, From 7d7b50434b5455f8d36e2682770eabd6d37f88a4 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 12 Jan 2023 20:36:06 -0800 Subject: [PATCH 09/22] Fix unit tests --- .../pages/detection_engine/rules/helpers.test.tsx | 15 +++++++++++++++ .../signals/single_search_after.ts | 10 ++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index 1c7433c9caabd..22ba4e03dbf38 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -116,6 +116,11 @@ describe('rule helpers', () => { tiebreakerField: undefined, }, groupByFields: ['host.name'], + groupByDuration: { + value: 5, + unit: 'm', + }, + groupByRadioSelection: 'per-rule-execution', newTermsFields: ['host.name'], historyWindowSize: '7d', }; @@ -260,6 +265,11 @@ describe('rule helpers', () => { tiebreakerField: undefined, }, groupByFields: [], + groupByDuration: { + value: 5, + unit: 'm', + }, + groupByRadioSelection: 'per-rule-execution', newTermsFields: [], historyWindowSize: '7d', shouldLoadQueryDynamically: true, @@ -315,6 +325,11 @@ describe('rule helpers', () => { tiebreakerField: undefined, }, groupByFields: [], + groupByDuration: { + value: 5, + unit: 'm', + }, + groupByRadioSelection: 'per-rule-execution', newTermsFields: [], historyWindowSize: '7d', shouldLoadQueryDynamically: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 859e0c34574eb..814cba856aad9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -79,10 +79,12 @@ export const singleSearchAfter = async < }); const start = performance.now(); - const nextSearchAfterResult = await services.scopedClusterClient.asCurrentUser.search< - SignalSource, - TAggregations - >(searchAfterQuery as estypes.SearchRequest); + const { body: nextSearchAfterResult } = + await services.scopedClusterClient.asCurrentUser.search( + searchAfterQuery as estypes.SearchRequest, + { meta: true } + ); + const end = performance.now(); const searchErrors = createErrorsFromShard({ From 60b849011be2b578c2670de66b5af9c714d0cd5e Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 12 Jan 2023 22:42:17 -0800 Subject: [PATCH 10/22] Fix test data --- x-pack/test/functional/es_archives/entity/host_risk/data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/es_archives/entity/host_risk/data.json b/x-pack/test/functional/es_archives/entity/host_risk/data.json index d17ae4cfe8353..50979283a0582 100644 --- a/x-pack/test/functional/es_archives/entity/host_risk/data.json +++ b/x-pack/test/functional/es_archives/entity/host_risk/data.json @@ -101,7 +101,7 @@ { "type": "doc", "value": { - "id": "5", + "id": "6", "index": "ml_host_risk_score_latest_default", "source": { "host": { From 14037fd26d6521003c013cb90cd9c2c65c104f57 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Fri, 13 Jan 2023 11:03:32 -0800 Subject: [PATCH 11/22] Add real rule execution test for suppression --- .../normalization/rule_converters.ts | 6 +- .../rule_execution_logic/non_ecs_fields.ts | 13 +-- .../rule_execution_logic/query.ts | 105 +++++++++++++++++- .../utils/data_generator/index_documents.ts | 21 +++- .../utils/get_open_signals.ts | 5 +- .../utils/patch_rule.ts | 40 +++++++ .../ecs_compliant/mappings.json | 64 +++++++++++ 7 files changed, 236 insertions(+), 18 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/utils/patch_rule.ts create mode 100644 x-pack/test/functional/es_archives/security_solution/ecs_compliant/mappings.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index e15c88ccf3aa1..2861478767975 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -246,7 +246,8 @@ const patchQueryParams = ( responseActions: params.response_actions?.map(transformRuleToAlertResponseAction) ?? existingRule.responseActions, - alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), + alertSuppression: + convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; @@ -265,7 +266,8 @@ const patchSavedQueryParams = ( responseActions: params.response_actions?.map(transformRuleToAlertResponseAction) ?? existingRule.responseActions, - alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), + alertSuppression: + convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts index 880a35082e535..46650ea0b8eb9 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts @@ -39,6 +39,7 @@ export default ({ getService }: FtrProviderContext) => { const indexDocuments = indexDocumentsFactory({ es, index: 'ecs_non_compliant', + log, }); /** @@ -50,17 +51,7 @@ export default ({ getService }: FtrProviderContext) => { const indexAndCreatePreviewAlert = async (document: Record) => { const documentId = uuidv4(); - const { items } = await indexDocuments([getDocument(documentId, document)]); - - // throw error if document wasn't indexed, so test will be terminated earlier and no false positives can happen - items.some(({ index } = {}) => { - if (index?.error) { - log.error( - `Failed to index document in non_ecs_fields test suits: "${index.error?.reason}"` - ); - throw Error(index.error.message); - } - }); + await indexDocuments([getDocument(documentId, document)]); const { previewId, logs } = await previewRule({ supertest, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts index bd68a3c84c966..b5922e26d185d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -22,8 +22,10 @@ import { import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { orderBy } from 'lodash'; +import { v4 as uuidv4 } from 'uuid'; import { QueryRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; +import { RuleExecutionStatus } from '@kbn/security-solution-plugin/common/detection_engine/rule_monitoring'; import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/types'; import { ALERT_ANCESTORS, @@ -42,6 +44,8 @@ import { previewRule, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { indexDocumentsFactory } from '../../utils/data_generator'; +import { patchRule } from '../../utils/patch_rule'; /** * Specific _id to use for some of the tests. If the archiver changes and you see errors @@ -78,7 +82,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllAlerts(supertest, log); }); - // First test creates a real rule - remaining tests use preview API + // First test creates a real rule - most remaining tests use preview API it('should have the specific audit record for _id or none of these tests below will pass', async () => { const rule: QueryRuleCreateProps = { ...getRuleForSignalTesting(['auditbeat-*']), @@ -721,6 +725,105 @@ export default ({ getService }: FtrProviderContext) => { }); describe('with a suppression time window', async () => { + const indexDocuments = indexDocumentsFactory({ + es, + index: 'ecs_compliant', + log, + }); + + before(async () => { + await esArchiver.load( + 'x-pack/test/functional/es_archives/security_solution/ecs_compliant' + ); + }); + + after(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/ecs_compliant' + ); + }); + + it('should update an alert using real rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + const firstDocument = { + id, + '@timestamp': firstTimestamp, + agent: { + name: 'agent-1', + }, + }; + await indexDocuments([firstDocument, firstDocument]); + + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['ecs_compliant']), + rule_id: 'rule-2', + query: `id:${id}`, + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + }, + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).eql(1); + expect(alerts.hits.hits[0]._source).to.eql({ + ...alerts.hits.hits[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + const secondTimestamp = new Date(); + const secondTimestampISOString = secondTimestamp.toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestampISOString, + agent: { + name: 'agent-1', + }, + }; + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexDocuments([secondDocument, secondDocument]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const secondAlerts = await getOpenSignals( + supertest, + log, + es, + createdRule, + RuleExecutionStatus.succeeded, + undefined, + secondTimestamp + ); + expect(secondAlerts.hits.hits.length).eql(1); + expect(secondAlerts.hits.hits[0]._source).to.eql({ + ...secondAlerts.hits.hits[0]._source, + [TIMESTAMP]: alerts.hits.hits[0]._source?.[TIMESTAMP], + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestampISOString, + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }); + }); + it('should generate an alert per rule run when duration is less than rule interval', async () => { const rule: QueryRuleCreateProps = { ...getRuleForSignalTesting(['suppression-data']), diff --git a/x-pack/test/detection_engine_api_integration/utils/data_generator/index_documents.ts b/x-pack/test/detection_engine_api_integration/utils/data_generator/index_documents.ts index 46de5d9858101..6e850ee54aefb 100644 --- a/x-pack/test/detection_engine_api_integration/utils/data_generator/index_documents.ts +++ b/x-pack/test/detection_engine_api_integration/utils/data_generator/index_documents.ts @@ -7,6 +7,7 @@ import type { Client } from '@elastic/elasticsearch'; import type { BulkResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { ToolingLog } from '@kbn/tooling-log'; interface IndexDocumentsParams { es: Client; documents: Array>; @@ -24,6 +25,22 @@ export const indexDocuments: IndexDocuments = async ({ es, documents, index }) = return es.bulk({ refresh: true, operations }); }; -export const indexDocumentsFactory = ({ es, index }: Omit) => { - return (documents: Array>) => indexDocuments({ es, index, documents }); +export const indexDocumentsFactory = ({ + es, + index, + log, +}: Omit & { log: ToolingLog }) => { + return async (documents: Array>) => { + const response = await indexDocuments({ es, index, documents }); + // throw error if document wasn't indexed, so test will be terminated earlier and no false positives can happen + response.items.some(({ index: responseIndex } = {}) => { + if (responseIndex?.error) { + log.error( + `Failed to index document in non_ecs_fields test suits: "${responseIndex.error?.reason}"` + ); + throw Error(responseIndex.error.message); + } + }); + return response; + }; }; diff --git a/x-pack/test/detection_engine_api_integration/utils/get_open_signals.ts b/x-pack/test/detection_engine_api_integration/utils/get_open_signals.ts index ae370cb5886ea..952336a265114 100644 --- a/x-pack/test/detection_engine_api_integration/utils/get_open_signals.ts +++ b/x-pack/test/detection_engine_api_integration/utils/get_open_signals.ts @@ -21,9 +21,10 @@ export const getOpenSignals = async ( es: Client, rule: RuleResponse, status: RuleExecutionStatus = RuleExecutionStatus.succeeded, - size?: number + size?: number, + afterDate?: Date ) => { - await waitForRuleSuccessOrStatus(supertest, log, rule.id, status); + await waitForRuleSuccessOrStatus(supertest, log, rule.id, status, afterDate); // Critically important that we wait for rule success AND refresh the write index in that order before we // assert that no signals were created. Otherwise, signals could be written but not available to query yet // when we search, causing tests that check that signals are NOT created to pass when they should fail. diff --git a/x-pack/test/detection_engine_api_integration/utils/patch_rule.ts b/x-pack/test/detection_engine_api_integration/utils/patch_rule.ts new file mode 100644 index 0000000000000..ec079c289e1fd --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/patch_rule.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import type SuperTest from 'supertest'; + +import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; +import { + RulePatchProps, + RuleResponse, +} from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; + +/** + * Helper to cut down on the noise in some of the tests. This checks for + * an expected 200 still and does not do any retries. + * @param supertest The supertest deps + * @param rule The rule to create + */ +export const patchRule = async ( + supertest: SuperTest.SuperTest, + log: ToolingLog, + patchedRule: RulePatchProps +): Promise => { + const response = await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(patchedRule); + if (response.status !== 200) { + log.error( + `Did not get an expected 200 "ok" when patching a rule (patchRule). CI issues could happen. Suspect this line if you are seeing CI issues. body: ${JSON.stringify( + response.body + )}, status: ${JSON.stringify(response.status)}` + ); + } + return response.body; +}; diff --git a/x-pack/test/functional/es_archives/security_solution/ecs_compliant/mappings.json b/x-pack/test/functional/es_archives/security_solution/ecs_compliant/mappings.json new file mode 100644 index 0000000000000..c9dfdf1c4e913 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/ecs_compliant/mappings.json @@ -0,0 +1,64 @@ +{ + "type": "index", + "value": { + "index": "ecs_compliant", + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "name": { + "type": "keyword" + }, + "type": { + "type": "long" + } + } + }, + "container": { + "properties": { + "image": { + "type": "keyword" + } + } + }, + "client": { + "properties": { + "ip": { + "type": "keyword" + } + } + }, + "event": { + "properties": { + "created": { + "type": "keyword" + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "valid": { + "type": "keyword" + } + } + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} From feb371ff9e7fc1b79059a7badc2e8e8e5ffbe2c2 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 18 Jan 2023 11:59:04 -0800 Subject: [PATCH 12/22] Move suppression types to rule registry --- .../common/schemas/8.6.0/index.ts | 33 +++++++++++++++++++ .../common/schemas/8.7.0/index.ts | 19 ++++------- .../rule_registry/common/schemas/index.ts | 4 +++ .../create_persistence_rule_type_wrapper.ts | 9 +++-- .../server/utils/persistence_types.ts | 4 +-- .../schemas/alerts/8.6.0/index.ts | 19 ++--------- .../schemas/alerts/8.7.0/index.ts | 15 +++------ .../detection_engine/schemas/alerts/index.ts | 3 +- .../rules/step_define_rule/index.tsx | 2 +- .../factories/bulk_create_with_suppression.ts | 10 ++++-- .../factories/utils/wrap_suppressed_alerts.ts | 4 +-- 11 files changed, 69 insertions(+), 53 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/schemas/8.6.0/index.ts diff --git a/x-pack/plugins/rule_registry/common/schemas/8.6.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.6.0/index.ts new file mode 100644 index 0000000000000..af19ee57cd03a --- /dev/null +++ b/x-pack/plugins/rule_registry/common/schemas/8.6.0/index.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_DOCS_COUNT, +} from '@kbn/rule-data-utils'; +import { AlertWithCommonFields800 } from '../8.0.0'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.0.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.0.0. + +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. + +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export interface SuppressionFields860 { + [ALERT_SUPPRESSION_TERMS]: Array<{ field: string; value: string | number | null }>; + [ALERT_SUPPRESSION_START]: Date; + [ALERT_SUPPRESSION_END]: Date; + [ALERT_SUPPRESSION_DOCS_COUNT]: number; +} + +export type AlertWithSuppressionFields860 = AlertWithCommonFields800 & SuppressionFields860; diff --git a/x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts index fb91c91829b89..46cc398ab85ba 100644 --- a/x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts @@ -5,13 +5,10 @@ * 2.0. */ -import { - ALERT_SUPPRESSION_TERMS, - ALERT_SUPPRESSION_START, - ALERT_SUPPRESSION_END, - ALERT_SUPPRESSION_DOCS_COUNT, - ALERT_INSTANCE_ID, -} from '@kbn/rule-data-utils'; +import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; +import { AlertWithCommonFields800 } from '../8.0.0'; + +import { SuppressionFields860 } from '../8.6.0'; /* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.0.0. Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.0.0. @@ -23,10 +20,8 @@ Then, update `../index.ts` to import from the new folder that has the latest sch new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. */ -export interface SuppressionFields { - [ALERT_SUPPRESSION_TERMS]: Array<{ field: string; value: string | number | null }>; - [ALERT_SUPPRESSION_START]: Date; - [ALERT_SUPPRESSION_END]: Date; - [ALERT_SUPPRESSION_DOCS_COUNT]: number; +export interface SuppressionFields870 extends SuppressionFields860 { [ALERT_INSTANCE_ID]: string; } + +export type AlertWithSuppressionFields870 = AlertWithCommonFields800 & SuppressionFields870; diff --git a/x-pack/plugins/rule_registry/common/schemas/index.ts b/x-pack/plugins/rule_registry/common/schemas/index.ts index e12c59617fd00..f6b71c9e792a8 100644 --- a/x-pack/plugins/rule_registry/common/schemas/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/index.ts @@ -12,9 +12,13 @@ import type { AlertWithCommonFields800, } from './8.0.0'; +import type { AlertWithSuppressionFields870, SuppressionFields870 } from './8.7.0'; + export type { CommonAlertFieldName800 as CommonAlertFieldNameLatest, CommonAlertIdFieldName800 as CommonAlertIdFieldNameLatest, CommonAlertFields800 as CommonAlertFieldsLatest, AlertWithCommonFields800 as AlertWithCommonFieldsLatest, + AlertWithSuppressionFields870 as AlertWithSuppressionFieldsLatest, + SuppressionFields870 as SuppressionFieldsLatest, }; diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 07b65aab9bdf4..6e4458938d70b 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -21,7 +21,7 @@ import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; import { errorAggregator } from './utils'; import { createGetSummarizedAlertsFn } from './create_get_summarized_alerts_fn'; -import { SuppressionFields } from '../../common/schemas/8.7.0'; +import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0'; export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper = ({ logger, ruleDataClient }) => @@ -221,14 +221,17 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, }; + // We use AlertWithSuppressionFields870 explicitly here as the type instead of + // AlertWithSuppressionFieldsLatest since we're reading alerts rather than writing, + // so future versions of Kibana may read 8.7.0 version alerts and need to update them const response = await ruleDataClient .getReader({ namespace: options.spaceId }) - .search( + .search>( suppressionAlertSearchRequest ); const existingAlertsByInstanceId = response.hits.hits.reduce< - Record> + Record>> >((acc, hit) => { acc[hit._source['kibana.alert.instance.id']] = hit; return acc; diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 4f2eee5d60a0f..d941bbc641732 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -18,7 +18,7 @@ import { WithoutReservedActionGroups } from '@kbn/alerting-plugin/common'; import { IRuleDataClient } from '../rule_data_client'; import { BulkResponseErrorAggregation } from './utils'; import { AlertWithCommonFieldsLatest } from '../../common/schemas'; -import { SuppressionFields } from '../../common/schemas/8.7.0'; +import { SuppressionFieldsLatest } from '../../common/schemas'; export type PersistenceAlertService = ( alerts: Array<{ @@ -41,7 +41,7 @@ export type PersistenceAlertService = ( > ) => Promise>; -export type SuppressedAlertService = ( +export type SuppressedAlertService = ( alerts: Array<{ _id: string; _source: T; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.6.0/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.6.0/index.ts index 3bf66b5e31ec6..402c2ae4fbbaf 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.6.0/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.6.0/index.ts @@ -5,13 +5,7 @@ * 2.0. */ -import type { AlertWithCommonFields800 } from '@kbn/rule-registry-plugin/common/schemas/8.0.0'; -import type { - ALERT_SUPPRESSION_TERMS, - ALERT_SUPPRESSION_START, - ALERT_SUPPRESSION_END, - ALERT_SUPPRESSION_DOCS_COUNT, -} from '@kbn/rule-data-utils'; +import type { AlertWithSuppressionFields860 } from '@kbn/rule-registry-plugin/common/schemas/8.6.0'; import type { BaseFields840, DetectionAlert840 } from '../8.4.0'; @@ -23,13 +17,4 @@ Then, update `../index.ts` to import from the new folder that has the latest sch new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. */ -export interface SuppressionFields860 extends BaseFields840 { - [ALERT_SUPPRESSION_TERMS]: Array<{ field: string; value: string | number | null }>; - [ALERT_SUPPRESSION_START]: Date; - [ALERT_SUPPRESSION_END]: Date; - [ALERT_SUPPRESSION_DOCS_COUNT]: number; -} - -export type SuppressionAlert860 = AlertWithCommonFields800; - -export type DetectionAlert860 = DetectionAlert840 | SuppressionAlert860; +export type DetectionAlert860 = DetectionAlert840 | AlertWithSuppressionFields860; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts index 160589a6b4270..c56453a602dcf 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/8.7.0/index.ts @@ -5,11 +5,10 @@ * 2.0. */ -import type { AlertWithCommonFields800 } from '@kbn/rule-registry-plugin/common/schemas/8.0.0'; -import type { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; +import type { AlertWithSuppressionFields870 } from '@kbn/rule-registry-plugin/common/schemas/8.7.0'; +import type { BaseFields840 } from '../8.4.0'; -import type { DetectionAlert840 } from '../8.4.0'; -import type { SuppressionFields860 } from '../8.6.0'; +import type { DetectionAlert860 } from '../8.6.0'; /* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.6.0. Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.6.0. @@ -19,10 +18,4 @@ Then, update `../index.ts` to import from the new folder that has the latest sch new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. */ -export interface SuppressionFields870 extends SuppressionFields860 { - [ALERT_INSTANCE_ID]: string; -} - -export type SuppressionAlert870 = AlertWithCommonFields800; - -export type DetectionAlert870 = DetectionAlert840 | SuppressionAlert870; +export type DetectionAlert870 = DetectionAlert860 | AlertWithSuppressionFields870; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts index 128675b4620c3..2fdf426f0aea0 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/alerts/index.ts @@ -18,7 +18,7 @@ import type { } from './8.4.0'; import type { DetectionAlert860 } from './8.6.0'; -import type { DetectionAlert870, SuppressionFields870 } from './8.7.0'; +import type { DetectionAlert870 } from './8.7.0'; // When new Alert schemas are created for new Kibana versions, add the DetectionAlert type from the new version // here, e.g. `export type DetectionAlert = DetectionAlert800 | DetectionAlert820` if a new schema is created in 8.2.0 @@ -36,5 +36,4 @@ export type { EqlBuildingBlockFields840 as EqlBuildingBlockFieldsLatest, EqlShellFields840 as EqlShellFieldsLatest, NewTermsFields840 as NewTermsFieldsLatest, - SuppressionFields870 as SuppressionFieldsLatest, }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 59d73742b0e21..78ce12774325f 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -149,7 +149,7 @@ const StepDefineRuleComponent: FC = ({ const { form } = useForm({ defaultValue: initialState, - options: { stripEmptyFields: true }, + options: { stripEmptyFields: false }, schema, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts index 36ae3051547f8..fb49a498805f9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_with_suppression.ts @@ -9,12 +9,14 @@ import { performance } from 'perf_hooks'; import { isEmpty } from 'lodash'; import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; -import type { AlertWithCommonFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import type { + AlertWithCommonFieldsLatest, + SuppressionFieldsLatest, +} from '@kbn/rule-registry-plugin/common/schemas'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { makeFloatString } from '../../signals/utils'; import type { BaseFieldsLatest, - SuppressionFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/detection_engine/schemas/alerts'; import type { RuleServices } from '../../signals/types'; @@ -29,7 +31,9 @@ export interface GenericBulkCreateResponse { errors: string[]; } -export const bulkCreateWithSuppression = async ({ +export const bulkCreateWithSuppression = async < + T extends SuppressionFieldsLatest & BaseFieldsLatest +>({ alertWithSuppression, ruleExecutionLogger, wrappedDocs, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts index af4b9287ed27c..581d6a256723e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/wrap_suppressed_alerts.ts @@ -15,9 +15,9 @@ import { ALERT_SUPPRESSION_START, ALERT_INSTANCE_ID, } from '@kbn/rule-data-utils'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { BaseFieldsLatest, - SuppressionFieldsLatest, WrappedFieldsLatest, } from '../../../../../../common/detection_engine/schemas/alerts'; import type { ConfigType } from '../../../../../config'; @@ -65,7 +65,7 @@ export const wrapSuppressedAlerts = ({ buildReasonMessage: BuildReasonMessage; alertTimestampOverride: Date | undefined; ruleExecutionLogger: IRuleExecutionLogForExecutors; -}): Array> => { +}): Array> => { return suppressionBuckets.map((bucket) => { const id = objectHash([ bucket.event._index, From 79e81a91d9e30e20981f3c3a75955c40f796e6ff Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 24 Jan 2023 11:03:54 -0800 Subject: [PATCH 13/22] Apply Garrett's fix --- .../detections/components/rules/step_define_rule/index.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 78ce12774325f..c61abf1703705 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -521,7 +521,10 @@ const StepDefineRuleComponent: FC = ({ From d2c586bc69eaa5e814435b11c957df8759a88c46 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 24 Jan 2023 15:44:17 -0800 Subject: [PATCH 14/22] Add timestamp override and max_signals tests, move schema --- .../specific_attributes/query_attributes.ts | 14 ++ .../rule_schema/model/rule_schemas.ts | 17 +- .../rule_execution_logic/query.ts | 146 ++++++++++++++++++ 3 files changed, 161 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts index 4e4c8a8a2a486..d83a187fe516a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/query_attributes.ts @@ -45,4 +45,18 @@ export const AlertSuppression = t.intersection([ ), ]); +export type AlertSuppressionCamel = t.TypeOf; +export const AlertSuppressionCamel = t.intersection([ + t.exact( + t.type({ + groupBy: AlertSuppressionGroupBy, + }) + ), + t.exact( + t.partial({ + duration: AlertSuppressionDuration, + }) + ), +]); + export const minimumLicenseForSuppression = 'platinum'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index 7e0c7568a3d67..e204aeb7bc50f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -39,8 +39,7 @@ import type { SanitizedRuleConfig } from '@kbn/alerting-plugin/common'; import { AlertsIndex, AlertsIndexNamespace, - AlertSuppressionDuration, - AlertSuppressionGroupBy, + AlertSuppressionCamel, BuildingBlockType, DataViewId, EventCategoryOverride, @@ -87,20 +86,6 @@ import { ResponseActionRuleParamsOrUndefined } from '../../../../../common/detec const nonEqlLanguages = t.keyof({ kuery: null, lucene: null }); -export type AlertSuppressionCamel = t.TypeOf; -const AlertSuppressionCamel = t.intersection([ - t.exact( - t.type({ - groupBy: AlertSuppressionGroupBy, - }) - ), - t.exact( - t.partial({ - duration: AlertSuppressionDuration, - }) - ), -]); - export const baseRuleParams = t.exact( t.type({ author: RuleAuthorArray, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts index b5922e26d185d..caaf0a3459202 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -1045,6 +1045,152 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('should correctly deduplicate when using a timestamp override', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:00:00.000Z'; + const docWithoutOverride = { + id, + '@timestamp': timestamp, + agent: { + name: 'agent-1', + }, + }; + const docWithOverride = { + ...docWithoutOverride, + // This doc simulates a very late arriving doc + '@timestamp': '2020-10-28T03:00:00.000Z', + event: { + ingested: '2020-10-28T06:10:00.000Z', + }, + }; + await indexDocuments([docWithoutOverride, docWithOverride]); + + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['ecs_compliant']), + query: `id:${id}`, + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + }, + from: 'now-2h', + interval: '1h', + timestamp_override: 'event.ingested', + }; + + // Here we want to check that (1) the suppression end time is set correctly based on the override, + // and (2) despite the large lookback, the docs are not counted again in the second invocation + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).to.eql(1); + expect(previewAlerts[0]._source).to.eql({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: '2020-10-28T06:10:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should generate and update up to max_signals alerts', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:00:00.000Z'; + const docs = Array(150) + .fill({}) + .map((_, i) => ({ + id, + '@timestamp': timestamp, + agent: { + name: `agent-${i}`, + }, + })); + const laterTimestamp = '2020-10-28T07:00:00.000Z'; + const laterDocs = Array(150) + .fill({}) + .map((_, i) => ({ + id, + '@timestamp': laterTimestamp, + agent: { + name: `agent-${i}`, + }, + })); + await indexDocuments([...docs, ...laterDocs]); + + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['ecs_compliant']), + query: `id:${id}`, + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + }, + from: 'now-1h', + interval: '1h', + timestamp_override: 'event.ingested', + max_signals: 150, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 1000, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).to.eql(150); + expect(previewAlerts[0]._source).to.eql({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-0', + }, + ], + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[149]._source).to.eql({ + ...previewAlerts[149]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + // Values are sorted with string comparison, so 'agent-99' is last (not 'agent-149') + value: 'agent-99', + }, + ], + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + describe('with host risk index', async () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); From 0c0b157023f57976b3d56fff15d7183ac2de94e2 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 24 Jan 2023 16:29:26 -0800 Subject: [PATCH 15/22] Fix import --- .../server/lib/detection_engine/rule_management/utils/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts index 7ea95389cdae1..e811dbd29cade 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts @@ -19,12 +19,13 @@ import type { RuleExecutionSummary } from '../../../../../common/detection_engin import type { AlertSuppression, RuleResponse, + AlertSuppressionCamel, } from '../../../../../common/detection_engine/rule_schema'; // eslint-disable-next-line no-restricted-imports import type { LegacyRulesActionsSavedObject } from '../../rule_actions_legacy'; import type { RuleExecutionSummariesByRuleId } from '../../rule_monitoring'; -import type { AlertSuppressionCamel, RuleAlertType, RuleParams } from '../../rule_schema'; +import type { RuleAlertType, RuleParams } from '../../rule_schema'; import { isAlertType } from '../../rule_schema'; import type { BulkError, OutputError } from '../../routes/utils'; import { createBulkErrorObject } from '../../routes/utils'; From ce3a624511aaea8d6f7d12ca4ab01a6dd6185011 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 24 Jan 2023 16:32:57 -0800 Subject: [PATCH 16/22] Fix technical field names post merge --- packages/kbn-rule-data-utils/src/technical_field_names.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts index b02ecf707da0c..9a0038b5a6eb5 100644 --- a/packages/kbn-rule-data-utils/src/technical_field_names.ts +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -61,7 +61,6 @@ const ALERT_SUPPRESSION_VALUE = `${ALERT_SUPPRESSION_TERMS}.value` as const; const ALERT_SUPPRESSION_START = `${ALERT_SUPPRESSION_META}.start` as const; const ALERT_SUPPRESSION_END = `${ALERT_SUPPRESSION_META}.end` as const; const ALERT_SUPPRESSION_DOCS_COUNT = `${ALERT_SUPPRESSION_META}.docs_count` as const; -const ALERT_SUPPRESSION_GROUP_ID = `${ALERT_SUPPRESSION_META}.group_id` as const; // Fields pertaining to the cases associated with the alert const ALERT_CASE_IDS = `${ALERT_NAMESPACE}.case_ids` as const; @@ -181,7 +180,6 @@ const fields = { ALERT_SUPPRESSION_START, ALERT_SUPPRESSION_END, ALERT_SUPPRESSION_DOCS_COUNT, - ALERT_SUPPRESSION_GROUP_ID, SPACE_IDS, VERSION, }; @@ -192,10 +190,6 @@ export { ALERT_EVALUATION_VALUE, ALERT_INSTANCE_ID, ALERT_LAST_DETECTED, - ALERT_NAMESPACE, - ALERT_RULE_NAMESPACE, - ALERT_RULE_CONSUMER, - ALERT_RULE_PRODUCER, ALERT_RISK_SCORE, ALERT_WORKFLOW_REASON, ALERT_WORKFLOW_USER, @@ -241,7 +235,6 @@ export { ALERT_SUPPRESSION_START, ALERT_SUPPRESSION_END, ALERT_SUPPRESSION_DOCS_COUNT, - ALERT_SUPPRESSION_GROUP_ID, TAGS, TIMESTAMP, }; From 6e6c572bbc15b990a7ac2b107341a2b822c47b57 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 25 Jan 2023 15:52:34 -0800 Subject: [PATCH 17/22] Helper funcs in persistence wrapper, fix license check in UI --- .../create_persistence_rule_type_wrapper.ts | 79 +++++++++++-------- .../rules/step_define_rule/index.tsx | 1 + 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 6e4458938d70b..82e92d539edfc 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -7,6 +7,7 @@ import dateMath from '@elastic/datemath'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { RuleExecutorOptions } from '@kbn/alerting-plugin/server'; import { chunk, partition } from 'lodash'; import { ALERT_INSTANCE_ID, @@ -23,6 +24,37 @@ import { errorAggregator } from './utils'; import { createGetSummarizedAlertsFn } from './create_get_summarized_alerts_fn'; import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0'; +const augmentAlerts = ({ + alerts, + options, + kibanaVersion, + includeLastDetected, + currentTimeOverride, +}: { + alerts: Array<{ _id: string; _source: T }>; + options: RuleExecutorOptions; + kibanaVersion: string; + includeLastDetected: boolean; + currentTimeOverride: Date | undefined; +}) => { + const commonRuleFields = getCommonAlertFields(options); + return alerts.map((alert) => { + return { + ...alert, + _source: { + [ALERT_LAST_DETECTED]: includeLastDetected ? currentTimeOverride ?? new Date() : undefined, + [VERSION]: kibanaVersion, + ...commonRuleFields, + ...alert._source, + }, + }; + }); +}; + +const mapAlertsToBulkCreate = (alerts: Array<{ _id: string; _source: T }>) => { + return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, alert._source]); +}; + export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper = ({ logger, ruleDataClient }) => (type) => { @@ -51,8 +83,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); if (writeAlerts && numAlerts) { - const commonRuleFields = getCommonAlertFields(options); - const CHUNK_SIZE = 10000; const alertChunks = chunk(alerts, CHUNK_SIZE); const filteredAlerts: typeof alerts = []; @@ -114,22 +144,16 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alertsWereTruncated = true; } - const augmentedAlerts = enrichedAlerts.map((alert) => { - return { - ...alert, - _source: { - [VERSION]: ruleDataClient.kibanaVersion, - ...commonRuleFields, - ...alert._source, - }, - }; + const augmentedAlerts = augmentAlerts({ + alerts: enrichedAlerts, + options, + kibanaVersion: ruleDataClient.kibanaVersion, + includeLastDetected: false, + currentTimeOverride: undefined, }); const response = await ruleDataClientWriter.bulk({ - body: augmentedAlerts.flatMap((alert) => [ - { create: { _id: alert._id } }, - alert._source, - ]), + body: mapAlertsToBulkCreate(augmentedAlerts), refresh, }); @@ -176,8 +200,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); if (writeAlerts && alerts.length > 0) { - const commonRuleFields = getCommonAlertFields(options); - const suppressionWindowStart = dateMath.parse(suppressionWindow, { forceNow: currentTimeOverride, }); @@ -279,25 +301,16 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper } } - const augmentedAlerts = enrichedAlerts.map((alert) => { - return { - ...alert, - _source: { - [VERSION]: ruleDataClient.kibanaVersion, - [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), - ...commonRuleFields, - ...alert._source, - }, - }; + const augmentedAlerts = augmentAlerts({ + alerts: enrichedAlerts, + options, + kibanaVersion: ruleDataClient.kibanaVersion, + includeLastDetected: true, + currentTimeOverride, }); - const newAlertCreates = augmentedAlerts.flatMap((alert) => [ - { create: { _id: alert._id } }, - alert._source, - ]); - const bulkResponse = await ruleDataClientWriter.bulk({ - body: [...duplicateAlertUpdates, ...newAlertCreates], + body: [...duplicateAlertUpdates, ...mapAlertsToBulkCreate(augmentedAlerts)], refresh: true, }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index c61abf1703705..3bf4aea61d656 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -522,6 +522,7 @@ const StepDefineRuleComponent: FC = ({ durationValueField={groupByDurationValue} durationUnitField={groupByDurationUnit} isDisabled={ + !license.isAtLeast(minimumLicenseForSuppression) || groupByFields?.length === 0 || groupByRadioSelection.value !== GroupByOptions.PerTimePeriod } From 969253f23ccc666bb26311d9544f98c2a8466ef4 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 26 Jan 2023 13:51:54 -0800 Subject: [PATCH 18/22] Exclude closed alerts from suppression update logic --- .../create_persistence_rule_type_wrapper.ts | 10 ++ .../rule_execution_logic/query.ts | 96 ++++++++++++++++++- .../utils/get_query_signals_ids.ts | 1 + 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 82e92d539edfc..8735b1f92d8fb 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -15,6 +15,7 @@ import { ALERT_SUPPRESSION_DOCS_COUNT, ALERT_SUPPRESSION_END, ALERT_UUID, + ALERT_WORKFLOW_STATUS, TIMESTAMP, VERSION, } from '@kbn/rule-data-utils'; @@ -227,6 +228,15 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ), }, }, + { + bool: { + must_not: { + term: { + [ALERT_WORKFLOW_STATUS]: 'closed', + }, + }, + }, + }, ], }, }, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts index caaf0a3459202..41b30798c6eed 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -33,6 +33,7 @@ import { ALERT_ORIGINAL_TIME, ALERT_ORIGINAL_EVENT, } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { DETECTION_ENGINE_SIGNALS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; import { createRule, deleteAllAlerts, @@ -42,6 +43,7 @@ import { getRuleForSignalTesting, getSimpleRule, previewRule, + setSignalStatus, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { indexDocumentsFactory } from '../../utils/data_generator'; @@ -810,7 +812,7 @@ export default ({ getService }: FtrProviderContext) => { expect(secondAlerts.hits.hits.length).eql(1); expect(secondAlerts.hits.hits[0]._source).to.eql({ ...secondAlerts.hits.hits[0]._source, - [TIMESTAMP]: alerts.hits.hits[0]._source?.[TIMESTAMP], + [TIMESTAMP]: secondAlerts.hits.hits[0]._source?.[TIMESTAMP], [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', @@ -824,6 +826,98 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('should NOT update an alert if the alert is closed', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + const firstDocument = { + id, + '@timestamp': firstTimestamp, + agent: { + name: 'agent-1', + }, + }; + await indexDocuments([firstDocument, firstDocument]); + + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['ecs_compliant']), + rule_id: 'rule-2', + query: `id:${id}`, + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + }, + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + + // Close the alert. Subsequent rule executions should ignore this closed alert + // for suppression purposes. + const alertIds = alerts.hits.hits.map((alert) => alert._id); + await supertest + .post(DETECTION_ENGINE_SIGNALS_STATUS_URL) + .set('kbn-xsrf', 'true') + .send(setSignalStatus({ signalIds: alertIds, status: 'closed' })) + .expect(200); + + const secondTimestamp = new Date(); + const secondTimestampISOString = secondTimestamp.toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestampISOString, + agent: { + name: 'agent-1', + }, + }; + // Add new documents, then disable and re-enable to trigger another rule run. The second doc should + // trigger a new alert since the first one is now closed. + await indexDocuments([secondDocument, secondDocument]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const secondAlerts = await getOpenSignals( + supertest, + log, + es, + createdRule, + RuleExecutionStatus.succeeded, + undefined, + secondTimestamp + ); + expect(secondAlerts.hits.hits.length).eql(2); + expect(secondAlerts.hits.hits[0]._source).to.eql({ + ...secondAlerts.hits.hits[0]._source, + [TIMESTAMP]: secondAlerts.hits.hits[0]._source?.[TIMESTAMP], + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_WORKFLOW_STATUS]: 'closed', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(secondAlerts.hits.hits[1]._source).to.eql({ + ...secondAlerts.hits.hits[1]._source, + [TIMESTAMP]: secondAlerts.hits.hits[1]._source?.[TIMESTAMP], + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-1', + }, + ], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_ORIGINAL_TIME]: secondTimestampISOString, + [ALERT_SUPPRESSION_START]: secondTimestampISOString, + [ALERT_SUPPRESSION_END]: secondTimestampISOString, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + it('should generate an alert per rule run when duration is less than rule interval', async () => { const rule: QueryRuleCreateProps = { ...getRuleForSignalTesting(['suppression-data']), diff --git a/x-pack/test/detection_engine_api_integration/utils/get_query_signals_ids.ts b/x-pack/test/detection_engine_api_integration/utils/get_query_signals_ids.ts index 2a94be5897749..75b8696625301 100644 --- a/x-pack/test/detection_engine_api_integration/utils/get_query_signals_ids.ts +++ b/x-pack/test/detection_engine_api_integration/utils/get_query_signals_ids.ts @@ -14,6 +14,7 @@ import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; */ export const getQuerySignalsId = (ids: string[], size = 10) => ({ size, + sort: ['@timestamp'], query: { terms: { [ALERT_RULE_UUID]: ids, From 9b982374b8285602a9d186d0ac4d7c6683a6f756 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Fri, 27 Jan 2023 08:42:56 -0800 Subject: [PATCH 19/22] PR comments --- .../kbn-rule-data-utils/src/default_alerts_as_data.ts | 5 +++++ .../kbn-rule-data-utils/src/technical_field_names.ts | 3 --- .../utils/create_persistence_rule_type_wrapper.ts | 11 ++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts index b428bea94cdcd..3a982124b58e6 100644 --- a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts +++ b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts @@ -34,6 +34,9 @@ const ALERT_FLAPPING = `${ALERT_NAMESPACE}.flapping` as const; // kibana.alert.id - alert ID, also known as alert instance ID const ALERT_ID = `${ALERT_NAMESPACE}.id` as const; +// kibana.alert.last_detected - timestamp when the alert was last seen +const ALERT_LAST_DETECTED = `${ALERT_NAMESPACE}.last_detected` as const; + // kibana.alert.reason - human readable reason that this alert is active const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const; @@ -91,6 +94,7 @@ const fields = { ALERT_END, ALERT_FLAPPING, ALERT_ID, + ALERT_LAST_DETECTED, ALERT_REASON, ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, @@ -116,6 +120,7 @@ export { ALERT_END, ALERT_FLAPPING, ALERT_ID, + ALERT_LAST_DETECTED, ALERT_REASON, ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts index 9a0038b5a6eb5..89eca0f923046 100644 --- a/packages/kbn-rule-data-utils/src/technical_field_names.ts +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -48,7 +48,6 @@ const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as co const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as const; const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const; const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const; -const ALERT_LAST_DETECTED = `${ALERT_NAMESPACE}.last_detected_at` as const; const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const; const ALERT_SEVERITY = `${ALERT_NAMESPACE}.severity` as const; const ALERT_SYSTEM_STATUS = `${ALERT_NAMESPACE}.system_status` as const; @@ -123,7 +122,6 @@ const fields = { ALERT_EVALUATION_VALUE, ALERT_FLAPPING, ALERT_INSTANCE_ID, - ALERT_LAST_DETECTED, ALERT_RULE_CONSUMER, ALERT_RULE_PRODUCER, ALERT_REASON, @@ -189,7 +187,6 @@ export { ALERT_EVALUATION_THRESHOLD, ALERT_EVALUATION_VALUE, ALERT_INSTANCE_ID, - ALERT_LAST_DETECTED, ALERT_RISK_SCORE, ALERT_WORKFLOW_REASON, ALERT_WORKFLOW_USER, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 8735b1f92d8fb..22e93aa44dab5 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -12,11 +12,11 @@ import { chunk, partition } from 'lodash'; import { ALERT_INSTANCE_ID, ALERT_LAST_DETECTED, + ALERT_START, ALERT_SUPPRESSION_DOCS_COUNT, ALERT_SUPPRESSION_END, ALERT_UUID, ALERT_WORKFLOW_STATUS, - TIMESTAMP, VERSION, } from '@kbn/rule-data-utils'; import { getCommonAlertFields } from './get_common_alert_fields'; @@ -29,13 +29,11 @@ const augmentAlerts = ({ alerts, options, kibanaVersion, - includeLastDetected, currentTimeOverride, }: { alerts: Array<{ _id: string; _source: T }>; options: RuleExecutorOptions; kibanaVersion: string; - includeLastDetected: boolean; currentTimeOverride: Date | undefined; }) => { const commonRuleFields = getCommonAlertFields(options); @@ -43,7 +41,8 @@ const augmentAlerts = ({ return { ...alert, _source: { - [ALERT_LAST_DETECTED]: includeLastDetected ? currentTimeOverride ?? new Date() : undefined, + [ALERT_START]: currentTimeOverride ?? new Date(), + [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), [VERSION]: kibanaVersion, ...commonRuleFields, ...alert._source, @@ -149,7 +148,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alerts: enrichedAlerts, options, kibanaVersion: ruleDataClient.kibanaVersion, - includeLastDetected: false, currentTimeOverride: undefined, }); @@ -216,7 +214,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper filter: [ { range: { - [TIMESTAMP]: { + [ALERT_START]: { gte: suppressionWindowStart.toISOString(), }, }, @@ -315,7 +313,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alerts: enrichedAlerts, options, kibanaVersion: ruleDataClient.kibanaVersion, - includeLastDetected: true, currentTimeOverride, }); From 3f1ecb2d070821baf422be7543ac1b90716953e2 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:24:08 -0800 Subject: [PATCH 20/22] Update x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mike Côté --- .../server/utils/create_persistence_rule_type_wrapper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 22e93aa44dab5..6109eabb87e96 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -243,7 +243,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, sort: [ { - '@timestamp': { + [ALERT_START]: { order: 'desc' as const, }, }, From 487f56b8aa1f91cc370b403d6170a8744d32a8e8 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Fri, 27 Jan 2023 13:27:21 -0800 Subject: [PATCH 21/22] Modify tests to account for alert start and enable rule timestamp bug --- .../group6/alerts/alerts_compatibility.ts | 3 +++ .../rule_execution_logic/new_terms.ts | 2 ++ .../rule_execution_logic/query.ts | 24 +++++++++---------- .../utils/wait_for_rule_success_or_status.ts | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts index 3b710bdfd4287..7c7d1933751c3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts @@ -20,6 +20,7 @@ import { ThreatMatchRuleCreateProps, ThresholdRuleCreateProps, } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; +import { ALERT_START } from '@kbn/rule-data-utils'; import { createRule, createSignalsIndex, @@ -237,6 +238,7 @@ export default ({ getService }: FtrProviderContext) => { 'kibana.alert.rule.updated_at': updatedAt, 'kibana.alert.rule.execution.uuid': executionUuid, 'kibana.alert.uuid': alertId, + [ALERT_START]: alertStart, ...source } = hit._source!; expect(source).to.eql({ @@ -406,6 +408,7 @@ export default ({ getService }: FtrProviderContext) => { 'kibana.alert.rule.updated_at': updatedAt, 'kibana.alert.rule.execution.uuid': executionUuid, 'kibana.alert.uuid': alertId, + [ALERT_START]: alertStart, ...source } = hit._source!; expect(source).to.eql({ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index fe21bae00a30b..ae086baffdfca 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -15,6 +15,7 @@ import { getNewTermsRuntimeMappings, AGG_FIELD_NAME, } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/new_terms/utils'; +import { ALERT_START } from '@kbn/rule-data-utils'; import { createRule, deleteAllAlerts, @@ -42,6 +43,7 @@ const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { 'kibana.alert.rule.created_at': createdAt, 'kibana.alert.rule.updated_at': updatedAt, 'kibana.alert.uuid': alertUuid, + [ALERT_START]: alertStart, ...restOfAlert } = alert; return restOfAlert; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts index 41b30798c6eed..e25f0099a608f 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -786,11 +786,10 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); - const secondTimestamp = new Date(); - const secondTimestampISOString = secondTimestamp.toISOString(); + const secondTimestamp = new Date().toISOString(); const secondDocument = { id, - '@timestamp': secondTimestampISOString, + '@timestamp': secondTimestamp, agent: { name: 'agent-1', }, @@ -800,6 +799,7 @@ export default ({ getService }: FtrProviderContext) => { await indexDocuments([secondDocument, secondDocument]); await patchRule(supertest, log, { id: createdRule.id, enabled: false }); await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); const secondAlerts = await getOpenSignals( supertest, log, @@ -807,7 +807,7 @@ export default ({ getService }: FtrProviderContext) => { createdRule, RuleExecutionStatus.succeeded, undefined, - secondTimestamp + afterTimestamp ); expect(secondAlerts.hits.hits.length).eql(1); expect(secondAlerts.hits.hits[0]._source).to.eql({ @@ -821,7 +821,7 @@ export default ({ getService }: FtrProviderContext) => { ], [ALERT_ORIGINAL_TIME]: firstTimestamp, [ALERT_SUPPRESSION_START]: firstTimestamp, - [ALERT_SUPPRESSION_END]: secondTimestampISOString, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 3, }); }); @@ -862,11 +862,10 @@ export default ({ getService }: FtrProviderContext) => { .send(setSignalStatus({ signalIds: alertIds, status: 'closed' })) .expect(200); - const secondTimestamp = new Date(); - const secondTimestampISOString = secondTimestamp.toISOString(); + const secondTimestamp = new Date().toISOString(); const secondDocument = { id, - '@timestamp': secondTimestampISOString, + '@timestamp': secondTimestamp, agent: { name: 'agent-1', }, @@ -876,6 +875,7 @@ export default ({ getService }: FtrProviderContext) => { await indexDocuments([secondDocument, secondDocument]); await patchRule(supertest, log, { id: createdRule.id, enabled: false }); await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); const secondAlerts = await getOpenSignals( supertest, log, @@ -883,7 +883,7 @@ export default ({ getService }: FtrProviderContext) => { createdRule, RuleExecutionStatus.succeeded, undefined, - secondTimestamp + afterTimestamp ); expect(secondAlerts.hits.hits.length).eql(2); expect(secondAlerts.hits.hits[0]._source).to.eql({ @@ -911,9 +911,9 @@ export default ({ getService }: FtrProviderContext) => { }, ], [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_ORIGINAL_TIME]: secondTimestampISOString, - [ALERT_SUPPRESSION_START]: secondTimestampISOString, - [ALERT_SUPPRESSION_END]: secondTimestampISOString, + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); diff --git a/x-pack/test/detection_engine_api_integration/utils/wait_for_rule_success_or_status.ts b/x-pack/test/detection_engine_api_integration/utils/wait_for_rule_success_or_status.ts index aed1ca76b6a78..f2506552771d5 100644 --- a/x-pack/test/detection_engine_api_integration/utils/wait_for_rule_success_or_status.ts +++ b/x-pack/test/detection_engine_api_integration/utils/wait_for_rule_success_or_status.ts @@ -48,7 +48,7 @@ export const waitForRuleSuccessOrStatus = async ( log.debug( `Did not get an expected status of ${status} while waiting for a rule success or status for rule id ${id} (waitForRuleSuccessOrStatus). Will continue retrying until status is found. body: ${JSON.stringify( response.body - )}, status: ${JSON.stringify(response.status)}` + )}, status: ${JSON.stringify(ruleStatus)}` ); } return ( From 5650d6f3baba4d707bc9f971e7069d62113e3607 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 30 Jan 2023 09:21:38 -0800 Subject: [PATCH 22/22] Fix tests --- .../group6/alerts/alerts_compatibility.ts | 26 ++--------------- .../rule_execution_logic/new_terms.ts | 21 +------------- .../rule_execution_logic/utils.ts | 28 +++++++++++++++++++ 3 files changed, 32 insertions(+), 43 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/utils.ts diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts index 7c7d1933751c3..c7d33dda982a0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group6/alerts/alerts_compatibility.ts @@ -20,7 +20,6 @@ import { ThreatMatchRuleCreateProps, ThresholdRuleCreateProps, } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; -import { ALERT_START } from '@kbn/rule-data-utils'; import { createRule, createSignalsIndex, @@ -39,6 +38,7 @@ import { waitForSignalsToBePresent, } from '../../../utils'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { removeRandomValuedProperties } from '../../rule_execution_logic/utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { @@ -231,23 +231,13 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).greaterThan(0); const hit = signalsOpen.hits.hits[0]; expect(hit._source?.kibana).to.eql(undefined); - const { - '@timestamp': timestamp, - 'kibana.version': kibanaVersion, - 'kibana.alert.rule.created_at': createdAt, - 'kibana.alert.rule.updated_at': updatedAt, - 'kibana.alert.rule.execution.uuid': executionUuid, - 'kibana.alert.uuid': alertId, - [ALERT_START]: alertStart, - ...source - } = hit._source!; + const source = removeRandomValuedProperties(hit._source); expect(source).to.eql({ 'kibana.alert.rule.category': 'Custom Query Rule', 'kibana.alert.rule.consumer': 'siem', 'kibana.alert.rule.name': 'Signal Testing Query', 'kibana.alert.rule.producer': 'siem', 'kibana.alert.rule.rule_type_id': 'siem.queryRule', - 'kibana.alert.rule.uuid': id, 'kibana.space_ids': ['default'], 'kibana.alert.rule.tags': [], agent: { @@ -401,23 +391,13 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).greaterThan(0); const hit = signalsOpen.hits.hits[0]; expect(hit._source?.kibana).to.eql(undefined); - const { - '@timestamp': timestamp, - 'kibana.version': kibanaVersion, - 'kibana.alert.rule.created_at': createdAt, - 'kibana.alert.rule.updated_at': updatedAt, - 'kibana.alert.rule.execution.uuid': executionUuid, - 'kibana.alert.uuid': alertId, - [ALERT_START]: alertStart, - ...source - } = hit._source!; + const source = removeRandomValuedProperties(hit._source); expect(source).to.eql({ 'kibana.alert.rule.category': 'Custom Query Rule', 'kibana.alert.rule.consumer': 'siem', 'kibana.alert.rule.name': 'Signal Testing Query', 'kibana.alert.rule.producer': 'siem', 'kibana.alert.rule.rule_type_id': 'siem.queryRule', - 'kibana.alert.rule.uuid': id, 'kibana.space_ids': ['default'], 'kibana.alert.rule.tags': [], agent: { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index ae086baffdfca..647bc8f0c82a0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -10,12 +10,10 @@ import expect from '@kbn/expect'; import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; import { orderBy } from 'lodash'; import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema/mocks'; -import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; import { getNewTermsRuntimeMappings, AGG_FIELD_NAME, } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/new_terms/utils'; -import { ALERT_START } from '@kbn/rule-data-utils'; import { createRule, deleteAllAlerts, @@ -30,24 +28,7 @@ import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_e import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { largeArraysBuckets } from './mocks/new_terms'; - -const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { - if (!alert) { - return undefined; - } - const { - 'kibana.version': version, - 'kibana.alert.rule.execution.uuid': execUuid, - 'kibana.alert.rule.uuid': uuid, - '@timestamp': timestamp, - 'kibana.alert.rule.created_at': createdAt, - 'kibana.alert.rule.updated_at': updatedAt, - 'kibana.alert.uuid': alertUuid, - [ALERT_START]: alertStart, - ...restOfAlert - } = alert; - return restOfAlert; -}; +import { removeRandomValuedProperties } from './utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/utils.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/utils.ts new file mode 100644 index 0000000000000..f17aae89f5810 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/utils.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; +import { ALERT_LAST_DETECTED, ALERT_START } from '@kbn/rule-data-utils'; + +export const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { + if (!alert) { + return undefined; + } + const { + 'kibana.version': version, + 'kibana.alert.rule.execution.uuid': execUuid, + 'kibana.alert.rule.uuid': uuid, + '@timestamp': timestamp, + 'kibana.alert.rule.created_at': createdAt, + 'kibana.alert.rule.updated_at': updatedAt, + 'kibana.alert.uuid': alertUuid, + [ALERT_START]: alertStart, + [ALERT_LAST_DETECTED]: lastDetected, + ...restOfAlert + } = alert; + return restOfAlert; +};