From 5814aa95db8be2470f46a1acb5f7367030b0256f Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 19 Mar 2024 18:57:07 -0400 Subject: [PATCH] [8.13] [Security Solution] Incorporates EQL options in EQL query validation on both Rule Creation and Timeline (#178468) (#178994) # Backport This will backport the following commits from `main` to `8.13`: - [[Security Solution] Incorporates EQL options in EQL query validation on both Rule Creation and Timeline (#178468)](https://github.com/elastic/kibana/pull/178468) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --------- Co-authored-by: Ryland Herrick --- .../search/strategies/eql_search/types.ts | 5 +- .../eql_search/eql_search_strategy.test.ts | 43 +++++- .../public/common/hooks/eql/api.ts | 6 + .../eql_query_bar/eql_query_bar.test.tsx | 35 +++++ .../eql_query_bar/eql_query_bar.tsx | 8 +- .../components/eql_query_bar/validators.ts | 11 +- .../components/step_define_rule/index.tsx | 94 +++++++------ .../components/step_define_rule/schema.tsx | 5 +- .../public/shared_imports.ts | 2 +- .../timeline/query_bar/eql/index.tsx | 21 ++- .../no_at_timestamp_field/data.json | 38 +++++ .../no_at_timestamp_field/mappings.json | 32 +++++ .../execution_logic/eql.ts | 132 ++++++++++++++++++ .../event_correlation_rule.cy.ts | 39 ++++++ .../rule_creation/indicator_match_rule.cy.ts | 4 +- .../cypress/screens/create_new_rule.ts | 7 +- .../cypress/tasks/create_new_rule.ts | 10 +- .../no_at_timestamp_field/data.json | 38 +++++ .../no_at_timestamp_field/mappings.json | 32 +++++ 19 files changed, 494 insertions(+), 68 deletions(-) create mode 100644 x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/mappings.json create mode 100644 x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json create mode 100644 x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json diff --git a/src/plugins/data/common/search/strategies/eql_search/types.ts b/src/plugins/data/common/search/strategies/eql_search/types.ts index cb325e0a4ec33..732eb77a01927 100644 --- a/src/plugins/data/common/search/strategies/eql_search/types.ts +++ b/src/plugins/data/common/search/strategies/eql_search/types.ts @@ -6,14 +6,15 @@ * Side Public License, v 1. */ -import type { EqlSearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { EqlSearchRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { EqlSearchRequest as EqlSearchRequestWithBody } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { TransportRequestOptions } from '@elastic/elasticsearch'; import { IKibanaSearchRequest, IKibanaSearchResponse } from '../../types'; export const EQL_SEARCH_STRATEGY = 'eql'; -export type EqlRequestParams = EqlSearchRequest; +export type EqlRequestParams = EqlSearchRequest | EqlSearchRequestWithBody; export interface EqlSearchStrategyRequest extends IKibanaSearchRequest { /** diff --git a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts index 475c43a5daed6..d78f152a31cc2 100644 --- a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts +++ b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts @@ -190,7 +190,6 @@ describe('EQL search strategy', () => { options, params: { ...params, - // @ts-expect-error not allowed at top level when using `typesWithBodyKey` wait_for_completion_timeout: '5ms', keep_on_completion: false, }, @@ -277,6 +276,48 @@ describe('EQL search strategy', () => { expect(requestOptions).toEqual({ ignore: [400], meta: true, signal: undefined }); }); + + describe('EQL-specific arguments', () => { + it('passes along a timestamp_field argument', async () => { + const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger); + const request: EqlSearchStrategyRequest = { + params: { index: 'all', timestamp_field: 'timestamp' }, + }; + + await firstValueFrom(eqlSearch.search(request, {}, mockDeps)); + const [[actualParams]] = mockEqlSearch.mock.calls; + + expect(actualParams).toEqual(expect.objectContaining({ timestamp_field: 'timestamp' })); + }); + + it('passes along an event_category_field argument', async () => { + const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger); + const request: EqlSearchStrategyRequest = { + params: { index: 'all', event_category_field: 'event_category' }, + }; + + await firstValueFrom(eqlSearch.search(request, {}, mockDeps)); + const [[actualParams]] = mockEqlSearch.mock.calls; + + expect(actualParams).toEqual( + expect.objectContaining({ event_category_field: 'event_category' }) + ); + }); + + it('passes along a tiebreaker_field argument', async () => { + const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger); + const request: EqlSearchStrategyRequest = { + params: { index: 'all', tiebreaker_field: 'event_category' }, + }; + + await firstValueFrom(eqlSearch.search(request, {}, mockDeps)); + const [[actualParams]] = mockEqlSearch.mock.calls; + + expect(actualParams).toEqual( + expect.objectContaining({ tiebreaker_field: 'event_category' }) + ); + }); + }); }); describe('response', () => { diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts index a707946be7625..46d01bf15d577 100644 --- a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts +++ b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts @@ -11,6 +11,7 @@ import type { EqlSearchStrategyRequest, EqlSearchStrategyResponse } from '@kbn/d import { EQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { EqlOptionsSelected } from '../../../../common/search_strategy'; import { getValidationErrors, isErrorResponse, @@ -23,6 +24,7 @@ interface Params { data: DataPublicPluginStart; signal: AbortSignal; runtimeMappings: estypes.MappingRuntimeFields | undefined; + options: Omit | undefined; } export const validateEql = async ({ @@ -31,6 +33,7 @@ export const validateEql = async ({ query, signal, runtimeMappings, + options, }: Params): Promise<{ valid: boolean; errors: string[] }> => { const { rawResponse: response } = await firstValueFrom( data.search.search( @@ -38,6 +41,9 @@ export const validateEql = async ({ params: { index: dataViewTitle, body: { query, runtime_mappings: runtimeMappings, size: 0 }, + timestamp_field: options?.timestampField, + tiebreaker_field: options?.tiebreakerField || undefined, + event_category_field: options?.eventCategoryField, }, options: { ignore: [400] }, }, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx index aec082224e044..a9a520e802460 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx @@ -13,6 +13,7 @@ import { mockQueryBar } from '../../../rule_management_ui/components/rules_table import type { EqlQueryBarProps } from './eql_query_bar'; import { EqlQueryBar } from './eql_query_bar'; import { getEqlValidationError } from './validators.mock'; +import { fireEvent, render, within } from '@testing-library/react'; jest.mock('../../../../common/lib/kibana'); @@ -117,4 +118,38 @@ describe('EqlQueryBar', () => { expect(wrapper.find('[data-test-subj="eql-validation-errors-popover"]').exists()).toEqual(true); }); + + describe('EQL options interaction', () => { + const mockOptionsData = { + keywordFields: [], + dateFields: [{ label: 'timestamp', value: 'timestamp' }], + nonDateFields: [], + }; + + it('invokes onOptionsChange when the EQL options change', () => { + const onOptionsChangeMock = jest.fn(); + + const { getByTestId, getByText } = render( + + + + ); + + // open options popover + fireEvent.click(getByTestId('eql-settings-trigger')); + // display combobox options + within(getByTestId(`eql-timestamp-field`)).getByRole('combobox').focus(); + // select timestamp + getByText('timestamp').click(); + + expect(onOptionsChangeMock).toHaveBeenCalledWith('timestampField', 'timestamp'); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx index fe1962a71f6c6..607b7a3ef2bb2 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx @@ -11,7 +11,7 @@ import { Subscription } from 'rxjs'; import styled from 'styled-components'; import deepEqual from 'fast-deep-equal'; import { EuiFormRow, EuiSpacer, EuiTextArea } from '@elastic/eui'; -import type { DataViewBase, Filter, Query } from '@kbn/es-query'; +import type { DataViewBase } from '@kbn/es-query'; import { FilterManager } from '@kbn/data-plugin/public'; import type { FieldHook } from '../../../../shared_imports'; @@ -58,12 +58,6 @@ const StyledFormRow = styled(EuiFormRow)` } `; -export interface FieldValueQueryBar { - filters: Filter[]; - query: Query; - saved_id?: string; -} - export interface EqlQueryBarProps { dataTestSubj: string; field: FieldHook; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts index 83b610eaaff7f..b3d6abc62d0b1 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts @@ -58,7 +58,7 @@ export const eqlValidator = async ( const [{ value, formData }] = args; const { query: queryValue } = value as FieldValueQueryBar; const query = queryValue.query as string; - const { dataViewId, index, ruleType } = formData as DefineStepRule; + const { dataViewId, index, ruleType, eqlOptions } = formData as DefineStepRule; const needsValidation = (ruleType === undefined && !isEmpty(query)) || (isEqlRule(ruleType) && !isEmpty(query)); @@ -82,7 +82,14 @@ export const eqlValidator = async ( } const signal = new AbortController().signal; - const response = await validateEql({ data, query, signal, dataViewTitle, runtimeMappings }); + const response = await validateEql({ + data, + query, + signal, + dataViewTitle, + runtimeMappings, + options: eqlOptions, + }); if (response?.valid === false) { return { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 87551441136ca..d8b4d2694dcb0 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -59,7 +59,14 @@ import { StepContentWrapper } from '../../../rule_creation/components/step_conte import { ThresholdInput } from '../threshold_input'; import { SuppressionInfoIcon } from '../suppression_info_icon'; import { EsqlInfoIcon } from '../../../rule_creation/components/esql_info_icon'; -import { Field, Form, getUseField, UseField, UseMultiFields } from '../../../../shared_imports'; +import { + Field, + Form, + getUseField, + HiddenField, + UseField, + UseMultiFields, +} from '../../../../shared_imports'; import type { FormHook } from '../../../../shared_imports'; import { schema } from './schema'; import { getTermsAggregationFields } from './utils'; @@ -768,14 +775,20 @@ const StepDefineRuleComponent: FC = ({ onOpenTimeline, ] ); + const onOptionsChange = useCallback( (field: FieldsEqlOptions, value: string | undefined) => { - setOptionsSelected((prevOptions) => ({ - ...prevOptions, - [field]: value, - })); + setOptionsSelected((prevOptions) => { + const newOptions = { + ...prevOptions, + [field]: value, + }; + + setFieldValue('eqlOptions', newOptions); + return newOptions; + }); }, - [setOptionsSelected] + [setFieldValue, setOptionsSelected] ); const optionsData = useMemo( @@ -814,17 +827,16 @@ const StepDefineRuleComponent: FC = ({ <>
- - - + = ({ {isEqlRule(ruleType) ? ( - + <> + + + ) : isEsqlRule(ruleType) ? ( EsqlQueryBarMemo ) : ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index 8a43698ab5ad4..49eba2d124d31 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -130,9 +130,10 @@ export const schema: FormSchema = { ), validations: [], }, - eqlOptions: {}, + eqlOptions: { + fieldsToValidateOnChange: ['eqlOptions', 'queryBar'], + }, queryBar: { - fieldsToValidateOnChange: ['queryBar'], validations: [ { validator: ( diff --git a/x-pack/plugins/security_solution/public/shared_imports.ts b/x-pack/plugins/security_solution/public/shared_imports.ts index 00993cfd7c6da..8686aecb3b99f 100644 --- a/x-pack/plugins/security_solution/public/shared_imports.ts +++ b/x-pack/plugins/security_solution/public/shared_imports.ts @@ -28,6 +28,6 @@ export { useFormData, VALIDATION_TYPES, } from '@kbn/es-ui-shared-plugin/static/forms/hook_form_lib'; -export { Field, SelectField } from '@kbn/es-ui-shared-plugin/static/forms/components'; +export { Field, SelectField, HiddenField } from '@kbn/es-ui-shared-plugin/static/forms/components'; export { fieldValidators } from '@kbn/es-ui-shared-plugin/static/forms/helpers'; export type { ERROR_CODE } from '@kbn/es-ui-shared-plugin/static/forms/helpers/field_validators/types'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx index b1e1e0934d1d6..8e165a2303036 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx @@ -10,7 +10,10 @@ import React, { memo, useCallback, useEffect, useMemo, useRef, useState } from ' import { useDispatch } from 'react-redux'; import styled from 'styled-components'; -import type { FieldsEqlOptions } from '../../../../../../common/search_strategy'; +import type { + EqlOptionsSelected, + FieldsEqlOptions, +} from '../../../../../../common/search_strategy'; import { useSourcererDataView } from '../../../../../common/containers/sourcerer'; import { useDeepEqualSelector } from '../../../../../common/hooks/use_selector'; import { SourcererScopeName } from '../../../../../common/store/sourcerer/model'; @@ -31,6 +34,7 @@ import { getEqlOptions } from './selectors'; interface TimelineEqlQueryBar { index: string[]; eqlQueryBar: FieldValueQueryBar; + eqlOptions: EqlOptionsSelected; } const defaultValues = { @@ -40,6 +44,7 @@ const defaultValues = { filters: [], saved_id: null, }, + eqlOptions: {}, }; const schema: FormSchema = { @@ -47,6 +52,9 @@ const schema: FormSchema = { fieldsToValidateOnChange: ['index', 'eqlQueryBar'], validations: [], }, + eqlOptions: { + fieldsToValidateOnChange: ['eqlOptions', 'eqlQueryBar'], + }, eqlQueryBar: { validations: [ { @@ -89,18 +97,20 @@ export const EqlQueryBarTimeline = memo(({ timelineId }: { timelineId: string }) options: { stripEmptyFields: false }, schema, }); - const { getFields } = form; + const { getFields, setFieldValue } = form; const onOptionsChange = useCallback( - (field: FieldsEqlOptions, value: string | undefined) => + (field: FieldsEqlOptions, value: string | undefined) => { dispatch( timelineActions.updateEqlOptions({ id: timelineId, field, value, }) - ), - [dispatch, timelineId] + ); + setFieldValue('eqlOptions', { ...optionsSelected, [field]: value }); + }, + [dispatch, optionsSelected, setFieldValue, timelineId] ); const [{ eqlQueryBar: formEqlQueryBar }] = useFormData({ @@ -179,6 +189,7 @@ export const EqlQueryBarTimeline = memo(({ timelineId }: { timelineId: string }) return ( + { expect(fullAlert?.['host.asset.criticality']).to.eql('high_impact'); }); }); + + describe('using data with a @timestamp field', () => { + const expectedWarning = + 'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.'; + + it('specifying only timestamp_field results in alert creation with an expected warning', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['auditbeat-*']), + timestamp_field: 'event.created', + }; + + const { + previewId, + logs: [_log], + } = await previewRule({ supertest, rule }); + + expect(_log.errors).to.be.empty(); + expect(_log.warnings).to.eql([expectedWarning]); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).to.be.greaterThan(0); + }); + + it('specifying only timestamp_override results in alert creation with an expected warning', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['auditbeat-*']), + timestamp_override: 'event.created', + }; + + const { + previewId, + logs: [_log], + } = await previewRule({ supertest, rule }); + + expect(_log.errors).to.be.empty(); + expect(_log.warnings).to.eql([expectedWarning]); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).to.be.greaterThan(0); + }); + + it('specifying both timestamp_override and timestamp_field results in alert creation with an expected warning', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['auditbeat-*']), + timestamp_field: 'event.created', + timestamp_override: 'event.created', + }; + + const { + previewId, + logs: [_log], + } = await previewRule({ supertest, rule }); + + expect(_log.errors).to.be.empty(); + expect(_log.warnings).to.eql([expectedWarning]); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).to.be.greaterThan(0); + }); + }); + + describe('using data without a @timestamp field', () => { + before(async () => { + await esArchiver.load( + 'x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field' + ); + }); + + after(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field' + ); + }); + + it('specifying only timestamp_field results in a warning, and no alerts are generated', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['no_at_timestamp_field']), + timestamp_field: 'event.ingested', + }; + + const { + previewId, + logs: [_log], + } = await previewRule({ supertest, rule }); + + expect(_log.errors).to.be.empty(); + expect(_log.warnings).to.contain( + 'The following indices are missing the timestamp field "@timestamp": ["no_at_timestamp_field"]' + ); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts).to.be.empty(); + }); + + it('specifying only timestamp_override results in an error, and no alerts are generated', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['no_at_timestamp_field']), + timestamp_override: 'event.ingested', + }; + + const { + previewId, + logs: [_log], + } = await previewRule({ supertest, rule }); + + expect(_log.errors).to.contain( + 'An error occurred during rule execution: message: "verification_exception\n\tRoot causes:\n\t\tverification_exception: Found 1 problem\nline -1:-1: Unknown column [@timestamp]"' + ); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts).to.be.empty(); + }); + + it('specifying both timestamp_override and timestamp_field results in alert creation with no warnings or errors', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['no_at_timestamp_field']), + timestamp_field: 'event.ingested', + timestamp_override: 'event.ingested', + }; + + const { + previewId, + logs: [_log], + } = await previewRule({ supertest, rule }); + + expect(_log.errors).to.be.empty(); + expect(_log.warnings).to.be.empty(); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts).to.have.length(3); + }); + }); }); }; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts index f02ec20ffd685..05ff0d9bfb92a 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts @@ -49,6 +49,8 @@ import { fillAboutRuleAndContinue, fillDefineEqlRuleAndContinue, fillScheduleRuleAndContinue, + getIndexPatternClearButton, + getRuleIndexInput, selectEqlRuleType, waitForAlertsToPopulate, } from '../../../../tasks/create_new_rule'; @@ -56,6 +58,13 @@ import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; import { openRuleManagementPageViaBreadcrumbs } from '../../../../tasks/rules_management'; import { CREATE_RULE_URL } from '../../../../urls/navigation'; +import { + EQL_OPTIONS_POPOVER_TRIGGER, + EQL_OPTIONS_TIMESTAMP_INPUT, + EQL_QUERY_INPUT, + EQL_QUERY_VALIDATION_ERROR, + RULES_CREATION_FORM, +} from '../../../../screens/create_new_rule'; describe('EQL rules', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { @@ -174,4 +183,34 @@ describe('EQL rules', { tags: ['@ess', '@serverless'] }, () => { }); }); }); + + describe('with source data requiring EQL overrides', () => { + before(() => { + cy.task('esArchiverLoad', { archiveName: 'no_at_timestamp_field' }); + }); + + after(() => { + cy.task('esArchiverUnload', 'no_at_timestamp_field'); + }); + + it('includes EQL options in query validation', () => { + login(); + visit(CREATE_RULE_URL); + selectEqlRuleType(); + getIndexPatternClearButton().click(); + getRuleIndexInput().type(`no_at_timestamp_field{enter}`); + + cy.get(RULES_CREATION_FORM).find(EQL_QUERY_INPUT).should('exist'); + cy.get(RULES_CREATION_FORM).find(EQL_QUERY_INPUT).should('be.visible'); + cy.get(RULES_CREATION_FORM).find(EQL_QUERY_INPUT).type('any where true'); + + cy.get(EQL_QUERY_VALIDATION_ERROR).should('be.visible'); + cy.get(EQL_QUERY_VALIDATION_ERROR).should('have.text', '1'); + + cy.get(RULES_CREATION_FORM).find(EQL_OPTIONS_POPOVER_TRIGGER).click(); + cy.get(EQL_OPTIONS_TIMESTAMP_INPUT).type('event.ingested{enter}'); + + cy.get(EQL_QUERY_VALIDATION_ERROR).should('not.exist'); + }); + }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts index 9ce0bbb5be44a..c62c13144b5ab 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts @@ -84,7 +84,7 @@ import { getIndicatorAndButton, getIndicatorAtLeastOneInvalidationText, getIndicatorDeleteButton, - getIndicatorIndex, + getRuleIndexInput, getIndicatorIndexComboField, getIndicatorIndicatorIndex, getIndicatorInvalidationText, @@ -141,7 +141,7 @@ describe('indicator match', { tags: ['@ess', '@serverless'] }, () => { }); it('Contains a predefined index pattern', () => { - getIndicatorIndex().should('have.text', getIndexPatterns().join('')); + getRuleIndexInput().should('have.text', getIndexPatterns().join('')); }); it('Does NOT show invalidation text on initial page load if indicator index pattern is filled out', () => { diff --git a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts index 992b937135773..ac474a56bd5b3 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts @@ -72,7 +72,7 @@ export const THREAT_MATCH_CUSTOM_QUERY_INPUT = export const THREAT_MATCH_QUERY_INPUT = '[data-test-subj="detectionEngineStepDefineThreatRuleQueryBar"] [data-test-subj="queryInput"]'; -export const THREAT_MATCH_INDICATOR_INDEX = +export const CUSTOM_INDEX_PATTERN_INPUT = '[data-test-subj="detectionEngineStepDefineRuleIndices"] [data-test-subj="comboBoxInput"]'; export const THREAT_MATCH_INDICATOR_INDICATOR_INDEX = @@ -117,6 +117,11 @@ export const EQL_QUERY_VALIDATION_SPINNER = '[data-test-subj="eql-validation-loa export const EQL_QUERY_VALIDATION_ERROR = '[data-test-subj="eql-validation-errors-popover-button"]'; +export const EQL_OPTIONS_POPOVER_TRIGGER = '[data-test-subj="eql-settings-trigger"]'; + +export const EQL_OPTIONS_TIMESTAMP_INPUT = + '[data-test-subj="eql-timestamp-field"] [data-test-subj="comboBoxInput"]'; + export const IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK = '[data-test-subj="importQueryFromSavedTimeline"]'; diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts index d8c1c60bc7639..1144613246e51 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts @@ -105,7 +105,7 @@ import { THREAT_MAPPING_COMBO_BOX_INPUT, THREAT_MATCH_AND_BUTTON, THREAT_MATCH_CUSTOM_QUERY_INPUT, - THREAT_MATCH_INDICATOR_INDEX, + CUSTOM_INDEX_PATTERN_INPUT, THREAT_MATCH_INDICATOR_INDICATOR_INDEX, THREAT_MATCH_OR_BUTTON, THREAT_MATCH_QUERY_INPUT, @@ -657,7 +657,7 @@ export const fillIndexAndIndicatorIndexPattern = ( ) => { getIndexPatternClearButton().click(); - getIndicatorIndex().type(`${indexPattern}{enter}`); + getRuleIndexInput().type(`${indexPattern}{enter}`); getIndicatorIndicatorIndex().type(`{backspace}{enter}${indicatorIndex}{enter}`); }; @@ -694,9 +694,9 @@ const getAboutContinueButton = () => cy.get(ABOUT_CONTINUE_BTN); /** Returns the continue button on the step of define */ export const getDefineContinueButton = () => cy.get(DEFINE_CONTINUE_BUTTON); -/** Returns the indicator index pattern */ -export const getIndicatorIndex = () => { - return cy.get(THREAT_MATCH_INDICATOR_INDEX).eq(0); +/** Returns the custom rule index pattern input */ +export const getRuleIndexInput = () => { + return cy.get(CUSTOM_INDEX_PATTERN_INPUT).eq(0); }; /** Returns the indicator's indicator index */ diff --git a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json new file mode 100644 index 0000000000000..3ee1387b01e5c --- /dev/null +++ b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json @@ -0,0 +1,38 @@ +{ + "type": "doc", + "value": { + "index": "no_at_timestamp_field", + "source": { + "locale": "pt", + "event.category": "configuration", + "event.ingested": 1608131778 + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "index": "no_at_timestamp_field", + "source": { + "locale": "es", + "event.category": "configuration", + "event.ingested": 1608131778 + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "index": "no_at_timestamp_field", + "source": { + "locale": "ua", + "event.category": "configuration", + "event.ingested": 1608131779 + }, + "type": "_doc" + } +} diff --git a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json new file mode 100644 index 0000000000000..76f0aa8521fc3 --- /dev/null +++ b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json @@ -0,0 +1,32 @@ +{ + "type": "index", + "value": { + "index": "no_at_timestamp_field", + "mappings": { + "dynamic": "strict", + "properties": { + "locale": { + "type": "keyword" + }, + "event": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date", + "format": "epoch_second" + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}