From 610622c4894542d5c48794cbc8a2e35c1de85574 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 12 Nov 2024 18:24:47 +0100 Subject: [PATCH 01/45] add ES|QL Query edit component --- .../esql/compute_if_esql_query_aggregating.ts | 10 +- .../esql_info_icon.tsx} | 0 .../esql_query_edit/esql_query_edit.tsx | 76 ++++++++ .../translations.ts | 6 +- .../logic/esql_validator.test.ts | 121 ------------ .../rule_creation/logic/esql_validator.ts | 160 --------------- .../use_esql_fields_options.ts | 3 +- .../components/step_define_rule/index.tsx | 55 +----- .../components/step_define_rule/schema.tsx | 4 +- .../step_define_rule/translations.tsx | 7 - .../rule_creation_ui/pages/form.tsx | 2 +- .../validators/debounce_async.ts | 37 ++++ .../esql_query_validator_factory.ts | 184 ++++++++++++++++++ .../validators/get_esql_query_config.ts | 40 ++++ .../final_edit/esql_rule_field_edit.tsx | 10 +- .../esql_query/esql_query_edit_adapter.tsx | 38 ++++ .../esql_query/esql_query_edit_form.tsx | 72 +++++++ .../final_edit/fields/esql_query/index.ts | 8 + 18 files changed, 484 insertions(+), 349 deletions(-) rename x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/{esql_info_icon/index.tsx => esql_query_edit/esql_info_icon.tsx} (100%) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx rename x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/{esql_info_icon => esql_query_edit}/translations.ts (77%) delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/debounce_async.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_adapter.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_form.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/index.ts diff --git a/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts b/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts index 4daf793b32d9d..83d4b6cf51e63 100644 --- a/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts +++ b/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts @@ -7,10 +7,10 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { ESQLAst, getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import { type ESQLAstQueryExpression, parse } from '@kbn/esql-ast'; -export const isAggregatingQuery = (ast: ESQLAst): boolean => { - return ast.some((astItem) => astItem.type === 'command' && astItem.name === 'stats'); +export const isAggregatingQuery = (astExpression: ESQLAstQueryExpression): boolean => { + return astExpression.commands.some((command) => command.name === 'stats'); }; /** @@ -19,6 +19,6 @@ export const isAggregatingQuery = (ast: ESQLAst): boolean => { * @returns boolean */ export const computeIsESQLQueryAggregating = (esqlQuery: string): boolean => { - const { ast } = getAstAndSyntaxErrors(esqlQuery); - return isAggregatingQuery(ast); + const { root } = parse(esqlQuery); + return isAggregatingQuery(root); }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx similarity index 100% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/index.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx new file mode 100644 index 0000000000000..bdc0900204289 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo, useMemo } from 'react'; +import type { DataViewBase } from '@kbn/es-query'; +import type { FieldConfig } from '../../../../shared_imports'; +import { UseField } from '../../../../shared_imports'; +import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar'; +import { QueryBarDefineRule } from '../../../rule_creation_ui/components/query_bar'; +import { queryRequiredValidatorFactory } from '../../../rule_creation_ui/validators/query_required_validator_factory'; +import { debounceAsync } from '../../../rule_creation_ui/validators/debounce_async'; +import { esqlQueryValidatorFactory } from '../../../rule_creation_ui/validators/esql_query_validator_factory'; +import { EsqlInfoIcon } from './esql_info_icon'; +import * as i18n from './translations'; + +interface EsqlQueryEditProps { + path: string; + dataView: DataViewBase; + required?: boolean; + loading?: boolean; + disabled?: boolean; + onValidityChange?: (arg: boolean) => void; +} + +export const EsqlQueryEdit = memo(function EsqlQueryEdit({ + path, + dataView, + required = false, + loading = false, + disabled = false, + onValidityChange, +}: EsqlQueryEditProps): JSX.Element { + const componentProps = useMemo( + () => ({ + isDisabled: disabled, + isLoading: loading, + indexPattern: dataView, + idAria: 'ruleEsqlQueryBar', + dataTestSubj: 'ruleEsqlQueryBar', + onValidityChange, + }), + [dataView, loading, disabled, onValidityChange] + ); + const fieldConfig: FieldConfig = useMemo( + () => ({ + label: i18n.ESQL_QUERY, + labelAppend: , + validations: [ + ...(required + ? [ + { + validator: queryRequiredValidatorFactory('esql'), + }, + ] + : []), + { + validator: debounceAsync(esqlQueryValidatorFactory(), 300), + }, + ], + }), + [required] + ); + + return ( + + ); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts similarity index 77% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/translations.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts index 8729f7b0dd3bc..6e9e15f77a6e3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/translations.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts @@ -7,9 +7,9 @@ import { i18n } from '@kbn/i18n'; -export const ARIA_LABEL = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel', +export const ESQL_QUERY = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel', { - defaultMessage: `Open help popover`, + defaultMessage: 'ES|QL query', } ); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts deleted file mode 100644 index 808597ff36495..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; -import { parseEsqlQuery, computeHasMetadataOperator } from './esql_validator'; - -import { isAggregatingQuery } from '@kbn/securitysolution-utils'; - -jest.mock('@kbn/securitysolution-utils', () => ({ isAggregatingQuery: jest.fn() })); - -const isAggregatingQueryMock = isAggregatingQuery as jest.Mock; - -const getQeryAst = (query: string) => { - const { ast } = getAstAndSyntaxErrors(query); - return ast; -}; - -describe('computeHasMetadataOperator', () => { - it('should be false if query does not have operator', () => { - expect(computeHasMetadataOperator(getQeryAst('from test*'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata id'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from metadata*'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* | keep metadata'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="metadata _id"'))).toBe( - false - ); - }); - it('should be true if query has operator', () => { - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id, _index'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _index, _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id | limit 10'))).toBe( - true - ); - expect( - computeHasMetadataOperator( - getQeryAst(`from packetbeat* metadata - - _id - | limit 100`) - ) - ).toBe(true); - - // still validates deprecated square bracket syntax - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id, _index'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _index, _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id | limit 10'))).toBe( - true - ); - expect( - computeHasMetadataOperator( - getQeryAst(`from packetbeat* metadata - - _id - | limit 100`) - ) - ).toBe(true); - }); -}); - -describe('parseEsqlQuery', () => { - it('returns isMissingMetadataOperator true when query is not aggregating and does not have metadata operator', () => { - isAggregatingQueryMock.mockReturnValueOnce(false); - - expect(parseEsqlQuery('from test*')).toEqual({ - errors: [], - isEsqlQueryAggregating: false, - isMissingMetadataOperator: true, - }); - }); - - it('returns isMissingMetadataOperator false when query is not aggregating and has metadata operator', () => { - isAggregatingQueryMock.mockReturnValueOnce(false); - - expect(parseEsqlQuery('from test* metadata _id')).toEqual({ - errors: [], - isEsqlQueryAggregating: false, - isMissingMetadataOperator: false, - }); - }); - - it('returns isMissingMetadataOperator false when query is aggregating', () => { - isAggregatingQueryMock.mockReturnValue(true); - - expect(parseEsqlQuery('from test*')).toEqual({ - errors: [], - isEsqlQueryAggregating: true, - isMissingMetadataOperator: false, - }); - - expect(parseEsqlQuery('from test* metadata _id')).toEqual({ - errors: [], - isEsqlQueryAggregating: true, - isMissingMetadataOperator: false, - }); - }); - - it('returns error when query is syntactically invalid', () => { - isAggregatingQueryMock.mockReturnValueOnce(false); - - expect(parseEsqlQuery('aaa bbbb ssdasd')).toEqual({ - errors: expect.arrayContaining([ - expect.objectContaining({ - message: - "SyntaxError: mismatched input 'aaa' expecting {'explain', 'from', 'row', 'show'}", - }), - ]), - isEsqlQueryAggregating: false, - isMissingMetadataOperator: true, - }); - }); -}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts deleted file mode 100644 index c508676cae92c..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isEmpty } from 'lodash'; -import type { QueryClient } from '@tanstack/react-query'; -import { isAggregatingQuery } from '@kbn/securitysolution-utils'; - -import type { ESQLAst } from '@kbn/esql-ast'; -import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; -import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete'; -import { KibanaServices } from '../../../common/lib/kibana'; - -import type { ValidationError, ValidationFunc } from '../../../shared_imports'; -import { isEsqlRule } from '../../../../common/detection_engine/utils'; -import type { DefineStepRule } from '../../../detections/pages/detection_engine/rules/types'; -import type { FieldValueQueryBar } from '../../rule_creation_ui/components/query_bar'; -import * as i18n from './translations'; -import { getEsqlQueryConfig } from './get_esql_query_config'; -export type FieldType = 'string'; - -export enum ERROR_CODES { - INVALID_ESQL = 'ERR_INVALID_ESQL', - INVALID_SYNTAX = 'ERR_INVALID_SYNTAX', - ERR_MISSING_ID_FIELD_FROM_RESULT = 'ERR_MISSING_ID_FIELD_FROM_RESULT', -} - -const constructValidationError = (error: Error) => { - return { - code: ERROR_CODES.INVALID_ESQL, - message: error?.message - ? i18n.esqlValidationErrorMessage(error.message) - : i18n.ESQL_VALIDATION_UNKNOWN_ERROR, - error, - }; -}; - -const constructSyntaxError = (error: Error) => { - return { - code: ERROR_CODES.INVALID_SYNTAX, - message: error?.message - ? i18n.esqlValidationErrorMessage(error.message) - : i18n.ESQL_VALIDATION_UNKNOWN_ERROR, - error, - }; -}; - -const getMetadataOption = (ast: ESQLAst) => { - const fromCommand = ast.find((astItem) => astItem.type === 'command' && astItem.name === 'from'); - - if (!fromCommand?.args) { - return undefined; - } - - // Check whether the `from` command has `metadata` operator - for (const fromArg of fromCommand.args) { - if (isOptionItem(fromArg) && fromArg.name === 'metadata') { - return fromArg; - } - } - - return undefined; -}; - -/** - * checks whether query has metadata _id operator - */ -export const computeHasMetadataOperator = (ast: ESQLAst) => { - // Check whether the `from` command has `metadata` operator - const metadataOption = getMetadataOption(ast); - if (!metadataOption) { - return false; - } - - // Check whether the `metadata` operator has `_id` argument - const idColumnItem = metadataOption.args.find( - (fromArg) => isColumnItem(fromArg) && fromArg.name === '_id' - ); - if (!idColumnItem) { - return false; - } - - return true; -}; - -/** - * form validator for ES|QL queryBar - */ -export const esqlValidator = async ( - ...args: Parameters -): Promise | void | undefined> => { - const [{ value, formData, customData }] = args; - const { query: queryValue } = value as FieldValueQueryBar; - const query = queryValue.query as string; - const { ruleType } = formData as DefineStepRule; - - const needsValidation = isEsqlRule(ruleType) && !isEmpty(query); - if (!needsValidation) { - return; - } - - try { - const queryClient = (customData.value as { queryClient: QueryClient | undefined })?.queryClient; - - const services = KibanaServices.get(); - const { isEsqlQueryAggregating, isMissingMetadataOperator, errors } = parseEsqlQuery(query); - - // Check if there are any syntax errors - if (errors.length) { - return constructSyntaxError(new Error(errors[0].message)); - } - - if (isMissingMetadataOperator) { - return { - code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, - message: i18n.ESQL_VALIDATION_MISSING_METADATA_OPERATOR_IN_QUERY_ERROR, - }; - } - - const columns = await queryClient?.fetchQuery( - getEsqlQueryConfig({ esqlQuery: query, search: services.data.search.search }) - ); - - if (columns && 'error' in columns) { - return constructValidationError(columns.error); - } - - // check whether _id field is present in response - const isIdFieldPresent = (columns ?? []).find(({ id }) => '_id' === id); - // for non-aggregating query, we want to disable queries w/o _id property returned in response - if (!isEsqlQueryAggregating && !isIdFieldPresent) { - return { - code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, - message: i18n.ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR, - }; - } - } catch (error) { - return constructValidationError(error); - } -}; - -/** - * check if esql query valid for Security rule: - * - if it's non aggregation query it must have metadata operator - */ -export const parseEsqlQuery = (query: string) => { - const { ast, errors } = getAstAndSyntaxErrors(query); - - const isEsqlQueryAggregating = isAggregatingQuery(ast); - - return { - errors, - isEsqlQueryAggregating, - // non-aggregating query which does not have metadata, is not a valid one - isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(ast), - }; -}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts index b29d44c0b855f..951accbf9b80f 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts @@ -14,7 +14,8 @@ import { useQuery } from '@tanstack/react-query'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import { getEsqlQueryConfig } from '../../../rule_creation/logic/get_esql_query_config'; -import type { FieldType } from '../../../rule_creation/logic/esql_validator'; + +type FieldType = 'string'; export const esqlToOptions = ( columns: { error: unknown } | DatatableColumn[] | undefined | null, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 053544e6fdbb5..d9c91b325a69b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -24,7 +24,6 @@ import { isEqual } from 'lodash'; import type { FieldSpec } from '@kbn/data-plugin/common'; import usePrevious from 'react-use/lib/usePrevious'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; -import { useQueryClient } from '@tanstack/react-query'; import type { SavedQuery } from '@kbn/data-plugin/public'; import type { DataViewBase } from '@kbn/es-query'; @@ -48,7 +47,6 @@ import { MlJobSelect } from '../../../rule_creation/components/ml_job_select'; import { PickTimeline } from '../../../rule_creation/components/pick_timeline'; import { StepContentWrapper } from '../../../rule_creation/components/step_content_wrapper'; import { ThresholdInput } from '../threshold_input'; -import { EsqlInfoIcon } from '../../../rule_creation/components/esql_info_icon'; import { Field, Form, @@ -96,6 +94,7 @@ import { } from '../../../rule_creation/components/alert_suppression_edit'; import { ThresholdAlertSuppressionEdit } from '../../../rule_creation/components/threshold_alert_suppression_edit'; import { usePersistentAlertSuppressionState } from './use_persistent_alert_suppression_state'; +import { EsqlQueryEdit } from '../../../rule_creation/components/esql_query_edit/esql_query_edit'; const CommonUseField = getUseField({ component: Field }); @@ -173,8 +172,6 @@ const StepDefineRuleComponent: FC = ({ threatIndicesConfig, thresholdFields, }) => { - const queryClient = useQueryClient(); - const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); @@ -621,46 +618,6 @@ const StepDefineRuleComponent: FC = ({ handleResetIndices, ]); - const queryBarProps = useMemo( - () => - ({ - idAria: 'detectionEngineStepDefineRuleQueryBar', - indexPattern, - isDisabled: isLoading, - isLoading, - dataTestSubj: 'detectionEngineStepDefineRuleQueryBar', - onValidityChange: setIsQueryBarValid, - } as QueryBarDefineRuleProps), - [indexPattern, isLoading, setIsQueryBarValid] - ); - - const esqlQueryBarConfig = useMemo( - () => ({ - ...schema.queryBar, - label: i18n.ESQL_QUERY, - labelAppend: , - }), - [] - ); - - const EsqlQueryBarMemo = useMemo( - () => ( - - ), - [queryBarProps, esqlQueryBarConfig, queryClient] - ); - const QueryBarMemo = useMemo( () => ( = ({ /> ) : isEsqlRule(ruleType) ? ( - EsqlQueryBarMemo + ) : ( QueryBarMemo )} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index 835cab7a4cfc2..46ab154e76428 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -31,6 +31,7 @@ import { FIELD_TYPES, fieldValidators } from '../../../../shared_imports'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; import { DataSourceType } from '../../../../detections/pages/detection_engine/rules/types'; import { esqlValidator } from '../../../rule_creation/logic/esql_validator'; +import { debounceAsync, eqlValidator } from '../eql_query_bar/validators'; import { dataViewIdValidatorFactory } from '../../validators/data_view_id_validator_factory'; import { indexPatternValidatorFactory } from '../../validators/index_pattern_validator_factory'; import { alertSuppressionFieldsValidatorFactory } from '../../validators/alert_suppression_fields_validator_factory'; @@ -134,9 +135,6 @@ export const schema: FormSchema = { { validator: kueryValidatorFactory(), }, - { - validator: debounceAsync(esqlValidator, 300), - }, ], }, ruleType: { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx index 7b8063b23e306..897331d20da73 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx @@ -139,13 +139,6 @@ export const ALERT_SUPPRESSION_MISSING_FIELDS_DO_NOT_SUPPRESS_OPTION = i18n.tran } ); -export const ESQL_QUERY = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel', - { - defaultMessage: 'ES|QL query', - } -); - export const ALERT_SUPPRESSION_PER_RULE_EXECUTION = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.alertSuppressionOptions.perRuleExecutionLabel', { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.tsx index 64230fd3a8a23..9c1ff4eecfbeb 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.tsx @@ -23,10 +23,10 @@ import { schema as aboutRuleSchema, threatMatchAboutSchema, } from '../components/step_about_rule/schema'; +import { ESQL_ERROR_CODES } from '../validators/esql_query_validator_factory'; import { schema as scheduleRuleSchema } from '../components/step_schedule_rule/schema'; import { getSchema as getActionsRuleSchema } from '../../rule_creation/components/step_rule_actions/get_schema'; import { useFetchIndex } from '../../../common/containers/source'; -import { ERROR_CODES as ESQL_ERROR_CODES } from '../../rule_creation/logic/esql_validator'; import { EQL_ERROR_CODES } from '../../../common/hooks/eql/api'; import * as i18n from './translations'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/debounce_async.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/debounce_async.ts new file mode 100644 index 0000000000000..f9de51e4a2e65 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/debounce_async.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Unlike lodash's debounce, which resolves intermediate calls with the most + * recent value, this implementation waits to resolve intermediate calls until + * the next invocation resolves. + * + * @param fn an async function + * + * @returns A debounced async function that resolves on the next invocation + */ +export const debounceAsync = ( + fn: (...args: Args) => Result, + interval: number +): ((...args: Args) => Result) => { + let handle: ReturnType | undefined; + let resolves: Array<(value?: Result) => void> = []; + + return (...args: Args): Result => { + if (handle) { + clearTimeout(handle); + } + + handle = setTimeout(() => { + const result = fn(...args); + resolves.forEach((resolve) => resolve(result)); + resolves = []; + }, interval); + + return new Promise((resolve) => resolves.push(resolve)) as Result; + }; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts new file mode 100644 index 0000000000000..9b789b88a976c --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryClient } from '@tanstack/react-query'; +import { isEmpty } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import type { ESQLAstQueryExpression, ESQLCommandOption } from '@kbn/esql-ast'; +import { parse } from '@kbn/esql-ast'; +import { isAggregatingQuery } from '@kbn/securitysolution-utils'; +import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete'; +import { getESQLQueryColumns } from '@kbn/esql-utils'; +import type { FormData, ValidationError, ValidationFunc } from '../../../shared_imports'; +import { KibanaServices } from '../../../common/lib/kibana'; +import type { FieldValueQueryBar } from '../components/query_bar'; + +export enum ESQL_ERROR_CODES { + INVALID_ESQL = 'ERR_INVALID_ESQL', + INVALID_SYNTAX = 'ERR_INVALID_SYNTAX', + ERR_MISSING_ID_FIELD_FROM_RESULT = 'ERR_MISSING_ID_FIELD_FROM_RESULT', +} + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 60 * 1000, + }, + }, +}); + +export function esqlQueryValidatorFactory(): ValidationFunc { + return async (...args) => { + const [{ value }] = args; + const esqlQuery = value.query.query as string; + + if (isEmpty(esqlQuery)) { + return; + } + + try { + const services = KibanaServices.get(); + const { isEsqlQueryAggregating, isMissingMetadataOperator, errors } = + parseEsqlQuery(esqlQuery); + + // Check if there are any syntax errors + if (errors.length) { + return constructSyntaxError(new Error(errors[0].message)); + } + + if (isMissingMetadataOperator) { + return { + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + message: ESQL_VALIDATION_MISSING_METADATA_OPERATOR_IN_QUERY_ERROR, + }; + } + + const columns = await queryClient.fetchQuery({ + queryKey: [esqlQuery.trim()], + queryFn: () => + getESQLQueryColumns({ + esqlQuery, + search: services.data.search.search, + }), + }); + + // check whether _id field is present in response + const isIdFieldPresent = columns.find(({ id }) => '_id' === id); + + // for non-aggregating query, we want to disable queries w/o _id property returned in response + if (!isEsqlQueryAggregating && !isIdFieldPresent) { + return { + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + message: ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR, + }; + } + } catch (error) { + return constructValidationError(error); + } + }; +} + +/** + * check if esql query valid for Security rule: + * - if it's non aggregation query it must have metadata operator + */ +function parseEsqlQuery(query: string) { + const { root, errors } = parse(query); + const isEsqlQueryAggregating = isAggregatingQuery(root); + + return { + errors, + isEsqlQueryAggregating, + // non-aggregating query which does not have metadata, is not a valid one + isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(root), + }; +} + +/** + * checks whether query has metadata _id operator + */ +function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean { + // Check whether the `from` command has `metadata` operator + const metadataOption = getMetadataOption(astExpression); + if (!metadataOption) { + return false; + } + + // Check whether the `metadata` operator has `_id` argument + const idColumnItem = metadataOption.args.find( + (fromArg) => isColumnItem(fromArg) && fromArg.name === '_id' + ); + if (!idColumnItem) { + return false; + } + + return true; +} + +function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOption | undefined { + const fromCommand = astExpression.commands.find((x) => x.name === 'from'); + + if (!fromCommand?.args) { + return undefined; + } + + // Check whether the `from` command has `metadata` operator + for (const fromArg of fromCommand.args) { + if (isOptionItem(fromArg) && fromArg.name === 'metadata') { + return fromArg; + } + } + + return undefined; +} + +function constructSyntaxError(error: Error): ValidationError { + return { + code: ESQL_ERROR_CODES.INVALID_SYNTAX, + message: error?.message + ? esqlValidationErrorMessage(error.message) + : ESQL_VALIDATION_UNKNOWN_ERROR, + error, + }; +} + +function constructValidationError(error: Error): ValidationError { + return { + code: ESQL_ERROR_CODES.INVALID_ESQL, + message: error?.message + ? esqlValidationErrorMessage(error.message) + : ESQL_VALIDATION_UNKNOWN_ERROR, + error, + }; +} + +const ESQL_VALIDATION_UNKNOWN_ERROR = i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlValidation.unknownError', + { + defaultMessage: 'Unknown error while validating ES|QL', + } +); + +const esqlValidationErrorMessage = (message: string) => + i18n.translate('xpack.securitySolution.detectionEngine.esqlValidation.errorMessage', { + values: { message }, + defaultMessage: 'Error validating ES|QL: "{message}"', + }); + +const ESQL_VALIDATION_MISSING_METADATA_OPERATOR_IN_QUERY_ERROR = i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlValidation.missingMetadataOperatorInQueryError', + { + defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`, + } +); + +const ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR = i18n.translate( + 'xpack.securitySolution.detectionEngine.esqlValidation.missingIdFieldInQueryError', + { + defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index. In addition, the metadata properties (_id, _version, and _index) must be returned in the query response.`, + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts new file mode 100644 index 0000000000000..c7d4f9184f181 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getESQLQueryColumns } from '@kbn/esql-utils'; +import type { ISearchGeneric } from '@kbn/search-types'; + +/** + * react-query configuration to be used to fetch ES|QL fields + * it sets limit in query to 0, so we don't fetch unnecessary results, only fields + */ +export const getEsqlQueryConfig = ({ + esqlQuery, + search, +}: { + esqlQuery: string | undefined; + search: ISearchGeneric; +}) => { + return { + queryKey: [(esqlQuery ?? '').trim()], + queryFn: async () => { + if (!esqlQuery) { + return null; + } + try { + const res = await getESQLQueryColumns({ + esqlQuery, + search, + }); + return res; + } catch (e) { + return { error: e }; + } + }, + staleTime: 60 * 1000, + }; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/esql_rule_field_edit.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/esql_rule_field_edit.tsx index 745d25da38394..12777222736cd 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/esql_rule_field_edit.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/esql_rule_field_edit.tsx @@ -7,17 +7,21 @@ import React from 'react'; import type { UpgradeableEsqlFields } from '../../../../model/prebuilt_rule_upgrade/fields'; +import { assertUnreachable } from '../../../../../../../common/utility_types'; +import { EsqlQueryEditForm } from './fields/esql_query'; import { AlertSuppressionEditForm } from './fields/alert_suppression'; -interface EqQlRuleFieldEditProps { +interface EsqlRuleFieldEditProps { fieldName: UpgradeableEsqlFields; } -export function EsqlRuleFieldEdit({ fieldName }: EqQlRuleFieldEditProps) { +export function EsqlRuleFieldEdit({ fieldName }: EsqlRuleFieldEditProps) { switch (fieldName) { + case 'esql_query': + return ; case 'alert_suppression': return ; default: - return null; // Will be replaced with `assertUnreachable(fieldName)` once all fields are implemented + return assertUnreachable(fieldName); } } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_adapter.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_adapter.tsx new file mode 100644 index 0000000000000..23efa9a91b046 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_adapter.tsx @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import type { DataViewBase } from '@kbn/es-query'; +import { EsqlQueryEdit } from '../../../../../../../rule_creation/components/esql_query_edit/esql_query_edit'; +import type { RuleFieldEditComponentProps } from '../rule_field_edit_component_props'; +import { useDiffableRuleDataView } from '../hooks/use_diffable_rule_data_view'; + +export function EsqlQueryEditAdapter({ + finalDiffableRule, +}: RuleFieldEditComponentProps): JSX.Element | null { + const { dataView, isLoading } = useDiffableRuleDataView(finalDiffableRule); + + // Wait for dataView to be defined to trigger validation with the correct index patterns + if (!dataView) { + return null; + } + + return ( + + ); +} + +const DEFAULT_DATA_VIEW_BASE: DataViewBase = { + title: '', + fields: [], +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_form.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_form.tsx new file mode 100644 index 0000000000000..76022c2132b74 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_form.tsx @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import type { FormData, FormSchema } from '../../../../../../../../shared_imports'; +import { RuleFieldEditFormWrapper } from '../rule_field_edit_form_wrapper'; +import type { FieldValueQueryBar } from '../../../../../../../rule_creation_ui/components/query_bar'; +import { + type DiffableRule, + QueryLanguageEnum, + RuleEsqlQuery, +} from '../../../../../../../../../common/api/detection_engine'; +import { EsqlQueryEditAdapter } from './esql_query_edit_adapter'; + +export function EsqlQueryEditForm(): JSX.Element { + return ( + + ); +} + +const formSchema = {} as FormSchema<{ + esqlQuery: RuleEsqlQuery; +}>; + +function deserializer( + fieldValue: FormData, + finalDiffableRule: DiffableRule +): { + esqlQuery: FieldValueQueryBar; +} { + const parsedEsqlQuery = + 'esql_query' in finalDiffableRule + ? RuleEsqlQuery.parse(fieldValue.esql_query) + : { + query: '', + language: QueryLanguageEnum.esql, + filters: [], + }; + + return { + esqlQuery: { + query: { + query: parsedEsqlQuery.query, + language: parsedEsqlQuery.language, + }, + filters: [], + saved_id: null, + }, + }; +} + +function serializer(formData: FormData): { + esql_query: RuleEsqlQuery; +} { + const formValue = formData as { esqlQuery: FieldValueQueryBar }; + + return { + esql_query: { + query: formValue.esqlQuery.query.query as string, + language: QueryLanguageEnum.esql, + }, + }; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/index.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/index.ts new file mode 100644 index 0000000000000..309a56c96d16d --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './esql_query_edit_form'; From 5d1a70bf99e8abeec8a9e81f0096686483623fa3 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Tue, 12 Nov 2024 20:11:17 +0100 Subject: [PATCH 02/45] add fieldsToValidateOnChange --- .../components/esql_query_edit/esql_query_edit.tsx | 7 ++++++- .../rule_creation_ui/components/step_define_rule/index.tsx | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx index bdc0900204289..e9eb3cd996eee 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx @@ -19,6 +19,7 @@ import * as i18n from './translations'; interface EsqlQueryEditProps { path: string; + fieldsToValidateOnChange?: string | string[]; dataView: DataViewBase; required?: boolean; loading?: boolean; @@ -28,6 +29,7 @@ interface EsqlQueryEditProps { export const EsqlQueryEdit = memo(function EsqlQueryEdit({ path, + fieldsToValidateOnChange, dataView, required = false, loading = false, @@ -49,6 +51,9 @@ export const EsqlQueryEdit = memo(function EsqlQueryEdit({ () => ({ label: i18n.ESQL_QUERY, labelAppend: , + fieldsToValidateOnChange: fieldsToValidateOnChange + ? [path, fieldsToValidateOnChange].flat() + : undefined, validations: [ ...(required ? [ @@ -62,7 +67,7 @@ export const EsqlQueryEdit = memo(function EsqlQueryEdit({ }, ], }), - [required] + [required, path, fieldsToValidateOnChange] ); return ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index d9c91b325a69b..f2bb255339446 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -726,6 +726,7 @@ const StepDefineRuleComponent: FC = ({ Date: Wed, 13 Nov 2024 00:04:40 +0100 Subject: [PATCH 03/45] fix query persistence --- .../components/step_define_rule/index.tsx | 96 +--------------- .../step_define_rule/use_persistent_query.ts | 107 ++++++++++++++++++ .../rule_creation_ui/pages/form.test.ts | 2 +- .../pages/detection_engine/rules/utils.ts | 21 ---- 4 files changed, 111 insertions(+), 115 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index f2bb255339446..50fc0a138d66d 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -16,7 +16,7 @@ import { EuiText, } from '@elastic/eui'; import type { FC } from 'react'; -import React, { memo, useCallback, useState, useEffect, useMemo, useRef } from 'react'; +import React, { memo, useCallback, useState, useEffect, useMemo } from 'react'; import styled from 'styled-components'; import { i18n as i18nCore } from '@kbn/i18n'; @@ -79,7 +79,6 @@ import { NewTermsFields } from '../new_terms_fields'; import { ScheduleItem } from '../../../rule_creation/components/schedule_item_form'; import { RequiredFields } from '../../../rule_creation/components/required_fields'; import { DocLink } from '../../../../common/components/links_to_docs/doc_link'; -import { defaultCustomQuery } from '../../../../detections/pages/detection_engine/rules/utils'; import { useLicense } from '../../../../common/hooks/use_license'; import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants'; import { useUpsellingMessage } from '../../../../common/hooks/use_upselling'; @@ -95,6 +94,7 @@ import { import { ThresholdAlertSuppressionEdit } from '../../../rule_creation/components/threshold_alert_suppression_edit'; import { usePersistentAlertSuppressionState } from './use_persistent_alert_suppression_state'; import { EsqlQueryEdit } from '../../../rule_creation/components/esql_query_edit/esql_query_edit'; +import { usePersistentQuery } from './use_persistent_query'; const CommonUseField = getUseField({ component: Field }); @@ -193,8 +193,6 @@ const StepDefineRuleComponent: FC = ({ const isMlSuppressionIncomplete = isMlRule(ruleType) && machineLearningJobId?.length > 0 && !allJobsStarted; - const esqlQueryRef = useRef(undefined); - const isAlertSuppressionLicenseValid = license.isAtLeast(MINIMUM_LICENSE_FOR_SUPPRESSION); const isThresholdRule = getIsThresholdRule(ruleType); @@ -278,94 +276,7 @@ const StepDefineRuleComponent: FC = ({ setThreatIndexModified(!isEqual(threatIndex, threatIndicesConfig)); }, [threatIndex, threatIndicesConfig]); - /** - * When the user changes rule type to or from "threat_match" this will modify the - * default "Custom query" string to either: - * * from '' to '*:*' if the type is switched to "threat_match" - * * from '*:*' back to '' if the type is switched back from "threat_match" to another one - */ - useEffect(() => { - const { queryBar: currentQuery } = getFields(); - if (currentQuery == null) { - return; - } - - // NOTE: Below this code does two things that are worth commenting. - - // 1. If the user enters some text in the "Custom query" form field, we want - // to keep it even if the user switched to another rule type. So we want to - // be able to figure out if the field has been modified. - // - The forms library provides properties (isPristine, isModified, isDirty) - // for that but they can't be used in our case: their values can be reset - // if you go to step 2 and then back to step 1 or the form is reset in another way. - // - That's why we compare the actual value of the field with default ones. - // NOTE: It's important to do a deep object comparison by value. - // Don't do it by reference because the forms lib can change it internally. - - // 2. We call currentQuery.reset() in both cases to not trigger validation errors - // as the user has not entered data into those areas yet. - - // If the user switched rule type to "threat_match" from any other one, - // but hasn't changed the custom query used for normal rules (''), - // we reset the custom query to the default used for "threat_match" rules ('*:*'). - if (isThreatMatchRule(ruleType) && !isThreatMatchRule(previousRuleType)) { - if (isEqual(currentQuery.value, defaultCustomQuery.forNormalRules)) { - currentQuery.reset({ - defaultValue: defaultCustomQuery.forThreatMatchRules, - }); - return; - } - } - - // If the user switched rule type from "threat_match" to any other one, - // but hasn't changed the custom query used for "threat_match" rules ('*:*'), - // we reset the custom query to another default value (''). - if (!isThreatMatchRule(ruleType) && isThreatMatchRule(previousRuleType)) { - if (isEqual(currentQuery.value, defaultCustomQuery.forThreatMatchRules)) { - currentQuery.reset({ - defaultValue: defaultCustomQuery.forNormalRules, - }); - } - } - }, [ruleType, previousRuleType, getFields]); - - /** - * ensures when user switches between rule types, written ES|QL query is not getting lost - * additional work is required in this code area, as currently switching to EQL will result in query lose - * https://github.com/elastic/kibana/issues/166933 - */ - useEffect(() => { - const { queryBar: currentQuery } = getFields(); - if (currentQuery == null) { - return; - } - - const currentQueryValue = currentQuery.value as DefineStepRule['queryBar']; - - // sets ES|QL query to a default value or earlier added one, when switching to ES|QL rule type - if (isEsqlRule(ruleType)) { - if (previousRuleType && !isEsqlRule(previousRuleType)) { - currentQuery.reset({ - defaultValue: esqlQueryRef.current ?? defaultCustomQuery.forEsqlRules, - }); - } - // otherwise reset it to default values of other rule types - } else if (isEsqlRule(previousRuleType)) { - // sets ES|QL query value to reference, so it can be used when user switch back from one rule type to another - if (currentQueryValue?.query?.language === 'esql') { - esqlQueryRef.current = currentQueryValue; - } - - const defaultValue = isThreatMatchRule(ruleType) - ? defaultCustomQuery.forThreatMatchRules - : defaultCustomQuery.forNormalRules; - - currentQuery.reset({ - defaultValue, - }); - } - }, [ruleType, previousRuleType, getFields]); - + usePersistentQuery({ form }); usePersistentAlertSuppressionState({ form }); // if saved query failed to load: @@ -724,7 +635,6 @@ const StepDefineRuleComponent: FC = ({ ) : isEsqlRule(ruleType) ? ( ; +} + +export function usePersistentQuery({ form }: UsePersistentQueryParams): void { + const [{ ruleType, queryBar: currentQuery }] = useFormData({ + form, + watch: ['ruleType', 'queryBar'], + }); + const previousRuleType = usePrevious(ruleType); + const queryRef = useRef(); + const eqlQueryRef = useRef(); + const esqlQueryRef = useRef(); + + useEffect(() => { + if (isEqlRule(ruleType)) { + eqlQueryRef.current = currentQuery; + } else if (isEsqlRule(ruleType)) { + esqlQueryRef.current = currentQuery; + } else { + queryRef.current = currentQuery; + } + }, [ruleType, currentQuery]); + + useEffect(() => { + if (ruleType === previousRuleType) { + return; + } + + const queryField = form.getFields().queryBar; + + if (isEsqlRule(ruleType)) { + queryField.reset({ + defaultValue: esqlQueryRef.current ?? DEFAULT_ESQL_QUERY, + }); + + return; + } + + if (isEqlRule(ruleType)) { + queryField.reset({ + defaultValue: eqlQueryRef.current ?? DEFAULT_EQL_QUERY, + }); + + return; + } + + if (isThreatMatchRule(ruleType)) { + queryField.reset({ + defaultValue: isEqual(queryRef.current, DEFAULT_KQL_QUERY) + ? DEFAULT_THREAT_MATCH_KQL_QUERY + : queryRef.current, + }); + + return; + } + + queryField.reset({ + defaultValue: isEqual(queryRef.current, DEFAULT_THREAT_MATCH_KQL_QUERY) + ? DEFAULT_KQL_QUERY + : queryRef.current, + }); + }, [ruleType, previousRuleType, form]); +} + +const DEFAULT_KQL_QUERY = { + query: { query: '', language: 'kuery' }, + filters: [], + saved_id: null, +}; + +const DEFAULT_THREAT_MATCH_KQL_QUERY = { + query: { query: '*:*', language: 'kuery' }, + filters: [], + saved_id: null, +}; + +const DEFAULT_EQL_QUERY = { + query: { query: '', language: 'eql' }, + filters: [], + saved_id: null, +}; + +const DEFAULT_ESQL_QUERY = { + query: { query: '', language: 'esql' }, + filters: [], + saved_id: null, +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.test.ts index d2bbe9edb1160..9d606ffd8dfcd 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/form.test.ts @@ -7,7 +7,7 @@ import { renderHook } from '@testing-library/react-hooks'; import type { FormData, FormHook, ValidationError } from '../../../shared_imports'; -import { ERROR_CODES as ESQL_ERROR_CODES } from '../../rule_creation/logic/esql_validator'; +import { ESQL_ERROR_CODES } from '../validators/esql_query_validator_factory'; import { EQL_ERROR_CODES } from '../../../common/hooks/eql/api'; import type { AboutStepRule, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts index 55a461d946834..d0d21a9e39b2b 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts @@ -125,24 +125,3 @@ export const getStepScheduleDefaultValue = (ruleType: Type | undefined): Schedul from: isThreatMatchRule(ruleType) ? THREAT_MATCH_FROM : DEFAULT_FROM, }; }; - -/** - * This default query will be used for threat query/indicator matches - * as the default when the user swaps to using it by changing their - * rule type from any rule type to the "threatMatchRule" type. Only - * difference is that "*:*" is used instead of '' for its query. - */ -const threatQueryBarDefaultValue: DefineStepRule['queryBar'] = { - ...stepDefineDefaultValue.queryBar, - query: { ...stepDefineDefaultValue.queryBar.query, query: '*:*' }, -}; - -export const defaultCustomQuery = { - forNormalRules: stepDefineDefaultValue.queryBar, - forThreatMatchRules: threatQueryBarDefaultValue, - forEsqlRules: { - query: { query: '', language: 'esql' }, - filters: [], - saved_id: null, - }, -}; From 5630e75c31cc83f8bef5f7d287b9f116423f9e91 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 13 Nov 2024 09:25:14 +0100 Subject: [PATCH 04/45] move default queries to query bar folder --- .../components/query_bar/default_queries.ts | 32 ++++++++++ .../components/query_bar/index.ts | 10 +++ .../{index.test.tsx => query_field.test.tsx} | 0 .../query_bar/{index.tsx => query_field.tsx} | 12 ++-- .../components/query_bar/types.ts | 15 +++++ .../components/step_define_rule/index.tsx | 7 +-- .../step_define_rule/use_persistent_query.ts | 63 ++++++++----------- .../validators/get_esql_query_config.ts | 40 ------------ .../pages/detection_engine/rules/utils.ts | 7 +-- 9 files changed, 92 insertions(+), 94 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/default_queries.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.ts rename x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/{index.test.tsx => query_field.test.tsx} (100%) rename x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/{index.tsx => query_field.tsx} (98%) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/types.ts delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/default_queries.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/default_queries.ts new file mode 100644 index 0000000000000..c1f3150a3aa75 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/default_queries.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FieldValueQueryBar } from './types'; + +export const DEFAULT_KQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '', language: 'kuery' }, + filters: [], + saved_id: null, +}; + +export const DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '*:*', language: 'kuery' }, + filters: [], + saved_id: null, +}; + +export const DEFAULT_EQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '', language: 'eql' }, + filters: [], + saved_id: null, +}; + +export const DEFAULT_ESQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '', language: 'esql' }, + filters: [], + saved_id: null, +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.ts new file mode 100644 index 0000000000000..07f6e408c5574 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; +export * from './default_queries'; +export * from './query_field'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/query_field.test.tsx similarity index 100% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.test.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/query_field.test.tsx diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/query_field.tsx similarity index 98% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/query_field.tsx index 43330bdd858e2..bfaff04c4da89 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/query_field.tsx @@ -9,7 +9,7 @@ import { EuiFormRow, EuiMutationObserver } from '@elastic/eui'; import React, { useCallback, useEffect, useState } from 'react'; import { Subscription } from 'rxjs'; import deepEqual from 'fast-deep-equal'; -import type { DataViewBase, Filter, Query } from '@kbn/es-query'; +import type { DataViewBase, Query } from '@kbn/es-query'; import type { SavedQuery } from '@kbn/data-plugin/public'; import { FilterManager } from '@kbn/data-plugin/public'; @@ -21,14 +21,12 @@ import type { TimelineModel } from '../../../../timelines/store/model'; import { useSavedQueryServices } from '../../../../common/utils/saved_query_services'; import type { FieldHook } from '../../../../shared_imports'; import { getFieldValidityAndErrorMessage } from '../../../../shared_imports'; +import type { FieldValueQueryBar } from './types'; import * as i18n from './translations'; -export interface FieldValueQueryBar { - filters: Filter[]; - query: Query; - saved_id: string | null; - title?: string; -} +export * from './types'; +export * from './default_queries'; + export interface QueryBarDefineRuleProps { dataTestSubj: string; field: FieldHook; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/types.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/types.ts new file mode 100644 index 0000000000000..9807be907209a --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/types.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Filter, Query } from '@kbn/es-query'; + +export interface FieldValueQueryBar { + filters: Filter[]; + query: Query; + saved_id: string | null; + title?: string; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 50fc0a138d66d..e8cf9121620c1 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -22,7 +22,6 @@ import styled from 'styled-components'; import { i18n as i18nCore } from '@kbn/i18n'; import { isEqual } from 'lodash'; import type { FieldSpec } from '@kbn/data-plugin/common'; -import usePrevious from 'react-use/lib/usePrevious'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import type { SavedQuery } from '@kbn/data-plugin/public'; @@ -224,10 +223,6 @@ const StepDefineRuleComponent: FC = ({ const { onOpenTimeline, loading: timelineQueryLoading } = useRuleFromTimeline(handleSetRuleFromTimeline); - // if 'index' is selected, use these browser fields - // otherwise use the dataview browserfields - const previousRuleType = usePrevious(ruleType); - // Callback for when user toggles between Data Views and Index Patterns const onChangeDataSource = useCallback( (optionId: string) => { @@ -276,7 +271,7 @@ const StepDefineRuleComponent: FC = ({ setThreatIndexModified(!isEqual(threatIndex, threatIndicesConfig)); }, [threatIndex, threatIndicesConfig]); - usePersistentQuery({ form }); + usePersistentQuery({ form, ruleTypePath: 'ruleType', queryPath: 'queryBar' }); usePersistentAlertSuppressionState({ form }); // if saved query failed to load: diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts index 6bf2790b45053..23238fd54fad5 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts @@ -15,16 +15,31 @@ import { isEsqlRule, isThreatMatchRule, } from '../../../../../common/detection_engine/utils'; -import type { FieldValueQueryBar } from '../query_bar'; +import { + DEFAULT_EQL_QUERY_FIELD_VALUE, + DEFAULT_ESQL_QUERY_FIELD_VALUE, + DEFAULT_KQL_QUERY_FIELD_VALUE, + DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE, + type FieldValueQueryBar, +} from '../query_bar'; interface UsePersistentQueryParams { form: FormHook; + ruleTypePath: string; + queryPath: string; } -export function usePersistentQuery({ form }: UsePersistentQueryParams): void { - const [{ ruleType, queryBar: currentQuery }] = useFormData({ +/** + * Persists query when switching between different rule types using different queries (kuery, EQL, ES|QL). + */ +export function usePersistentQuery({ + form, + ruleTypePath, + queryPath, +}: UsePersistentQueryParams): void { + const [{ [ruleTypePath]: ruleType, [queryPath]: currentQuery }] = useFormData({ form, - watch: ['ruleType', 'queryBar'], + watch: [ruleTypePath, queryPath], }); const previousRuleType = usePrevious(ruleType); const queryRef = useRef(); @@ -46,11 +61,11 @@ export function usePersistentQuery({ form }: UsePersistentQueryParams): void { return; } - const queryField = form.getFields().queryBar; + const queryField = form.getFields()[queryPath]; if (isEsqlRule(ruleType)) { queryField.reset({ - defaultValue: esqlQueryRef.current ?? DEFAULT_ESQL_QUERY, + defaultValue: esqlQueryRef.current ?? DEFAULT_ESQL_QUERY_FIELD_VALUE, }); return; @@ -58,7 +73,7 @@ export function usePersistentQuery({ form }: UsePersistentQueryParams): void { if (isEqlRule(ruleType)) { queryField.reset({ - defaultValue: eqlQueryRef.current ?? DEFAULT_EQL_QUERY, + defaultValue: eqlQueryRef.current ?? DEFAULT_EQL_QUERY_FIELD_VALUE, }); return; @@ -66,8 +81,8 @@ export function usePersistentQuery({ form }: UsePersistentQueryParams): void { if (isThreatMatchRule(ruleType)) { queryField.reset({ - defaultValue: isEqual(queryRef.current, DEFAULT_KQL_QUERY) - ? DEFAULT_THREAT_MATCH_KQL_QUERY + defaultValue: isEqual(queryRef.current, DEFAULT_KQL_QUERY_FIELD_VALUE) + ? DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE : queryRef.current, }); @@ -75,33 +90,9 @@ export function usePersistentQuery({ form }: UsePersistentQueryParams): void { } queryField.reset({ - defaultValue: isEqual(queryRef.current, DEFAULT_THREAT_MATCH_KQL_QUERY) - ? DEFAULT_KQL_QUERY + defaultValue: isEqual(queryRef.current, DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE) + ? DEFAULT_KQL_QUERY_FIELD_VALUE : queryRef.current, }); - }, [ruleType, previousRuleType, form]); + }, [queryPath, ruleType, previousRuleType, form]); } - -const DEFAULT_KQL_QUERY = { - query: { query: '', language: 'kuery' }, - filters: [], - saved_id: null, -}; - -const DEFAULT_THREAT_MATCH_KQL_QUERY = { - query: { query: '*:*', language: 'kuery' }, - filters: [], - saved_id: null, -}; - -const DEFAULT_EQL_QUERY = { - query: { query: '', language: 'eql' }, - filters: [], - saved_id: null, -}; - -const DEFAULT_ESQL_QUERY = { - query: { query: '', language: 'esql' }, - filters: [], - saved_id: null, -}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts deleted file mode 100644 index c7d4f9184f181..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/get_esql_query_config.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getESQLQueryColumns } from '@kbn/esql-utils'; -import type { ISearchGeneric } from '@kbn/search-types'; - -/** - * react-query configuration to be used to fetch ES|QL fields - * it sets limit in query to 0, so we don't fetch unnecessary results, only fields - */ -export const getEsqlQueryConfig = ({ - esqlQuery, - search, -}: { - esqlQuery: string | undefined; - search: ISearchGeneric; -}) => { - return { - queryKey: [(esqlQuery ?? '').trim()], - queryFn: async () => { - if (!esqlQuery) { - return null; - } - try { - const res = await getESQLQueryColumns({ - esqlQuery, - search, - }); - return res; - } catch (e) { - return { error: e }; - } - }, - staleTime: 60 * 1000, - }; -}; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts index d0d21a9e39b2b..c33aaebe8fab5 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts @@ -14,6 +14,7 @@ import { ALERT_SUPPRESSION_DEFAULT_DURATION, } from '../../../../detection_engine/rule_creation/components/alert_suppression_edit'; import { THRESHOLD_ALERT_SUPPRESSION_ENABLED } from '../../../../detection_engine/rule_creation/components/threshold_alert_suppression_edit'; +import { DEFAULT_KQL_QUERY_FIELD_VALUE } from '../../../../detection_engine/rule_creation_ui/components/query_bar'; import { isThreatMatchRule } from '../../../../../common/detection_engine/utils'; import { DEFAULT_TIMELINE_TITLE } from '../../../../timelines/components/timeline/translations'; import { DEFAULT_MAX_SIGNALS, DEFAULT_THREAT_MATCH_QUERY } from '../../../../../common/constants'; @@ -44,11 +45,7 @@ export const stepDefineDefaultValue: DefineStepRule = { machineLearningJobId: [], ruleType: 'query', threatIndex: [], - queryBar: { - query: { query: '', language: 'kuery' }, - filters: [], - saved_id: null, - }, + queryBar: DEFAULT_KQL_QUERY_FIELD_VALUE, threatQueryBar: { query: { query: DEFAULT_THREAT_MATCH_QUERY, language: 'kuery' }, filters: [], From 0fb0ce71195e473953cc3942121ba0b9cab23e3f Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 13 Nov 2024 11:58:36 +0100 Subject: [PATCH 05/45] add ES|QL Query validator unit tests --- .../esql/compute_if_esql_query_aggregating.ts | 5 +- .../esql_query_validator_factory.test.ts | 156 ++++++++++++++++++ .../esql_query_validator_factory.ts | 28 ++-- 3 files changed, 170 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.test.ts diff --git a/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts b/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts index 83d4b6cf51e63..c9b8474b55969 100644 --- a/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts +++ b/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts @@ -9,9 +9,8 @@ import { type ESQLAstQueryExpression, parse } from '@kbn/esql-ast'; -export const isAggregatingQuery = (astExpression: ESQLAstQueryExpression): boolean => { - return astExpression.commands.some((command) => command.name === 'stats'); -}; +export const isAggregatingQuery = (astExpression: ESQLAstQueryExpression): boolean => + astExpression.commands.some((command) => command.name === 'stats'); /** * compute if esqlQuery is aggregating/grouping, i.e. using STATS...BY command diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.test.ts new file mode 100644 index 0000000000000..7e2b5f1bafc79 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getESQLQueryColumns } from '@kbn/esql-utils'; +import type { ValidationFuncArg } from '../../../shared_imports'; +import type { FieldValueQueryBar } from '../components/query_bar'; +import { ESQL_ERROR_CODES, esqlQueryValidatorFactory } from './esql_query_validator_factory'; + +jest.mock('@kbn/esql-utils', () => ({ + getESQLQueryColumns: jest.fn().mockResolvedValue([{ id: '_id' }]), +})); +jest.mock('../../../common/lib/kibana'); + +describe('esqlQueryValidator', () => { + describe('ES|QL query syntax', () => { + const validator = esqlQueryValidatorFactory(); + + it.each([['incorrect syntax'], ['from test* metadata']])( + 'reports incorrect syntax in "%s"', + (esqlQuery) => + expect( + validator({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.INVALID_SYNTAX, + }) + ); + + it.each([ + ['from test* metadata _id'], + [ + 'FROM kibana_sample_data_logs | STATS total_bytes = SUM(bytes) BY host | WHERE total_bytes > 200000 | SORT total_bytes DESC | LIMIT 10', + ], + ])('succeeds validation for correct syntax in "%s"', (esqlQuery) => + expect( + validator({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined() + ); + }); + + describe('METADATA operator validation', () => { + const validator = esqlQueryValidatorFactory(); + + it.each([ + ['from test*'], + ['from metadata*'], + ['from test* | keep metadata'], + ['from test* | eval x="metadata _id"'], + ])('reports when METADATA operator is missing in a NON aggregating query "%s"', (esqlQuery) => + expect( + validator({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + }) + ); + + it.each([ + ['from test* metadata _id'], + ['from test* metadata _id, _index'], + ['from test* metadata _index, _id'], + ['from test* metadata _id '], + ['from test* metadata _id | limit 10'], + ])( + 'succeeds validation when METADATA operator EXISTS in a NON aggregating query "%s"', + (esqlQuery) => + expect( + validator({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined() + ); + + it.each([['from test* | stats c = count(*) by fieldA']])( + 'succeeds validation when METADATA operator is missing in an aggregating query "%s"', + (esqlQuery) => + expect( + validator({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined() + ); + }); + + describe('METADATA _id field validation for NON aggregating queries', () => { + const validator = esqlQueryValidatorFactory(); + + it('reports when METADATA "_id" field is missing', () => { + (getESQLQueryColumns as jest.Mock).mockResolvedValue([{ id: 'column1' }, { id: 'column2' }]); + + return expect( + validator({ + value: createEsqlQueryFieldValue('from test*'), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + }); + }); + + it('succeeds validation when METADATA "_id" field EXISTS', async () => { + (getESQLQueryColumns as jest.Mock).mockResolvedValue([{ id: '_id' }, { id: 'column1' }]); + + return expect( + validator({ + value: createEsqlQueryFieldValue('from test* metadata _id'), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined(); + }); + + it('succeeds validation when METADATA operator with "_id" field is missing in an aggregating query "%s"', () => { + (getESQLQueryColumns as jest.Mock).mockResolvedValue([{ id: 'column1' }, { id: 'column2' }]); + + return expect( + validator({ + value: createEsqlQueryFieldValue( + 'from test* metadata someField | stats c = count(*) by fieldA' + ), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined(); + }); + }); + + describe('when getESQLQueryColumns fails', () => { + const validator = esqlQueryValidatorFactory(); + + it('reports an error message', () => { + // suppress the expected error messages + jest.spyOn(console, 'error').mockReturnValue(); + + (getESQLQueryColumns as jest.Mock).mockRejectedValue(new Error('some error')); + + return expect( + validator({ + value: createEsqlQueryFieldValue('from test* metadata _id'), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.INVALID_ESQL, + message: 'Error validating ES|QL: "some error"', + }); + }); + }); +}); + +type EsqlQueryValidatorArgs = ValidationFuncArg; + +function createEsqlQueryFieldValue(esqlQuery: string): Readonly { + return { query: { query: esqlQuery, language: 'esql' }, filters: [], saved_id: null }; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts index 9b789b88a976c..93e548f8dd8bf 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts @@ -23,15 +23,15 @@ export enum ESQL_ERROR_CODES { ERR_MISSING_ID_FIELD_FROM_RESULT = 'ERR_MISSING_ID_FIELD_FROM_RESULT', } -const queryClient = new QueryClient({ - defaultOptions: { - queries: { - staleTime: 60 * 1000, +export function esqlQueryValidatorFactory(): ValidationFunc { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 60 * 1000, + }, }, - }, -}); + }); -export function esqlQueryValidatorFactory(): ValidationFunc { return async (...args) => { const [{ value }] = args; const esqlQuery = value.query.query as string; @@ -42,15 +42,15 @@ export function esqlQueryValidatorFactory(): ValidationFunc '_id' === id); - // for non-aggregating query, we want to disable queries w/o _id property returned in response - if (!isEsqlQueryAggregating && !isIdFieldPresent) { + if (!isEsqlQueryAggregating && !columns.some(({ id }) => '_id' === id)) { return { code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, message: ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR, @@ -93,8 +90,7 @@ function parseEsqlQuery(query: string) { return { errors, isEsqlQueryAggregating, - // non-aggregating query which does not have metadata, is not a valid one - isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(root), + hasMetadataOperator: computeHasMetadataOperator(root), }; } From 8565b788c6b98fdcbb1c3f7f409383a81a736903 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 13 Nov 2024 13:18:13 +0100 Subject: [PATCH 06/45] revert aria-label --- .../esql_query_edit/esql_info_icon.tsx | 22 +++++++------------ .../esql_query_edit/translations.ts | 10 ++++++--- .../esql_query_validator_factory.ts | 8 +++---- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx index d0b4cee6752ad..933a45004fc70 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx @@ -5,21 +5,19 @@ * 2.0. */ -import React from 'react'; +import React, { memo } from 'react'; import { EuiPopover, EuiText, EuiButtonIcon, EuiLink } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; -import * as i18n from './translations'; - -import { useBoolState } from '../../../../common/hooks/use_bool_state'; +import { useBoolean } from '@kbn/react-hooks'; import { useKibana } from '../../../../common/lib/kibana'; +import * as i18n from './translations'; /** * Icon and popover that gives hint to users how to get started with ES|QL rules */ -const EsqlInfoIconComponent = () => { +export const EsqlInfoIcon = memo(function EsqlInfoIcon(): JSX.Element { const { docLinks } = useKibana().services; - - const [isPopoverOpen, , closePopover, togglePopover] = useBoolState(); + const [isPopoverOpen, { off: closePopover, on: togglePopover }] = useBoolean(false); const button = ( @@ -29,13 +27,13 @@ const EsqlInfoIconComponent = () => { @@ -45,8 +43,4 @@ const EsqlInfoIconComponent = () => { ); -}; - -export const EsqlInfoIcon = React.memo(EsqlInfoIconComponent); - -EsqlInfoIcon.displayName = 'EsqlInfoIcon'; +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts index 6e9e15f77a6e3..6e997057667e6 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts @@ -7,9 +7,13 @@ import { i18n } from '@kbn/i18n'; -export const ESQL_QUERY = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel', +export const ESQL_QUERY = i18n.translate('xpack.securitySolution.ruleManagement.esqlQuery.label', { + defaultMessage: 'ES|QL query', +}); + +export const ARIA_LABEL = i18n.translate( + 'xpack.securitySolution.ruleManagement.esqlQuery.ariaLabel', { - defaultMessage: 'ES|QL query', + defaultMessage: `Open help popover`, } ); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts index 93e548f8dd8bf..baa6ab7a87614 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/validators/esql_query_validator_factory.ts @@ -153,27 +153,27 @@ function constructValidationError(error: Error): ValidationError { } const ESQL_VALIDATION_UNKNOWN_ERROR = i18n.translate( - 'xpack.securitySolution.detectionEngine.esqlValidation.unknownError', + 'xpack.securitySolution.ruleManagement.esqlValidation.unknownError', { defaultMessage: 'Unknown error while validating ES|QL', } ); const esqlValidationErrorMessage = (message: string) => - i18n.translate('xpack.securitySolution.detectionEngine.esqlValidation.errorMessage', { + i18n.translate('xpack.securitySolution.ruleManagement.esqlValidation.errorMessage', { values: { message }, defaultMessage: 'Error validating ES|QL: "{message}"', }); const ESQL_VALIDATION_MISSING_METADATA_OPERATOR_IN_QUERY_ERROR = i18n.translate( - 'xpack.securitySolution.detectionEngine.esqlValidation.missingMetadataOperatorInQueryError', + 'xpack.securitySolution.ruleManagement.esqlValidation.missingMetadataOperatorInQueryError', { defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`, } ); const ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR = i18n.translate( - 'xpack.securitySolution.detectionEngine.esqlValidation.missingIdFieldInQueryError', + 'xpack.securitySolution.ruleManagement.esqlValidation.missingIdFieldInQueryError', { defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index. In addition, the metadata properties (_id, _version, and _index) must be returned in the query response.`, } From ab8989e3779560e9afa123033bb073c98f67e89d Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 13 Nov 2024 21:02:17 +0100 Subject: [PATCH 07/45] remove unused translation keys --- x-pack/plugins/translations/translations/fr-FR.json | 4 ---- x-pack/plugins/translations/translations/ja-JP.json | 4 ---- x-pack/plugins/translations/translations/zh-CN.json | 4 ---- 3 files changed, 12 deletions(-) diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 9866bea029a98..8e34342479303 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -38033,10 +38033,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "Une requête EQL est requise.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "La suppression n'est pas prise en charge pour les requêtes de séquence EQL.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} Remplacez la requête EQL par une requête non séquentielle ou supprimez les champs de suppression.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "Ouvrir une fenêtre contextuelle d'aide", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "Consultez {createEsqlRuleTypeLink} pour commencer à utiliser les règles ES|QL.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "documentation", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel": "Requête ES|QL", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "Seuil de score d'anomalie", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "Tâche de Machine Learning", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "Requête personnalisée", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 08befe63cfa0a..5c78514b70eae 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -38004,10 +38004,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQLクエリは必須です。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQLシーケンスクエリでは抑制はサポートされていません。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} EQLクエリを非シーケンスクエリに変更するか、抑制フィールドを削除してください。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "ヘルプポップオーバーを開く", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "ES|QL ルールの使用を開始するには、{createEsqlRuleTypeLink}を確認してください。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "ドキュメンテーション", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel": "ES|QLクエリ", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "異常スコアしきい値", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "機械学習ジョブ", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "カスタムクエリー", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index e357618355f9b..7cf6f27d50ba9 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -37397,10 +37397,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQL 查询必填。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQL 序列查询不支持阻止。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} 将 EQL 查询更改为非序列查询，或移除阻止字段。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "打开帮助弹出框", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "请访问我们的{createEsqlRuleTypeLink}以开始使用 ES|QL 规则。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "文档", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel": "ES|QL 查询", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "异常分数阈值", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "Machine Learning 作业", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "定制查询", From 80879b52e3534cb1d9d6623a09854426d7bda8d5 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Wed, 13 Nov 2024 21:08:17 +0100 Subject: [PATCH 08/45] fix usePersistentQuery implementation --- .../step_define_rule/use_persistent_query.ts | 55 ++++++++++++------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts index 23238fd54fad5..5a79a580a7e1c 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts @@ -8,6 +8,7 @@ import { useEffect, useRef } from 'react'; import { isEqual } from 'lodash'; import usePrevious from 'react-use/lib/usePrevious'; +import type { FieldHook } from '../../../../shared_imports'; import { useFormData, type FormHook } from '../../../../shared_imports'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; import { @@ -23,6 +24,9 @@ import { type FieldValueQueryBar, } from '../query_bar'; +const EQL_QUERY_LANGUAGE = 'eql'; +const ESQL_QUERY_LANGIAGE = 'esql'; + interface UsePersistentQueryParams { form: FormHook; ruleTypePath: string; @@ -42,14 +46,18 @@ export function usePersistentQuery({ watch: [ruleTypePath, queryPath], }); const previousRuleType = usePrevious(ruleType); - const queryRef = useRef(); - const eqlQueryRef = useRef(); - const esqlQueryRef = useRef(); + const queryRef = useRef(DEFAULT_KQL_QUERY_FIELD_VALUE); + const eqlQueryRef = useRef(DEFAULT_EQL_QUERY_FIELD_VALUE); + const esqlQueryRef = useRef(DEFAULT_ESQL_QUERY_FIELD_VALUE); useEffect(() => { - if (isEqlRule(ruleType)) { + if (!ruleType) { + return; + } + + if (currentQuery?.query?.language === EQL_QUERY_LANGUAGE) { eqlQueryRef.current = currentQuery; - } else if (isEsqlRule(ruleType)) { + } else if (currentQuery?.query?.language === ESQL_QUERY_LANGIAGE) { esqlQueryRef.current = currentQuery; } else { queryRef.current = currentQuery; @@ -57,42 +65,49 @@ export function usePersistentQuery({ }, [ruleType, currentQuery]); useEffect(() => { - if (ruleType === previousRuleType) { + if (ruleType === previousRuleType || !ruleType) { return; } - const queryField = form.getFields()[queryPath]; + const queryField = form.getFields()[queryPath] as FieldHook; - if (isEsqlRule(ruleType)) { + if (isEqlRule(ruleType) && queryField.value?.query?.language !== EQL_QUERY_LANGUAGE) { queryField.reset({ - defaultValue: esqlQueryRef.current ?? DEFAULT_ESQL_QUERY_FIELD_VALUE, + defaultValue: eqlQueryRef.current, }); return; } - if (isEqlRule(ruleType)) { + if (isEsqlRule(ruleType) && queryField.value?.query?.language !== ESQL_QUERY_LANGIAGE) { queryField.reset({ - defaultValue: eqlQueryRef.current ?? DEFAULT_EQL_QUERY_FIELD_VALUE, + defaultValue: esqlQueryRef.current, }); return; } - if (isThreatMatchRule(ruleType)) { + if (isThreatMatchRule(ruleType) && isEqual(queryRef.current, DEFAULT_KQL_QUERY_FIELD_VALUE)) { queryField.reset({ - defaultValue: isEqual(queryRef.current, DEFAULT_KQL_QUERY_FIELD_VALUE) - ? DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE - : queryRef.current, + defaultValue: DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE, }); return; } - queryField.reset({ - defaultValue: isEqual(queryRef.current, DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE) - ? DEFAULT_KQL_QUERY_FIELD_VALUE - : queryRef.current, - }); + if ( + isThreatMatchRule(previousRuleType) && + isEqual(queryRef.current, DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE) + ) { + queryField.reset({ + defaultValue: DEFAULT_KQL_QUERY_FIELD_VALUE, + }); + } + + if (isEqlRule(previousRuleType) || isEsqlRule(previousRuleType)) { + queryField.reset({ + defaultValue: queryRef.current ?? DEFAULT_KQL_QUERY_FIELD_VALUE, + }); + } }, [queryPath, ruleType, previousRuleType, form]); } From a67d481ea036035bed4da345dde24e4dd6cb408f Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Thu, 14 Nov 2024 13:11:01 +0100 Subject: [PATCH 09/45] update changed test id --- .../public/management/cypress/tasks/response_actions.ts | 2 +- .../cypress/screens/create_new_rule.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/response_actions.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/response_actions.ts index 03d9ab6590167..51530fd0d7a7a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/response_actions.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/response_actions.ts @@ -80,7 +80,7 @@ export const fillUpNewEsqlRule = (name = 'Test', description = 'Test', query: st cy.getByTestSubj('create-new-rule').click(); cy.getByTestSubj('stepDefineRule').within(() => { cy.getByTestSubj('esqlRuleType').click(); - cy.getByTestSubj('detectionEngineStepDefineRuleEsqlQueryBar').within(() => { + cy.getByTestSubj('ruleEsqlQueryBar').within(() => { cy.getByTestSubj('globalQueryBar').click(); cy.getByTestSubj('kibanaCodeEditor').type(query); }); diff --git a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts index 315e89564aae2..903b463d6f585 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts @@ -268,10 +268,9 @@ export const NEW_TERMS_TYPE = '[data-test-subj="newTermsRuleType"]'; export const ESQL_TYPE = '[data-test-subj="esqlRuleType"]'; -export const ESQL_QUERY_BAR_INPUT_AREA = - '[data-test-subj="detectionEngineStepDefineRuleEsqlQueryBar"] textarea'; +export const ESQL_QUERY_BAR_INPUT_AREA = '[data-test-subj="ruleEsqlQueryBar"] textarea'; -export const ESQL_QUERY_BAR = '[data-test-subj="detectionEngineStepDefineRuleEsqlQueryBar"]'; +export const ESQL_QUERY_BAR = '[data-test-subj="ruleEsqlQueryBar"]'; export const NEW_TERMS_INPUT_AREA = '[data-test-subj="newTermsInput"]'; From f87563285268bbf3c0efdbc9a4a5d4c8314c2870 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Thu, 14 Nov 2024 13:11:41 +0100 Subject: [PATCH 10/45] add query persistence tests --- .../select_rule_type/test_helpers.ts | 29 ++ .../step_define_rule/index.test.tsx | 383 +++++++++++++----- .../components/step_define_rule/index.tsx | 38 +- .../step_define_rule/use_persistent_query.ts | 57 ++- .../pages/rule_creation/index.tsx | 1 - .../pages/rule_editing/index.tsx | 1 - 6 files changed, 388 insertions(+), 121 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/select_rule_type/test_helpers.ts diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/select_rule_type/test_helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/select_rule_type/test_helpers.ts new file mode 100644 index 0000000000000..a4d5fca3a2ffe --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/select_rule_type/test_helpers.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// eslint-disable-next-line import/no-extraneous-dependencies +import { act, fireEvent, within, screen } from '@testing-library/react'; +import type { Type as RuleType } from '@kbn/securitysolution-io-ts-alerting-types'; + +export async function selectRuleType(ruleType: RuleType): Promise { + const testId = RULE_TYPE_TEST_ID_MAP[ruleType]; + + await act(async () => fireEvent.click(screen.getByTestId(testId))); + + expect(within(screen.getByTestId(testId)).getByRole('switch')).toBeChecked(); +} + +const RULE_TYPE_TEST_ID_MAP = { + query: 'customRuleType', + saved_query: 'customRuleType', + eql: 'eqlRuleType', + machine_learning: 'machineLearningRuleType', + threshold: 'thresholdRuleType', + threat_match: 'threatMatchRuleType', + new_terms: 'newTermsRuleType', + esql: 'esqlRuleType', +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx index 364f1b7705732..281645310c313 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx @@ -6,6 +6,7 @@ */ import React, { useEffect } from 'react'; +import type { ChangeEvent } from 'react'; import { screen, fireEvent, render, within, act, waitFor } from '@testing-library/react'; import type { Type as RuleType } from '@kbn/securitysolution-io-ts-alerting-types'; import type { DataViewBase } from '@kbn/es-query'; @@ -42,13 +43,43 @@ import { addRelatedIntegrationRow, setVersion, } from '../../../rule_creation/components/related_integrations/test_helpers'; +import { useEsqlAvailability } from '../../../../common/hooks/esql/use_esql_availability'; +import { useMLRuleConfig } from '../../../../common/components/ml/hooks/use_ml_rule_config'; +import { selectRuleType } from '../select_rule_type/test_helpers'; + +// Set the extended default timeout for all define rule step form test +jest.setTimeout(10 * 1000); // Mocks integrations jest.mock('../../../fleet_integrations/api'); + +const MOCKED_QUERY_BAR_TEST_ID = 'mockedQueryBar'; +const MOCKED_LANGUAGE_INPUT_TEST_ID = 'languageInput'; + +// Mocking QueryBar to avoid pulling and mocking a ton of dependencies jest.mock('../../../../common/components/query_bar', () => { return { - QueryBar: jest.fn(({ filterQuery }) => { - return
{`${filterQuery.query} ${filterQuery.language}`}
; + QueryBar: jest.fn().mockImplementation(({ filterQuery, onSubmitQuery }) => { + const handleQueryChange = (event: ChangeEvent) => { + onSubmitQuery({ query: event.target.value, language: filterQuery.language }); + }; + + const handleLanguageChange = (event: ChangeEvent) => { + onSubmitQuery({ query: filterQuery.query, language: event.target.value }); + }; + + return ( +
+