From e55232f87732594eb9502993766203a219be44db Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Thu, 28 Nov 2024 15:48:48 +0100 Subject: [PATCH] [Security Solution] Add ES|QL Query editable component (#199887) **Partially addresses:** https://github.com/elastic/kibana/issues/171520 ## Summary This PR adds is built on top of https://github.com/elastic/kibana/pull/193828 and https://github.com/elastic/kibana/pull/196948 and adds an ES|QL Query editable component for Three Way Diff tab's final edit side of the upgrade prebuilt rule workflow. ## Details This PR extracts ES|QL Query edit component from Define rule form step and makes it reusable. The following changes were made - ES|QL validator was refactored and covered by unit tests - Query persistence was addressed and covered by tests (previous functionality didn't work out of the box and didn't have tests) ## How to test The simplest way to test is via patching installed prebuilt rules (a.k.a. downgrading a prebuilt rule) via Rule Patch API. Please follow steps below - Ensure the `prebuiltRulesCustomizationEnabled` feature flag is enabled - Run Kibana locally - Install an ES|QL prebuilt rule, e.g. `AWS Bedrock Guardrails Detected Multiple Violations by a Single User Over a Session` with rule_id `0cd2f3e6-41da-40e6-b28b-466f688f00a6` - Patch the installed rule by running a query below ```bash curl -X PATCH --user elastic:changeme -H 'Content-Type: application/json' -H 'kbn-xsrf: 123' -H "elastic-api-version: 2023-10-31" -d '{"rule_id":"0cd2f3e6-41da-40e6-b28b-466f688f00a6","version":1,"query":"from logs-*","language":"esql"}' http://localhost:5601/kbn/api/detection_engine/rules ``` - Open `Detection Rules (SIEM)` Page -> `Rule Updates` -> click on `AWS Bedrock Guardrails Detected Multiple Violations by a Single User Over a Session` rule -> expand `EQL Query` to see EQL Query -> press `Edit` button ## Screenshots image image --- .../esql/compute_if_esql_query_aggregating.ts | 11 +- .../utils/use_set_field_value_cb.test.ts | 50 --- .../common/utils/use_set_field_value_cb.ts | 42 -- .../eql_query_edit/eql_query_bar.tsx | 2 +- .../eql_query_edit/eql_query_edit.tsx | 2 +- .../eql_query_validator_factory.ts | 2 +- .../esql_info_icon.tsx} | 22 +- .../esql_query_edit/esql_query_edit.tsx | 88 ++++ .../components/esql_query_edit/index.ts | 9 + .../translations.ts | 9 +- .../esql_query_edit/validators/error_codes.ts | 12 + .../esql_query_required_validator.ts | 22 + .../esql_query_validator_factory.test.ts | 182 ++++++++ .../esql_query_validator_factory.ts | 153 +++++++ .../validators}/translations.ts | 15 +- .../rule_creation/logic/esql_query_columns.ts | 84 ++++ .../logic/esql_validator.test.ts | 121 ------ .../rule_creation/logic/esql_validator.ts | 160 ------- .../logic/get_esql_query_config.ts | 40 -- .../components/description_step/index.tsx | 12 +- .../use_esql_fields_options.test.ts | 9 - .../use_esql_fields_options.ts | 31 +- .../query_bar_field/default_queries.ts | 32 ++ .../components/query_bar_field/index.ts | 10 + .../query_field.test.tsx} | 8 +- .../query_field.tsx} | 15 +- .../translations.tsx | 0 .../components/query_bar_field/types.ts | 15 + .../components/rule_preview/helpers.ts | 2 +- .../step_define_rule/index.test.tsx | 404 ++++++++++++++---- .../components/step_define_rule/index.tsx | 270 ++++-------- .../components/step_define_rule/schema.tsx | 18 +- .../step_define_rule/translations.tsx | 7 - .../step_define_rule/use_persistent_query.ts | 169 ++++++++ .../components/threatmatch_input/index.tsx | 4 +- .../hooks/use_all_esql_rule_fields.ts | 37 +- .../rule_creation_ui/pages/form.test.ts | 5 +- .../rule_creation_ui/pages/form.tsx | 2 +- .../pages/rule_creation/index.tsx | 1 - .../pages/rule_editing/index.tsx | 1 - .../validators/kuery_validator_factory.ts | 2 +- .../query_required_validator_factory.ts | 2 +- .../final_edit/esql_rule_field_edit.tsx | 10 +- .../fields/eql_query/eql_query_edit_form.tsx | 2 +- .../esql_query/esql_query_edit_adapter.tsx | 34 ++ .../esql_query/esql_query_edit_form.tsx | 72 ++++ .../final_edit/fields/esql_query/index.ts | 8 + .../fields/kql_query/kql_query_edit.tsx | 4 +- .../fields/kql_query/kql_query_edit_form.tsx | 2 +- .../components/rules_table/__mocks__/mock.ts | 2 +- .../rules/use_rule_from_timeline.tsx | 2 +- .../pages/detection_engine/rules/types.ts | 2 +- .../pages/detection_engine/rules/utils.ts | 28 +- .../cypress/tasks/response_actions.ts | 2 +- .../timeline/query_bar/eql/index.tsx | 2 +- .../translations/translations/fr-FR.json | 8 - .../translations/translations/ja-JP.json | 8 - .../translations/translations/zh-CN.json | 7 - .../cypress/screens/create_new_rule.ts | 5 +- 59 files changed, 1375 insertions(+), 905 deletions(-) delete mode 100644 x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.test.ts delete mode 100644 x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.ts rename x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/{esql_info_icon/index.tsx => esql_query_edit/esql_info_icon.tsx} (71%) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/index.ts rename x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/{esql_info_icon => esql_query_edit}/translations.ts (62%) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/error_codes.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_required_validator.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.test.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts rename x-pack/plugins/security_solution/public/detection_engine/rule_creation/{logic => components/esql_query_edit/validators}/translations.ts (72%) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_query_columns.ts delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/get_esql_query_config.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/default_queries.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/index.ts rename x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/{query_bar/index.test.tsx => query_bar_field/query_field.test.tsx} (97%) rename x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/{query_bar/index.tsx => query_bar_field/query_field.tsx} (96%) rename x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/{query_bar => query_bar_field}/translations.tsx (100%) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/types.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_persistent_query.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_adapter.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/esql_query_edit_form.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/three_way_diff/final_edit/fields/esql_query/index.ts diff --git a/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts b/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts index 4daf793b32d9d..c9b8474b55969 100644 --- a/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts +++ b/packages/kbn-securitysolution-utils/src/esql/compute_if_esql_query_aggregating.ts @@ -7,11 +7,10 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { ESQLAst, getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import { type ESQLAstQueryExpression, parse } from '@kbn/esql-ast'; -export const isAggregatingQuery = (ast: ESQLAst): boolean => { - return ast.some((astItem) => astItem.type === 'command' && astItem.name === 'stats'); -}; +export const isAggregatingQuery = (astExpression: ESQLAstQueryExpression): boolean => + astExpression.commands.some((command) => command.name === 'stats'); /** * compute if esqlQuery is aggregating/grouping, i.e. using STATS...BY command @@ -19,6 +18,6 @@ export const isAggregatingQuery = (ast: ESQLAst): boolean => { * @returns boolean */ export const computeIsESQLQueryAggregating = (esqlQuery: string): boolean => { - const { ast } = getAstAndSyntaxErrors(esqlQuery); - return isAggregatingQuery(ast); + const { root } = parse(esqlQuery); + return isAggregatingQuery(root); }; diff --git a/x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.test.ts b/x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.test.ts deleted file mode 100644 index 5879e2a256cce..0000000000000 --- a/x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { act, renderHook } from '@testing-library/react-hooks'; -import { useSetFieldValueWithCallback } from './use_set_field_value_cb'; - -const initialValue = 'initial value'; -const newValue = 'new value'; -const callback = jest.fn(); -const initialProps = { field: 'theField', setFieldValue: () => {}, value: initialValue }; - -describe('set field value callback', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - it('invokes the callback after value is set', () => { - const { result, rerender } = renderHook((props) => useSetFieldValueWithCallback(props), { - initialProps, - }); - act(() => { - result.current(newValue, callback); - }); - rerender({ ...initialProps, value: newValue }); - expect(callback).toHaveBeenCalled(); - }); - it('invokes the callback after value is set to equal value', () => { - const { result, rerender } = renderHook((props) => useSetFieldValueWithCallback(props), { - initialProps, - }); - act(() => { - result.current(initialValue, callback); - }); - rerender(); - expect(callback).toHaveBeenCalled(); - }); - it('does not invoke the callback if value does not update', () => { - const { result, rerender } = renderHook((props) => useSetFieldValueWithCallback(props), { - initialProps, - }); - act(() => { - result.current(newValue, callback); - }); - rerender(); - expect(callback).not.toHaveBeenCalled(); - }); -}); diff --git a/x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.ts b/x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.ts deleted file mode 100644 index c067e1747b4ff..0000000000000 --- a/x-pack/plugins/security_solution/public/common/utils/use_set_field_value_cb.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { FormHook } from '@kbn/es-ui-shared-plugin/static/forms/hook_form_lib'; -import { useCallback, useEffect, useRef, useState } from 'react'; - -export const useSetFieldValueWithCallback = ({ - field, - setFieldValue, - value, -}: { - field: string; - value: unknown; - setFieldValue: FormHook['setFieldValue']; -}) => { - const isWaitingRef = useRef(false); - const valueRef = useRef(); - const [callback, setCallback] = useState<() => void>(() => null); - - useEffect(() => { - if (isWaitingRef.current && value === valueRef.current) { - isWaitingRef.current = false; - valueRef.current = undefined; - callback(); - } - }, [value, callback]); - - return useCallback( - (v: unknown, cb: () => void) => { - setFieldValue(field, v); - - setCallback(() => cb); - valueRef.current = v; - isWaitingRef.current = true; - }, - [field, setFieldValue] - ); -}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_bar.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_bar.tsx index 079735aab3c48..c7ef28f5ed909 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_bar.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_bar.tsx @@ -18,8 +18,8 @@ import type { FieldHook } from '../../../../shared_imports'; import { FilterBar } from '../../../../common/components/filter_bar'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import type { EqlOptions } from '../../../../../common/search_strategy'; +import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar_field'; import { useKibana } from '../../../../common/lib/kibana'; -import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar'; import type { EqlQueryBarFooterProps } from './footer'; import { EqlQueryBarFooter } from './footer'; import { getValidationResults } from './validators'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_edit.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_edit.tsx index 5b519cb43c841..75d3412705fde 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_edit.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_edit.tsx @@ -11,11 +11,11 @@ import { debounceAsync } from '@kbn/securitysolution-utils'; import type { FormData, FieldConfig, ValidationFuncArg } from '../../../../shared_imports'; import { UseMultiFields } from '../../../../shared_imports'; import type { EqlFieldsComboBoxOptions, EqlOptions } from '../../../../../common/search_strategy'; +import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar_field'; import { queryRequiredValidatorFactory } from '../../../rule_creation_ui/validators/query_required_validator_factory'; import { eqlQueryValidatorFactory } from './eql_query_validator_factory'; import { EqlQueryBar } from './eql_query_bar'; import * as i18n from './translations'; -import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar'; interface EqlQueryEditProps { path: string; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_validator_factory.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_validator_factory.ts index 54a0b3e3b6a65..284d0670dfbc3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_validator_factory.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/eql_query_edit/eql_query_validator_factory.ts @@ -8,8 +8,8 @@ import { isEmpty } from 'lodash'; import type { FormData, ValidationError, ValidationFunc } from '../../../../shared_imports'; import { KibanaServices } from '../../../../common/lib/kibana'; +import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar_field'; import type { EqlOptions } from '../../../../../common/search_strategy'; -import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar'; import type { EqlResponseError } from '../../../../common/hooks/eql/api'; import { EQL_ERROR_CODES, validateEql } from '../../../../common/hooks/eql/api'; import { EQL_VALIDATION_REQUEST_ERROR } from './translations'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx similarity index 71% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/index.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx index d0b4cee6752ad..933a45004fc70 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_info_icon.tsx @@ -5,21 +5,19 @@ * 2.0. */ -import React from 'react'; +import React, { memo } from 'react'; import { EuiPopover, EuiText, EuiButtonIcon, EuiLink } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; -import * as i18n from './translations'; - -import { useBoolState } from '../../../../common/hooks/use_bool_state'; +import { useBoolean } from '@kbn/react-hooks'; import { useKibana } from '../../../../common/lib/kibana'; +import * as i18n from './translations'; /** * Icon and popover that gives hint to users how to get started with ES|QL rules */ -const EsqlInfoIconComponent = () => { +export const EsqlInfoIcon = memo(function EsqlInfoIcon(): JSX.Element { const { docLinks } = useKibana().services; - - const [isPopoverOpen, , closePopover, togglePopover] = useBoolState(); + const [isPopoverOpen, { off: closePopover, on: togglePopover }] = useBoolean(false); const button = ( @@ -29,13 +27,13 @@ const EsqlInfoIconComponent = () => { @@ -45,8 +43,4 @@ const EsqlInfoIconComponent = () => { ); -}; - -export const EsqlInfoIcon = React.memo(EsqlInfoIconComponent); - -EsqlInfoIcon.displayName = 'EsqlInfoIcon'; +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx new file mode 100644 index 0000000000000..695a3d121c9a6 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/esql_query_edit.tsx @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo, useMemo } from 'react'; +import { useQueryClient } from '@tanstack/react-query'; +import type { DataViewBase } from '@kbn/es-query'; +import { debounceAsync } from '@kbn/securitysolution-utils'; +import type { FieldConfig } from '../../../../shared_imports'; +import { UseField } from '../../../../shared_imports'; +import type { FieldValueQueryBar } from '../../../rule_creation_ui/components/query_bar_field'; +import { QueryBarField } from '../../../rule_creation_ui/components/query_bar_field'; +import { esqlQueryRequiredValidator } from './validators/esql_query_required_validator'; +import { esqlQueryValidatorFactory } from './validators/esql_query_validator_factory'; +import { EsqlInfoIcon } from './esql_info_icon'; +import * as i18n from './translations'; + +interface EsqlQueryEditProps { + path: string; + fieldsToValidateOnChange?: string | string[]; + dataView: DataViewBase; + required?: boolean; + loading?: boolean; + disabled?: boolean; + skipIdColumnCheck?: boolean; + onValidityChange?: (arg: boolean) => void; +} + +export const EsqlQueryEdit = memo(function EsqlQueryEdit({ + path, + fieldsToValidateOnChange, + dataView, + required = false, + loading = false, + disabled = false, + skipIdColumnCheck, + onValidityChange, +}: EsqlQueryEditProps): JSX.Element { + const queryClient = useQueryClient(); + const componentProps = useMemo( + () => ({ + isDisabled: disabled, + isLoading: loading, + indexPattern: dataView, + idAria: 'ruleEsqlQueryBar', + dataTestSubj: 'ruleEsqlQueryBar', + onValidityChange, + }), + [dataView, loading, disabled, onValidityChange] + ); + const fieldConfig: FieldConfig = useMemo( + () => ({ + label: i18n.ESQL_QUERY, + labelAppend: , + fieldsToValidateOnChange: fieldsToValidateOnChange + ? [path, fieldsToValidateOnChange].flat() + : undefined, + validations: [ + ...(required + ? [ + { + validator: esqlQueryRequiredValidator, + }, + ] + : []), + { + validator: debounceAsync( + esqlQueryValidatorFactory({ queryClient, skipIdColumnCheck }), + 300 + ), + }, + ], + }), + [required, path, fieldsToValidateOnChange, queryClient, skipIdColumnCheck] + ); + + return ( + + ); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/index.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/index.ts new file mode 100644 index 0000000000000..43eef8c21183d --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './esql_query_edit'; +export * from './validators/error_codes'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts similarity index 62% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/translations.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts index 8729f7b0dd3bc..9e48318c1c8bd 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_info_icon/translations.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/translations.ts @@ -7,8 +7,15 @@ import { i18n } from '@kbn/i18n'; +export const ESQL_QUERY = i18n.translate( + 'xpack.securitySolution.ruleManagement.fields.esqlQuery.label', + { + defaultMessage: 'ES|QL query', + } +); + export const ARIA_LABEL = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel', + 'xpack.securitySolution.ruleManagement.fields.esqlQuery.ariaLabel', { defaultMessage: `Open help popover`, } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/error_codes.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/error_codes.ts new file mode 100644 index 0000000000000..5455dd1db5f0a --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/error_codes.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum ESQL_ERROR_CODES { + INVALID_ESQL = 'ERR_INVALID_ESQL', + INVALID_SYNTAX = 'ERR_INVALID_SYNTAX', + ERR_MISSING_ID_FIELD_FROM_RESULT = 'ERR_MISSING_ID_FIELD_FROM_RESULT', +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_required_validator.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_required_validator.ts new file mode 100644 index 0000000000000..d3f71eaaf5a86 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_required_validator.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fieldValidators, type FormData, type ValidationFunc } from '../../../../../shared_imports'; +import type { FieldValueQueryBar } from '../../../../rule_creation_ui/components/query_bar_field'; +import * as i18n from './translations'; + +export const esqlQueryRequiredValidator: ValidationFunc = ( + data +) => { + const { value } = data; + const esqlQuery = value.query.query as string; + + return fieldValidators.emptyField(i18n.ESQL_QUERY_VALIDATION_REQUIRED)({ + ...data, + value: esqlQuery, + }); +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.test.ts new file mode 100644 index 0000000000000..2799a606d3d9b --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.test.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryClient } from '@tanstack/react-query'; +import { getESQLQueryColumns } from '@kbn/esql-utils'; +import type { FormData, ValidationFunc, ValidationFuncArg } from '../../../../../shared_imports'; +import type { FieldValueQueryBar } from '../../../../rule_creation_ui/components/query_bar_field'; +import { esqlQueryValidatorFactory } from './esql_query_validator_factory'; +import { ESQL_ERROR_CODES } from './error_codes'; + +jest.mock('@kbn/esql-utils', () => ({ + getESQLQueryColumns: jest.fn().mockResolvedValue([{ id: '_id' }]), +})); +jest.mock('../../../../../common/lib/kibana'); + +describe('esqlQueryValidator', () => { + describe('ES|QL query syntax', () => { + it.each([['incorrect syntax'], ['from test* metadata']])( + 'reports incorrect syntax in "%s"', + (esqlQuery) => + expect( + createValidator()({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.INVALID_SYNTAX, + }) + ); + + it.each([ + ['from test* metadata _id'], + [ + 'FROM kibana_sample_data_logs | STATS total_bytes = SUM(bytes) BY host | WHERE total_bytes > 200000 | SORT total_bytes DESC | LIMIT 10', + ], + [ + `from packetbeat* metadata + _id + | limit 100`, + ], + [ + `FROM kibana_sample_data_logs | + STATS total_bytes = SUM(bytes) BY host | + WHERE total_bytes > 200000 | + SORT total_bytes DESC | + LIMIT 10`, + ], + ])('succeeds validation for correct syntax in "%s"', (esqlQuery) => + expect( + createValidator()({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined() + ); + }); + + describe('METADATA operator validation', () => { + it.each([ + ['from test*'], + ['from metadata*'], + ['from test* | keep metadata'], + ['from test* | eval x="metadata _id"'], + ])('reports when METADATA operator is missing in a NON aggregating query "%s"', (esqlQuery) => + expect( + createValidator()({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + }) + ); + + it.each([ + ['from test* metadata _id'], + ['from test* metadata _id, _index'], + ['from test* metadata _index, _id'], + ['from test* metadata _id '], + ['from test* metadata _id '], + ['from test* metadata _id | limit 10'], + [ + `from packetbeat* metadata + + _id + | limit 100`, + ], + ])( + 'succeeds validation when METADATA operator EXISTS in a NON aggregating query "%s"', + (esqlQuery) => + expect( + createValidator()({ + value: createEsqlQueryFieldValue(esqlQuery), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined() + ); + + it('succeeds validation when METADATA operator is missing in an aggregating query "from test* | stats c = count(*) by fieldA"', () => + expect( + createValidator()({ + value: createEsqlQueryFieldValue('from test* | stats c = count(*) by fieldA'), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined()); + }); + + describe('METADATA _id field validation for NON aggregating queries', () => { + it('reports when METADATA "_id" field is missing', () => { + getESQLQueryColumnsMock.mockResolvedValue([{ id: 'column1' }, { id: 'column2' }]); + + return expect( + createValidator()({ + value: createEsqlQueryFieldValue('from test*'), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + }); + }); + + it('succeeds validation when METADATA "_id" field EXISTS', async () => { + getESQLQueryColumnsMock.mockResolvedValue([{ id: '_id' }, { id: 'column1' }]); + + return expect( + createValidator()({ + value: createEsqlQueryFieldValue('from test* metadata _id'), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined(); + }); + }); + + describe('METADATA _id field validation for aggregating queries', () => { + it('succeeds validation when METADATA operator with "_id" field is missing', () => { + getESQLQueryColumnsMock.mockResolvedValue([{ id: 'column1' }, { id: 'column2' }]); + + return expect( + createValidator()({ + value: createEsqlQueryFieldValue( + 'from test* metadata someField | stats c = count(*) by fieldA' + ), + } as EsqlQueryValidatorArgs) + ).resolves.toBeUndefined(); + }); + }); + + describe('when getESQLQueryColumns fails', () => { + it('returns a validation error', () => { + // suppress the expected error messages + jest.spyOn(console, 'error').mockReturnValue(); + + getESQLQueryColumnsMock.mockRejectedValue(new Error('some error')); + + return expect( + createValidator()({ + value: createEsqlQueryFieldValue('from test* metadata _id'), + } as EsqlQueryValidatorArgs) + ).resolves.toMatchObject({ + code: ESQL_ERROR_CODES.INVALID_ESQL, + message: 'Error validating ES|QL: "some error"', + }); + }); + }); +}); + +type EsqlQueryValidatorArgs = ValidationFuncArg; + +const getESQLQueryColumnsMock = getESQLQueryColumns as jest.Mock; + +function createValidator(): ValidationFunc { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 60 * 1000, + }, + }, + }); + + return esqlQueryValidatorFactory({ queryClient }); +} + +function createEsqlQueryFieldValue(esqlQuery: string): Readonly { + return { query: { query: esqlQuery, language: 'esql' }, filters: [], saved_id: null }; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts new file mode 100644 index 0000000000000..90cdaff14cc9b --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryClient } from '@tanstack/react-query'; +import type { DatatableColumn } from '@kbn/expressions-plugin/common'; +import type { ESQLAstQueryExpression, ESQLCommandOption } from '@kbn/esql-ast'; +import { parse } from '@kbn/esql-ast'; +import { isAggregatingQuery } from '@kbn/securitysolution-utils'; +import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete'; +import type { FormData, ValidationError, ValidationFunc } from '../../../../../shared_imports'; +import type { FieldValueQueryBar } from '../../../../rule_creation_ui/components/query_bar_field'; +import { fetchEsqlQueryColumns } from '../../../logic/esql_query_columns'; +import { ESQL_ERROR_CODES } from './error_codes'; +import * as i18n from './translations'; + +interface EsqlQueryValidatorFactoryParams { + queryClient: QueryClient; + /** + * This is a temporal fix to unlock prebuilt rule customization workflow + */ + skipIdColumnCheck?: boolean; +} + +export function esqlQueryValidatorFactory({ + queryClient, + skipIdColumnCheck, +}: EsqlQueryValidatorFactoryParams): ValidationFunc { + return async (...args) => { + const [{ value }] = args; + const esqlQuery = value.query.query as string; + + if (esqlQuery.trim() === '') { + return; + } + + try { + const { isEsqlQueryAggregating, hasMetadataOperator, errors } = parseEsqlQuery(esqlQuery); + + // Check if there are any syntax errors + if (errors.length) { + return constructSyntaxError(new Error(errors[0].message)); + } + + // non-aggregating query which does not have metadata, is not a valid one + if (!isEsqlQueryAggregating && !hasMetadataOperator) { + return { + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + message: i18n.ESQL_VALIDATION_MISSING_METADATA_OPERATOR_IN_QUERY_ERROR, + }; + } + + if (skipIdColumnCheck) { + return; + } + + const columns = await fetchEsqlQueryColumns({ + esqlQuery, + queryClient, + }); + + // for non-aggregating query, we want to disable queries w/o _id property returned in response + if (!isEsqlQueryAggregating && !hasIdColumn(columns)) { + return { + code: ESQL_ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, + message: i18n.ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR, + }; + } + } catch (error) { + return constructValidationError(error); + } + }; +} + +function hasIdColumn(columns: DatatableColumn[]): boolean { + return columns.some(({ id }) => '_id' === id); +} + +/** + * check if esql query valid for Security rule: + * - if it's non aggregation query it must have metadata operator + */ +function parseEsqlQuery(query: string) { + const { root, errors } = parse(query); + const isEsqlQueryAggregating = isAggregatingQuery(root); + + return { + errors, + isEsqlQueryAggregating, + hasMetadataOperator: computeHasMetadataOperator(root), + }; +} + +/** + * checks whether query has metadata _id operator + */ +function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean { + // Check whether the `from` command has `metadata` operator + const metadataOption = getMetadataOption(astExpression); + if (!metadataOption) { + return false; + } + + // Check whether the `metadata` operator has `_id` argument + const idColumnItem = metadataOption.args.find( + (fromArg) => isColumnItem(fromArg) && fromArg.name === '_id' + ); + if (!idColumnItem) { + return false; + } + + return true; +} + +function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOption | undefined { + const fromCommand = astExpression.commands.find((x) => x.name === 'from'); + + if (!fromCommand?.args) { + return undefined; + } + + // Check whether the `from` command has `metadata` operator + for (const fromArg of fromCommand.args) { + if (isOptionItem(fromArg) && fromArg.name === 'metadata') { + return fromArg; + } + } + + return undefined; +} + +function constructSyntaxError(error: Error): ValidationError { + return { + code: ESQL_ERROR_CODES.INVALID_SYNTAX, + message: error?.message + ? i18n.esqlValidationErrorMessage(error.message) + : i18n.ESQL_VALIDATION_UNKNOWN_ERROR, + error, + }; +} + +function constructValidationError(error: Error): ValidationError { + return { + code: ESQL_ERROR_CODES.INVALID_ESQL, + message: error?.message + ? i18n.esqlValidationErrorMessage(error.message) + : i18n.ESQL_VALIDATION_UNKNOWN_ERROR, + error, + }; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/translations.ts similarity index 72% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/translations.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/translations.ts index 4d6dde31fd5b4..21948a8863ec2 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/translations.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/translations.ts @@ -7,28 +7,35 @@ import { i18n } from '@kbn/i18n'; +export const ESQL_QUERY_VALIDATION_REQUIRED = i18n.translate( + 'xpack.securitySolution.ruleManagement.esqlValidation.requiredError', + { + defaultMessage: 'An ES|QL query is required.', + } +); + export const ESQL_VALIDATION_UNKNOWN_ERROR = i18n.translate( - 'xpack.securitySolution.detectionEngine.esqlValidation.unknownError', + 'xpack.securitySolution.ruleManagement.esqlValidation.unknownError', { defaultMessage: 'Unknown error while validating ES|QL', } ); export const esqlValidationErrorMessage = (message: string) => - i18n.translate('xpack.securitySolution.detectionEngine.esqlValidation.errorMessage', { + i18n.translate('xpack.securitySolution.ruleManagement.esqlValidation.errorMessage', { values: { message }, defaultMessage: 'Error validating ES|QL: "{message}"', }); export const ESQL_VALIDATION_MISSING_METADATA_OPERATOR_IN_QUERY_ERROR = i18n.translate( - 'xpack.securitySolution.detectionEngine.esqlValidation.missingMetadataOperatorInQueryError', + 'xpack.securitySolution.ruleManagement.esqlValidation.missingMetadataOperatorInQueryError', { defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index.`, } ); export const ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR = i18n.translate( - 'xpack.securitySolution.detectionEngine.esqlValidation.missingIdFieldInQueryError', + 'xpack.securitySolution.ruleManagement.esqlValidation.missingIdFieldInQueryError', { defaultMessage: `Queries that don’t use the STATS...BY function (non-aggregating queries) must include the "metadata _id, _version, _index" operator after the source command. For example: FROM logs* metadata _id, _version, _index. In addition, the metadata properties (_id, _version, and _index) must be returned in the query response.`, } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_query_columns.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_query_columns.ts new file mode 100644 index 0000000000000..be4c53a31cef8 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_query_columns.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + FetchQueryOptions, + QueryClient, + QueryFunction, + QueryKey, +} from '@tanstack/react-query'; +import { useQuery } from '@tanstack/react-query'; +import type { DatatableColumn } from '@kbn/expressions-plugin/common'; +import { getESQLQueryColumns } from '@kbn/esql-utils'; +import { KibanaServices } from '../../../common/lib/kibana'; + +const DEFAULT_STALE_TIME = 60 * 1000; + +interface FetchEsqlQueryColumnsParams { + esqlQuery: string; + queryClient: QueryClient; +} + +export async function fetchEsqlQueryColumns({ + esqlQuery, + queryClient, +}: FetchEsqlQueryColumnsParams): Promise { + const data = await queryClient.fetchQuery(createSharedTanstackQueryOptions(esqlQuery)); + + if (data instanceof Error) { + throw data; + } + + return data; +} + +interface UseEsqlQueryColumnsResult { + columns: DatatableColumn[]; + isLoading: boolean; +} + +export function useEsqlQueryColumns(esqlQuery: string): UseEsqlQueryColumnsResult { + const { data, isLoading } = useQuery({ + ...createSharedTanstackQueryOptions(esqlQuery), + retryOnMount: false, + refetchOnMount: false, + refetchOnWindowFocus: false, + }); + + return { columns: !data || data instanceof Error ? [] : data, isLoading }; +} + +function createSharedTanstackQueryOptions( + esqlQuery: string +): FetchQueryOptions { + return { + queryKey: [esqlQuery.trim()], + queryFn: queryEsqlColumnsFactory(esqlQuery), + staleTime: DEFAULT_STALE_TIME, + retry: false, + }; +} + +function queryEsqlColumnsFactory( + esqlQuery: string +): QueryFunction { + return async ({ signal }) => { + if (esqlQuery.trim() === '') { + return []; + } + + try { + return await getESQLQueryColumns({ + esqlQuery, + search: KibanaServices.get().data.search.search, + signal, + }); + } catch (e) { + return e; + } + }; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts deleted file mode 100644 index 808597ff36495..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; -import { parseEsqlQuery, computeHasMetadataOperator } from './esql_validator'; - -import { isAggregatingQuery } from '@kbn/securitysolution-utils'; - -jest.mock('@kbn/securitysolution-utils', () => ({ isAggregatingQuery: jest.fn() })); - -const isAggregatingQueryMock = isAggregatingQuery as jest.Mock; - -const getQeryAst = (query: string) => { - const { ast } = getAstAndSyntaxErrors(query); - return ast; -}; - -describe('computeHasMetadataOperator', () => { - it('should be false if query does not have operator', () => { - expect(computeHasMetadataOperator(getQeryAst('from test*'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata id'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from metadata*'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* | keep metadata'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="metadata _id"'))).toBe( - false - ); - }); - it('should be true if query has operator', () => { - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id, _index'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _index, _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id | limit 10'))).toBe( - true - ); - expect( - computeHasMetadataOperator( - getQeryAst(`from packetbeat* metadata - - _id - | limit 100`) - ) - ).toBe(true); - - // still validates deprecated square bracket syntax - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id, _index'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _index, _id'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id | limit 10'))).toBe( - true - ); - expect( - computeHasMetadataOperator( - getQeryAst(`from packetbeat* metadata - - _id - | limit 100`) - ) - ).toBe(true); - }); -}); - -describe('parseEsqlQuery', () => { - it('returns isMissingMetadataOperator true when query is not aggregating and does not have metadata operator', () => { - isAggregatingQueryMock.mockReturnValueOnce(false); - - expect(parseEsqlQuery('from test*')).toEqual({ - errors: [], - isEsqlQueryAggregating: false, - isMissingMetadataOperator: true, - }); - }); - - it('returns isMissingMetadataOperator false when query is not aggregating and has metadata operator', () => { - isAggregatingQueryMock.mockReturnValueOnce(false); - - expect(parseEsqlQuery('from test* metadata _id')).toEqual({ - errors: [], - isEsqlQueryAggregating: false, - isMissingMetadataOperator: false, - }); - }); - - it('returns isMissingMetadataOperator false when query is aggregating', () => { - isAggregatingQueryMock.mockReturnValue(true); - - expect(parseEsqlQuery('from test*')).toEqual({ - errors: [], - isEsqlQueryAggregating: true, - isMissingMetadataOperator: false, - }); - - expect(parseEsqlQuery('from test* metadata _id')).toEqual({ - errors: [], - isEsqlQueryAggregating: true, - isMissingMetadataOperator: false, - }); - }); - - it('returns error when query is syntactically invalid', () => { - isAggregatingQueryMock.mockReturnValueOnce(false); - - expect(parseEsqlQuery('aaa bbbb ssdasd')).toEqual({ - errors: expect.arrayContaining([ - expect.objectContaining({ - message: - "SyntaxError: mismatched input 'aaa' expecting {'explain', 'from', 'row', 'show'}", - }), - ]), - isEsqlQueryAggregating: false, - isMissingMetadataOperator: true, - }); - }); -}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts deleted file mode 100644 index c508676cae92c..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isEmpty } from 'lodash'; -import type { QueryClient } from '@tanstack/react-query'; -import { isAggregatingQuery } from '@kbn/securitysolution-utils'; - -import type { ESQLAst } from '@kbn/esql-ast'; -import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; -import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete'; -import { KibanaServices } from '../../../common/lib/kibana'; - -import type { ValidationError, ValidationFunc } from '../../../shared_imports'; -import { isEsqlRule } from '../../../../common/detection_engine/utils'; -import type { DefineStepRule } from '../../../detections/pages/detection_engine/rules/types'; -import type { FieldValueQueryBar } from '../../rule_creation_ui/components/query_bar'; -import * as i18n from './translations'; -import { getEsqlQueryConfig } from './get_esql_query_config'; -export type FieldType = 'string'; - -export enum ERROR_CODES { - INVALID_ESQL = 'ERR_INVALID_ESQL', - INVALID_SYNTAX = 'ERR_INVALID_SYNTAX', - ERR_MISSING_ID_FIELD_FROM_RESULT = 'ERR_MISSING_ID_FIELD_FROM_RESULT', -} - -const constructValidationError = (error: Error) => { - return { - code: ERROR_CODES.INVALID_ESQL, - message: error?.message - ? i18n.esqlValidationErrorMessage(error.message) - : i18n.ESQL_VALIDATION_UNKNOWN_ERROR, - error, - }; -}; - -const constructSyntaxError = (error: Error) => { - return { - code: ERROR_CODES.INVALID_SYNTAX, - message: error?.message - ? i18n.esqlValidationErrorMessage(error.message) - : i18n.ESQL_VALIDATION_UNKNOWN_ERROR, - error, - }; -}; - -const getMetadataOption = (ast: ESQLAst) => { - const fromCommand = ast.find((astItem) => astItem.type === 'command' && astItem.name === 'from'); - - if (!fromCommand?.args) { - return undefined; - } - - // Check whether the `from` command has `metadata` operator - for (const fromArg of fromCommand.args) { - if (isOptionItem(fromArg) && fromArg.name === 'metadata') { - return fromArg; - } - } - - return undefined; -}; - -/** - * checks whether query has metadata _id operator - */ -export const computeHasMetadataOperator = (ast: ESQLAst) => { - // Check whether the `from` command has `metadata` operator - const metadataOption = getMetadataOption(ast); - if (!metadataOption) { - return false; - } - - // Check whether the `metadata` operator has `_id` argument - const idColumnItem = metadataOption.args.find( - (fromArg) => isColumnItem(fromArg) && fromArg.name === '_id' - ); - if (!idColumnItem) { - return false; - } - - return true; -}; - -/** - * form validator for ES|QL queryBar - */ -export const esqlValidator = async ( - ...args: Parameters -): Promise | void | undefined> => { - const [{ value, formData, customData }] = args; - const { query: queryValue } = value as FieldValueQueryBar; - const query = queryValue.query as string; - const { ruleType } = formData as DefineStepRule; - - const needsValidation = isEsqlRule(ruleType) && !isEmpty(query); - if (!needsValidation) { - return; - } - - try { - const queryClient = (customData.value as { queryClient: QueryClient | undefined })?.queryClient; - - const services = KibanaServices.get(); - const { isEsqlQueryAggregating, isMissingMetadataOperator, errors } = parseEsqlQuery(query); - - // Check if there are any syntax errors - if (errors.length) { - return constructSyntaxError(new Error(errors[0].message)); - } - - if (isMissingMetadataOperator) { - return { - code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, - message: i18n.ESQL_VALIDATION_MISSING_METADATA_OPERATOR_IN_QUERY_ERROR, - }; - } - - const columns = await queryClient?.fetchQuery( - getEsqlQueryConfig({ esqlQuery: query, search: services.data.search.search }) - ); - - if (columns && 'error' in columns) { - return constructValidationError(columns.error); - } - - // check whether _id field is present in response - const isIdFieldPresent = (columns ?? []).find(({ id }) => '_id' === id); - // for non-aggregating query, we want to disable queries w/o _id property returned in response - if (!isEsqlQueryAggregating && !isIdFieldPresent) { - return { - code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, - message: i18n.ESQL_VALIDATION_MISSING_ID_FIELD_IN_QUERY_ERROR, - }; - } - } catch (error) { - return constructValidationError(error); - } -}; - -/** - * check if esql query valid for Security rule: - * - if it's non aggregation query it must have metadata operator - */ -export const parseEsqlQuery = (query: string) => { - const { ast, errors } = getAstAndSyntaxErrors(query); - - const isEsqlQueryAggregating = isAggregatingQuery(ast); - - return { - errors, - isEsqlQueryAggregating, - // non-aggregating query which does not have metadata, is not a valid one - isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(ast), - }; -}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/get_esql_query_config.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/get_esql_query_config.ts deleted file mode 100644 index c7d4f9184f181..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/get_esql_query_config.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getESQLQueryColumns } from '@kbn/esql-utils'; -import type { ISearchGeneric } from '@kbn/search-types'; - -/** - * react-query configuration to be used to fetch ES|QL fields - * it sets limit in query to 0, so we don't fetch unnecessary results, only fields - */ -export const getEsqlQueryConfig = ({ - esqlQuery, - search, -}: { - esqlQuery: string | undefined; - search: ISearchGeneric; -}) => { - return { - queryKey: [(esqlQuery ?? '').trim()], - queryFn: async () => { - if (!esqlQuery) { - return null; - } - try { - const res = await getESQLQueryColumns({ - esqlQuery, - search, - }); - return res; - } catch (e) { - return { error: e }; - } - }, - staleTime: 60 * 1000, - }; -}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx index 24ad5f4135a14..26c81f325ea18 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx @@ -72,6 +72,7 @@ import { ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, } from '../../../rule_creation/components/alert_suppression_edit'; import { THRESHOLD_ALERT_SUPPRESSION_ENABLED } from '../../../rule_creation/components/threshold_alert_suppression_edit'; +import type { FieldValueQueryBar } from '../query_bar_field'; const DescriptionListContainer = styled(EuiDescriptionList)` max-width: 600px; @@ -206,11 +207,12 @@ export const getDescriptionItem = ( indexPatterns?: DataViewBase ): ListItems[] => { if (field === 'queryBar') { - const filters = addFilterStateIfNotThere(get('queryBar.filters', data) ?? []); - const query = get('queryBar.query.query', data); - const savedId = get('queryBar.saved_id', data); - const savedQueryName = get('queryBar.title', data); - const ruleType: Type = get('ruleType', data); + const queryBar = get('queryBar', data) as FieldValueQueryBar; + const filters = addFilterStateIfNotThere(queryBar.filters ?? []); + const query = queryBar.query.query as string; + const savedId = queryBar.saved_id ?? ''; + const savedQueryName = queryBar.title; + const ruleType: Type = get('ruleType', data) as Type; const queryLabel = getQueryLabel(ruleType); return buildQueryBarDescription({ field, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.test.ts index e6a6b3d4a3d78..fa00914bced13 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.test.ts @@ -8,15 +8,6 @@ import { esqlToOptions } from './use_esql_fields_options'; describe('esqlToOptions', () => { - it('should return empty array if data is undefined', () => { - expect(esqlToOptions(undefined)).toEqual([]); - }); - it('should return empty array if data is null', () => { - expect(esqlToOptions(null)).toEqual([]); - }); - it('should return empty array if data has error', () => { - expect(esqlToOptions({ error: Error })).toEqual([]); - }); it('should transform all columns if fieldTYpe is not passed', () => { expect( esqlToOptions([ diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts index b29d44c0b855f..f3a3128c8842e 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/esql_autocomplete/use_esql_fields_options.ts @@ -7,24 +7,12 @@ import { useMemo } from 'react'; import type { EuiComboBoxOptionOption } from '@elastic/eui'; import type { DatatableColumn } from '@kbn/expressions-plugin/public'; -import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import { useEsqlQueryColumns } from '../../../rule_creation/logic/esql_query_columns'; -import { useQuery } from '@tanstack/react-query'; +type FieldType = 'string'; -import { useKibana } from '@kbn/kibana-react-plugin/public'; - -import { getEsqlQueryConfig } from '../../../rule_creation/logic/get_esql_query_config'; -import type { FieldType } from '../../../rule_creation/logic/esql_validator'; - -export const esqlToOptions = ( - columns: { error: unknown } | DatatableColumn[] | undefined | null, - fieldType?: FieldType -) => { - if (columns && 'error' in columns) { - return []; - } - - const options = (columns ?? []).reduce>((acc, { id, meta }) => { +export const esqlToOptions = (columns: DatatableColumn[], fieldType?: FieldType) => { + const options = columns.reduce>((acc, { id, meta }) => { // if fieldType absent, we do not filter columns by type if (!fieldType || fieldType === meta.type) { acc.push({ label: id }); @@ -47,16 +35,11 @@ type UseEsqlFieldOptions = ( * fetches ES|QL fields and convert them to Combobox options */ export const useEsqlFieldOptions: UseEsqlFieldOptions = (esqlQuery, fieldType) => { - const kibana = useKibana<{ data: DataPublicPluginStart }>(); - - const { data: dataService } = kibana.services; - - const queryConfig = getEsqlQueryConfig({ esqlQuery, search: dataService.search.search }); - const { data, isLoading } = useQuery(queryConfig); + const { columns, isLoading } = useEsqlQueryColumns(esqlQuery ?? ''); const options = useMemo(() => { - return esqlToOptions(data, fieldType); - }, [data, fieldType]); + return esqlToOptions(columns, fieldType); + }, [columns, fieldType]); return { options, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/default_queries.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/default_queries.ts new file mode 100644 index 0000000000000..c1f3150a3aa75 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/default_queries.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FieldValueQueryBar } from './types'; + +export const DEFAULT_KQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '', language: 'kuery' }, + filters: [], + saved_id: null, +}; + +export const DEFAULT_THREAT_MATCH_KQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '*:*', language: 'kuery' }, + filters: [], + saved_id: null, +}; + +export const DEFAULT_EQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '', language: 'eql' }, + filters: [], + saved_id: null, +}; + +export const DEFAULT_ESQL_QUERY_FIELD_VALUE: Readonly = { + query: { query: '', language: 'esql' }, + filters: [], + saved_id: null, +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/index.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/index.ts new file mode 100644 index 0000000000000..07f6e408c5574 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; +export * from './default_queries'; +export * from './query_field'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/query_field.test.tsx similarity index 97% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.test.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/query_field.test.tsx index 7b757f8fc621d..190d10fe0a3b1 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/query_field.test.tsx @@ -7,7 +7,7 @@ import React from 'react'; -import { QueryBarDefineRule } from '.'; +import { QueryBarField } from '.'; import { TestProviders, useFormFieldMock, @@ -74,7 +74,7 @@ describe('QueryBarDefineRule', () => { const { getByTestId } = render( - { const { queryByTestId } = render( - { const { getByTestId } = render( - ( title: savedQuery.attributes.title, }); -export const QueryBarDefineRule = ({ +export const QueryBarField = ({ defaultSavedQuery, dataTestSubj, field, @@ -85,7 +80,7 @@ export const QueryBarDefineRule = ({ resetToSavedQuery, onOpenTimeline, onSavedQueryError, -}: QueryBarDefineRuleProps) => { +}: QueryBarFieldProps) => { const { value: fieldValue, setValue: setFieldValue } = field as FieldHook; const [originalHeight, setOriginalHeight] = useState(-1); const [loadingTimeline, setLoadingTimeline] = useState(false); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/translations.tsx similarity index 100% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar/translations.tsx rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/translations.tsx diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/types.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/types.ts new file mode 100644 index 0000000000000..9807be907209a --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/query_bar_field/types.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Filter, Query } from '@kbn/es-query'; + +export interface FieldValueQueryBar { + filters: Filter[]; + query: Query; + saved_id: string | null; + title?: string; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/helpers.ts index e6f9945737444..045e957c4e129 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/rule_preview/helpers.ts @@ -10,7 +10,7 @@ import type { EuiSelectOption } from '@elastic/eui'; import type { Type, ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types'; import * as i18n from './translations'; -import type { FieldValueQueryBar } from '../query_bar'; +import type { FieldValueQueryBar } from '../query_bar_field'; import type { TimeframePreviewOptions } from '../../../../detections/pages/detection_engine/rules/types'; import { DataSourceType } from '../../../../detections/pages/detection_engine/rules/types'; import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../../common/constants'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx index 364f1b7705732..83aa6a114362a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx @@ -6,6 +6,7 @@ */ import React, { useEffect } from 'react'; +import type { ChangeEvent } from 'react'; import { screen, fireEvent, render, within, act, waitFor } from '@testing-library/react'; import type { Type as RuleType } from '@kbn/securitysolution-io-ts-alerting-types'; import type { DataViewBase } from '@kbn/es-query'; @@ -42,13 +43,42 @@ import { addRelatedIntegrationRow, setVersion, } from '../../../rule_creation/components/related_integrations/test_helpers'; +import { useEsqlAvailability } from '../../../../common/hooks/esql/use_esql_availability'; +import { useMLRuleConfig } from '../../../../common/components/ml/hooks/use_ml_rule_config'; + +// Set the extended default timeout for all define rule step form test +jest.setTimeout(10 * 1000); // Mocks integrations jest.mock('../../../fleet_integrations/api'); + +const MOCKED_QUERY_BAR_TEST_ID = 'mockedQueryBar'; +const MOCKED_LANGUAGE_INPUT_TEST_ID = 'languageInput'; + +// Mocking QueryBar to avoid pulling and mocking a ton of dependencies jest.mock('../../../../common/components/query_bar', () => { return { - QueryBar: jest.fn(({ filterQuery }) => { - return
{`${filterQuery.query} ${filterQuery.language}`}
; + QueryBar: jest.fn().mockImplementation(({ filterQuery, onSubmitQuery }) => { + const handleQueryChange = (event: ChangeEvent) => { + onSubmitQuery({ query: event.target.value, language: filterQuery.language }); + }; + + const handleLanguageChange = (event: ChangeEvent) => { + onSubmitQuery({ query: filterQuery.query, language: event.target.value }); + }; + + return ( +
+