From 77cfc0cbc4b050c2a870501e3ca2200ddba60cc2 Mon Sep 17 00:00:00 2001 From: "Quynh Nguyen (Quinn)" <43350163+qn895@users.noreply.github.com> Date: Thu, 14 Nov 2024 03:05:36 -0600 Subject: [PATCH] [ES|QL] Add autocomplete and validation to support MATCH and QSRT (#199032) ## Summary Closes https://github.com/elastic/kibana/issues/196995. This PR adds autocomplete and validation to support MATCH and QSRT https://github.com/user-attachments/assets/f6be7108-cc6c-480f-b7cf-8c7953d06e7a https://github.com/user-attachments/assets/0549e044-90d6-4619-a00b-c9a2c8f94c04 ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels) - [ ] This will appear in the **Release Notes** and follow the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Stratoula Kalafateli --- .../scripts/generate_function_definitions.ts | 27 ++- .../autocomplete.command.where.test.ts | 2 +- .../autocomplete.suggest.eval.test.ts | 2 +- .../src/autocomplete/__tests__/helpers.ts | 8 +- .../src/autocomplete/autocomplete.ts | 46 ++-- .../src/autocomplete/commands/where/index.ts | 24 +- .../src/code_actions/actions.test.ts | 13 ++ .../definitions/generated/scalar_functions.ts | 14 +- .../src/definitions/types.ts | 8 +- .../src/shared/constants.ts | 16 ++ .../validation.functions.full_text.test.ts | 78 +++++++ .../src/validation/errors.ts | 25 +++ .../src/validation/types.ts | 8 + .../src/validation/validation.ts | 209 ++++++++++++++++-- 14 files changed, 422 insertions(+), 58 deletions(-) create mode 100644 packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.functions.full_text.test.ts diff --git a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts index 4f649b44e44b8..8462f9e2a050b 100644 --- a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts +++ b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts @@ -13,7 +13,7 @@ import { join } from 'path'; import _ from 'lodash'; import type { RecursivePartial } from '@kbn/utility-types'; import { FunctionDefinition } from '../src/definitions/types'; - +import { FULL_TEXT_SEARCH_FUNCTIONS } from '../src/shared/constants'; const aliasTable: Record = { to_version: ['to_ver'], to_unsigned_long: ['to_ul', 'to_ulong'], @@ -246,12 +246,25 @@ const convertDateTime = (s: string) => (s === 'datetime' ? 'date' : s); * @returns */ function getFunctionDefinition(ESFunctionDefinition: Record): FunctionDefinition { + let supportedCommandsAndOptions: Pick< + FunctionDefinition, + 'supportedCommands' | 'supportedOptions' + > = + ESFunctionDefinition.type === 'eval' + ? scalarSupportedCommandsAndOptions + : aggregationSupportedCommandsAndOptions; + + // MATCH and QSRT has limited supported for where commands only + if (FULL_TEXT_SEARCH_FUNCTIONS.includes(ESFunctionDefinition.name)) { + supportedCommandsAndOptions = { + supportedCommands: ['where'], + supportedOptions: [], + }; + } const ret = { type: ESFunctionDefinition.type, name: ESFunctionDefinition.name, - ...(ESFunctionDefinition.type === 'eval' - ? scalarSupportedCommandsAndOptions - : aggregationSupportedCommandsAndOptions), + ...supportedCommandsAndOptions, description: ESFunctionDefinition.description, alias: aliasTable[ESFunctionDefinition.name], ignoreAsSuggestion: ESFunctionDefinition.snapshot_only, @@ -259,10 +272,14 @@ function getFunctionDefinition(ESFunctionDefinition: Record): Funct signatures: _.uniqBy( ESFunctionDefinition.signatures.map((signature: any) => ({ ...signature, - params: signature.params.map((param: any) => ({ + params: signature.params.map((param: any, idx: number) => ({ ...param, type: convertDateTime(param.type), description: undefined, + ...(idx === 0 && FULL_TEXT_SEARCH_FUNCTIONS.includes(ESFunctionDefinition.name) + ? // Default to false. If set to true, this parameter does not accept a function or literal, only fields. + { fieldsOnly: true } + : {}), })), returnType: convertDateTime(signature.returnType), variadic: undefined, // we don't support variadic property diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.where.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.where.test.ts index 3345f7646e2ff..3931480d739a4 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.where.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.where.test.ts @@ -39,7 +39,7 @@ describe('WHERE ', () => { .map((name) => `${name} `) .map(attachTriggerCommand), attachTriggerCommand('var0 '), - ...allEvalFns, + ...allEvalFns.filter((fn) => fn.label !== 'QSTR'), ], { callbacks: { diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts index 5c67bfedbae75..aae715ee66749 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts @@ -371,7 +371,7 @@ describe('autocomplete.suggest', () => { for (const fn of scalarFunctionDefinitions) { // skip this fn for the moment as it's quite hard to test // Add match in the text when the autocomplete is ready https://github.com/elastic/kibana/issues/196995 - if (!['bucket', 'date_extract', 'date_diff', 'case', 'match'].includes(fn.name)) { + if (!['bucket', 'date_extract', 'date_diff', 'case', 'match', 'qstr'].includes(fn.name)) { test(`${fn.name}`, async () => { const testedCases = new Set(); diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts index 9964fc96d00ca..2221f4dc1582f 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts @@ -121,7 +121,7 @@ export const policies = [ * @returns */ export function getFunctionSignaturesByReturnType( - command: string, + command: string | string[], _expectedReturnType: Readonly>, { agg, @@ -165,12 +165,16 @@ export function getFunctionSignaturesByReturnType( const deduped = Array.from(new Set(list)); + const commands = Array.isArray(command) ? command : [command]; return deduped .filter(({ signatures, ignoreAsSuggestion, supportedCommands, supportedOptions, name }) => { if (ignoreAsSuggestion) { return false; } - if (!supportedCommands.includes(command) && !supportedOptions?.includes(option || '')) { + if ( + !commands.some((c) => supportedCommands.includes(c)) && + !supportedOptions?.includes(option || '') + ) { return false; } const filteredByReturnType = signatures.filter( diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts index bae10b4c321f4..2a37155358e85 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts @@ -10,6 +10,7 @@ import { uniq, uniqBy } from 'lodash'; import type { AstProviderFn, + ESQLAst, ESQLAstItem, ESQLCommand, ESQLCommandOption, @@ -151,14 +152,16 @@ export async function suggest( astProvider: AstProviderFn, resourceRetriever?: ESQLCallbacks ): Promise { + // Partition out to inner ast / ast context for the latest command const innerText = fullText.substring(0, offset); - const correctedQuery = correctQuerySyntax(innerText, context); - const { ast } = await astProvider(correctedQuery); - const astContext = getAstContext(innerText, ast, offset); + // But we also need the full ast for the full query + const correctedFullQuery = correctQuerySyntax(fullText, context); + const { ast: fullAst } = await astProvider(correctedFullQuery); + if (astContext.type === 'comment') { return []; } @@ -216,7 +219,8 @@ export async function suggest( getFieldsMap, getPolicies, getPolicyMetadata, - resourceRetriever?.getPreferences + resourceRetriever?.getPreferences, + fullAst ); } if (astContext.type === 'setting') { @@ -394,7 +398,8 @@ async function getSuggestionsWithinCommandExpression( getFieldsMap: GetFieldsMapFn, getPolicies: GetPoliciesFn, getPolicyMetadata: GetPolicyMetadataFn, - getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined> + getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>, + fullAst?: ESQLAst ) { const commandDef = getCommandDefinition(command.name); @@ -413,7 +418,8 @@ async function getSuggestionsWithinCommandExpression( () => findNewVariable(anyVariables), (expression: ESQLAstItem | undefined) => getExpressionType(expression, references.fields, references.variables), - getPreferences + getPreferences, + fullAst ); } else { // The deprecated path. @@ -1173,19 +1179,21 @@ async function getFunctionArgsSuggestions( ); // Functions - suggestions.push( - ...getFunctionSuggestions({ - command: command.name, - option: option?.name, - returnTypes: canBeBooleanCondition - ? ['any'] - : (getTypesFromParamDefs(typesToSuggestNext) as string[]), - ignored: fnToIgnore, - }).map((suggestion) => ({ - ...suggestion, - text: addCommaIf(shouldAddComma, suggestion.text), - })) - ); + if (typesToSuggestNext.every((d) => !d.fieldsOnly)) { + suggestions.push( + ...getFunctionSuggestions({ + command: command.name, + option: option?.name, + returnTypes: canBeBooleanCondition + ? ['any'] + : (getTypesFromParamDefs(typesToSuggestNext) as string[]), + ignored: fnToIgnore, + }).map((suggestion) => ({ + ...suggestion, + text: addCommaIf(shouldAddComma, suggestion.text), + })) + ); + } // could also be in stats (bucket) but our autocomplete is not great yet if ( (getTypesFromParamDefs(typesToSuggestNext).includes('date') && diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts index dc2ab341e961e..a7d381538f738 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/commands/where/index.ts @@ -13,6 +13,7 @@ import { type ESQLCommand, type ESQLSingleAstItem, type ESQLFunction, + ESQLAst, } from '@kbn/esql-ast'; import { logicalOperators } from '../../../definitions/builtin'; import { isParameterType, type SupportedDataType } from '../../../definitions/types'; @@ -27,6 +28,10 @@ import { import { getOverlapRange, getSuggestionsToRightOfOperatorExpression } from '../../helper'; import { getPosition } from './util'; import { pipeCompleteItem } from '../../complete_items'; +import { + UNSUPPORTED_COMMANDS_BEFORE_MATCH, + UNSUPPORTED_COMMANDS_BEFORE_QSTR, +} from '../../../shared/constants'; export async function suggest( innerText: string, @@ -35,7 +40,8 @@ export async function suggest( _columnExists: (column: string) => boolean, _getSuggestedVariableName: () => string, getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown', - _getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined> + _getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>, + fullTextAst?: ESQLAst ): Promise { const suggestions: SuggestionRawDefinition[] = []; @@ -154,11 +160,25 @@ export async function suggest( break; case 'empty_expression': + // Don't suggest MATCH or QSTR after unsupported commands + const priorCommands = fullTextAst?.map((a) => a.name) ?? []; + const ignored = []; + if (priorCommands.some((c) => UNSUPPORTED_COMMANDS_BEFORE_MATCH.has(c))) { + ignored.push('match'); + } + if (priorCommands.some((c) => UNSUPPORTED_COMMANDS_BEFORE_QSTR.has(c))) { + ignored.push('qstr'); + } + const columnSuggestions = await getColumnsByType('any', [], { advanceCursor: true, openSuggestions: true, }); - suggestions.push(...columnSuggestions, ...getFunctionSuggestions({ command: 'where' })); + + suggestions.push( + ...columnSuggestions, + ...getFunctionSuggestions({ command: 'where', ignored }) + ); break; } diff --git a/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts b/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts index 4563379642767..665c3df0df060 100644 --- a/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts @@ -15,6 +15,7 @@ import type { CodeActionOptions } from './types'; import type { ESQLRealField } from '../validation/types'; import type { FieldType } from '../definitions/types'; import type { ESQLCallbacks, PartialFieldsMetadataClient } from '../shared/types'; +import { FULL_TEXT_SEARCH_FUNCTIONS } from '../shared/constants'; function getCallbackMocks(): jest.Mocked { return { @@ -285,6 +286,16 @@ describe('quick fixes logic', () => { { relaxOnMissingCallbacks: false }, ]) { for (const fn of getAllFunctions({ type: 'eval' })) { + if (FULL_TEXT_SEARCH_FUNCTIONS.includes(fn.name)) { + testQuickFixes( + `FROM index | WHERE ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + } + } + for (const fn of getAllFunctions({ type: 'eval' })) { + if (FULL_TEXT_SEARCH_FUNCTIONS.includes(fn.name)) continue; // add an A to the function name to make it invalid testQuickFixes( `FROM index | EVAL ${BROKEN_PREFIX}${fn.name}()`, @@ -313,6 +324,8 @@ describe('quick fixes logic', () => { ); } for (const fn of getAllFunctions({ type: 'agg' })) { + if (FULL_TEXT_SEARCH_FUNCTIONS.includes(fn.name)) continue; + // add an A to the function name to make it invalid testQuickFixes( `FROM index | STATS ${BROKEN_PREFIX}${fn.name}()`, diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts index 7e9019aeb905a..257753d036aa7 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts @@ -3259,6 +3259,7 @@ const matchDefinition: FunctionDefinition = { name: 'field', type: 'keyword', optional: false, + fieldsOnly: true, }, { name: 'query', @@ -3274,6 +3275,7 @@ const matchDefinition: FunctionDefinition = { name: 'field', type: 'keyword', optional: false, + fieldsOnly: true, }, { name: 'query', @@ -3289,6 +3291,7 @@ const matchDefinition: FunctionDefinition = { name: 'field', type: 'text', optional: false, + fieldsOnly: true, }, { name: 'query', @@ -3304,6 +3307,7 @@ const matchDefinition: FunctionDefinition = { name: 'field', type: 'text', optional: false, + fieldsOnly: true, }, { name: 'query', @@ -3314,8 +3318,8 @@ const matchDefinition: FunctionDefinition = { returnType: 'boolean', }, ], - supportedCommands: ['stats', 'inlinestats', 'metrics', 'eval', 'where', 'row', 'sort'], - supportedOptions: ['by'], + supportedCommands: ['where'], + supportedOptions: [], validate: undefined, examples: [ 'from books \n| where match(author, "Faulkner")\n| keep book_no, author \n| sort book_no \n| limit 5;', @@ -5912,6 +5916,7 @@ const qstrDefinition: FunctionDefinition = { name: 'query', type: 'keyword', optional: false, + fieldsOnly: true, }, ], returnType: 'boolean', @@ -5922,13 +5927,14 @@ const qstrDefinition: FunctionDefinition = { name: 'query', type: 'text', optional: false, + fieldsOnly: true, }, ], returnType: 'boolean', }, ], - supportedCommands: ['stats', 'inlinestats', 'metrics', 'eval', 'where', 'row', 'sort'], - supportedOptions: ['by'], + supportedCommands: ['where'], + supportedOptions: [], validate: undefined, examples: [ 'from books \n| where qstr("author: Faulkner")\n| keep book_no, author \n| sort book_no \n| limit 5;', diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/types.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/types.ts index ba0a50c4a71b9..ce649acec5b44 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/types.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/types.ts @@ -8,6 +8,7 @@ */ import type { + ESQLAst, ESQLAstItem, ESQLCommand, ESQLCommandOption, @@ -136,6 +137,10 @@ export interface FunctionDefinition { * though a function can be used to create the value. (e.g. now() for dates or concat() for strings) */ constantOnly?: boolean; + /** + * Default to false. If set to true, this parameter does not accept a function or literal, only fields. + */ + fieldsOnly?: boolean; /** * if provided this means that the value must be one * of the options in the array iff the value is a literal. @@ -181,7 +186,8 @@ export interface CommandBaseDefinition { columnExists: (column: string) => boolean, getSuggestedVariableName: () => string, getExpressionType: (expression: ESQLAstItem | undefined) => SupportedDataType | 'unknown', - getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined> + getPreferences?: () => Promise<{ histogramBarTarget: number } | undefined>, + fullTextAst?: ESQLAst ) => Promise; /** @deprecated this property will disappear in the future */ signature: { diff --git a/packages/kbn-esql-validation-autocomplete/src/shared/constants.ts b/packages/kbn-esql-validation-autocomplete/src/shared/constants.ts index 1a9f382d32a6d..9e073e5329cac 100644 --- a/packages/kbn-esql-validation-autocomplete/src/shared/constants.ts +++ b/packages/kbn-esql-validation-autocomplete/src/shared/constants.ts @@ -16,3 +16,19 @@ export const DOUBLE_BACKTICK = '``'; export const SINGLE_BACKTICK = '`'; export const METADATA_FIELDS = ['_version', '_id', '_index', '_source', '_ignored', '_index_mode']; + +export const FULL_TEXT_SEARCH_FUNCTIONS = ['match', 'qstr']; +export const UNSUPPORTED_COMMANDS_BEFORE_QSTR = new Set([ + 'show', + 'row', + 'dissect', + 'enrich', + 'eval', + 'grok', + 'keep', + 'mv_expand', + 'rename', + 'stats', + 'limit', +]); +export const UNSUPPORTED_COMMANDS_BEFORE_MATCH = new Set(['limit']); diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.functions.full_text.test.ts b/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.functions.full_text.test.ts new file mode 100644 index 0000000000000..b7d962ffb4a42 --- /dev/null +++ b/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.functions.full_text.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { setup } from './helpers'; + +describe('validation', () => { + describe('MATCH function', () => { + it('no error if valid', async () => { + const { expectErrors } = await setup(); + await expectErrors('FROM index | WHERE MATCH(keywordField, "value") | LIMIT 10 ', []); + await expectErrors( + 'FROM index | EVAL a=CONCAT(keywordField, "_") | WHERE MATCH(a, "value") | LIMIT 10 ', + [] + ); + }); + + it('shows errors if after incompatible commands ', async () => { + const { expectErrors } = await setup(); + await expectErrors('FROM index | LIMIT 10 | WHERE MATCH(keywordField, "value")', [ + '[MATCH] function cannot be used after LIMIT', + ]); + + await expectErrors(`FROM index | EVAL MATCH(a, "value")`, [ + 'EVAL does not support function match', + '[MATCH] function is only supported in WHERE commands', + ]); + }); + + it('shows errors if argument is not an index field ', async () => { + const { expectErrors } = await setup(); + await expectErrors( + 'FROM index | LIMIT 10 | where MATCH(`kubernetes.something.something`, "value")', + [ + 'Argument of [match] must be [keyword], found value [kubernetes.something.something] type [double]', + '[MATCH] function cannot be used after LIMIT', + ] + ); + }); + }); + describe('QSRT function', () => { + it('no error if valid', async () => { + const { expectErrors } = await setup(); + await expectErrors('FROM index | WHERE QSTR("keywordField:value") | LIMIT 10 ', []); + }); + + it('shows errors if comes after incompatible functions or commands ', async () => { + const { expectErrors } = await setup(); + await expectErrors('ROW a = 1, b = "two", c = null | WHERE QSTR("keywordField:value")', [ + '[QSTR] function cannot be used after ROW', + ]); + for (const clause of [ + { command: 'LIMIT', clause: 'LIMIT 10' }, + { command: 'EVAL', clause: 'EVAL a=CONCAT(keywordField, "_")' }, + { command: 'KEEP', clause: 'KEEP keywordField' }, + { command: 'RENAME', clause: 'RENAME keywordField as a' }, + { command: 'STATS', clause: 'STATS avg(doubleField) by keywordField' }, + ]) { + await expectErrors(`FROM index | ${clause.clause} | WHERE QSTR("keywordField:value")`, [ + `[QSTR] function cannot be used after ${clause.command}`, + ]); + } + await expectErrors(`FROM index | EVAL QSTR("keywordField:value")`, [ + `EVAL does not support function qstr`, + '[QSTR] function cannot be used after EVAL', + ]); + + await expectErrors(`FROM index | STATS avg(doubleField) by QSTR("keywordField:value")`, [ + `STATS BY does not support function qstr`, + ]); + }); + }); +}); diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/errors.ts b/packages/kbn-esql-validation-autocomplete/src/validation/errors.ts index 0f82d7fe4aad9..abf4db6e7fe69 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/errors.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/errors.ts @@ -190,6 +190,21 @@ function getMessageAndTypeFromId({ } ), }; + case 'fnUnsupportedAfterCommand': + return { + type: 'error', + message: i18n.translate( + 'kbn-esql-validation-autocomplete.esql.validation.fnUnsupportedAfterCommand', + { + defaultMessage: '[{function}] function cannot be used after {command}', + values: { + function: out.function, + command: out.command, + }, + } + ), + }; + case 'unknownInterval': return { message: i18n.translate( @@ -418,6 +433,16 @@ function getMessageAndTypeFromId({ } ), }; + case 'onlyWhereCommandSupported': + return { + message: i18n.translate( + 'kbn-esql-validation-autocomplete.esql.validation.onlyWhereCommandSupported', + { + defaultMessage: '[{fn}] function is only supported in WHERE commands', + values: { fn: out.fn.toUpperCase() }, + } + ), + }; } return { message: '' }; } diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/types.ts b/packages/kbn-esql-validation-autocomplete/src/validation/types.ts index 7aac9f16ad032..2beffbfa26425 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/types.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/types.ts @@ -160,6 +160,10 @@ export interface ValidationErrors { message: string; type: { command: string; value: string; expected: string }; }; + fnUnsupportedAfterCommand: { + message: string; + type: { function: string; command: string }; + }; expectedConstant: { message: string; type: { fn: string; given: string }; @@ -196,6 +200,10 @@ export interface ValidationErrors { nestedAgg: string; }; }; + onlyWhereCommandSupported: { + message: string; + type: { fn: string }; + }; } export type ErrorTypes = keyof ValidationErrors; diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts b/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts index b43a9e5c336b5..b4d095e2c0442 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts @@ -10,6 +10,7 @@ import uniqBy from 'lodash/uniqBy'; import { AstProviderFn, + ESQLAst, ESQLAstItem, ESQLAstMetricsCommand, ESQLColumn, @@ -79,9 +80,14 @@ import { } from './resources'; import { collapseWrongArgumentTypeMessages, getMaxMinNumberOfParams } from './helpers'; import { getParamAtPosition } from '../shared/helpers'; -import { METADATA_FIELDS } from '../shared/constants'; +import { + METADATA_FIELDS, + UNSUPPORTED_COMMANDS_BEFORE_MATCH, + UNSUPPORTED_COMMANDS_BEFORE_QSTR, +} from '../shared/constants'; import { compareTypesWithLiterals } from '../shared/esql_types'; +const NO_MESSAGE: ESQLMessage[] = []; function validateFunctionLiteralArg( astFunction: ESQLFunction, actualArg: ESQLAstItem, @@ -320,27 +326,146 @@ function removeInlineCasts(arg: ESQLAstItem): ESQLAstItem { return arg; } -function validateFunction( +function validateIfHasUnsupportedCommandPrior( fn: ESQLFunction, - parentCommand: string, - parentOption: string | undefined, - references: ReferenceMaps, - forceConstantOnly: boolean = false, - isNested?: boolean -): ESQLMessage[] { + parentAst: ESQLCommand[] = [], + unsupportedCommands: Set, + currentCommandIndex?: number +) { + if (currentCommandIndex === undefined) { + return NO_MESSAGE; + } + const unsupportedCommandsPrior = parentAst.filter( + (cmd, idx) => idx <= currentCommandIndex && unsupportedCommands.has(cmd.name) + ); + + if (unsupportedCommandsPrior.length > 0) { + return [ + getMessageFromId({ + messageId: 'fnUnsupportedAfterCommand', + values: { + function: fn.name.toUpperCase(), + command: unsupportedCommandsPrior[0].name.toUpperCase(), + }, + locations: fn.location, + }), + ]; + } + return NO_MESSAGE; +} + +const validateMatchFunction: FunctionValidator = ({ + fn, + parentCommand, + parentOption, + references, + forceConstantOnly = false, + isNested, + parentAst, + currentCommandIndex, +}) => { + if (fn.name === 'match') { + if (parentCommand !== 'where') { + return [ + getMessageFromId({ + messageId: 'onlyWhereCommandSupported', + values: { fn: fn.name }, + locations: fn.location, + }), + ]; + } + return validateIfHasUnsupportedCommandPrior( + fn, + parentAst, + UNSUPPORTED_COMMANDS_BEFORE_MATCH, + currentCommandIndex + ); + } + return NO_MESSAGE; +}; + +type FunctionValidator = (args: { + fn: ESQLFunction; + parentCommand: string; + parentOption?: string; + references: ReferenceMaps; + forceConstantOnly?: boolean; + isNested?: boolean; + parentAst?: ESQLCommand[]; + currentCommandIndex?: number; +}) => ESQLMessage[]; + +const validateQSTRFunction: FunctionValidator = ({ + fn, + parentCommand, + parentOption, + references, + forceConstantOnly = false, + isNested, + parentAst, + currentCommandIndex, +}) => { + if (fn.name === 'qstr') { + return validateIfHasUnsupportedCommandPrior( + fn, + parentAst, + UNSUPPORTED_COMMANDS_BEFORE_QSTR, + currentCommandIndex + ); + } + return NO_MESSAGE; +}; + +const textSearchFunctionsValidators: Record = { + match: validateMatchFunction, + qstr: validateQSTRFunction, +}; + +function validateFunction({ + fn, + parentCommand, + parentOption, + references, + forceConstantOnly = false, + isNested, + parentAst, + currentCommandIndex, +}: { + fn: ESQLFunction; + parentCommand: string; + parentOption?: string; + references: ReferenceMaps; + forceConstantOnly?: boolean; + isNested?: boolean; + parentAst?: ESQLCommand[]; + currentCommandIndex?: number; +}): ESQLMessage[] { const messages: ESQLMessage[] = []; if (fn.incomplete) { return messages; } - if (isFunctionOperatorParam(fn)) { return messages; } - const fnDefinition = getFunctionDefinition(fn.name)!; + const isFnSupported = isSupportedFunction(fn.name, parentCommand, parentOption); + if (typeof textSearchFunctionsValidators[fn.name] === 'function') { + const validator = textSearchFunctionsValidators[fn.name]; + messages.push( + ...validator({ + fn, + parentCommand, + parentOption, + references, + isNested, + parentAst, + currentCommandIndex, + }) + ); + } if (!isFnSupported.supported) { if (isFnSupported.reason === 'unknownFunction') { messages.push(errors.unknownFunction(fn)); @@ -430,8 +555,8 @@ function validateFunction( const subArg = removeInlineCasts(_subArg); if (isFunctionItem(subArg)) { - const messagesFromArg = validateFunction( - subArg, + const messagesFromArg = validateFunction({ + fn: subArg, parentCommand, parentOption, references, @@ -450,13 +575,14 @@ function validateFunction( * Because of this, the abs function's arguments inherit the constraint * and each should be validated as if each were constantOnly. */ - allMatchingArgDefinitionsAreConstantOnly || forceConstantOnly, + forceConstantOnly: allMatchingArgDefinitionsAreConstantOnly || forceConstantOnly, // use the nesting flag for now just for stats and metrics // TODO: revisit this part later on to make it more generic - ['stats', 'inlinestats', 'metrics'].includes(parentCommand) + isNested: ['stats', 'inlinestats', 'metrics'].includes(parentCommand) ? isNested || !isAssignment(fn) - : false - ); + : false, + parentAst, + }); if (messagesFromArg.some(({ code }) => code === 'expectedConstant')) { const consolidatedMessage = getMessageFromId({ @@ -668,7 +794,14 @@ const validateAggregates = ( for (const aggregate of aggregates) { if (isFunctionItem(aggregate)) { - messages.push(...validateFunction(aggregate, command.name, undefined, references)); + messages.push( + ...validateFunction({ + fn: aggregate, + parentCommand: command.name, + parentOption: undefined, + references, + }) + ); let hasAggregationFunction = false; @@ -742,7 +875,14 @@ const validateByGrouping = ( messages.push(...validateColumnForCommand(field, commandName, referenceMaps)); } if (isFunctionItem(field)) { - messages.push(...validateFunction(field, commandName, 'by', referenceMaps)); + messages.push( + ...validateFunction({ + fn: field, + parentCommand: commandName, + parentOption: 'by', + references: referenceMaps, + }) + ); } } } @@ -788,7 +928,14 @@ function validateOption( messages.push(...validateColumnForCommand(arg, command.name, referenceMaps)); } if (isFunctionItem(arg)) { - messages.push(...validateFunction(arg, command.name, option.name, referenceMaps)); + messages.push( + ...validateFunction({ + fn: arg, + parentCommand: command.name, + parentOption: option.name, + references: referenceMaps, + }) + ); } } } @@ -957,7 +1104,12 @@ const validateMetricsCommand = ( return messages; }; -function validateCommand(command: ESQLCommand, references: ReferenceMaps): ESQLMessage[] { +function validateCommand( + command: ESQLCommand, + references: ReferenceMaps, + ast: ESQLAst, + currentCommandIndex: number +): ESQLMessage[] { const messages: ESQLMessage[] = []; if (command.incomplete) { return messages; @@ -981,7 +1133,16 @@ function validateCommand(command: ESQLCommand, references: ReferenceMaps): ESQLM const wrappedArg = Array.isArray(commandArg) ? commandArg : [commandArg]; for (const arg of wrappedArg) { if (isFunctionItem(arg)) { - messages.push(...validateFunction(arg, command.name, undefined, references)); + messages.push( + ...validateFunction({ + fn: arg, + parentCommand: command.name, + parentOption: undefined, + references, + parentAst: ast, + currentCommandIndex, + }) + ); } if (isSettingItem(arg)) { @@ -1058,6 +1219,7 @@ function validateFieldsShadowing( } } } + return messages; } @@ -1153,6 +1315,7 @@ async function validateAst( const messages: ESQLMessage[] = []; const parsingResult = await astProvider(queryString); + const { ast } = parsingResult; const [sources, availableFields, availablePolicies] = await Promise.all([ @@ -1189,7 +1352,7 @@ async function validateAst( messages.push(...validateFieldsShadowing(availableFields, variables)); messages.push(...validateUnsupportedTypeFields(availableFields)); - for (const command of ast) { + for (const [index, command] of ast.entries()) { const references: ReferenceMaps = { sources, fields: availableFields, @@ -1197,7 +1360,7 @@ async function validateAst( variables, query: queryString, }; - const commandMessages = validateCommand(command, references); + const commandMessages = validateCommand(command, references, ast, index); messages.push(...commandMessages); }