From f029ee9dfc51a501a0f8dbee226b82442f485580 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 12 Oct 2024 03:39:31 +1100 Subject: [PATCH] [8.x] [ES|QL] handle snapshot-only functions (#195691) (#195937) # Backport This will backport the following commits from `main` to `8.x`: - [[ES|QL] handle snapshot-only functions (#195691)](https://github.com/elastic/kibana/pull/195691) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Drew Tate --- .../scripts/generate_function_definitions.ts | 3 +- .../hidden_functions_and_commands.test.ts | 124 ++++++++++++++++++ .../src/autocomplete/complete_items.ts | 3 +- .../src/autocomplete/factories.ts | 20 ++- .../src/definitions/commands.ts | 12 ++ .../definitions/generated/scalar_functions.ts | 10 +- .../scripts/generate_esql_docs.ts | 2 +- .../sections/generated/scalar_functions.tsx | 76 +---------- 8 files changed, 164 insertions(+), 86 deletions(-) create mode 100644 packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/hidden_functions_and_commands.test.ts diff --git a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts index 144ba21f51ecb..13f0b2c66ce32 100644 --- a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts +++ b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts @@ -254,6 +254,7 @@ function getFunctionDefinition(ESFunctionDefinition: Record): Funct : aggregationSupportedCommandsAndOptions), description: ESFunctionDefinition.description, alias: aliasTable[ESFunctionDefinition.name], + ignoreAsSuggestion: ESFunctionDefinition.snapshot_only, signatures: _.uniqBy( ESFunctionDefinition.signatures.map((signature: any) => ({ ...signature, @@ -332,7 +333,7 @@ function printGeneratedFunctionsFile( name: '${name}', description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.${name}', { defaultMessage: ${JSON.stringify( removeAsciiDocInternalCrossReferences(removeInlineAsciiDocLinks(description), functionNames) - )} }), + )} }),${functionDefinition.ignoreAsSuggestion ? 'ignoreAsSuggestion: true,\n' : ''} alias: ${alias ? `['${alias.join("', '")}']` : 'undefined'}, signatures: ${JSON.stringify(signatures, null, 2)}, supportedCommands: ${JSON.stringify(functionDefinition.supportedCommands)}, diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/hidden_functions_and_commands.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/hidden_functions_and_commands.test.ts new file mode 100644 index 0000000000000..cc4562f999fe3 --- /dev/null +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/hidden_functions_and_commands.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { setTestFunctions } from '../../shared/test_functions'; +import { setup } from './helpers'; + +describe('hidden commands', () => { + it('does not suggest hidden commands', async () => { + const { suggest } = await setup(); + const suggestedCommands = (await suggest('FROM index | /')).map((s) => s.text); + expect(suggestedCommands).not.toContain('HIDDEN_COMMAND $0'); + expect(suggestedCommands).toContain('EVAL $0'); + expect(suggestedCommands.every((s) => !s.toLowerCase().includes('HIDDEN_COMMAND'))).toBe(true); + }); +}); + +describe('hidden functions', () => { + afterEach(() => { + setTestFunctions([]); + }); + + it('does not suggest hidden scalar functions', async () => { + setTestFunctions([ + { + type: 'eval', + name: 'HIDDEN_FUNCTION', + description: 'This is a hidden function', + signatures: [{ params: [], returnType: 'text' }], + supportedCommands: ['eval'], + ignoreAsSuggestion: true, + }, + { + type: 'eval', + name: 'VISIBLE_FUNCTION', + description: 'This is a visible function', + signatures: [{ params: [], returnType: 'text' }], + supportedCommands: ['eval'], + ignoreAsSuggestion: false, + }, + ]); + + const { suggest } = await setup(); + const suggestedFunctions = (await suggest('FROM index | EVAL /')).map((s) => s.text); + expect(suggestedFunctions).toContain('VISIBLE_FUNCTION($0)'); + expect(suggestedFunctions).not.toContain('HIDDEN_FUNCTION($0)'); + }); + it('does not suggest hidden agg functions', async () => { + setTestFunctions([ + { + type: 'agg', + name: 'HIDDEN_FUNCTION', + description: 'This is a hidden function', + signatures: [{ params: [], returnType: 'text' }], + supportedCommands: ['stats'], + ignoreAsSuggestion: true, + }, + { + type: 'agg', + name: 'VISIBLE_FUNCTION', + description: 'This is a visible function', + signatures: [{ params: [], returnType: 'text' }], + supportedCommands: ['stats'], + ignoreAsSuggestion: false, + }, + ]); + + const { suggest } = await setup(); + const suggestedFunctions = (await suggest('FROM index | STATS /')).map((s) => s.text); + expect(suggestedFunctions).toContain('VISIBLE_FUNCTION($0)'); + expect(suggestedFunctions).not.toContain('HIDDEN_FUNCTION($0)'); + }); + + it('does not suggest hidden operators', async () => { + setTestFunctions([ + { + type: 'builtin', + name: 'HIDDEN_OPERATOR', + description: 'This is a hidden function', + supportedCommands: ['eval', 'where', 'row', 'sort'], + ignoreAsSuggestion: true, + supportedOptions: ['by'], + signatures: [ + { + params: [ + { name: 'left', type: 'keyword' as const }, + { name: 'right', type: 'keyword' as const }, + ], + returnType: 'boolean', + }, + ], + }, + { + type: 'builtin', + name: 'VISIBLE_OPERATOR', + description: 'This is a visible function', + supportedCommands: ['eval', 'where', 'row', 'sort'], + ignoreAsSuggestion: false, + supportedOptions: ['by'], + signatures: [ + { + params: [ + { name: 'left', type: 'keyword' as const }, + { name: 'right', type: 'keyword' as const }, + ], + returnType: 'boolean', + }, + ], + }, + ]); + + const { suggest } = await setup(); + const suggestedFunctions = (await suggest('FROM index | EVAL keywordField /')).map( + (s) => s.text + ); + expect(suggestedFunctions).toContain('VISIBLE_OPERATOR $0'); + expect(suggestedFunctions).not.toContain('HIDDEN_OPERATOR $0'); + }); +}); diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts index 7a3df2d578f58..42bb02058023b 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts @@ -18,6 +18,7 @@ import { buildConstantsDefinitions, } from './factories'; import { FunctionParameterType, FunctionReturnType } from '../definitions/types'; +import { getTestFunctions } from '../shared/test_functions'; export function getAssignmentDefinitionCompletitionItem() { const assignFn = builtinFunctions.find(({ name }) => name === '=')!; @@ -60,7 +61,7 @@ export const getBuiltinCompatibleFunctionDefinition = ( returnTypes?: FunctionReturnType[], { skipAssign }: { skipAssign?: boolean } = {} ): SuggestionRawDefinition[] => { - const compatibleFunctions = builtinFunctions.filter( + const compatibleFunctions = [...builtinFunctions, ...getTestFunctions()].filter( ({ name, supportedCommands, supportedOptions, signatures, ignoreAsSuggestion }) => !ignoreAsSuggestion && !/not_/.test(name) && diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts index 72f5759d0d555..43f6f8ccff365 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts @@ -8,6 +8,7 @@ */ import { i18n } from '@kbn/i18n'; +import { memoize } from 'lodash'; import { SuggestionRawDefinition } from './types'; import { groupingFunctionDefinitions } from '../definitions/grouping'; import { aggregationFunctionDefinitions } from '../definitions/generated/aggregation_functions'; @@ -25,10 +26,16 @@ import { buildDocumentation, buildFunctionDocumentation } from './documentation_ import { DOUBLE_BACKTICK, SINGLE_TICK_REGEX } from '../shared/constants'; import { ESQLRealField } from '../validation/types'; import { isNumericType } from '../shared/esql_types'; +import { getTestFunctions } from '../shared/test_functions'; -const allFunctions = aggregationFunctionDefinitions - .concat(scalarFunctionDefinitions) - .concat(groupingFunctionDefinitions); +const allFunctions = memoize( + () => + aggregationFunctionDefinitions + .concat(scalarFunctionDefinitions) + .concat(groupingFunctionDefinitions) + .concat(getTestFunctions()), + () => getTestFunctions() +); export const TIME_SYSTEM_PARAMS = ['?_tstart', '?_tend']; @@ -94,11 +101,12 @@ export const getCompatibleFunctionDefinition = ( returnTypes?: string[], ignored: string[] = [] ): SuggestionRawDefinition[] => { - const fnSupportedByCommand = allFunctions + const fnSupportedByCommand = allFunctions() .filter( - ({ name, supportedCommands, supportedOptions }) => + ({ name, supportedCommands, supportedOptions, ignoreAsSuggestion }) => (option ? supportedOptions?.includes(option) : supportedCommands.includes(command)) && - !ignored.includes(name) + !ignored.includes(name) && + !ignoreAsSuggestion ) .sort((a, b) => a.name.localeCompare(b.name)); if (!returnTypes) { diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts index e02024968306b..54504ac1a2a18 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts @@ -466,4 +466,16 @@ export const commandDefinitions: CommandDefinition[] = [ params: [{ name: 'policyName', type: 'source', innerTypes: ['policy'] }], }, }, + { + name: 'hidden_command', + description: 'A test fixture to test hidden-ness', + hidden: true, + examples: [], + modes: [], + options: [], + signature: { + params: [], + multipleParams: false, + }, + }, ]; diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts index 9f75e8dcab429..e860ff932f3dc 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts @@ -524,6 +524,8 @@ const categorizeDefinition: FunctionDefinition = { description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.categorize', { defaultMessage: 'Categorizes text messages.', }), + ignoreAsSuggestion: true, + alias: undefined, signatures: [ { @@ -3607,7 +3609,7 @@ const mvFirstDefinition: FunctionDefinition = { name: 'mv_first', description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mv_first', { defaultMessage: - "Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like `SPLIT`.\n\nThe order that multivalued fields are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the minimum value use `MV_MIN` instead of\n`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_FIRST`.", + 'Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like `SPLIT`.', }), alias: undefined, signatures: [ @@ -3764,7 +3766,7 @@ const mvLastDefinition: FunctionDefinition = { name: 'mv_last', description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mv_last', { defaultMessage: - "Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like `SPLIT`.\n\nThe order that multivalued fields are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the maximum value use `MV_MAX` instead of\n`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_LAST`.", + 'Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like `SPLIT`.', }), alias: undefined, signatures: [ @@ -4474,7 +4476,7 @@ const mvSliceDefinition: FunctionDefinition = { name: 'mv_slice', description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mv_slice', { defaultMessage: - 'Returns a subset of the multivalued field using the start and end index values.', + 'Returns a subset of the multivalued field using the start and end index values.\nThis is most useful when reading from a function that emits multivalued columns\nin a known order like `SPLIT` or `MV_SORT`.', }), alias: undefined, signatures: [ @@ -5511,6 +5513,8 @@ const qstrDefinition: FunctionDefinition = { defaultMessage: 'Performs a query string query. Returns true if the provided query string matches the row.', }), + ignoreAsSuggestion: true, + alias: undefined, signatures: [ { diff --git a/packages/kbn-language-documentation/scripts/generate_esql_docs.ts b/packages/kbn-language-documentation/scripts/generate_esql_docs.ts index ff4baf499c8db..072d53a02753a 100644 --- a/packages/kbn-language-documentation/scripts/generate_esql_docs.ts +++ b/packages/kbn-language-documentation/scripts/generate_esql_docs.ts @@ -52,7 +52,7 @@ function loadFunctionDocs(pathToElasticsearch: string) { (def) => def.name === path.basename(file, '.md') ); - if (!functionDefinition) { + if (!functionDefinition || functionDefinition.snapshot_only) { continue; } diff --git a/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx b/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx index 8028a6e51f95c..a93d21905da0b 100644 --- a/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx +++ b/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx @@ -256,35 +256,6 @@ export const functions = { ), }, // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts - { - label: i18n.translate('languageDocumentation.documentationESQL.categorize', { - defaultMessage: 'CATEGORIZE', - }), - description: ( - - - ### CATEGORIZE - Categorizes text messages. - - `, - description: - 'Text is in markdown. Do not translate function names, special characters, or field names like sum(bytes)', - ignoreTag: true, - } - )} - /> - ), - }, - // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts { label: i18n.translate('languageDocumentation.documentationESQL.cbrt', { defaultMessage: 'CBRT', @@ -1327,12 +1298,6 @@ export const functions = { first value. This is most useful when reading from a function that emits multivalued columns in a known order like \`SPLIT\`. - The order that multivalued fields are read from - underlying storage is not guaranteed. It is *frequently* ascending, but don't - rely on that. If you need the minimum value use \`MV_MIN\` instead of - \`MV_FIRST\`. \`MV_MIN\` has optimizations for sorted values so there isn't a - performance benefit to \`MV_FIRST\`. - \`\`\` ROW a="foo;bar;baz" | EVAL first_a = MV_FIRST(SPLIT(a, ";")) @@ -1368,12 +1333,6 @@ export const functions = { value. This is most useful when reading from a function that emits multivalued columns in a known order like \`SPLIT\`. - The order that multivalued fields are read from - underlying storage is not guaranteed. It is *frequently* ascending, but don't - rely on that. If you need the maximum value use \`MV_MAX\` instead of - \`MV_LAST\`. \`MV_MAX\` has optimizations for sorted values so there isn't a - performance benefit to \`MV_LAST\`. - \`\`\` ROW a="foo;bar;baz" | EVAL last_a = MV_LAST(SPLIT(a, ";")) @@ -1611,6 +1570,8 @@ export const functions = { ### MV_SLICE Returns a subset of the multivalued field using the start and end index values. + This is most useful when reading from a function that emits multivalued columns + in a known order like \`SPLIT\` or \`MV_SORT\`. \`\`\` row a = [1, 2, 2, 3] @@ -1806,39 +1767,6 @@ export const functions = { | EVAL result = POW(base, exponent) \`\`\` Note: It is still possible to overflow a double result here; in that case, null will be returned. - `, - description: - 'Text is in markdown. Do not translate function names, special characters, or field names like sum(bytes)', - ignoreTag: true, - })} - /> - ), - }, - // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts - { - label: i18n.translate('languageDocumentation.documentationESQL.qstr', { - defaultMessage: 'QSTR', - }), - description: ( - - - ### QSTR - Performs a query string query. Returns true if the provided query string matches the row. - - \`\`\` - from books - | where qstr("author: Faulkner") - | keep book_no, author - | sort book_no - | limit 5; - \`\`\` `, description: 'Text is in markdown. Do not translate function names, special characters, or field names like sum(bytes)',