[8.x] [ES|QL] handle snapshot-only functions (#195691) (#195937)

# Backport This will backport the following commits from `main` to `8.x`: - [[ES|QL] handle snapshot-only functions (#195691)](#195691)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Drew Tate <[email protected]>
elastic · Oct 11, 2024 · f029ee9 · f029ee9
1 parent 7b9ace8
commit f029ee9
Show file tree

Hide file tree

Showing 8 changed files with 164 additions and 86 deletions.
diff --git a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts
@@ -254,6 +254,7 @@ function getFunctionDefinition(ESFunctionDefinition: Record<string, any>): Funct
       : aggregationSupportedCommandsAndOptions),
     description: ESFunctionDefinition.description,
     alias: aliasTable[ESFunctionDefinition.name],
+    ignoreAsSuggestion: ESFunctionDefinition.snapshot_only,
     signatures: _.uniqBy(
       ESFunctionDefinition.signatures.map((signature: any) => ({
         ...signature,
@@ -332,7 +333,7 @@ function printGeneratedFunctionsFile(
     name: '${name}',
     description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.${name}', { defaultMessage: ${JSON.stringify(
       removeAsciiDocInternalCrossReferences(removeInlineAsciiDocLinks(description), functionNames)
-    )} }),
+    )} }),${functionDefinition.ignoreAsSuggestion ? 'ignoreAsSuggestion: true,\n' : ''}
     alias: ${alias ? `['${alias.join("', '")}']` : 'undefined'},
     signatures: ${JSON.stringify(signatures, null, 2)},
     supportedCommands: ${JSON.stringify(functionDefinition.supportedCommands)},

diff --git a/...-validation-autocomplete/src/autocomplete/__tests__/hidden_functions_and_commands.test.ts b/...-validation-autocomplete/src/autocomplete/__tests__/hidden_functions_and_commands.test.ts
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { setTestFunctions } from '../../shared/test_functions';
+import { setup } from './helpers';
+
+describe('hidden commands', () => {
+  it('does not suggest hidden commands', async () => {
+    const { suggest } = await setup();
+    const suggestedCommands = (await suggest('FROM index | /')).map((s) => s.text);
+    expect(suggestedCommands).not.toContain('HIDDEN_COMMAND $0');
+    expect(suggestedCommands).toContain('EVAL $0');
+    expect(suggestedCommands.every((s) => !s.toLowerCase().includes('HIDDEN_COMMAND'))).toBe(true);
+  });
+});
+
+describe('hidden functions', () => {
+  afterEach(() => {
+    setTestFunctions([]);
+  });
+
+  it('does not suggest hidden scalar functions', async () => {
+    setTestFunctions([
+      {
+        type: 'eval',
+        name: 'HIDDEN_FUNCTION',
+        description: 'This is a hidden function',
+        signatures: [{ params: [], returnType: 'text' }],
+        supportedCommands: ['eval'],
+        ignoreAsSuggestion: true,
+      },
+      {
+        type: 'eval',
+        name: 'VISIBLE_FUNCTION',
+        description: 'This is a visible function',
+        signatures: [{ params: [], returnType: 'text' }],
+        supportedCommands: ['eval'],
+        ignoreAsSuggestion: false,
+      },
+    ]);
+
+    const { suggest } = await setup();
+    const suggestedFunctions = (await suggest('FROM index | EVAL /')).map((s) => s.text);
+    expect(suggestedFunctions).toContain('VISIBLE_FUNCTION($0)');
+    expect(suggestedFunctions).not.toContain('HIDDEN_FUNCTION($0)');
+  });
+  it('does not suggest hidden agg functions', async () => {
+    setTestFunctions([
+      {
+        type: 'agg',
+        name: 'HIDDEN_FUNCTION',
+        description: 'This is a hidden function',
+        signatures: [{ params: [], returnType: 'text' }],
+        supportedCommands: ['stats'],
+        ignoreAsSuggestion: true,
+      },
+      {
+        type: 'agg',
+        name: 'VISIBLE_FUNCTION',
+        description: 'This is a visible function',
+        signatures: [{ params: [], returnType: 'text' }],
+        supportedCommands: ['stats'],
+        ignoreAsSuggestion: false,
+      },
+    ]);
+
+    const { suggest } = await setup();
+    const suggestedFunctions = (await suggest('FROM index | STATS /')).map((s) => s.text);
+    expect(suggestedFunctions).toContain('VISIBLE_FUNCTION($0)');
+    expect(suggestedFunctions).not.toContain('HIDDEN_FUNCTION($0)');
+  });
+
+  it('does not suggest hidden operators', async () => {
+    setTestFunctions([
+      {
+        type: 'builtin',
+        name: 'HIDDEN_OPERATOR',
+        description: 'This is a hidden function',
+        supportedCommands: ['eval', 'where', 'row', 'sort'],
+        ignoreAsSuggestion: true,
+        supportedOptions: ['by'],
+        signatures: [
+          {
+            params: [
+              { name: 'left', type: 'keyword' as const },
+              { name: 'right', type: 'keyword' as const },
+            ],
+            returnType: 'boolean',
+          },
+        ],
+      },
+      {
+        type: 'builtin',
+        name: 'VISIBLE_OPERATOR',
+        description: 'This is a visible function',
+        supportedCommands: ['eval', 'where', 'row', 'sort'],
+        ignoreAsSuggestion: false,
+        supportedOptions: ['by'],
+        signatures: [
+          {
+            params: [
+              { name: 'left', type: 'keyword' as const },
+              { name: 'right', type: 'keyword' as const },
+            ],
+            returnType: 'boolean',
+          },
+        ],
+      },
+    ]);
+
+    const { suggest } = await setup();
+    const suggestedFunctions = (await suggest('FROM index | EVAL keywordField /')).map(
+      (s) => s.text
+    );
+    expect(suggestedFunctions).toContain('VISIBLE_OPERATOR $0');
+    expect(suggestedFunctions).not.toContain('HIDDEN_OPERATOR $0');
+  });
+});
diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts
@@ -18,6 +18,7 @@ import {
   buildConstantsDefinitions,
 } from './factories';
 import { FunctionParameterType, FunctionReturnType } from '../definitions/types';
+import { getTestFunctions } from '../shared/test_functions';
 
 export function getAssignmentDefinitionCompletitionItem() {
   const assignFn = builtinFunctions.find(({ name }) => name === '=')!;
@@ -60,7 +61,7 @@ export const getBuiltinCompatibleFunctionDefinition = (
   returnTypes?: FunctionReturnType[],
   { skipAssign }: { skipAssign?: boolean } = {}
 ): SuggestionRawDefinition[] => {
-  const compatibleFunctions = builtinFunctions.filter(
+  const compatibleFunctions = [...builtinFunctions, ...getTestFunctions()].filter(
     ({ name, supportedCommands, supportedOptions, signatures, ignoreAsSuggestion }) =>
       !ignoreAsSuggestion &&
       !/not_/.test(name) &&

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts
@@ -8,6 +8,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { memoize } from 'lodash';
 import { SuggestionRawDefinition } from './types';
 import { groupingFunctionDefinitions } from '../definitions/grouping';
 import { aggregationFunctionDefinitions } from '../definitions/generated/aggregation_functions';
@@ -25,10 +26,16 @@ import { buildDocumentation, buildFunctionDocumentation } from './documentation_
 import { DOUBLE_BACKTICK, SINGLE_TICK_REGEX } from '../shared/constants';
 import { ESQLRealField } from '../validation/types';
 import { isNumericType } from '../shared/esql_types';
+import { getTestFunctions } from '../shared/test_functions';
 
-const allFunctions = aggregationFunctionDefinitions
-  .concat(scalarFunctionDefinitions)
-  .concat(groupingFunctionDefinitions);
+const allFunctions = memoize(
+  () =>
+    aggregationFunctionDefinitions
+      .concat(scalarFunctionDefinitions)
+      .concat(groupingFunctionDefinitions)
+      .concat(getTestFunctions()),
+  () => getTestFunctions()
+);
 
 export const TIME_SYSTEM_PARAMS = ['?_tstart', '?_tend'];
 
@@ -94,11 +101,12 @@ export const getCompatibleFunctionDefinition = (
   returnTypes?: string[],
   ignored: string[] = []
 ): SuggestionRawDefinition[] => {
-  const fnSupportedByCommand = allFunctions
+  const fnSupportedByCommand = allFunctions()
     .filter(
-      ({ name, supportedCommands, supportedOptions }) =>
+      ({ name, supportedCommands, supportedOptions, ignoreAsSuggestion }) =>
         (option ? supportedOptions?.includes(option) : supportedCommands.includes(command)) &&
-        !ignored.includes(name)
+        !ignored.includes(name) &&
+        !ignoreAsSuggestion
     )
     .sort((a, b) => a.name.localeCompare(b.name));
   if (!returnTypes) {

diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/commands.ts
@@ -466,4 +466,16 @@ export const commandDefinitions: CommandDefinition[] = [
       params: [{ name: 'policyName', type: 'source', innerTypes: ['policy'] }],
     },
   },
+  {
+    name: 'hidden_command',
+    description: 'A test fixture to test hidden-ness',
+    hidden: true,
+    examples: [],
+    modes: [],
+    options: [],
+    signature: {
+      params: [],
+      multipleParams: false,
+    },
+  },
 ];
diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts
@@ -524,6 +524,8 @@ const categorizeDefinition: FunctionDefinition = {
   description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.categorize', {
     defaultMessage: 'Categorizes text messages.',
   }),
+  ignoreAsSuggestion: true,
+
   alias: undefined,
   signatures: [
     {
@@ -3607,7 +3609,7 @@ const mvFirstDefinition: FunctionDefinition = {
   name: 'mv_first',
   description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mv_first', {
     defaultMessage:
-      "Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like `SPLIT`.\n\nThe order that  multivalued fields are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the minimum value use `MV_MIN` instead of\n`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_FIRST`.",
+      'Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like `SPLIT`.',
   }),
   alias: undefined,
   signatures: [
@@ -3764,7 +3766,7 @@ const mvLastDefinition: FunctionDefinition = {
   name: 'mv_last',
   description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mv_last', {
     defaultMessage:
-      "Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like `SPLIT`.\n\nThe order that  multivalued fields are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the maximum value use `MV_MAX` instead of\n`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_LAST`.",
+      'Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like `SPLIT`.',
   }),
   alias: undefined,
   signatures: [
@@ -4474,7 +4476,7 @@ const mvSliceDefinition: FunctionDefinition = {
   name: 'mv_slice',
   description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mv_slice', {
     defaultMessage:
-      'Returns a subset of the multivalued field using the start and end index values.',
+      'Returns a subset of the multivalued field using the start and end index values.\nThis is most useful when reading from a function that emits multivalued columns\nin a known order like `SPLIT` or `MV_SORT`.',
   }),
   alias: undefined,
   signatures: [
@@ -5511,6 +5513,8 @@ const qstrDefinition: FunctionDefinition = {
     defaultMessage:
       'Performs a query string query. Returns true if the provided query string matches the row.',
   }),
+  ignoreAsSuggestion: true,
+
   alias: undefined,
   signatures: [
     {

diff --git a/packages/kbn-language-documentation/scripts/generate_esql_docs.ts b/packages/kbn-language-documentation/scripts/generate_esql_docs.ts
@@ -52,7 +52,7 @@ function loadFunctionDocs(pathToElasticsearch: string) {
         (def) => def.name === path.basename(file, '.md')
       );
 
-      if (!functionDefinition) {
+      if (!functionDefinition || functionDefinition.snapshot_only) {
         continue;
       }
 

diff --git a/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx b/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx
@@ -256,35 +256,6 @@ export const functions = {
       ),
     },
     // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts
-    {
-      label: i18n.translate('languageDocumentation.documentationESQL.categorize', {
-        defaultMessage: 'CATEGORIZE',
-      }),
-      description: (
-        <Markdown
-          openLinksInNewTab
-          readOnly
-          enableSoftLineBreaks
-          markdownContent={i18n.translate(
-            'languageDocumentation.documentationESQL.categorize.markdown',
-            {
-              defaultMessage: `<!--
-  This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
-  -->
-
-  ### CATEGORIZE
-  Categorizes text messages.
-
-  `,
-              description:
-                'Text is in markdown. Do not translate function names, special characters, or field names like sum(bytes)',
-              ignoreTag: true,
-            }
-          )}
-        />
-      ),
-    },
-    // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts
     {
       label: i18n.translate('languageDocumentation.documentationESQL.cbrt', {
         defaultMessage: 'CBRT',
@@ -1327,12 +1298,6 @@ export const functions = {
   first value. This is most useful when reading from a function that emits
   multivalued columns in a known order like \`SPLIT\`.
 
-  The order that  multivalued fields are read from
-  underlying storage is not guaranteed. It is *frequently* ascending, but don't
-  rely on that. If you need the minimum value use \`MV_MIN\` instead of
-  \`MV_FIRST\`. \`MV_MIN\` has optimizations for sorted values so there isn't a
-  performance benefit to \`MV_FIRST\`.
-
   \`\`\`
   ROW a="foo;bar;baz"
   | EVAL first_a = MV_FIRST(SPLIT(a, ";"))
@@ -1368,12 +1333,6 @@ export const functions = {
   value. This is most useful when reading from a function that emits multivalued
   columns in a known order like \`SPLIT\`.
 
-  The order that  multivalued fields are read from
-  underlying storage is not guaranteed. It is *frequently* ascending, but don't
-  rely on that. If you need the maximum value use \`MV_MAX\` instead of
-  \`MV_LAST\`. \`MV_MAX\` has optimizations for sorted values so there isn't a
-  performance benefit to \`MV_LAST\`.
-
   \`\`\`
   ROW a="foo;bar;baz"
   | EVAL last_a = MV_LAST(SPLIT(a, ";"))
@@ -1611,6 +1570,8 @@ export const functions = {
 
   ### MV_SLICE
   Returns a subset of the multivalued field using the start and end index values.
+  This is most useful when reading from a function that emits multivalued columns
+  in a known order like \`SPLIT\` or \`MV_SORT\`.
 
   \`\`\`
   row a = [1, 2, 2, 3]
@@ -1806,39 +1767,6 @@ export const functions = {
   | EVAL result = POW(base, exponent)
   \`\`\`
   Note: It is still possible to overflow a double result here; in that case, null will be returned.
-  `,
-            description:
-              'Text is in markdown. Do not translate function names, special characters, or field names like sum(bytes)',
-            ignoreTag: true,
-          })}
-        />
-      ),
-    },
-    // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts
-    {
-      label: i18n.translate('languageDocumentation.documentationESQL.qstr', {
-        defaultMessage: 'QSTR',
-      }),
-      description: (
-        <Markdown
-          openLinksInNewTab
-          readOnly
-          enableSoftLineBreaks
-          markdownContent={i18n.translate('languageDocumentation.documentationESQL.qstr.markdown', {
-            defaultMessage: `<!--
-  This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
-  -->
-
-  ### QSTR
-  Performs a query string query. Returns true if the provided query string matches the row.
-
-  \`\`\`
-  from books 
-  | where qstr("author: Faulkner")
-  | keep book_no, author 
-  | sort book_no 
-  | limit 5;
-  \`\`\`
   `,
             description:
               'Text is in markdown. Do not translate function names, special characters, or field names like sum(bytes)',