[ES|QL] open suggestions automatically in sources lists and ENRICH (e…

…lastic#191312)

## Summary

Part of elastic#189662

Also, a follow-on to elastic#187184
with certain source names (e.g. `foo$bar`, `my-policy`) and fields in
the `ENRICH` command.

During this effort I discovered
elastic#191321 and a bug with the
validation of field names in the "ENRICH ... WITH" list (documented
[here](elastic#177699)), both of which
will be addressed separately.

## Sources

### General flow

https://github.com/user-attachments/assets/4b103621-0e66-4c36-807f-4932f0cb8faf

### Works with wild-card matches

https://github.com/user-attachments/assets/6b47fffc-e922-4e2d-b6aa-3d9a2fc2236c

### `METADATA` field list

https://github.com/user-attachments/assets/d3bdf4dc-1d0c-4d56-81d7-af6bc4e25a4a

## ENRICH

Autosuggest now helps you along

https://github.com/user-attachments/assets/d627484c-e729-4dc7-9e7b-795395a31d4f

Also, fixed this bug (follow on to
elastic#187184 )

https://github.com/user-attachments/assets/aa62a0c3-6db5-434a-829a-59f14c5c4c85

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Stratoula Kalafateli <[email protected]>
(cherry picked from commit e404a39)

packages/kbn-esql-ast/src/ast_helpers.ts

@@ -405,9 +405,9 @@ export function createSource(
     index,
     name: text,
     sourceType: type,
-    text,
     location: getPosition(ctx.start, ctx.stop),
     incomplete: Boolean(ctx.exception || text === ''),
+    text: ctx?.getText(),
   };
 }
 

packages/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -24,7 +24,7 @@ export function getIndexPatternFromESQLQuery(esql?: string) {
   const sourceCommand = ast.find(({ name }) => ['from', 'metrics'].includes(name));
   const args = (sourceCommand?.args ?? []) as ESQLSource[];
   const indices = args.filter((arg) => arg.sourceType === 'index');
-  return indices?.map((index) => index.text).join(',');
+  return indices?.map((index) => index.name).join(',');
 }
 
 // For ES|QL we consider stats and keep transformational command

packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts

@@ -15,9 +15,6 @@ const visibleIndices = indexes
   .map(({ name, suggestedAs }) => suggestedAs || name)
   .sort();
 
-const addTrailingSpace = (strings: string[], predicate: (s: string) => boolean = (_s) => true) =>
-  strings.map((string) => (predicate(string) ? `${string} ` : string));
-
 const metadataFields = [...METADATA_FIELDS].sort();
 
 describe('autocomplete.suggest', () => {
@@ -37,17 +34,17 @@ describe('autocomplete.suggest', () => {
       test('suggests visible indices on space', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('from /', addTrailingSpace(visibleIndices));
-        await assertSuggestions('FROM /', addTrailingSpace(visibleIndices));
-        await assertSuggestions('from /index', addTrailingSpace(visibleIndices));
+        await assertSuggestions('from /', visibleIndices);
+        await assertSuggestions('FROM /', visibleIndices);
+        await assertSuggestions('from /index', visibleIndices);
       });
 
       test('suggests visible indices on comma', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('FROM a,/', addTrailingSpace(visibleIndices));
-        await assertSuggestions('FROM a, /', addTrailingSpace(visibleIndices));
-        await assertSuggestions('from *,/', addTrailingSpace(visibleIndices));
+        await assertSuggestions('FROM a,/', visibleIndices);
+        await assertSuggestions('FROM a, /', visibleIndices);
+        await assertSuggestions('from *,/', visibleIndices);
       });
 
       test('can suggest integration data sources', async () => {
@@ -56,10 +53,7 @@ describe('autocomplete.suggest', () => {
           .filter(({ hidden }) => !hidden)
           .map(({ name, suggestedAs }) => suggestedAs || name)
           .sort();
-        const expectedSuggestions = addTrailingSpace(
-          visibleDataSources,
-          (s) => !integrations.find(({ name }) => name === s)
-        );
+        const expectedSuggestions = visibleDataSources;
         const { assertSuggestions, callbacks } = await setup();
         const cb = {
           ...callbacks,
@@ -75,7 +69,7 @@ describe('autocomplete.suggest', () => {
     });
 
     describe('... METADATA <fields>', () => {
-      const metadataFieldsSandIndex = metadataFields.filter((field) => field !== '_index');
+      const metadataFieldsAndIndex = metadataFields.filter((field) => field !== '_index');
 
       test('on <kbd>SPACE</kbd> without comma ",", suggests adding metadata', async () => {
         const { assertSuggestions } = await setup();
@@ -103,8 +97,8 @@ describe('autocomplete.suggest', () => {
       test('filters out already used metadata fields', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('from a, b [metadata _index, /]', metadataFieldsSandIndex);
-        await assertSuggestions('from a, b metadata _index, /', metadataFieldsSandIndex);
+        await assertSuggestions('from a, b [metadata _index, /]', metadataFieldsAndIndex);
+        await assertSuggestions('from a, b metadata _index, /', metadataFieldsAndIndex);
       });
     });
   });

packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts

@@ -17,7 +17,7 @@ import { groupingFunctionDefinitions } from '../../definitions/grouping';
 import * as autocomplete from '../autocomplete';
 import type { ESQLCallbacks } from '../../shared/types';
 import type { EditorContext, SuggestionRawDefinition } from '../types';
-import { TIME_SYSTEM_PARAMS } from '../factories';
+import { TIME_SYSTEM_PARAMS, getSafeInsertText } from '../factories';
 import { getFunctionSignatures } from '../../definitions/helpers';
 import { ESQLRealField } from '../../validation/types';
 import {
@@ -280,10 +280,7 @@ export function createCompletionContext(triggerCharacter?: string) {
 export function getPolicyFields(policyName: string) {
   return policies
     .filter(({ name }) => name === policyName)
-    .flatMap(({ enrichFields }) =>
-      // ok, this is a bit of cheating as it's using the same logic as in the helper
-      enrichFields.map((field) => (/[^a-zA-Z\d_\.@]/.test(field) ? `\`${field}\`` : field))
-    );
+    .flatMap(({ enrichFields }) => enrichFields.map((field) => getSafeInsertText(field)));
 }
 
 export interface SuggestOptions {