[8.x] [ES|QL] Don't suggest unsupported fields (#200544) (#200598)

# Backport This will backport the following commits from `main` to `8.x`: - [[ES|QL] Don't suggest unsupported fields (#200544)](#200544)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Stratoula Kalafateli <[email protected]>
elastic · Nov 18, 2024 · dd67c8c · dd67c8c
1 parent 47d9e14
commit dd67c8c
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 21 deletions.
diff --git a/packages/kbn-esql-validation-autocomplete/src/__tests__/helpers.ts b/packages/kbn-esql-validation-autocomplete/src/__tests__/helpers.ts
@@ -12,9 +12,7 @@ import { ESQLRealField } from '../validation/types';
 import { fieldTypes } from '../definitions/types';
 
 export const fields: ESQLRealField[] = [
-  ...fieldTypes
-    .map((type) => ({ name: `${camelCase(type)}Field`, type }))
-    .filter((f) => f.type !== 'unsupported'),
+  ...fieldTypes.map((type) => ({ name: `${camelCase(type)}Field`, type })),
   { name: 'any#Char$Field', type: 'double' },
   { name: 'kubernetes.something.something', type: 'double' },
   { name: '@timestamp', type: 'date' },

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts
@@ -11,6 +11,7 @@ import { camelCase } from 'lodash';
 import { parse } from '@kbn/esql-ast';
 import { scalarFunctionDefinitions } from '../../definitions/generated/scalar_functions';
 import { builtinFunctions } from '../../definitions/builtin';
+import { NOT_SUGGESTED_TYPES } from '../../shared/resources_helpers';
 import { aggregationFunctionDefinitions } from '../../definitions/generated/aggregation_functions';
 import { timeUnitsToSuggest } from '../../definitions/literals';
 import { groupingFunctionDefinitions } from '../../definitions/grouping';
@@ -229,7 +230,11 @@ export function getFieldNamesByType(
 ) {
   const requestedType = Array.isArray(_requestedType) ? _requestedType : [_requestedType];
   return fields
-    .filter(({ type }) => requestedType.includes('any') || requestedType.includes(type))
+    .filter(
+      ({ type }) =>
+        (requestedType.includes('any') || requestedType.includes(type)) &&
+        !NOT_SUGGESTED_TYPES.includes(type)
+    )
     .map(({ name, suggestedAs }) => suggestedAs || name);
 }
 
@@ -267,7 +272,9 @@ export function createCustomCallbackMocks(
     enrichFields: string[];
   }>
 ) {
-  const finalColumnsSinceLastCommand = customColumnsSinceLastCommand || fields;
+  const finalColumnsSinceLastCommand =
+    customColumnsSinceLastCommand ||
+    fields.filter(({ type }) => !NOT_SUGGESTED_TYPES.includes(type));
   const finalSources = customSources || indexes;
   const finalPolicies = customPolicies || policies;
   return {

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
@@ -887,6 +887,7 @@ describe('autocomplete', () => {
         'FROM a | ENRICH policy /',
         ['ON $0', 'WITH $0', '| '].map(attachTriggerCommand)
       );
+
       testSuggestions(
         'FROM a | ENRICH policy ON /',
         getFieldNamesByType('any')

diff --git a/packages/kbn-esql-validation-autocomplete/src/shared/resources_helpers.ts b/packages/kbn-esql-validation-autocomplete/src/shared/resources_helpers.ts
@@ -12,6 +12,8 @@ import type { ESQLCallbacks } from './types';
 import type { ESQLRealField } from '../validation/types';
 import { enrichFieldsWithECSInfo } from '../autocomplete/utils/ecs_metadata_helper';
 
+export const NOT_SUGGESTED_TYPES = ['unsupported'];
+
 export function buildQueryUntilPreviousCommand(ast: ESQLAst, queryString: string) {
   const prevCommand = ast[Math.max(ast.length - 2, 0)];
   return prevCommand ? queryString.substring(0, prevCommand.location.max + 1) : queryString;
@@ -54,7 +56,11 @@ export function getFieldsByTypeHelper(queryText: string, resourceRetriever?: ESQ
       return (
         Array.from(cacheFields.values())?.filter(({ name, type }) => {
           const ts = Array.isArray(type) ? type : [type];
-          return !ignored.includes(name) && ts.some((t) => types[0] === 'any' || types.includes(t));
+          return (
+            !ignored.includes(name) &&
+            ts.some((t) => types[0] === 'any' || types.includes(t)) &&
+            !NOT_SUGGESTED_TYPES.includes(type)
+          );
         }) || []
       );
     },

diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json b/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json
@@ -76,6 +76,10 @@
       "name": "counterDoubleField",
       "type": "counter_double"
     },
+    {
+      "name": "unsupportedField",
+      "type": "unsupported"
+    },
     {
       "name": "dateNanosField",
       "type": "date_nanos"
@@ -9690,6 +9694,13 @@
       ],
       "warning": []
     },
+    {
+      "query": "from a_index | keep unsupportedField",
+      "error": [],
+      "warning": [
+        "Field [unsupportedField] cannot be retrieved, it is unsupported or not indexed; returning null"
+      ]
+    },
     {
       "query": "f",
       "error": [

diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts b/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts
@@ -1695,6 +1695,16 @@ describe('validation logic', () => {
         ['Argument of [trim] must be [keyword], found value [doubleField] type [double]']
       );
     });
+
+    describe('unsupported fields', () => {
+      testErrorsAndWarnings(
+        `from a_index | keep unsupportedField`,
+        [],
+        [
+          'Field [unsupportedField] cannot be retrieved, it is unsupported or not indexed; returning null',
+        ]
+      );
+    });
   });
 
   describe('Ignoring errors based on callbacks', () => {

diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts b/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts
@@ -1223,21 +1223,24 @@ function validateFieldsShadowing(
   return messages;
 }
 
-function validateUnsupportedTypeFields(fields: Map<string, ESQLRealField>) {
+function validateUnsupportedTypeFields(fields: Map<string, ESQLRealField>, ast: ESQLAst) {
+  const usedColumnsInQuery: string[] = [];
+
+  walk(ast, {
+    visitColumn: (node) => usedColumnsInQuery.push(node.name),
+  });
   const messages: ESQLMessage[] = [];
-  for (const field of fields.values()) {
-    if (field.type === 'unsupported') {
-      // Removed temporarily to supress all these warnings
-      // Issue to re-enable in a better way: https://github.com/elastic/kibana/issues/189666
-      // messages.push(
-      //   getMessageFromId({
-      //     messageId: 'unsupportedFieldType',
-      //     values: {
-      //       field: field.name,
-      //     },
-      //     locations: { min: 1, max: 1 },
-      //   })
-      // );
+  for (const column of usedColumnsInQuery) {
+    if (fields.has(column) && fields.get(column)!.type === 'unsupported') {
+      messages.push(
+        getMessageFromId({
+          messageId: 'unsupportedFieldType',
+          values: {
+            field: column,
+          },
+          locations: { min: 1, max: 1 },
+        })
+      );
     }
   }
   return messages;
@@ -1350,7 +1353,7 @@ async function validateAst(
   const variables = collectVariables(ast, availableFields, queryString);
   // notify if the user is rewriting a column as variable with another type
   messages.push(...validateFieldsShadowing(availableFields, variables));
-  messages.push(...validateUnsupportedTypeFields(availableFields));
+  messages.push(...validateUnsupportedTypeFields(availableFields, ast));
 
   for (const [index, command] of ast.entries()) {
     const references: ReferenceMaps = {