[UnifiedFieldList] Add new fields ingested in background with valid mappings

packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx

@@ -22,7 +22,7 @@ import {
   useEuiTheme,
 } from '@elastic/eui';
 import { ToolbarButton } from '@kbn/shared-ux-button-toolbar';
-import { type DataViewField } from '@kbn/data-views-plugin/public';
+import { DataViewField, FieldSpec } from '@kbn/data-views-plugin/common';
 import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils';
 import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils';
 import { FieldList } from '../../components/field_list';
@@ -191,25 +191,6 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
     onSelectedFieldFilter,
   ]);
 
-  useEffect(() => {
-    if (
-      searchMode !== 'documents' ||
-      !useNewFieldsApi ||
-      stateService.creationOptions.disableMultiFieldsGroupingByParent
-    ) {
-      setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case
-    } else {
-      setMultiFieldsMap(calculateMultiFields(allFields, selectedFieldsState.selectedFieldsMap));
-    }
-  }, [
-    stateService.creationOptions.disableMultiFieldsGroupingByParent,
-    selectedFieldsState.selectedFieldsMap,
-    allFields,
-    useNewFieldsApi,
-    setMultiFieldsMap,
-    searchMode,
-  ]);
-
   const popularFieldsLimit = useMemo(
     () => core.uiSettings.get(FIELDS_LIMIT_SETTING),
     [core.uiSettings]
@@ -226,24 +207,47 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
       [searchMode, stateService.creationOptions.disableMultiFieldsGroupingByParent]
     );
 
-  const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields<DataViewField>({
-    dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries
-    allFields,
-    popularFieldsLimit:
-      searchMode !== 'documents' || stateService.creationOptions.disablePopularFields
-        ? 0
-        : popularFieldsLimit,
-    isAffectedByGlobalFilter,
-    services: {
-      dataViews,
-      core,
-    },
-    sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields,
-    onSelectedFieldFilter,
-    onSupportedFieldFilter:
-      stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter,
-    onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails,
-  });
+  const { fieldListFiltersProps, fieldListGroupedProps, allFieldsModified } =
+    useGroupedFields<DataViewField>({
+      dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries
+      allFields,
+      popularFieldsLimit:
+        searchMode !== 'documents' || stateService.creationOptions.disablePopularFields
+          ? 0
+          : popularFieldsLimit,
+      isAffectedByGlobalFilter,
+      services: {
+        dataViews,
+        core,
+      },
+      sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields,
+      onSelectedFieldFilter,
+      onSupportedFieldFilter:
+        stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter,
+      onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails,
+      getNewFieldsBySpec,
+    });
+
+  useEffect(() => {
+    if (
+      searchMode !== 'documents' ||
+      !useNewFieldsApi ||
+      stateService.creationOptions.disableMultiFieldsGroupingByParent
+    ) {
+      setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case
+    } else {
+      setMultiFieldsMap(
+        calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap)
+      );
+    }
+  }, [
+    stateService.creationOptions.disableMultiFieldsGroupingByParent,
+    selectedFieldsState.selectedFieldsMap,
+    allFieldsModified,
+    useNewFieldsApi,
+    setMultiFieldsMap,
+    searchMode,
+  ]);
 
   const renderFieldItem: FieldListGroupedProps<DataViewField>['renderFieldItem'] = useCallback(
     ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => (
@@ -456,3 +460,7 @@ function calculateMultiFields(
   });
   return map;
 }
+
+function getNewFieldsBySpec(fieldSpecArr: FieldSpec[]): DataViewField[] {
+  return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec));
+}

packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts

@@ -12,7 +12,7 @@ import { BehaviorSubject } from 'rxjs';
 import type { CoreStart } from '@kbn/core/public';
 import type { AggregateQuery, EsQueryConfig, Filter, Query } from '@kbn/es-query';
 import { type DataPublicPluginStart } from '@kbn/data-plugin/public';
-import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common';
 import { getEsQueryConfig } from '@kbn/data-service/src/es_query';
 import { reportPerformanceMetricEvent } from '@kbn/ebt-tools';
 import { loadFieldExisting } from '../services/field_existing';
@@ -24,6 +24,7 @@ const generateId = htmlIdGenerator();
 export interface ExistingFieldsInfo {
   fetchStatus: ExistenceFetchStatus;
   existingFieldsByFieldNameMap: Record<string, boolean>;
+  newFields?: FieldSpec[];
   numberOfFetches: number;
   hasDataViewRestrictions?: boolean;
 }
@@ -54,6 +55,7 @@ export interface ExistingFieldsReader {
   hasFieldData: (dataViewId: string, fieldName: string) => boolean;
   getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus;
   isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean;
+  getNewFields: (dataViewId: string) => FieldSpec[];
 }
 
 const initialData: ExistingFieldsByDataViewMap = {};
@@ -118,7 +120,7 @@ export const useExistingFieldsFetcher = (
 
       setActiveRequests((value) => value + 1);
 
-      const hasRestrictions = Boolean(dataView.getAggregationRestrictions?.());
+      const hasRestrictions = Boolean(dataView?.getAggregationRestrictions?.());
       const info: ExistingFieldsInfo = {
         ...unknownInfo,
         numberOfFetches,
@@ -157,6 +159,7 @@ export const useExistingFieldsFetcher = (
           }
 
           info.existingFieldsByFieldNameMap = booleanMap(existingFieldNames);
+          info.newFields = result.newFields;
           info.fetchStatus = ExistenceFetchStatus.succeeded;
         } catch (error) {
           info.fetchStatus = ExistenceFetchStatus.failed;
@@ -286,6 +289,19 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => {
     [existingFieldsByDataViewMap]
   );
 
+  const getNewFields = useCallback(
+    (dataViewId: string) => {
+      const info = existingFieldsByDataViewMap[dataViewId];
+
+      if (info?.fetchStatus === ExistenceFetchStatus.succeeded) {
+        return info?.newFields ?? [];
+      }
+
+      return [];
+    },
+    [existingFieldsByDataViewMap]
+  );
+
   const getFieldsExistenceInfo = useCallback(
     (dataViewId: string) => {
       return dataViewId ? existingFieldsByDataViewMap[dataViewId] : unknownInfo;
@@ -321,8 +337,9 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => {
       hasFieldData,
       getFieldsExistenceStatus,
       isFieldsExistenceInfoUnavailable,
+      getNewFields,
     }),
-    [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable]
+    [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, getNewFields]
   );
 };
 

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx

@@ -96,6 +96,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId,
+        getNewFields: () => [],
       })
     );
 
@@ -156,6 +157,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId,
+        getNewFields: () => [],
       })
     );
 
@@ -438,6 +440,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== knownDataViewId,
+        getNewFields: () => [],
       })
     );
 

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts

@@ -10,8 +10,9 @@ import { groupBy } from 'lodash';
 import { useEffect, useMemo, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { type CoreStart } from '@kbn/core-lifecycle-browser';
-import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/common';
 import { type DataViewsContract } from '@kbn/data-views-plugin/public';
+import { useNewFields } from './use_new_fields';
 import {
   type FieldListGroups,
   type FieldsGroup,
@@ -41,6 +42,7 @@ export interface GroupedFieldsParams<T extends FieldListItem> {
   onOverrideFieldGroupDetails?: OverrideFieldGroupDetails;
   onSupportedFieldFilter?: (field: T) => boolean;
   onSelectedFieldFilter?: (field: T) => boolean;
+  getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[];
 }
 
 export interface GroupedFieldsResult<T extends FieldListItem> {
@@ -52,6 +54,8 @@ export interface GroupedFieldsResult<T extends FieldListItem> {
     fieldsExistInIndex: boolean;
     screenReaderDescriptionId?: string;
   };
+  allFieldsModified: T[] | null; // `null` is for loading indicator
+  hasNewFields: boolean;
 }
 
 export function useGroupedFields<T extends FieldListItem = DataViewField>({
@@ -65,6 +69,7 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
   onOverrideFieldGroupDetails,
   onSupportedFieldFilter,
   onSelectedFieldFilter,
+  getNewFieldsBySpec,
 }: GroupedFieldsParams<T>): GroupedFieldsResult<T> {
   const fieldsExistenceReader = useExistingFieldsReader();
   const fieldListFilters = useFieldFilters<T>({
@@ -73,6 +78,7 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     getCustomFieldType,
     onSupportedFieldFilter,
   });
+
   const onFilterFieldList = fieldListFilters.onFilterField;
   const [dataView, setDataView] = useState<DataView | null>(null);
   const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName);
@@ -101,6 +107,13 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     // if field existence information changed, reload the data view too
   }, [dataViewId, services.dataViews, setDataView, hasFieldDataHandler]);
 
+  const { allFieldsModified, hasNewFields } = useNewFields<T>({
+    dataView,
+    allFields,
+    getNewFieldsBySpec,
+    fieldsExistenceReader,
+  });
+
   // important when switching from a known dataViewId to no data view (like in text-based queries)
   useEffect(() => {
     if (dataView && !dataViewId) {
@@ -120,13 +133,16 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     };
 
     const selectedFields = sortedSelectedFields || [];
-    const sortedFields = [...(allFields || [])].sort(sortFields);
+
+    const sortedFields = [...(allFieldsModified || [])].sort(sortFields);
+
     const groupedFields = {
       ...getDefaultFieldGroups(),
       ...groupBy(sortedFields, (field) => {
         if (!sortedSelectedFields && onSelectedFieldFilter && onSelectedFieldFilter(field)) {
           selectedFields.push(field);
         }
+
         if (onSupportedFieldFilter && !onSupportedFieldFilter(field)) {
           return 'skippedFields';
         }
@@ -311,7 +327,7 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
 
     return fieldGroupDefinitions;
   }, [
-    allFields,
+    allFieldsModified,
     onSupportedFieldFilter,
     onSelectedFieldFilter,
     onOverrideFieldGroupDetails,
@@ -381,6 +397,8 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
   return {
     fieldListGroupedProps,
     fieldListFiltersProps: fieldListFilters.fieldListFiltersProps,
+    allFieldsModified,
+    hasNewFields,
   };
 }
 

packages/kbn-unified-field-list/src/hooks/use_new_fields.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { useMemo } from 'react';
+import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
+import { ExistingFieldsReader, type FieldListItem, GroupedFieldsParams } from '../..';
+
+/**
+ * This hook is used to get the new fields of previous fields for wildcards request, and merges those
+ * with the existing fields.
+ * @param dataView
+ * @param allFields
+ * @param getNewFieldsBySpec
+ * @param fieldsExistenceReader
+ */
+export function useNewFields<T extends FieldListItem = DataViewField>({
+  dataView,
+  allFields,
+  getNewFieldsBySpec,
+  fieldsExistenceReader,
+}: {
+  dataView?: DataView | null;
+  allFields: GroupedFieldsParams<T>['allFields'];
+  getNewFieldsBySpec: GroupedFieldsParams<T>['getNewFieldsBySpec'];
+  fieldsExistenceReader: ExistingFieldsReader;
+}) {
+  const newFields =
+    allFields && dataView?.id && getNewFieldsBySpec
+      ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView)
+      : null;
+  const hasNewFields = Boolean(allFields && newFields && newFields.length > 0);
+
+  const allFieldsModified = useMemo(() => {
+    if (!allFields || !newFields?.length) return allFields;
+    // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request
+    // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way
+    const allFieldsExlNew =
+      allFields && newFields
+        ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name))
+        : allFields;
+
+    return newFields ? [...allFieldsExlNew, ...newFields] : allFields;
+  }, [newFields, allFields]);
+
+  return { allFieldsModified, hasNewFields };
+}