[UnifiedFieldList] Add new fields ingested in background with valid mappings

packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx

@@ -191,25 +191,6 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
     onSelectedFieldFilter,
   ]);
 
-  useEffect(() => {
-    if (
-      searchMode !== 'documents' ||
-      !useNewFieldsApi ||
-      stateService.creationOptions.disableMultiFieldsGroupingByParent
-    ) {
-      setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case
-    } else {
-      setMultiFieldsMap(calculateMultiFields(allFields, selectedFieldsState.selectedFieldsMap));
-    }
-  }, [
-    stateService.creationOptions.disableMultiFieldsGroupingByParent,
-    selectedFieldsState.selectedFieldsMap,
-    allFields,
-    useNewFieldsApi,
-    setMultiFieldsMap,
-    searchMode,
-  ]);
-
   const popularFieldsLimit = useMemo(
     () => core.uiSettings.get(FIELDS_LIMIT_SETTING),
     [core.uiSettings]
@@ -226,7 +207,11 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
       [searchMode, stateService.creationOptions.disableMultiFieldsGroupingByParent]
     );
 
-  const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields<DataViewField>({
+  const {
+    fieldListFiltersProps,
+    fieldListGroupedProps,
+    allFields: allFieldsModified,
+  } = useGroupedFields<DataViewField>({
     dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries
     allFields,
     popularFieldsLimit:
@@ -245,6 +230,27 @@ export const UnifiedFieldListSidebarComponent: React.FC<UnifiedFieldListSidebarP
     onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails,
   });
 
+  useEffect(() => {
+    if (
+      searchMode !== 'documents' ||
+      !useNewFieldsApi ||
+      stateService.creationOptions.disableMultiFieldsGroupingByParent
+    ) {
+      setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case
+    } else {
+      setMultiFieldsMap(
+        calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap)
+      );
+    }
+  }, [
+    stateService.creationOptions.disableMultiFieldsGroupingByParent,
+    selectedFieldsState.selectedFieldsMap,
+    allFieldsModified,
+    useNewFieldsApi,
+    setMultiFieldsMap,
+    searchMode,
+  ]);
+
   const renderFieldItem: FieldListGroupedProps<DataViewField>['renderFieldItem'] = useCallback(
     ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => (
       <li key={`field${field.name}`} data-attr-field={field.name}>

packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts

@@ -15,6 +15,7 @@ import { type DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
 import { getEsQueryConfig } from '@kbn/data-service/src/es_query';
 import { reportPerformanceMetricEvent } from '@kbn/ebt-tools';
+import { DataViewField } from '@kbn/data-views-plugin/common';
 import { loadFieldExisting } from '../services/field_existing';
 import { ExistenceFetchStatus } from '../types';
 
@@ -24,6 +25,7 @@ const generateId = htmlIdGenerator();
 export interface ExistingFieldsInfo {
   fetchStatus: ExistenceFetchStatus;
   existingFieldsByFieldNameMap: Record<string, boolean>;
+  newFields?: DataViewField[];
   numberOfFetches: number;
   hasDataViewRestrictions?: boolean;
 }
@@ -54,6 +56,7 @@ export interface ExistingFieldsReader {
   hasFieldData: (dataViewId: string, fieldName: string) => boolean;
   getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus;
   isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean;
+  getNewFields: (dataViewId: string) => DataViewField[];
 }
 
 const initialData: ExistingFieldsByDataViewMap = {};
@@ -157,6 +160,7 @@ export const useExistingFieldsFetcher = (
           }
 
           info.existingFieldsByFieldNameMap = booleanMap(existingFieldNames);
+          info.newFields = result.newFields;
           info.fetchStatus = ExistenceFetchStatus.succeeded;
         } catch (error) {
           info.fetchStatus = ExistenceFetchStatus.failed;
@@ -286,6 +290,19 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => {
     [existingFieldsByDataViewMap]
   );
 
+  const getNewFields = useCallback(
+    (dataViewId: string) => {
+      const info = existingFieldsByDataViewMap[dataViewId];
+
+      if (info?.fetchStatus === ExistenceFetchStatus.succeeded) {
+        return info?.newFields ?? [];
+      }
+
+      return [];
+    },
+    [existingFieldsByDataViewMap]
+  );
+
   const getFieldsExistenceInfo = useCallback(
     (dataViewId: string) => {
       return dataViewId ? existingFieldsByDataViewMap[dataViewId] : unknownInfo;
@@ -321,8 +338,9 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => {
       hasFieldData,
       getFieldsExistenceStatus,
       isFieldsExistenceInfoUnavailable,
+      getNewFields,
     }),
-    [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable]
+    [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, getNewFields]
   );
 };
 

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx

@@ -96,6 +96,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId,
+        getNewFields: () => [],
       })
     );
 
@@ -156,6 +157,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId,
+        getNewFields: () => [],
       })
     );
 
@@ -438,6 +440,7 @@ describe('UnifiedFieldList useGroupedFields()', () => {
             ? ExistenceFetchStatus.succeeded
             : ExistenceFetchStatus.unknown,
         isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== knownDataViewId,
+        getNewFields: () => [],
       })
     );
 

packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts

@@ -7,7 +7,7 @@
  */
 
 import { groupBy } from 'lodash';
-import { useEffect, useMemo, useState } from 'react';
+import { useEffect, useMemo, useState, useRef } from 'react';
 import { i18n } from '@kbn/i18n';
 import { type CoreStart } from '@kbn/core-lifecycle-browser';
 import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common';
@@ -41,6 +41,7 @@ export interface GroupedFieldsParams<T extends FieldListItem> {
   onOverrideFieldGroupDetails?: OverrideFieldGroupDetails;
   onSupportedFieldFilter?: (field: T) => boolean;
   onSelectedFieldFilter?: (field: T) => boolean;
+  isCompatibleField?: (fields: DataViewField) => boolean;
 }
 
 export interface GroupedFieldsResult<T extends FieldListItem> {
@@ -52,6 +53,8 @@ export interface GroupedFieldsResult<T extends FieldListItem> {
     fieldsExistInIndex: boolean;
     screenReaderDescriptionId?: string;
   };
+  allFields: T[] | null; // `null` is for loading indicator
+  hasNewFields: boolean;
 }
 
 export function useGroupedFields<T extends FieldListItem = DataViewField>({
@@ -65,6 +68,7 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
   onOverrideFieldGroupDetails,
   onSupportedFieldFilter,
   onSelectedFieldFilter,
+  isCompatibleField,
 }: GroupedFieldsParams<T>): GroupedFieldsResult<T> {
   const fieldsExistenceReader = useExistingFieldsReader();
   const fieldListFilters = useFieldFilters<T>({
@@ -73,6 +77,8 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     getCustomFieldType,
     onSupportedFieldFilter,
   });
+  const allFieldsInclNew = useRef(allFields);
+  const hasNewFields = useRef(false);
   const onFilterFieldList = fieldListFilters.onFilterField;
   const [dataView, setDataView] = useState<DataView | null>(null);
   const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName);
@@ -120,13 +126,30 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
     };
 
     const selectedFields = sortedSelectedFields || [];
-    const sortedFields = [...(allFields || [])].sort(sortFields);
+    const newFields = dataViewId
+      ? fieldsExistenceReader
+          .getNewFields(dataViewId)
+          .filter((field) => (isCompatibleField ? isCompatibleField(field) : true))
+      : [];
+    // remove fields from allFields that are available in newFields, because they can be provided in unmapped state
+    const allFieldsWithoutNewFields = !newFields.length
+      ? allFields
+      : allFields?.filter((field) => !newFields.find((newField) => newField.name === field.name));
+    // append new fields to the end of the list allFieldsWithoutNewFields
+    const allFieldsWithNewFields = allFieldsWithoutNewFields
+      ? [...allFieldsWithoutNewFields, ...newFields]
+      : newFields;
+
+    const sortedFields = [...((allFieldsWithNewFields as unknown as T[]) || [])].sort(sortFields);
+    allFieldsInclNew.current = sortedFields;
+    hasNewFields.current = Boolean(newFields.length);
     const groupedFields = {
       ...getDefaultFieldGroups(),
       ...groupBy(sortedFields, (field) => {
         if (!sortedSelectedFields && onSelectedFieldFilter && onSelectedFieldFilter(field)) {
           selectedFields.push(field);
         }
+
         if (onSupportedFieldFilter && !onSupportedFieldFilter(field)) {
           return 'skippedFields';
         }
@@ -311,18 +334,20 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
 
     return fieldGroupDefinitions;
   }, [
+    sortedSelectedFields,
     allFields,
-    onSupportedFieldFilter,
-    onSelectedFieldFilter,
-    onOverrideFieldGroupDetails,
-    dataView,
-    dataViewId,
-    hasFieldDataHandler,
-    fieldsExistenceInfoUnavailable,
+    popularFieldsLimit,
     isAffectedByGlobalFilter,
     isAffectedByTimeFilter,
-    popularFieldsLimit,
-    sortedSelectedFields,
+    dataViewId,
+    fieldsExistenceInfoUnavailable,
+    onOverrideFieldGroupDetails,
+    hasFieldDataHandler,
+    fieldsExistenceReader,
+    onSelectedFieldFilter,
+    onSupportedFieldFilter,
+    dataView,
+    isCompatibleField,
   ]);
 
   const fieldGroups: FieldListGroups<T> = useMemo(() => {
@@ -381,6 +406,8 @@ export function useGroupedFields<T extends FieldListItem = DataViewField>({
   return {
     fieldListGroupedProps,
     fieldListFiltersProps: fieldListFilters.fieldListFiltersProps,
+    allFields: allFieldsInclNew.current,
+    hasNewFields: hasNewFields.current,
   };
 }
 

packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts

@@ -7,7 +7,7 @@
  */
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { RuntimeField } from '@kbn/data-views-plugin/common';
+import { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common';
 import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common';
 import type { IKibanaSearchRequest } from '@kbn/data-plugin/common';
 
@@ -49,15 +49,22 @@ export async function fetchFieldExistence({
   metaFields: string[];
   dataViewsService: DataViewsContract;
 }) {
-  const allFields = buildFieldList(dataView, metaFields);
   const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
     // filled in by data views service
     pattern: '',
     indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
   });
+  const newFields = existingFieldList
+    .filter((field) => dataView.getFieldByName(field.name) === undefined)
+    .map((field) => dataView.getFieldByName(field.name) ?? new DataViewField(field));
+  if (newFields.length) {
+    await dataViewsService.refreshFields(dataView, false);
+  }
+  const allFields = buildFieldList(dataView, metaFields);
   return {
     indexPatternTitle: dataView.title,
     existingFieldNames: existingFields(existingFieldList, allFields),
+    newFields,
   };
 }
 

packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts

@@ -9,7 +9,7 @@
 import { IUiSettingsClient } from '@kbn/core/public';
 import { type DataPublicPluginStart } from '@kbn/data-plugin/public';
 import { UI_SETTINGS } from '@kbn/data-service/src/constants';
-import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
+import type { DataView, DataViewField, DataViewsContract } from '@kbn/data-views-plugin/common';
 import { lastValueFrom } from 'rxjs';
 import { fetchFieldExistence } from './field_existing_utils';
 
@@ -27,6 +27,7 @@ interface FetchFieldExistenceParams {
 export type LoadFieldExistingHandler = (params: FetchFieldExistenceParams) => Promise<{
   existingFieldNames: string[];
   indexPatternTitle: string;
+  newFields?: DataViewField[];
 }>;
 
 export const loadFieldExisting: LoadFieldExistingHandler = async ({