From 26d3d6de42c5b0fcdcc3df7c7643b6fa6f13cb69 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 30 Nov 2023 22:38:54 +0100 Subject: [PATCH 01/29] Add new fields to list with correct types --- .../src/hooks/use_existing_fields.ts | 22 ++++++++++++- .../src/hooks/use_grouped_fields.ts | 32 +++++++++++++------ .../field_existing/field_existing_utils.ts | 11 +++++-- .../field_existing/load_field_existing.ts | 3 +- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index d099a6f2b85e2..588e0f60384b3 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -15,6 +15,7 @@ import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; +import { DataViewField } from '@kbn/data-views-plugin/common'; import { loadFieldExisting } from '../services/field_existing'; import { ExistenceFetchStatus } from '../types'; @@ -24,6 +25,8 @@ const generateId = htmlIdGenerator(); export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; existingFieldsByFieldNameMap: Record; + newFieldsByFieldNameMap: Record; + newFields?: DataViewField[]; numberOfFetches: number; hasDataViewRestrictions?: boolean; } @@ -54,12 +57,14 @@ export interface ExistingFieldsReader { hasFieldData: (dataViewId: string, fieldName: string) => boolean; getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; + getNewFields: (dataViewId: string) => DataViewField[]; } const initialData: ExistingFieldsByDataViewMap = {}; const unknownInfo: ExistingFieldsInfo = { fetchStatus: ExistenceFetchStatus.unknown, existingFieldsByFieldNameMap: {}, + newFieldsByFieldNameMap: {}, numberOfFetches: 0, }; @@ -157,6 +162,7 @@ export const useExistingFieldsFetcher = ( } info.existingFieldsByFieldNameMap = booleanMap(existingFieldNames); + info.newFields = result.newFields; info.fetchStatus = ExistenceFetchStatus.succeeded; } catch (error) { info.fetchStatus = ExistenceFetchStatus.failed; @@ -286,6 +292,19 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { [existingFieldsByDataViewMap] ); + const getNewFields = useCallback( + (dataViewId: string) => { + const info = existingFieldsByDataViewMap[dataViewId]; + + if (info?.fetchStatus === ExistenceFetchStatus.succeeded) { + return info?.newFields ?? []; + } + + return []; + }, + [existingFieldsByDataViewMap] + ); + const getFieldsExistenceInfo = useCallback( (dataViewId: string) => { return dataViewId ? existingFieldsByDataViewMap[dataViewId] : unknownInfo; @@ -321,8 +340,9 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, + getNewFields, }), - [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable] + [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, getNewFields] ); }; diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index f51f9b0fffb2d..7822e71244d10 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -120,13 +120,23 @@ export function useGroupedFields({ }; const selectedFields = sortedSelectedFields || []; - const sortedFields = [...(allFields || [])].sort(sortFields); + const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; + console.log({ newFields }); + const fieldsToSort = + allFields && newFields.length + ? allFields.map((field) => { + return (dataView?.getFieldByName(field.name) as unknown as T) ?? field; + }) + : allFields; + const sortedFields = [...(fieldsToSort || [])].sort(sortFields); + const groupedFields = { ...getDefaultFieldGroups(), ...groupBy(sortedFields, (field) => { if (!sortedSelectedFields && onSelectedFieldFilter && onSelectedFieldFilter(field)) { selectedFields.push(field); } + if (onSupportedFieldFilter && !onSupportedFieldFilter(field)) { return 'skippedFields'; } @@ -311,18 +321,19 @@ export function useGroupedFields({ return fieldGroupDefinitions; }, [ + sortedSelectedFields, allFields, - onSupportedFieldFilter, - onSelectedFieldFilter, - onOverrideFieldGroupDetails, - dataView, - dataViewId, - hasFieldDataHandler, - fieldsExistenceInfoUnavailable, + popularFieldsLimit, isAffectedByGlobalFilter, isAffectedByTimeFilter, - popularFieldsLimit, - sortedSelectedFields, + dataViewId, + fieldsExistenceInfoUnavailable, + onOverrideFieldGroupDetails, + hasFieldDataHandler, + fieldsExistenceReader, + onSelectedFieldFilter, + onSupportedFieldFilter, + dataView, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -403,5 +414,6 @@ function getDefaultFieldGroups() { metaFields: [], unmappedFields: [], skippedFields: [], + newFields: [], }; } diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 2aa6137ffb1fb..0a487368748cf 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -7,7 +7,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { RuntimeField } from '@kbn/data-views-plugin/common'; +import { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; @@ -49,15 +49,22 @@ export async function fetchFieldExistence({ metaFields: string[]; dataViewsService: DataViewsContract; }) { - const allFields = buildFieldList(dataView, metaFields); const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, { // filled in by data views service pattern: '', indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery), }); + const newFields = existingFieldList + .filter((field) => dataView.getFieldByName(field.name) === undefined) + .map((field) => dataView.getFieldByName(field.name) ?? new DataViewField(field)); + if (newFields.length) { + await dataViewsService.refreshFields(dataView, false); + } + const allFields = buildFieldList(dataView, metaFields); return { indexPatternTitle: dataView.title, existingFieldNames: existingFields(existingFieldList, allFields), + newFields, }; } diff --git a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts index 796062bc60559..889c92008554b 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts @@ -9,7 +9,7 @@ import { IUiSettingsClient } from '@kbn/core/public'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import { UI_SETTINGS } from '@kbn/data-service/src/constants'; -import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField, DataViewsContract } from '@kbn/data-views-plugin/common'; import { lastValueFrom } from 'rxjs'; import { fetchFieldExistence } from './field_existing_utils'; @@ -27,6 +27,7 @@ interface FetchFieldExistenceParams { export type LoadFieldExistingHandler = (params: FetchFieldExistenceParams) => Promise<{ existingFieldNames: string[]; indexPatternTitle: string; + newFields?: DataViewField[]; }>; export const loadFieldExisting: LoadFieldExistingHandler = async ({ From 7b999db6641a1919a09d3ee4aec052be9d9200c6 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 1 Dec 2023 09:36:14 +0100 Subject: [PATCH 02/29] Improve code --- .../src/hooks/use_grouped_fields.test.tsx | 3 +++ .../kbn-unified-field-list/src/hooks/use_grouped_fields.ts | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx index 053e7d912d375..78b34329e0bf0 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx @@ -96,6 +96,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [], }) ); @@ -156,6 +157,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [], }) ); @@ -438,6 +440,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== knownDataViewId, + getNewFields: () => [], }) ); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 7822e71244d10..731eb63dd7934 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -121,7 +121,6 @@ export function useGroupedFields({ const selectedFields = sortedSelectedFields || []; const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; - console.log({ newFields }); const fieldsToSort = allFields && newFields.length ? allFields.map((field) => { From 3a6c239cb35b64871f1bea7080bec7c8183ce154 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 1 Dec 2023 09:46:44 +0100 Subject: [PATCH 03/29] Remove redundant newFieldsByFieldNameMap --- .../kbn-unified-field-list/src/hooks/use_existing_fields.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index 588e0f60384b3..f756b4cc19126 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -25,7 +25,6 @@ const generateId = htmlIdGenerator(); export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; existingFieldsByFieldNameMap: Record; - newFieldsByFieldNameMap: Record; newFields?: DataViewField[]; numberOfFetches: number; hasDataViewRestrictions?: boolean; @@ -64,7 +63,6 @@ const initialData: ExistingFieldsByDataViewMap = {}; const unknownInfo: ExistingFieldsInfo = { fetchStatus: ExistenceFetchStatus.unknown, existingFieldsByFieldNameMap: {}, - newFieldsByFieldNameMap: {}, numberOfFetches: 0, }; From 307a16d9fb4059aad0c3aaedd4ff88c9b689754b Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Mon, 4 Dec 2023 17:06:38 +0100 Subject: [PATCH 04/29] Add multi field support --- .../field_list_sidebar.tsx | 46 +++++++++++-------- .../src/hooks/use_grouped_fields.ts | 10 ++-- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 4bc54069336b0..e1e72ca0a9aa9 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -191,25 +191,6 @@ export const UnifiedFieldListSidebarComponent: React.FC { - if ( - searchMode !== 'documents' || - !useNewFieldsApi || - stateService.creationOptions.disableMultiFieldsGroupingByParent - ) { - setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case - } else { - setMultiFieldsMap(calculateMultiFields(allFields, selectedFieldsState.selectedFieldsMap)); - } - }, [ - stateService.creationOptions.disableMultiFieldsGroupingByParent, - selectedFieldsState.selectedFieldsMap, - allFields, - useNewFieldsApi, - setMultiFieldsMap, - searchMode, - ]); - const popularFieldsLimit = useMemo( () => core.uiSettings.get(FIELDS_LIMIT_SETTING), [core.uiSettings] @@ -226,7 +207,11 @@ export const UnifiedFieldListSidebarComponent: React.FC({ + const { + fieldListFiltersProps, + fieldListGroupedProps, + allFields: allFieldsModified, + } = useGroupedFields({ dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries allFields, popularFieldsLimit: @@ -245,6 +230,27 @@ export const UnifiedFieldListSidebarComponent: React.FC { + if ( + searchMode !== 'documents' || + !useNewFieldsApi || + stateService.creationOptions.disableMultiFieldsGroupingByParent + ) { + setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case + } else { + setMultiFieldsMap( + calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap) + ); + } + }, [ + stateService.creationOptions.disableMultiFieldsGroupingByParent, + selectedFieldsState.selectedFieldsMap, + allFieldsModified, + useNewFieldsApi, + setMultiFieldsMap, + searchMode, + ]); + const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => (
  • diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 731eb63dd7934..87b9ab5187139 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -7,7 +7,7 @@ */ import { groupBy } from 'lodash'; -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useState, useRef } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common'; @@ -52,6 +52,7 @@ export interface GroupedFieldsResult { fieldsExistInIndex: boolean; screenReaderDescriptionId?: string; }; + allFields: T[] | null; // `null` is for loading indicator } export function useGroupedFields({ @@ -73,6 +74,7 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); + const fieldsToSort = useRef(allFields); const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName); @@ -121,13 +123,14 @@ export function useGroupedFields({ const selectedFields = sortedSelectedFields || []; const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; - const fieldsToSort = + fieldsToSort.current = allFields && newFields.length ? allFields.map((field) => { return (dataView?.getFieldByName(field.name) as unknown as T) ?? field; }) : allFields; - const sortedFields = [...(fieldsToSort || [])].sort(sortFields); + + const sortedFields = [...(fieldsToSort.current || [])].sort(sortFields); const groupedFields = { ...getDefaultFieldGroups(), @@ -391,6 +394,7 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, + allFields: fieldsToSort.current, }; } From 999a6a9449e10651598d324ab6bd37dfa5f8d9ba Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Sat, 9 Dec 2023 14:30:04 +0100 Subject: [PATCH 05/29] Add support for Lens --- .../src/hooks/use_grouped_fields.ts | 36 ++++++++++++------- .../datasources/form_based/datapanel.tsx | 32 ++++++++++------- .../datasources/text_based/datapanel.tsx | 2 ++ 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 87b9ab5187139..8da8646da8e5c 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -41,6 +41,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; + isCompatibleField?: (fields: DataViewField) => boolean; } export interface GroupedFieldsResult { @@ -53,6 +54,7 @@ export interface GroupedFieldsResult { screenReaderDescriptionId?: string; }; allFields: T[] | null; // `null` is for loading indicator + hasNewFields: boolean; } export function useGroupedFields({ @@ -66,6 +68,7 @@ export function useGroupedFields({ onOverrideFieldGroupDetails, onSupportedFieldFilter, onSelectedFieldFilter, + isCompatibleField, }: GroupedFieldsParams): GroupedFieldsResult { const fieldsExistenceReader = useExistingFieldsReader(); const fieldListFilters = useFieldFilters({ @@ -74,7 +77,8 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); - const fieldsToSort = useRef(allFields); + const allFieldsInclNew = useRef(allFields); + const hasNewFields = useRef(false); const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName); @@ -122,16 +126,23 @@ export function useGroupedFields({ }; const selectedFields = sortedSelectedFields || []; - const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; - fieldsToSort.current = - allFields && newFields.length - ? allFields.map((field) => { - return (dataView?.getFieldByName(field.name) as unknown as T) ?? field; - }) - : allFields; - - const sortedFields = [...(fieldsToSort.current || [])].sort(sortFields); + const newFields = dataViewId + ? fieldsExistenceReader + .getNewFields(dataViewId) + .filter((field) => (isCompatibleField ? isCompatibleField(field) : true)) + : []; + // remove fields from allFields that are available in newFields, because they can be provided in unmapped state + const allFieldsWithoutNewFields = !newFields.length + ? allFields + : allFields?.filter((field) => !newFields.find((newField) => newField.name === field.name)); + // append new fields to the end of the list allFieldsWithoutNewFields + const allFieldsWithNewFields = allFieldsWithoutNewFields + ? [...allFieldsWithoutNewFields, ...newFields] + : newFields; + const sortedFields = [...((allFieldsWithNewFields as unknown as T[]) || [])].sort(sortFields); + allFieldsInclNew.current = sortedFields; + hasNewFields.current = Boolean(newFields.length); const groupedFields = { ...getDefaultFieldGroups(), ...groupBy(sortedFields, (field) => { @@ -336,6 +347,7 @@ export function useGroupedFields({ onSelectedFieldFilter, onSupportedFieldFilter, dataView, + isCompatibleField, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -394,7 +406,8 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFields: fieldsToSort.current, + allFields: allFieldsInclNew.current, + hasNewFields: hasNewFields.current, }; } @@ -417,6 +430,5 @@ function getDefaultFieldGroups() { metaFields: [], unmappedFields: [], skippedFields: [], - newFields: [], }; } diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 53a26693b7a7a..58c4e1bddaa27 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -28,6 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; +import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import type { DatasourceDataPanelProps, FramePublicAPI, @@ -249,18 +250,20 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ } }, []); - const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields({ - dataViewId: currentIndexPatternId, - allFields, - services: { - dataViews, - core, - }, - isAffectedByGlobalFilter: Boolean(filters.length), - onSupportedFieldFilter, - onSelectedFieldFilter, - onOverrideFieldGroupDetails, - }); + const { fieldListFiltersProps, fieldListGroupedProps, hasNewFields } = + useGroupedFields({ + dataViewId: currentIndexPatternId, + allFields, + services: { + dataViews, + core, + }, + isAffectedByGlobalFilter: Boolean(filters.length), + onSupportedFieldFilter, + onSelectedFieldFilter, + onOverrideFieldGroupDetails, + isCompatibleField: isFieldLensCompatible, + }); const closeFieldEditor = useRef<() => void | undefined>(); @@ -296,6 +299,11 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ frame.dataViews.indexPatterns, data.search.session, ]); + useEffect(() => { + if (hasNewFields) { + refreshFieldList(); + } + }, [hasNewFields, refreshFieldList]); const editField = useMemo( () => diff --git a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx index 113125484cddf..ac62636ad50c1 100644 --- a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx @@ -24,6 +24,7 @@ import { GetCustomFieldType, useGroupedFields, } from '@kbn/unified-field-list'; +import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import type { DatasourceDataPanelProps } from '../../types'; import type { TextBasedPrivateState } from './types'; import { getStateFromAggregateQuery } from './utils'; @@ -105,6 +106,7 @@ export function TextBasedDataPanel({ getCustomFieldType, onSelectedFieldFilter, onOverrideFieldGroupDetails, + isCompatibleField: isFieldLensCompatible, }); const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( From 15cc582977bb76e683d223e8a9167add0264471d Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 13 Dec 2023 19:11:49 +0100 Subject: [PATCH 06/29] Add functional test --- .../discover/group4/_field_list_new_fields.ts | 86 +++++++++++++++++++ test/functional/apps/discover/group4/index.ts | 1 + 2 files changed, 87 insertions(+) create mode 100644 test/functional/apps/discover/group4/_field_list_new_fields.ts diff --git a/test/functional/apps/discover/group4/_field_list_new_fields.ts b/test/functional/apps/discover/group4/_field_list_new_fields.ts new file mode 100644 index 0000000000000..3c24bcf613ae4 --- /dev/null +++ b/test/functional/apps/discover/group4/_field_list_new_fields.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const security = getService('security'); + const es = getService('es'); + const retry = getService('retry'); + const queryBar = getService('queryBar'); + const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'unifiedFieldList']); + + describe('Field list new fields in background handling', function () { + before(async () => { + await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); + await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); + + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setCommonlyUsedTime('This_week'); + }); + + after(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); + await es.transport.request({ + path: '/my-index-000001', + method: 'DELETE', + }); + }); + + it('Check that new ingested fields are added to the available fields section', async function () { + const initialPattern = 'my-index-'; + await es.transport.request({ + path: '/my-index-000001/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + a: 'GET /search HTTP/1.1 200 1070000', + }, + }); + + await PageObjects.discover.createAdHocDataView(initialPattern, true); + + await retry.waitFor('current data view to get updated', async () => { + return (await PageObjects.discover.getCurrentlySelectedDataView()) === `${initialPattern}*`; + }); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + + expect(await PageObjects.discover.getHitCountInt()).to.be(1); + expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([ + '@timestamp', + 'a', + ]); + + await es.transport.request({ + path: '/my-index-000001/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + b: 'GET /search HTTP/1.1 200 1070000', + }, + }); + + await retry.waitFor('the new record was found', async () => { + await queryBar.submitQuery(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + return (await PageObjects.discover.getHitCountInt()) === 2; + }); + + expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([ + '@timestamp', + 'a', + 'b', + ]); + }); + }); +} diff --git a/test/functional/apps/discover/group4/index.ts b/test/functional/apps/discover/group4/index.ts index 1aab3db2bfc43..656a116551db8 100644 --- a/test/functional/apps/discover/group4/index.ts +++ b/test/functional/apps/discover/group4/index.ts @@ -33,5 +33,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_context_encoded_url_params')); loadTestFile(require.resolve('./_hide_announcements')); loadTestFile(require.resolve('./_data_view_edit')); + loadTestFile(require.resolve('./_field_list_new_fields')); }); } From 065a50c9da9b3e1ae2e3c6e2aa4abf38d6e97acf Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 14 Dec 2023 06:06:19 +0100 Subject: [PATCH 07/29] Add documentation --- .../src/services/field_existing/field_existing_utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 0a487368748cf..e4dab136c5a52 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -54,15 +54,20 @@ export async function fetchFieldExistence({ pattern: '', indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery), }); + + // take care of fields of existingFieldList, that are not yet available + // in the given data view. Those fields we consider as new fields, + // that were ingested after the data view was loaded const newFields = existingFieldList .filter((field) => dataView.getFieldByName(field.name) === undefined) .map((field) => dataView.getFieldByName(field.name) ?? new DataViewField(field)); + // refresh the data view in case there are new fields if (newFields.length) { await dataViewsService.refreshFields(dataView, false); } const allFields = buildFieldList(dataView, metaFields); return { - indexPatternTitle: dataView.title, + indexPatternTitle: dataView.getIndexPattern(), existingFieldNames: existingFields(existingFieldList, allFields), newFields, }; From e9fd00eecc06b6cc287c039d302e307fec9c7831 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 14 Dec 2023 10:12:32 +0100 Subject: [PATCH 08/29] Improve code --- .../src/hooks/use_grouped_fields.ts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 8da8646da8e5c..c079b6b9503e9 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -77,7 +77,7 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); - const allFieldsInclNew = useRef(allFields); + const allFieldsToReturn = useRef(allFields); const hasNewFields = useRef(false); const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); @@ -126,22 +126,22 @@ export function useGroupedFields({ }; const selectedFields = sortedSelectedFields || []; + + // Taking care of new fields that were ingested after the selected data view was loaded + // Those replace existing fields if updated, or are added to the list const newFields = dataViewId - ? fieldsExistenceReader - .getNewFields(dataViewId) - .filter((field) => (isCompatibleField ? isCompatibleField(field) : true)) + ? fieldsExistenceReader.getNewFields(dataViewId).filter(isCompatibleField || (() => true)) + : []; + // Filtering out fields that e.g. Discover provides by analyzing the resultset, that are not part of the loaded DataView + // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way + const allFieldsExlNew = allFields + ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) : []; - // remove fields from allFields that are available in newFields, because they can be provided in unmapped state - const allFieldsWithoutNewFields = !newFields.length - ? allFields - : allFields?.filter((field) => !newFields.find((newField) => newField.name === field.name)); - // append new fields to the end of the list allFieldsWithoutNewFields - const allFieldsWithNewFields = allFieldsWithoutNewFields - ? [...allFieldsWithoutNewFields, ...newFields] - : newFields; - const sortedFields = [...((allFieldsWithNewFields as unknown as T[]) || [])].sort(sortFields); - allFieldsInclNew.current = sortedFields; + const allFieldsInclNew = [...allFieldsExlNew, ...newFields] as unknown as T[]; + + const sortedFields = allFieldsInclNew.sort(sortFields); + allFieldsToReturn.current = sortedFields; hasNewFields.current = Boolean(newFields.length); const groupedFields = { ...getDefaultFieldGroups(), @@ -406,7 +406,7 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFields: allFieldsInclNew.current, + allFields: allFieldsToReturn.current, hasNewFields: hasNewFields.current, }; } From 454592c884f3194c9149296feb6740e652c14417 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 15 Dec 2023 09:45:19 +0100 Subject: [PATCH 09/29] Add Lens test --- .../apps/lens/group2/fields_list.ts | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/lens/group2/fields_list.ts b/x-pack/test/functional/apps/lens/group2/fields_list.ts index 4e1f771d0b042..79baafe6100a6 100644 --- a/x-pack/test/functional/apps/lens/group2/fields_list.ts +++ b/x-pack/test/functional/apps/lens/group2/fields_list.ts @@ -9,13 +9,15 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header', 'timePicker']); const find = getService('find'); const log = getService('log'); const testSubjects = getService('testSubjects'); const filterBar = getService('filterBar'); const fieldEditor = getService('fieldEditor'); const retry = getService('retry'); + const es = getService('es'); + const queryBar = getService('queryBar'); describe('lens fields list tests', () => { for (const datasourceType of ['form-based', 'ad-hoc', 'ad-hoc-no-timefield']) { @@ -48,7 +50,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); }); }); - it('should show all fields as available', async () => { expect( await (await testSubjects.find('lnsIndexPatternAvailableFields-count')).getVisibleText() @@ -231,5 +232,50 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } }); } + + describe(`update field list test`, () => { + before(async () => { + await es.transport.request({ + path: '/field-update-test/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + oldField: 10, + }, + }); + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.timePicker.setCommonlyUsedTime('This_week'); + + await PageObjects.lens.createAdHocDataView('field-update-test', true); + await retry.try(async () => { + const selectedPattern = await PageObjects.lens.getDataPanelIndexPattern(); + expect(selectedPattern).to.eql('field-update-test*'); + }); + }); + after(async () => { + await es.transport.request({ + path: '/field-update-test', + method: 'DELETE', + }); + }); + + it('should show new fields Available fields', async () => { + await es.transport.request({ + path: '/field-update-test/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + oldField: 10, + newField: 20, + }, + }); + await PageObjects.lens.waitForField('oldField'); + await queryBar.setQuery('oldField: 10'); + await queryBar.submitQuery(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.waitForField('newField'); + }); + }); }); } From 75cc2355c61ac0dde3066972f11fd7b827adb2cc Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 15 Dec 2023 13:08:28 +0100 Subject: [PATCH 10/29] Improve Lens code preventing an endless loop of update --- .../plugins/lens/public/datasources/form_based/datapanel.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 58c4e1bddaa27..1d7b3603482c8 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -299,11 +299,14 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ frame.dataViews.indexPatterns, data.search.session, ]); + useEffect(() => { if (hasNewFields) { refreshFieldList(); } - }, [hasNewFields, refreshFieldList]); + // Preventing a race condition, making sure refreshFieldList is just executed once when hasNewFields is true + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hasNewFields]); const editField = useMemo( () => From 3a746002e7b11ce77277a2d75eb3ae3555aa7a0c Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 07:58:30 +0100 Subject: [PATCH 11/29] Update packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts Co-authored-by: Julia Rechkunova --- .../kbn-unified-field-list/src/hooks/use_existing_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index f756b4cc19126..dba018e59a86b 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -15,7 +15,7 @@ import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; -import { DataViewField } from '@kbn/data-views-plugin/common'; +import type { DataViewField } from '@kbn/data-views-plugin/common'; import { loadFieldExisting } from '../services/field_existing'; import { ExistenceFetchStatus } from '../types'; From 15e5168f8ea9ad54d0f404a3ef225ea0bbb53b1e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 07:58:57 +0100 Subject: [PATCH 12/29] Update packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts Co-authored-by: Julia Rechkunova --- packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index c079b6b9503e9..0d7f92983c230 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -41,7 +41,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - isCompatibleField?: (fields: DataViewField) => boolean; + isCompatibleField?: (field: DataViewField) => boolean; } export interface GroupedFieldsResult { From 139b65165a75a7eae384aa02091ce2e183da94d9 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 07:59:17 +0100 Subject: [PATCH 13/29] Update packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts Co-authored-by: Julia Rechkunova --- .../src/services/field_existing/field_existing_utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index e4dab136c5a52..912ef361dd9a8 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -7,7 +7,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; +import { type DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; From ab514e4c4fcc44459bfb7539dad4a3b0fb55650e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 08:00:34 +0100 Subject: [PATCH 14/29] Update packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts Co-authored-by: Julia Rechkunova --- .../src/services/field_existing/field_existing_utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 912ef361dd9a8..75ac7b159f113 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -59,8 +59,8 @@ export async function fetchFieldExistence({ // in the given data view. Those fields we consider as new fields, // that were ingested after the data view was loaded const newFields = existingFieldList - .filter((field) => dataView.getFieldByName(field.name) === undefined) - .map((field) => dataView.getFieldByName(field.name) ?? new DataViewField(field)); + .filter((field) => !dataView.getFieldByName(field.name)) + .map((field) => new DataViewField(field)); // refresh the data view in case there are new fields if (newFields.length) { await dataViewsService.refreshFields(dataView, false); From b424159790279852eec8d655172b2ee6060c5ef0 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 20 Dec 2023 09:54:32 +0100 Subject: [PATCH 15/29] Improve code --- .../field_list_sidebar.tsx | 46 ++++----- .../src/hooks/use_existing_fields.ts | 14 ++- .../src/hooks/use_grouped_fields.ts | 36 +++---- .../field_existing/field_existing_utils.ts | 30 +++--- .../field_existing/load_field_existing.ts | 4 +- .../lens/public/data_views_service/loader.ts | 94 ++++++++++--------- .../datasources/form_based/datapanel.tsx | 10 +- .../datasources/text_based/datapanel.tsx | 2 - 8 files changed, 129 insertions(+), 107 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index e1e72ca0a9aa9..256d6a8f148fc 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -22,7 +22,7 @@ import { useEuiTheme, } from '@elastic/eui'; import { ToolbarButton } from '@kbn/shared-ux-button-toolbar'; -import { type DataViewField } from '@kbn/data-views-plugin/public'; +import { DataViewField } from '@kbn/data-views-plugin/common'; import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils'; import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils'; import { FieldList } from '../../components/field_list'; @@ -207,28 +207,28 @@ export const UnifiedFieldListSidebarComponent: React.FC({ - dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries - allFields, - popularFieldsLimit: - searchMode !== 'documents' || stateService.creationOptions.disablePopularFields - ? 0 - : popularFieldsLimit, - isAffectedByGlobalFilter, - services: { - dataViews, - core, - }, - sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields, - onSelectedFieldFilter, - onSupportedFieldFilter: - stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter, - onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails, - }); + const { fieldListFiltersProps, fieldListGroupedProps, allFieldsModified } = + useGroupedFields({ + dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries + allFields, + popularFieldsLimit: + searchMode !== 'documents' || stateService.creationOptions.disablePopularFields + ? 0 + : popularFieldsLimit, + isAffectedByGlobalFilter, + services: { + dataViews, + core, + }, + sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields, + onSelectedFieldFilter, + onSupportedFieldFilter: + stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter, + onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails, + getNewFieldsBySpec: (fieldSpecArr) => { + return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)); + }, + }); useEffect(() => { if ( diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index dba018e59a86b..7826c815b9af3 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -12,10 +12,9 @@ import { BehaviorSubject } from 'rxjs'; import type { CoreStart } from '@kbn/core/public'; import type { AggregateQuery, EsQueryConfig, Filter, Query } from '@kbn/es-query'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; -import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; -import type { DataViewField } from '@kbn/data-views-plugin/common'; import { loadFieldExisting } from '../services/field_existing'; import { ExistenceFetchStatus } from '../types'; @@ -25,7 +24,7 @@ const generateId = htmlIdGenerator(); export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; existingFieldsByFieldNameMap: Record; - newFields?: DataViewField[]; + newFields?: FieldSpec[]; numberOfFetches: number; hasDataViewRestrictions?: boolean; } @@ -56,7 +55,7 @@ export interface ExistingFieldsReader { hasFieldData: (dataViewId: string, fieldName: string) => boolean; getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; - getNewFields: (dataViewId: string) => DataViewField[]; + getNewFields: (dataViewId: string) => FieldSpec[]; } const initialData: ExistingFieldsByDataViewMap = {}; @@ -255,7 +254,12 @@ export const useExistingFieldsFetcher = ( ); }; -export const useExistingFieldsReader: () => ExistingFieldsReader = () => { +export const useExistingFieldsReader: () => { + getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; + isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; + getNewFields: (dataViewId: string) => FieldSpec[]; + hasFieldData: (dataViewId: string, fieldName: string) => boolean; +} = () => { const mountedRef = useRef(true); const [existingFieldsByDataViewMap, setExistingFieldsByDataViewMap] = useState(globalMap$.getValue()); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 0d7f92983c230..bfc32be695120 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -10,7 +10,7 @@ import { groupBy } from 'lodash'; import { useEffect, useMemo, useState, useRef } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; -import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; import { type DataViewsContract } from '@kbn/data-views-plugin/public'; import { type FieldListGroups, @@ -41,7 +41,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - isCompatibleField?: (field: DataViewField) => boolean; + getNewFieldsBySpec?: (field: FieldSpec[], dataView: DataView | null) => T[]; } export interface GroupedFieldsResult { @@ -53,7 +53,7 @@ export interface GroupedFieldsResult { fieldsExistInIndex: boolean; screenReaderDescriptionId?: string; }; - allFields: T[] | null; // `null` is for loading indicator + allFieldsModified: T[] | null; // `null` is for loading indicator hasNewFields: boolean; } @@ -68,7 +68,7 @@ export function useGroupedFields({ onOverrideFieldGroupDetails, onSupportedFieldFilter, onSelectedFieldFilter, - isCompatibleField, + getNewFieldsBySpec, }: GroupedFieldsParams): GroupedFieldsResult { const fieldsExistenceReader = useExistingFieldsReader(); const fieldListFilters = useFieldFilters({ @@ -129,20 +129,24 @@ export function useGroupedFields({ // Taking care of new fields that were ingested after the selected data view was loaded // Those replace existing fields if updated, or are added to the list - const newFields = dataViewId - ? fieldsExistenceReader.getNewFields(dataViewId).filter(isCompatibleField || (() => true)) - : []; + const newFields = + dataViewId && getNewFieldsBySpec + ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataViewId), dataView) + : []; // Filtering out fields that e.g. Discover provides by analyzing the resultset, that are not part of the loaded DataView // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way - const allFieldsExlNew = allFields - ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) - : []; + const allFieldsExlNew = + allFields && newFields.length + ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) + : allFields; - const allFieldsInclNew = [...allFieldsExlNew, ...newFields] as unknown as T[]; - - const sortedFields = allFieldsInclNew.sort(sortFields); - allFieldsToReturn.current = sortedFields; + const allFieldsInclNew = newFields.length + ? [...(allFieldsExlNew || []), ...newFields] + : allFields || []; + allFieldsToReturn.current = newFields.length ? allFieldsInclNew : allFields; hasNewFields.current = Boolean(newFields.length); + const sortedFields = [...allFieldsInclNew].sort(sortFields); + const groupedFields = { ...getDefaultFieldGroups(), ...groupBy(sortedFields, (field) => { @@ -347,7 +351,7 @@ export function useGroupedFields({ onSelectedFieldFilter, onSupportedFieldFilter, dataView, - isCompatibleField, + getNewFieldsBySpec, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -406,7 +410,7 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFields: allFieldsToReturn.current, + allFieldsModified: allFieldsToReturn.current, hasNewFields: hasNewFields.current, }; } diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 75ac7b159f113..41a1e53a33849 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -7,7 +7,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { type DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; +import type { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; @@ -58,9 +58,7 @@ export async function fetchFieldExistence({ // take care of fields of existingFieldList, that are not yet available // in the given data view. Those fields we consider as new fields, // that were ingested after the data view was loaded - const newFields = existingFieldList - .filter((field) => !dataView.getFieldByName(field.name)) - .map((field) => new DataViewField(field)); + const newFields = existingFieldList.filter((field) => !dataView.getFieldByName(field.name)); // refresh the data view in case there are new fields if (newFields.length) { await dataViewsService.refreshFields(dataView, false); @@ -78,19 +76,23 @@ export async function fetchFieldExistence({ */ export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] { return indexPattern.fields.map((field) => { - return { - name: field.name, - isScript: !!field.scripted, - lang: field.lang, - script: field.script, - // id is a special case - it doesn't show up in the meta field list, - // but as it's not part of source, it has to be handled separately. - isMeta: metaFields?.includes(field.name) || field.name === '_id', - runtimeField: !field.isMapped ? field.runtimeField : undefined, - }; + return buildField(field, metaFields); }); } +export function buildField(field: DataViewField, metaFields: string[]): Field { + return { + name: field.name, + isScript: !!field.scripted, + lang: field.lang, + script: field.script, + // id is a special case - it doesn't show up in the meta field list, + // but as it's not part of source, it has to be handled separately. + isMeta: metaFields?.includes(field.name) || field.name === '_id', + runtimeField: !field.isMapped ? field.runtimeField : undefined, + }; +} + function toQuery( timeFieldName: string | undefined, fromDate: string | undefined, diff --git a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts index 889c92008554b..590bc46eff573 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts @@ -9,7 +9,7 @@ import { IUiSettingsClient } from '@kbn/core/public'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import { UI_SETTINGS } from '@kbn/data-service/src/constants'; -import type { DataView, DataViewField, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common'; import { lastValueFrom } from 'rxjs'; import { fetchFieldExistence } from './field_existing_utils'; @@ -27,7 +27,7 @@ interface FetchFieldExistenceParams { export type LoadFieldExistingHandler = (params: FetchFieldExistenceParams) => Promise<{ existingFieldNames: string[]; indexPatternTitle: string; - newFields?: DataViewField[]; + newFields?: FieldSpec[]; }>; export const loadFieldExisting: LoadFieldExistingHandler = async ({ diff --git a/x-pack/plugins/lens/public/data_views_service/loader.ts b/x-pack/plugins/lens/public/data_views_service/loader.ts index 784c97d832e34..00bd291ef9c24 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.ts @@ -6,7 +6,12 @@ */ import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; -import type { DataViewsContract, DataView, DataViewSpec } from '@kbn/data-views-plugin/public'; +import { + DataViewsContract, + DataView, + DataViewSpec, + DataViewField, +} from '@kbn/data-views-plugin/public'; import { keyBy } from 'lodash'; import { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef } from '../types'; import { documentField } from '../datasources/form_based/document_field'; @@ -29,49 +34,9 @@ export function convertDataViewIntoLensIndexPattern( dataView: DataView, restrictionRemapper: (name: string) => string = onRestrictionMapping ): IndexPattern { - const metaKeys = new Set(dataView.metaFields); const newFields = dataView.fields .filter(isFieldLensCompatible) - .map((field): IndexPatternField => { - // Convert the getters on the index pattern service into plain JSON - const base = { - name: field.name, - displayName: field.displayName, - type: field.type, - aggregatable: field.aggregatable, - filterable: field.filterable, - searchable: field.searchable, - meta: metaKeys.has(field.name), - esTypes: field.esTypes, - scripted: field.scripted, - isMapped: field.isMapped, - customLabel: field.customLabel, - runtimeField: field.runtimeField, - runtime: Boolean(field.runtimeField), - timeSeriesDimension: field.timeSeriesDimension, - timeSeriesMetric: field.timeSeriesMetric, - timeSeriesRollup: field.isRolledUpField, - partiallyApplicableFunctions: field.isRolledUpField - ? { - percentile: true, - percentile_rank: true, - median: true, - last_value: true, - unique_count: true, - standard_deviation: true, - } - : undefined, - }; - - // Simplifies tests by hiding optional properties instead of undefined - return base.scripted - ? { - ...base, - lang: field.lang, - script: field.script, - } - : base; - }) + .map((field) => buildIndexPatternField(field, dataView)) .concat(documentField); const { typeMeta, title, name, timeFieldName, fieldFormatMap } = dataView; @@ -113,6 +78,51 @@ export function convertDataViewIntoLensIndexPattern( }; } +export function buildIndexPatternField( + field: DataViewField, + dataView: DataView | null +): IndexPatternField { + const meta = dataView?.metaFields.includes(field.name); + // Convert the getters on the index pattern service into plain JSON + const base = { + name: field.name, + displayName: field.displayName, + type: field.type, + aggregatable: field.aggregatable, + filterable: field.filterable, + searchable: field.searchable, + meta, + esTypes: field.esTypes, + scripted: field.scripted, + isMapped: field.isMapped, + customLabel: field.customLabel, + runtimeField: field.runtimeField, + runtime: Boolean(field.runtimeField), + timeSeriesDimension: field.timeSeriesDimension, + timeSeriesMetric: field.timeSeriesMetric, + timeSeriesRollup: field.isRolledUpField, + partiallyApplicableFunctions: field.isRolledUpField + ? { + percentile: true, + percentile_rank: true, + median: true, + last_value: true, + unique_count: true, + standard_deviation: true, + } + : undefined, + }; + + // Simplifies tests by hiding optional properties instead of undefined + return base.scripted + ? { + ...base, + lang: field.lang, + script: field.script, + } + : base; +} + export async function loadIndexPatternRefs( dataViews: MinimalDataViewsContract ): Promise { diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 1d7b3603482c8..5108cfbf8019a 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -13,7 +13,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { CoreStart } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { type DataView } from '@kbn/data-plugin/common'; +import { type DataView, DataViewField } from '@kbn/data-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public'; import { VISUALIZE_GEO_FIELD_TRIGGER } from '@kbn/ui-actions-plugin/public'; @@ -28,7 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; -import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; +import { buildIndexPatternField } from '../../data_views_service/loader'; import type { DatasourceDataPanelProps, FramePublicAPI, @@ -262,7 +262,11 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ onSupportedFieldFilter, onSelectedFieldFilter, onOverrideFieldGroupDetails, - isCompatibleField: isFieldLensCompatible, + getNewFieldsBySpec: (spec, dataView) => { + return spec.map((fieldSpec) => + buildIndexPatternField(new DataViewField(fieldSpec), dataView) + ); + }, }); const closeFieldEditor = useRef<() => void | undefined>(); diff --git a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx index ac62636ad50c1..113125484cddf 100644 --- a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx @@ -24,7 +24,6 @@ import { GetCustomFieldType, useGroupedFields, } from '@kbn/unified-field-list'; -import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import type { DatasourceDataPanelProps } from '../../types'; import type { TextBasedPrivateState } from './types'; import { getStateFromAggregateQuery } from './utils'; @@ -106,7 +105,6 @@ export function TextBasedDataPanel({ getCustomFieldType, onSelectedFieldFilter, onOverrideFieldGroupDetails, - isCompatibleField: isFieldLensCompatible, }); const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( From 30dfec6cc0a971a298beb17c9eaa1587e3b8e0f8 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 20 Dec 2023 17:09:56 +0100 Subject: [PATCH 16/29] Add isFieldLensCompatible for new fields in Lens --- .../public/datasources/form_based/datapanel.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 5108cfbf8019a..6ee8483658e68 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -13,7 +13,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { CoreStart } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { type DataView, DataViewField } from '@kbn/data-plugin/common'; +import { type DataView, DataViewField, FieldSpec } from '@kbn/data-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public'; import { VISUALIZE_GEO_FIELD_TRIGGER } from '@kbn/ui-actions-plugin/public'; @@ -28,6 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; +import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import { buildIndexPatternField } from '../../data_views_service/loader'; import type { DatasourceDataPanelProps, @@ -263,9 +264,13 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ onSelectedFieldFilter, onOverrideFieldGroupDetails, getNewFieldsBySpec: (spec, dataView) => { - return spec.map((fieldSpec) => - buildIndexPatternField(new DataViewField(fieldSpec), dataView) - ); + return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { + const field = new DataViewField(fieldSpec); + if (isFieldLensCompatible(field)) { + result.push(buildIndexPatternField(field, dataView)); + } + return result; + }, []); }, }); From f747c2e945cb383f8ed04e54a82ff1abc95d20a5 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 21 Dec 2023 07:10:11 +0100 Subject: [PATCH 17/29] Improve multi field code --- .../field_list_item.tsx | 18 +++++--- .../field_list_sidebar.tsx | 41 +++++++++---------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx index 745d463b28386..306d348855202 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx @@ -67,9 +67,14 @@ function getCommonFieldItemButtonProps({ }; } +export interface FieldListMultiField { + field: DataViewField; + isSelected: boolean; +} + interface MultiFieldsProps { stateService: UnifiedFieldListSidebarContainerStateService; - multiFields: NonNullable; + multiFields: NonNullable; toggleDisplay: (field: DataViewField) => void; alwaysShowActionButton: boolean; size: FieldItemButtonProps['size']; @@ -163,9 +168,9 @@ export interface UnifiedFieldListItemProps { */ trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; /** - * Multi fields for the current field + * Get multi fields for the given field */ - multiFields?: Array<{ field: DataViewField; isSelected: boolean }>; + getMultiFields?: (field: DataViewField) => FieldListMultiField[] | undefined; /** * Callback to edit a field from data view * @param fieldName name of the field to edit @@ -212,7 +217,7 @@ function UnifiedFieldListItemComponent({ isEmpty, isSelected, trackUiMetric, - multiFields, + getMultiFields, onEditField, onDeleteField, workspaceSelectedFieldNames, @@ -253,8 +258,6 @@ function UnifiedFieldListItemComponent({ [onAddFieldToWorkspace, onRemoveFieldFromWorkspace, closePopover] ); - const rawMultiFields = useMemo(() => multiFields?.map((f) => f.field), [multiFields]); - const customPopoverHeaderProps: Partial = useMemo(() => { const dataTestSubjPrefix = stateService.creationOptions.dataTestSubj?.fieldListItemPopoverHeaderDataTestSubjPrefix; @@ -275,6 +278,9 @@ function UnifiedFieldListItemComponent({ }, [field.name, stateService.creationOptions]); const renderPopover = () => { + const multiFields = getMultiFields?.(field); + const rawMultiFields = multiFields?.map((f) => f.field); + return ( <> ( INITIAL_SELECTED_FIELDS_RESULT ); - const [multiFieldsMap, setMultiFieldsMap] = useState< - Map> | undefined - >(undefined); useEffect(() => { const result = getSelectedFields({ @@ -230,27 +228,28 @@ export const UnifiedFieldListSidebarComponent: React.FC { + const useMultiFields = useMemo(() => { if ( searchMode !== 'documents' || !useNewFieldsApi || stateService.creationOptions.disableMultiFieldsGroupingByParent ) { - setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case + return false; } else { - setMultiFieldsMap( - calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap) - ); + return true; } }, [ stateService.creationOptions.disableMultiFieldsGroupingByParent, - selectedFieldsState.selectedFieldsMap, - allFieldsModified, useNewFieldsApi, - setMultiFieldsMap, searchMode, ]); + const getMultiFieldsByField = useCallback( + (field: DataViewField) => + getMultiFieldsByParent(field, allFieldsModified, selectedFieldsState.selectedFieldsMap), + [allFieldsModified, selectedFieldsState.selectedFieldsMap] + ); + const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => (
  • @@ -267,7 +266,7 @@ export const UnifiedFieldListSidebarComponent: React.FC>(); + const result: FieldListMultiField[] = []; allFields.forEach((field) => { const subTypeMulti = getDataViewFieldSubtypeMulti(field); const parent = subTypeMulti?.multi.parent; - if (!parent) { + if (!parent || parent !== parentField.name) { return; } const multiField = { field, isSelected: Boolean(selectedFieldsMap?.[field.name]), }; - const value = map.get(parent) ?? []; - value.push(multiField); - map.set(parent, value); + result.push(multiField); }); - return map; + return result.length ? result : undefined; } From 9ed0c495247bbf63d742effa90260b31570c08ef Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 21 Dec 2023 08:50:45 +0100 Subject: [PATCH 18/29] Improve code --- .../field_list_sidebar.tsx | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 3ed0e7dcbf68b..a98b8567aad78 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -223,26 +223,14 @@ export const UnifiedFieldListSidebarComponent: React.FC { - return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)); - }, + getNewFieldsBySpec: (fieldSpecArr) => + fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)), }); - const useMultiFields = useMemo(() => { - if ( - searchMode !== 'documents' || - !useNewFieldsApi || - stateService.creationOptions.disableMultiFieldsGroupingByParent - ) { - return false; - } else { - return true; - } - }, [ - stateService.creationOptions.disableMultiFieldsGroupingByParent, - useNewFieldsApi, - searchMode, - ]); + const useMultiFields = + searchMode === 'documents' && + useNewFieldsApi && + !stateService.creationOptions.disableMultiFieldsGroupingByParent; const getMultiFieldsByField = useCallback( (field: DataViewField) => From 5420c7708910b945b85434bd5549f0a559301016 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 21 Dec 2023 08:51:05 +0100 Subject: [PATCH 19/29] Remove redundant field list refreshing --- .../public/dataview_picker/change_dataview.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx index 9076bcb37c7df..2d857de608834 100644 --- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx +++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx @@ -307,12 +307,6 @@ export function ChangeDataView({ isTextBasedLangSelected={isTextBasedLangSelected} setPopoverIsOpen={setPopoverIsOpen} onChangeDataView={async (newId) => { - try { - // refreshing the field list - await dataViews.get(newId, false, true); - } catch (e) { - // - } setSelectedDataViewId(newId); setPopoverIsOpen(false); if (isTextBasedLangSelected && !isTextLangTransitionModalDismissed) { From f340fae4cae5bfe9533fd3c8bcf082248c58b8b3 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 12:09:49 +0100 Subject: [PATCH 20/29] Undo multi field changes --- .../field_list_item.tsx | 18 +++---- .../field_list_sidebar.tsx | 54 +++++++++++-------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx index 306d348855202..745d463b28386 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx @@ -67,14 +67,9 @@ function getCommonFieldItemButtonProps({ }; } -export interface FieldListMultiField { - field: DataViewField; - isSelected: boolean; -} - interface MultiFieldsProps { stateService: UnifiedFieldListSidebarContainerStateService; - multiFields: NonNullable; + multiFields: NonNullable; toggleDisplay: (field: DataViewField) => void; alwaysShowActionButton: boolean; size: FieldItemButtonProps['size']; @@ -168,9 +163,9 @@ export interface UnifiedFieldListItemProps { */ trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; /** - * Get multi fields for the given field + * Multi fields for the current field */ - getMultiFields?: (field: DataViewField) => FieldListMultiField[] | undefined; + multiFields?: Array<{ field: DataViewField; isSelected: boolean }>; /** * Callback to edit a field from data view * @param fieldName name of the field to edit @@ -217,7 +212,7 @@ function UnifiedFieldListItemComponent({ isEmpty, isSelected, trackUiMetric, - getMultiFields, + multiFields, onEditField, onDeleteField, workspaceSelectedFieldNames, @@ -258,6 +253,8 @@ function UnifiedFieldListItemComponent({ [onAddFieldToWorkspace, onRemoveFieldFromWorkspace, closePopover] ); + const rawMultiFields = useMemo(() => multiFields?.map((f) => f.field), [multiFields]); + const customPopoverHeaderProps: Partial = useMemo(() => { const dataTestSubjPrefix = stateService.creationOptions.dataTestSubj?.fieldListItemPopoverHeaderDataTestSubjPrefix; @@ -278,9 +275,6 @@ function UnifiedFieldListItemComponent({ }, [field.name, stateService.creationOptions]); const renderPopover = () => { - const multiFields = getMultiFields?.(field); - const rawMultiFields = multiFields?.map((f) => f.field); - return ( <> ( INITIAL_SELECTED_FIELDS_RESULT ); + const [multiFieldsMap, setMultiFieldsMap] = useState< + Map> | undefined + >(undefined); useEffect(() => { const result = getSelectedFields({ @@ -227,16 +229,26 @@ export const UnifiedFieldListSidebarComponent: React.FC new DataViewField(fieldSpec)), }); - const useMultiFields = - searchMode === 'documents' && - useNewFieldsApi && - !stateService.creationOptions.disableMultiFieldsGroupingByParent; - - const getMultiFieldsByField = useCallback( - (field: DataViewField) => - getMultiFieldsByParent(field, allFieldsModified, selectedFieldsState.selectedFieldsMap), - [allFieldsModified, selectedFieldsState.selectedFieldsMap] - ); + useEffect(() => { + if ( + searchMode !== 'documents' || + !useNewFieldsApi || + stateService.creationOptions.disableMultiFieldsGroupingByParent + ) { + setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case + } else { + setMultiFieldsMap( + calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap) + ); + } + }, [ + stateService.creationOptions.disableMultiFieldsGroupingByParent, + selectedFieldsState.selectedFieldsMap, + allFieldsModified, + useNewFieldsApi, + setMultiFieldsMap, + searchMode, + ]); const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => ( @@ -254,7 +266,7 @@ export const UnifiedFieldListSidebarComponent: React.FC>(); allFields.forEach((field) => { const subTypeMulti = getDataViewFieldSubtypeMulti(field); const parent = subTypeMulti?.multi.parent; - if (!parent || parent !== parentField.name) { + if (!parent) { return; } const multiField = { field, isSelected: Boolean(selectedFieldsMap?.[field.name]), }; - result.push(multiField); + const value = map.get(parent) ?? []; + value.push(multiField); + map.set(parent, value); }); - return result.length ? result : undefined; + return map; } From 26c4d72c10ff4f4f3a70209a679625eba20d2bec Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:20:42 +0100 Subject: [PATCH 21/29] Address feedback --- .../field_list_sidebar.tsx | 9 ++- .../src/hooks/use_existing_fields.ts | 9 +-- .../src/hooks/use_grouped_fields.ts | 61 ++++++++----------- .../src/hooks/use_new_fields.ts | 51 ++++++++++++++++ 4 files changed, 83 insertions(+), 47 deletions(-) create mode 100644 packages/kbn-unified-field-list/src/hooks/use_new_fields.ts diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 7b4e69b3ea739..17ebe5b19286f 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -22,7 +22,7 @@ import { useEuiTheme, } from '@elastic/eui'; import { ToolbarButton } from '@kbn/shared-ux-button-toolbar'; -import { DataViewField } from '@kbn/data-views-plugin/common'; +import { DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils'; import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils'; import { FieldList } from '../../components/field_list'; @@ -225,8 +225,7 @@ export const UnifiedFieldListSidebarComponent: React.FC - fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)), + getNewFieldsBySpec, }); useEffect(() => { @@ -461,3 +460,7 @@ function calculateMultiFields( }); return map; } + +function getNewFieldsBySpec(fieldSpecArr: FieldSpec[]): DataViewField[] { + return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)); +} diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index 7826c815b9af3..a733643008b11 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -120,7 +120,7 @@ export const useExistingFieldsFetcher = ( setActiveRequests((value) => value + 1); - const hasRestrictions = Boolean(dataView.getAggregationRestrictions?.()); + const hasRestrictions = Boolean(dataView?.getAggregationRestrictions?.()); const info: ExistingFieldsInfo = { ...unknownInfo, numberOfFetches, @@ -254,12 +254,7 @@ export const useExistingFieldsFetcher = ( ); }; -export const useExistingFieldsReader: () => { - getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; - isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; - getNewFields: (dataViewId: string) => FieldSpec[]; - hasFieldData: (dataViewId: string, fieldName: string) => boolean; -} = () => { +export const useExistingFieldsReader: () => ExistingFieldsReader = () => { const mountedRef = useRef(true); const [existingFieldsByDataViewMap, setExistingFieldsByDataViewMap] = useState(globalMap$.getValue()); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index bfc32be695120..381803ac74345 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -7,11 +7,12 @@ */ import { groupBy } from 'lodash'; -import { useEffect, useMemo, useState, useRef } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; import type { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; import { type DataViewsContract } from '@kbn/data-views-plugin/public'; +import { useNewFields } from './use_new_fields'; import { type FieldListGroups, type FieldsGroup, @@ -41,7 +42,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - getNewFieldsBySpec?: (field: FieldSpec[], dataView: DataView | null) => T[]; + getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[]; } export interface GroupedFieldsResult { @@ -77,8 +78,7 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); - const allFieldsToReturn = useRef(allFields); - const hasNewFields = useRef(false); + const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName); @@ -107,6 +107,13 @@ export function useGroupedFields({ // if field existence information changed, reload the data view too }, [dataViewId, services.dataViews, setDataView, hasFieldDataHandler]); + const { allFieldsModified, hasNewFields } = useNewFields({ + dataView, + allFields, + getNewFieldsBySpec, + fieldsExistenceReader, + }); + // important when switching from a known dataViewId to no data view (like in text-based queries) useEffect(() => { if (dataView && !dataViewId) { @@ -127,25 +134,7 @@ export function useGroupedFields({ const selectedFields = sortedSelectedFields || []; - // Taking care of new fields that were ingested after the selected data view was loaded - // Those replace existing fields if updated, or are added to the list - const newFields = - dataViewId && getNewFieldsBySpec - ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataViewId), dataView) - : []; - // Filtering out fields that e.g. Discover provides by analyzing the resultset, that are not part of the loaded DataView - // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way - const allFieldsExlNew = - allFields && newFields.length - ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) - : allFields; - - const allFieldsInclNew = newFields.length - ? [...(allFieldsExlNew || []), ...newFields] - : allFields || []; - allFieldsToReturn.current = newFields.length ? allFieldsInclNew : allFields; - hasNewFields.current = Boolean(newFields.length); - const sortedFields = [...allFieldsInclNew].sort(sortFields); + const sortedFields = [...(allFieldsModified || [])].sort(sortFields); const groupedFields = { ...getDefaultFieldGroups(), @@ -338,20 +327,18 @@ export function useGroupedFields({ return fieldGroupDefinitions; }, [ - sortedSelectedFields, - allFields, - popularFieldsLimit, - isAffectedByGlobalFilter, - isAffectedByTimeFilter, - dataViewId, - fieldsExistenceInfoUnavailable, - onOverrideFieldGroupDetails, - hasFieldDataHandler, - fieldsExistenceReader, - onSelectedFieldFilter, + allFieldsModified, onSupportedFieldFilter, + onSelectedFieldFilter, + onOverrideFieldGroupDetails, dataView, - getNewFieldsBySpec, + dataViewId, + hasFieldDataHandler, + fieldsExistenceInfoUnavailable, + isAffectedByGlobalFilter, + isAffectedByTimeFilter, + popularFieldsLimit, + sortedSelectedFields, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -410,8 +397,8 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFieldsModified: allFieldsToReturn.current, - hasNewFields: hasNewFields.current, + allFieldsModified, + hasNewFields, }; } diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts new file mode 100644 index 0000000000000..5f16ad63a1101 --- /dev/null +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { useMemo } from 'react'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; +import { ExistingFieldsReader, type FieldListItem, GroupedFieldsParams } from '../..'; + +/** + * This hook is used to get the new fields of previous fields for wildcards request, and merges those + * with the existing fields. + * @param dataView + * @param allFields + * @param getNewFieldsBySpec + * @param fieldsExistenceReader + */ +export function useNewFields({ + dataView, + allFields, + getNewFieldsBySpec, + fieldsExistenceReader, +}: { + dataView?: DataView | null; + allFields: GroupedFieldsParams['allFields']; + getNewFieldsBySpec: GroupedFieldsParams['getNewFieldsBySpec']; + fieldsExistenceReader: ExistingFieldsReader; +}) { + const newFields = + allFields && dataView?.id && getNewFieldsBySpec + ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) + : null; + const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); + + const allFieldsModified = useMemo(() => { + if (!allFields || !newFields?.length) return allFields; + // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request + // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way + const allFieldsExlNew = + allFields && newFields + ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) + : allFields; + + return newFields ? [...allFieldsExlNew, ...newFields] : allFields; + }, [newFields, allFields]); + + return { allFieldsModified, hasNewFields }; +} From 921a59825b51f1d68e7c99438a78c660d4a3ec4b Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:48:44 +0100 Subject: [PATCH 22/29] Address review feedback --- packages/kbn-unified-field-list/src/hooks/use_new_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index 5f16ad63a1101..a03b319858738 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -36,7 +36,7 @@ export function useNewFields({ const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); const allFieldsModified = useMemo(() => { - if (!allFields || !newFields?.length) return allFields; + if (!allFields || !newFields?.length || !dataView) return allFields; // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way const allFieldsExlNew = From ec55f89b965903975b541f9d2bb7f7fadd446490 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:49:50 +0100 Subject: [PATCH 23/29] Fix error --- packages/kbn-unified-field-list/src/hooks/use_new_fields.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index a03b319858738..b866aea306d2b 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -13,10 +13,6 @@ import { ExistingFieldsReader, type FieldListItem, GroupedFieldsParams } from '. /** * This hook is used to get the new fields of previous fields for wildcards request, and merges those * with the existing fields. - * @param dataView - * @param allFields - * @param getNewFieldsBySpec - * @param fieldsExistenceReader */ export function useNewFields({ dataView, @@ -45,7 +41,7 @@ export function useNewFields({ : allFields; return newFields ? [...allFieldsExlNew, ...newFields] : allFields; - }, [newFields, allFields]); + }, [newFields, allFields, dataView]); return { allFieldsModified, hasNewFields }; } From c76c87e375c9e639e86b9fa4cf802f55d7ddc497 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:57:07 +0100 Subject: [PATCH 24/29] Add useMemo for new fields --- packages/kbn-unified-field-list/src/hooks/use_new_fields.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index b866aea306d2b..ee18f31363fab 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -25,10 +25,12 @@ export function useNewFields({ getNewFieldsBySpec: GroupedFieldsParams['getNewFieldsBySpec']; fieldsExistenceReader: ExistingFieldsReader; }) { - const newFields = - allFields && dataView?.id && getNewFieldsBySpec + const newFields = useMemo(() => { + return allFields && dataView?.id && getNewFieldsBySpec ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) : null; + }, [allFields, dataView, fieldsExistenceReader, getNewFieldsBySpec]); + const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); const allFieldsModified = useMemo(() => { From 165c3cde88970ed8f0794e4c1e8ffdc3bd4504f6 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 16:04:53 +0100 Subject: [PATCH 25/29] Address review feedback --- .../datasources/form_based/datapanel.tsx | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 6ee8483658e68..2674293da399a 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -263,15 +263,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ onSupportedFieldFilter, onSelectedFieldFilter, onOverrideFieldGroupDetails, - getNewFieldsBySpec: (spec, dataView) => { - return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { - const field = new DataViewField(fieldSpec); - if (isFieldLensCompatible(field)) { - result.push(buildIndexPatternField(field, dataView)); - } - return result; - }, []); - }, + getNewFieldsBySpec, }); const closeFieldEditor = useRef<() => void | undefined>(); @@ -428,4 +420,14 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ ); }; +function getNewFieldsBySpec(spec: FieldSpec[], dataView: DataView | null) { + return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { + const field = new DataViewField(fieldSpec); + if (isFieldLensCompatible(field)) { + result.push(buildIndexPatternField(field, dataView)); + } + return result; + }, []); +} + export const MemoizedDataPanel = memo(InnerFormBasedDataPanel); From 9170300a6a04c02cccadceb1dab38c27ed29a7ab Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:17:40 +0000 Subject: [PATCH 26/29] [CI] Auto-commit changed files from 'node scripts/notice' --- NOTICE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE.txt b/NOTICE.txt index 45af6e5231783..d02031c4b5a2b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Kibana source code with Kibana X-Pack source code -Copyright 2012-2023 Elasticsearch B.V. +Copyright 2012-2024 Elasticsearch B.V. --- Pretty handling of logarithmic axes. From 02a4537c466e228687adaf78d2281740f37faf04 Mon Sep 17 00:00:00 2001 From: Julia Rechkunova Date: Wed, 3 Jan 2024 14:33:18 +0100 Subject: [PATCH 27/29] [Discover] Small fixes for types and memo deps. Add tests. --- .../field_list_sidebar.tsx | 2 +- .../src/hooks/use_existing_fields.test.tsx | 48 +++++++++++ .../src/hooks/use_existing_fields.ts | 5 +- .../src/hooks/use_grouped_fields.test.tsx | 78 ++++++++++++++++++ .../src/hooks/use_grouped_fields.ts | 6 +- .../src/hooks/use_new_fields.test.tsx | 80 +++++++++++++++++++ .../src/hooks/use_new_fields.ts | 47 +++++++---- 7 files changed, 243 insertions(+), 23 deletions(-) create mode 100644 packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 17ebe5b19286f..f8438b0917577 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -22,7 +22,7 @@ import { useEuiTheme, } from '@elastic/eui'; import { ToolbarButton } from '@kbn/shared-ux-button-toolbar'; -import { DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; +import { DataViewField, type FieldSpec } from '@kbn/data-views-plugin/common'; import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils'; import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils'; import { FieldList } from '../../components/field_list'; diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx index 7cf22950a4a7d..0af0e4abc8e0d 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx @@ -125,6 +125,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(hookReader.result.current.getFieldsExistenceStatus(dataViewId)).toBe( ExistenceFetchStatus.succeeded ); + expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]); // does not have existence info => works less restrictive const anotherDataViewId = 'test-id'; @@ -140,6 +141,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(hookReader.result.current.getFieldsExistenceStatus(anotherDataViewId)).toBe( ExistenceFetchStatus.unknown ); + expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]); }); it('should work correctly with multiple readers', async () => { @@ -217,6 +219,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(currentResult.isFieldsExistenceInfoUnavailable(dataViewId)).toBe(true); expect(currentResult.hasFieldData(dataViewId, dataView.fields[0].name)).toBe(true); expect(currentResult.getFieldsExistenceStatus(dataViewId)).toBe(ExistenceFetchStatus.failed); + expect(currentResult.getNewFields(dataViewId)).toStrictEqual([]); }); it('should work correctly for multiple data views', async () => { @@ -533,4 +536,49 @@ describe('UnifiedFieldList useExistingFields', () => { expect(params.onNoData).toHaveBeenCalledTimes(1); // still 1 time }); + + it('should include newFields', async () => { + const newFields = [{ name: 'test', type: 'keyword', searchable: true, aggregatable: true }]; + + (ExistingFieldsServiceApi.loadFieldExisting as jest.Mock).mockImplementation( + async ({ dataView: currentDataView }) => { + return { + existingFieldNames: [currentDataView.fields[0].name], + newFields, + }; + } + ); + + const params: ExistingFieldsFetcherParams = { + dataViews: [dataView], + services: mockedServices, + fromDate: '2019-01-01', + toDate: '2020-01-01', + query: { query: '', language: 'lucene' }, + filters: [], + }; + const hookFetcher = renderHook(useExistingFieldsFetcher, { + initialProps: params, + }); + + const hookReader = renderHook(useExistingFieldsReader); + await hookFetcher.waitForNextUpdate(); + + expect(ExistingFieldsServiceApi.loadFieldExisting).toHaveBeenCalledWith( + expect.objectContaining({ + fromDate: '2019-01-01', + toDate: '2020-01-01', + dslQuery, + dataView, + timeFieldName: dataView.timeFieldName, + }) + ); + + expect(hookReader.result.current.getFieldsExistenceStatus(dataView.id!)).toBe( + ExistenceFetchStatus.succeeded + ); + + expect(hookReader.result.current.getNewFields(dataView.id!)).toBe(newFields); + expect(hookReader.result.current.getNewFields('another-id')).toStrictEqual([]); + }); }); diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index a733643008b11..e71d89d6b3626 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -20,6 +20,7 @@ import { ExistenceFetchStatus } from '../types'; const getBuildEsQueryAsync = async () => (await import('@kbn/es-query')).buildEsQuery; const generateId = htmlIdGenerator(); +const DEFAULT_EMPTY_NEW_FIELDS: FieldSpec[] = []; export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; @@ -294,10 +295,10 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { const info = existingFieldsByDataViewMap[dataViewId]; if (info?.fetchStatus === ExistenceFetchStatus.succeeded) { - return info?.newFields ?? []; + return info?.newFields ?? DEFAULT_EMPTY_NEW_FIELDS; } - return []; + return DEFAULT_EMPTY_NEW_FIELDS; }, [existingFieldsByDataViewMap] ); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx index 78b34329e0bf0..4a937f86bd5c0 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx @@ -187,6 +187,8 @@ describe('UnifiedFieldList useGroupedFields()', () => { expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); rerender({ ...props, @@ -200,6 +202,82 @@ describe('UnifiedFieldList useGroupedFields()', () => { expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe( scrollToTopResetCounter1 ); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); + + (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore(); + }); + + it('should work correctly with new fields', async () => { + const props: GroupedFieldsParams = { + dataViewId: dataView.id!, + allFields, + services: mockedServices, + getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)), + }; + + const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true }; + + jest.spyOn(ExistenceApi, 'useExistingFieldsReader').mockImplementation( + (): ExistingFieldsReader => ({ + hasFieldData: (dataViewId) => { + return dataViewId === props.dataViewId; + }, + getFieldsExistenceStatus: (dataViewId) => + dataViewId === props.dataViewId + ? ExistenceFetchStatus.succeeded + : ExistenceFetchStatus.unknown, + isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [newField], + }) + ); + + const { result, waitForNextUpdate, rerender } = renderHook(useGroupedFields, { + initialProps: props, + }); + + await waitForNextUpdate(); + + let fieldListGroupedProps = result.current.fieldListGroupedProps; + const fieldGroups = fieldListGroupedProps.fieldGroups; + const scrollToTopResetCounter1 = fieldListGroupedProps.scrollToTopResetCounter; + + expect( + Object.keys(fieldGroups!).map( + (key) => `${key}-${fieldGroups![key as FieldsGroupNames]?.fields.length}` + ) + ).toStrictEqual([ + 'SpecialFields-0', + 'SelectedFields-0', + 'PopularFields-0', + 'AvailableFields-25', + 'UnmappedFields-1', + 'EmptyFields-0', + 'MetaFields-3', + ]); + + expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); + expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.allFieldsModified).toStrictEqual([ + ...allFields, + new DataViewField(newField), + ]); + expect(result.current.hasNewFields).toBe(true); + + rerender({ + ...props, + dataViewId: null, // for text-based queries + allFields, + }); + + fieldListGroupedProps = result.current.fieldListGroupedProps; + expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); + expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe( + scrollToTopResetCounter1 + ); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore(); }); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index a11d897c37199..7853c7e67800b 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -10,9 +10,9 @@ import { groupBy } from 'lodash'; import { useEffect, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; -import type { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { type DataViewsContract } from '@kbn/data-views-plugin/public'; -import { useNewFields } from './use_new_fields'; +import { type UseNewFieldsParams, useNewFields } from './use_new_fields'; import { type FieldListGroups, type FieldsGroup, @@ -42,7 +42,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[]; + getNewFieldsBySpec?: UseNewFieldsParams['getNewFieldsBySpec']; } export interface GroupedFieldsResult { diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx new file mode 100644 index 0000000000000..2be5d50764bc9 --- /dev/null +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { stubLogstashDataView as dataView } from '@kbn/data-views-plugin/common/data_view.stub'; +import { DataViewField } from '@kbn/data-views-plugin/common'; +import { useNewFields, type UseNewFieldsParams } from './use_new_fields'; +import { type ExistingFieldsReader } from './use_existing_fields'; +import { ExistenceFetchStatus } from '../types'; + +const fieldsExistenceReader: ExistingFieldsReader = { + hasFieldData: (dataViewId) => { + return dataViewId === dataView.id; + }, + getFieldsExistenceStatus: (dataViewId) => + dataViewId === dataView.id ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, + isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== dataView.id, + getNewFields: () => [], +}; + +describe('UnifiedFieldList useNewFields()', () => { + const allFields = dataView.fields; + + it('should work correctly in loading state', async () => { + const props: UseNewFieldsParams = { + dataView, + allFields: null, + fieldsExistenceReader, + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toBe(null); + expect(result.current.hasNewFields).toBe(false); + }); + + it('should work correctly with empty new fields', async () => { + const props: UseNewFieldsParams = { + dataView, + allFields, + fieldsExistenceReader, + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); + }); + + it('should work correctly with new fields', async () => { + const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true }; + const newField2 = { ...newField, name: 'test2' }; + const props: UseNewFieldsParams = { + dataView, + allFields, + fieldsExistenceReader: { + ...fieldsExistenceReader, + getNewFields: () => [newField, newField2], + }, + getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)), + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toStrictEqual([ + ...allFields, + new DataViewField(newField), + new DataViewField(newField2), + ]); + expect(result.current.hasNewFields).toBe(true); + }); +}); diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index ee18f31363fab..51e143cc524c3 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -7,8 +7,22 @@ */ import { useMemo } from 'react'; +import type { FieldSpec } from '@kbn/data-views-plugin/common'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; -import { ExistingFieldsReader, type FieldListItem, GroupedFieldsParams } from '../..'; +import type { FieldListItem } from '../types'; +import type { ExistingFieldsReader } from './use_existing_fields'; + +export interface UseNewFieldsParams { + dataView?: DataView | null; + allFields: T[] | null; // `null` is for loading indicator + getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[]; + fieldsExistenceReader: ExistingFieldsReader; +} + +export interface UseNewFieldsResult { + allFieldsModified: T[] | null; + hasNewFields: boolean; +} /** * This hook is used to get the new fields of previous fields for wildcards request, and merges those @@ -19,31 +33,30 @@ export function useNewFields({ allFields, getNewFieldsBySpec, fieldsExistenceReader, -}: { - dataView?: DataView | null; - allFields: GroupedFieldsParams['allFields']; - getNewFieldsBySpec: GroupedFieldsParams['getNewFieldsBySpec']; - fieldsExistenceReader: ExistingFieldsReader; -}) { +}: UseNewFieldsParams): UseNewFieldsResult { + const dataViewId = dataView?.id; + const newFields = useMemo(() => { - return allFields && dataView?.id && getNewFieldsBySpec - ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) - : null; + const newLoadedFields = + allFields && dataView?.id && getNewFieldsBySpec + ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) + : null; + + return newLoadedFields?.length ? newLoadedFields : null; }, [allFields, dataView, fieldsExistenceReader, getNewFieldsBySpec]); const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); const allFieldsModified = useMemo(() => { - if (!allFields || !newFields?.length || !dataView) return allFields; + if (!allFields || !newFields?.length || !dataViewId) return allFields; // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way - const allFieldsExlNew = - allFields && newFields - ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) - : allFields; + const allFieldsExlNew = allFields.filter( + (field) => !newFields.some((newField) => newField.name === field.name) + ); - return newFields ? [...allFieldsExlNew, ...newFields] : allFields; - }, [newFields, allFields, dataView]); + return [...allFieldsExlNew, ...newFields]; + }, [newFields, allFields, dataViewId]); return { allFieldsModified, hasNewFields }; } From f751a4e6c691654ea3ddd006deb6cf232e81acc1 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 9 Jan 2024 10:56:27 +0100 Subject: [PATCH 28/29] Address review feedback --- .../src/hooks/use_existing_fields.ts | 2 +- .../public/data_views_service/loader.test.ts | 37 ++++++++++++++++++- .../lens/public/data_views_service/loader.ts | 7 ++-- .../datasources/form_based/datapanel.tsx | 4 +- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index e71d89d6b3626..cd4e9ef05206f 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -121,7 +121,7 @@ export const useExistingFieldsFetcher = ( setActiveRequests((value) => value + 1); - const hasRestrictions = Boolean(dataView?.getAggregationRestrictions?.()); + const hasRestrictions = Boolean(dataView.getAggregationRestrictions?.()); const info: ExistingFieldsInfo = { ...unknownInfo, numberOfFetches, diff --git a/x-pack/plugins/lens/public/data_views_service/loader.test.ts b/x-pack/plugins/lens/public/data_views_service/loader.test.ts index f91d236986b11..4c648a7782896 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.test.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.test.ts @@ -5,8 +5,13 @@ * 2.0. */ -import { DataViewsContract } from '@kbn/data-views-plugin/public'; -import { ensureIndexPattern, loadIndexPatternRefs, loadIndexPatterns } from './loader'; +import { DataViewsContract, DataViewField } from '@kbn/data-views-plugin/public'; +import { + ensureIndexPattern, + loadIndexPatternRefs, + loadIndexPatterns, + buildIndexPatternField, +} from './loader'; import { sampleIndexPatterns, mockDataViewsService } from './mocks'; import { documentField } from '../datasources/form_based/document_field'; @@ -313,4 +318,32 @@ describe('loader', () => { expect(onError).not.toHaveBeenCalled(); }); }); + + describe('buildIndexPatternField', () => { + it('should return a field with the correct name and derived parameters', async () => { + const field = buildIndexPatternField({ + name: 'foo', + displayName: 'Foo', + type: 'string', + aggregatable: true, + searchable: true, + } as DataViewField); + expect(field.name).toEqual('foo'); + expect(field.meta).toEqual(false); + expect(field.runtime).toEqual(false); + }); + it('should return return the right meta field value', async () => { + const field = buildIndexPatternField( + { + name: 'meta', + displayName: 'Meta', + type: 'string', + aggregatable: true, + searchable: true, + } as DataViewField, + new Set(['meta']) + ); + expect(field.meta).toEqual(true); + }); + }); }); diff --git a/x-pack/plugins/lens/public/data_views_service/loader.ts b/x-pack/plugins/lens/public/data_views_service/loader.ts index 00bd291ef9c24..8a52146991b8d 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.ts @@ -34,9 +34,10 @@ export function convertDataViewIntoLensIndexPattern( dataView: DataView, restrictionRemapper: (name: string) => string = onRestrictionMapping ): IndexPattern { + const metaKeys = new Set(dataView.metaFields); const newFields = dataView.fields .filter(isFieldLensCompatible) - .map((field) => buildIndexPatternField(field, dataView)) + .map((field) => buildIndexPatternField(field, metaKeys)) .concat(documentField); const { typeMeta, title, name, timeFieldName, fieldFormatMap } = dataView; @@ -80,9 +81,9 @@ export function convertDataViewIntoLensIndexPattern( export function buildIndexPatternField( field: DataViewField, - dataView: DataView | null + metaKeys?: Set ): IndexPatternField { - const meta = dataView?.metaFields.includes(field.name); + const meta = metaKeys ? metaKeys.has(field.name) : false; // Convert the getters on the index pattern service into plain JSON const base = { name: field.name, diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 2674293da399a..b2db83cf0e7c6 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -421,10 +421,12 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ }; function getNewFieldsBySpec(spec: FieldSpec[], dataView: DataView | null) { + const metaKeys = dataView ? new Set(dataView.metaFields) : undefined; + return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { const field = new DataViewField(fieldSpec); if (isFieldLensCompatible(field)) { - result.push(buildIndexPatternField(field, dataView)); + result.push(buildIndexPatternField(field, metaKeys)); } return result; }, []); From d024027cec062a59ae8bbb0b6f15f2a7f00a9320 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 9 Jan 2024 11:24:09 +0100 Subject: [PATCH 29/29] Apply useLatest feedback --- .../datasources/form_based/datapanel.tsx | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index b2db83cf0e7c6..5d7e928c95594 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -28,6 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; +import useLatest from 'react-use/lib/useLatest'; import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import { buildIndexPatternField } from '../../data_views_service/loader'; import type { @@ -277,7 +278,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ }; }, []); - const refreshFieldList = useCallback(async () => { + const refreshFieldList = useLatest(async () => { if (currentIndexPattern) { const newlyMappedIndexPattern = await indexPatternService.loadIndexPatterns({ patterns: [currentIndexPattern.id], @@ -293,21 +294,13 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ } // start a new session so all charts are refreshed data.search.session.start(); - }, [ - indexPatternService, - currentIndexPattern, - onIndexPatternRefresh, - frame.dataViews.indexPatterns, - data.search.session, - ]); + }); useEffect(() => { if (hasNewFields) { - refreshFieldList(); + refreshFieldList.current(); } - // Preventing a race condition, making sure refreshFieldList is just executed once when hasNewFields is true - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [hasNewFields]); + }, [hasNewFields, refreshFieldList]); const editField = useMemo( () => @@ -321,7 +314,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ fieldName, onSave: () => { if (indexPatternInstance.isPersisted()) { - refreshFieldList(); + refreshFieldList.current(); refetchFieldsExistenceInfo(indexPatternInstance.id); } else { indexPatternService.replaceDataViewId(indexPatternInstance); @@ -353,7 +346,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ fieldName, onDelete: () => { if (indexPatternInstance.isPersisted()) { - refreshFieldList(); + refreshFieldList.current(); refetchFieldsExistenceInfo(indexPatternInstance.id); } else { indexPatternService.replaceDataViewId(indexPatternInstance);