From 326bd1a92d8990e84839cb493c2c24f27f4925c4 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 9 Jan 2024 22:37:31 +0100 Subject: [PATCH] [UnifiedFieldList] Add new fields ingested in background with valid mappings (#172329) Improves UnifiedFieldList by adding a newly ingested field to the list with the right type. On top of that, it triggers a refresh of the selected DataView fields, so the new field be available by consumers of the DataView. Co-authored-by: Julia Rechkunova --- .../field_list_sidebar.tsx | 84 +++++++++-------- .../src/hooks/use_existing_fields.test.tsx | 48 ++++++++++ .../src/hooks/use_existing_fields.ts | 22 ++++- .../src/hooks/use_grouped_fields.test.tsx | 81 ++++++++++++++++ .../src/hooks/use_grouped_fields.ts | 24 ++++- .../src/hooks/use_new_fields.test.tsx | 80 ++++++++++++++++ .../src/hooks/use_new_fields.ts | 62 +++++++++++++ .../field_existing/field_existing_utils.ts | 40 +++++--- .../field_existing/load_field_existing.ts | 3 +- .../dataview_picker/change_dataview.tsx | 6 -- .../discover/group4/_field_list_new_fields.ts | 86 +++++++++++++++++ test/functional/apps/discover/group4/index.ts | 1 + .../public/data_views_service/loader.test.ts | 37 +++++++- .../lens/public/data_views_service/loader.ts | 93 +++++++++++-------- .../datasources/form_based/datapanel.tsx | 63 ++++++++----- .../apps/lens/group2/fields_list.ts | 50 +++++++++- 16 files changed, 649 insertions(+), 131 deletions(-) create mode 100644 packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx create mode 100644 packages/kbn-unified-field-list/src/hooks/use_new_fields.ts create mode 100644 test/functional/apps/discover/group4/_field_list_new_fields.ts diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 4bc54069336b0..f8438b0917577 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -22,7 +22,7 @@ import { useEuiTheme, } from '@elastic/eui'; import { ToolbarButton } from '@kbn/shared-ux-button-toolbar'; -import { type DataViewField } from '@kbn/data-views-plugin/public'; +import { DataViewField, type FieldSpec } from '@kbn/data-views-plugin/common'; import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils'; import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils'; import { FieldList } from '../../components/field_list'; @@ -191,25 +191,6 @@ export const UnifiedFieldListSidebarComponent: React.FC { - if ( - searchMode !== 'documents' || - !useNewFieldsApi || - stateService.creationOptions.disableMultiFieldsGroupingByParent - ) { - setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case - } else { - setMultiFieldsMap(calculateMultiFields(allFields, selectedFieldsState.selectedFieldsMap)); - } - }, [ - stateService.creationOptions.disableMultiFieldsGroupingByParent, - selectedFieldsState.selectedFieldsMap, - allFields, - useNewFieldsApi, - setMultiFieldsMap, - searchMode, - ]); - const popularFieldsLimit = useMemo( () => core.uiSettings.get(FIELDS_LIMIT_SETTING), [core.uiSettings] @@ -226,24 +207,47 @@ export const UnifiedFieldListSidebarComponent: React.FC({ - dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries - allFields, - popularFieldsLimit: - searchMode !== 'documents' || stateService.creationOptions.disablePopularFields - ? 0 - : popularFieldsLimit, - isAffectedByGlobalFilter, - services: { - dataViews, - core, - }, - sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields, - onSelectedFieldFilter, - onSupportedFieldFilter: - stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter, - onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails, - }); + const { fieldListFiltersProps, fieldListGroupedProps, allFieldsModified } = + useGroupedFields({ + dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries + allFields, + popularFieldsLimit: + searchMode !== 'documents' || stateService.creationOptions.disablePopularFields + ? 0 + : popularFieldsLimit, + isAffectedByGlobalFilter, + services: { + dataViews, + core, + }, + sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields, + onSelectedFieldFilter, + onSupportedFieldFilter: + stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter, + onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails, + getNewFieldsBySpec, + }); + + useEffect(() => { + if ( + searchMode !== 'documents' || + !useNewFieldsApi || + stateService.creationOptions.disableMultiFieldsGroupingByParent + ) { + setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case + } else { + setMultiFieldsMap( + calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap) + ); + } + }, [ + stateService.creationOptions.disableMultiFieldsGroupingByParent, + selectedFieldsState.selectedFieldsMap, + allFieldsModified, + useNewFieldsApi, + setMultiFieldsMap, + searchMode, + ]); const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => ( @@ -456,3 +460,7 @@ function calculateMultiFields( }); return map; } + +function getNewFieldsBySpec(fieldSpecArr: FieldSpec[]): DataViewField[] { + return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)); +} diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx index 7cf22950a4a7d..0af0e4abc8e0d 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx @@ -125,6 +125,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(hookReader.result.current.getFieldsExistenceStatus(dataViewId)).toBe( ExistenceFetchStatus.succeeded ); + expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]); // does not have existence info => works less restrictive const anotherDataViewId = 'test-id'; @@ -140,6 +141,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(hookReader.result.current.getFieldsExistenceStatus(anotherDataViewId)).toBe( ExistenceFetchStatus.unknown ); + expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]); }); it('should work correctly with multiple readers', async () => { @@ -217,6 +219,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(currentResult.isFieldsExistenceInfoUnavailable(dataViewId)).toBe(true); expect(currentResult.hasFieldData(dataViewId, dataView.fields[0].name)).toBe(true); expect(currentResult.getFieldsExistenceStatus(dataViewId)).toBe(ExistenceFetchStatus.failed); + expect(currentResult.getNewFields(dataViewId)).toStrictEqual([]); }); it('should work correctly for multiple data views', async () => { @@ -533,4 +536,49 @@ describe('UnifiedFieldList useExistingFields', () => { expect(params.onNoData).toHaveBeenCalledTimes(1); // still 1 time }); + + it('should include newFields', async () => { + const newFields = [{ name: 'test', type: 'keyword', searchable: true, aggregatable: true }]; + + (ExistingFieldsServiceApi.loadFieldExisting as jest.Mock).mockImplementation( + async ({ dataView: currentDataView }) => { + return { + existingFieldNames: [currentDataView.fields[0].name], + newFields, + }; + } + ); + + const params: ExistingFieldsFetcherParams = { + dataViews: [dataView], + services: mockedServices, + fromDate: '2019-01-01', + toDate: '2020-01-01', + query: { query: '', language: 'lucene' }, + filters: [], + }; + const hookFetcher = renderHook(useExistingFieldsFetcher, { + initialProps: params, + }); + + const hookReader = renderHook(useExistingFieldsReader); + await hookFetcher.waitForNextUpdate(); + + expect(ExistingFieldsServiceApi.loadFieldExisting).toHaveBeenCalledWith( + expect.objectContaining({ + fromDate: '2019-01-01', + toDate: '2020-01-01', + dslQuery, + dataView, + timeFieldName: dataView.timeFieldName, + }) + ); + + expect(hookReader.result.current.getFieldsExistenceStatus(dataView.id!)).toBe( + ExistenceFetchStatus.succeeded + ); + + expect(hookReader.result.current.getNewFields(dataView.id!)).toBe(newFields); + expect(hookReader.result.current.getNewFields('another-id')).toStrictEqual([]); + }); }); diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index d099a6f2b85e2..cd4e9ef05206f 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -12,7 +12,7 @@ import { BehaviorSubject } from 'rxjs'; import type { CoreStart } from '@kbn/core/public'; import type { AggregateQuery, EsQueryConfig, Filter, Query } from '@kbn/es-query'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; -import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; import { loadFieldExisting } from '../services/field_existing'; @@ -20,10 +20,12 @@ import { ExistenceFetchStatus } from '../types'; const getBuildEsQueryAsync = async () => (await import('@kbn/es-query')).buildEsQuery; const generateId = htmlIdGenerator(); +const DEFAULT_EMPTY_NEW_FIELDS: FieldSpec[] = []; export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; existingFieldsByFieldNameMap: Record; + newFields?: FieldSpec[]; numberOfFetches: number; hasDataViewRestrictions?: boolean; } @@ -54,6 +56,7 @@ export interface ExistingFieldsReader { hasFieldData: (dataViewId: string, fieldName: string) => boolean; getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; + getNewFields: (dataViewId: string) => FieldSpec[]; } const initialData: ExistingFieldsByDataViewMap = {}; @@ -157,6 +160,7 @@ export const useExistingFieldsFetcher = ( } info.existingFieldsByFieldNameMap = booleanMap(existingFieldNames); + info.newFields = result.newFields; info.fetchStatus = ExistenceFetchStatus.succeeded; } catch (error) { info.fetchStatus = ExistenceFetchStatus.failed; @@ -286,6 +290,19 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { [existingFieldsByDataViewMap] ); + const getNewFields = useCallback( + (dataViewId: string) => { + const info = existingFieldsByDataViewMap[dataViewId]; + + if (info?.fetchStatus === ExistenceFetchStatus.succeeded) { + return info?.newFields ?? DEFAULT_EMPTY_NEW_FIELDS; + } + + return DEFAULT_EMPTY_NEW_FIELDS; + }, + [existingFieldsByDataViewMap] + ); + const getFieldsExistenceInfo = useCallback( (dataViewId: string) => { return dataViewId ? existingFieldsByDataViewMap[dataViewId] : unknownInfo; @@ -321,8 +338,9 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, + getNewFields, }), - [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable] + [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, getNewFields] ); }; diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx index 053e7d912d375..4a937f86bd5c0 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx @@ -96,6 +96,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [], }) ); @@ -156,6 +157,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [], }) ); @@ -185,6 +187,8 @@ describe('UnifiedFieldList useGroupedFields()', () => { expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); rerender({ ...props, @@ -198,6 +202,82 @@ describe('UnifiedFieldList useGroupedFields()', () => { expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe( scrollToTopResetCounter1 ); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); + + (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore(); + }); + + it('should work correctly with new fields', async () => { + const props: GroupedFieldsParams = { + dataViewId: dataView.id!, + allFields, + services: mockedServices, + getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)), + }; + + const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true }; + + jest.spyOn(ExistenceApi, 'useExistingFieldsReader').mockImplementation( + (): ExistingFieldsReader => ({ + hasFieldData: (dataViewId) => { + return dataViewId === props.dataViewId; + }, + getFieldsExistenceStatus: (dataViewId) => + dataViewId === props.dataViewId + ? ExistenceFetchStatus.succeeded + : ExistenceFetchStatus.unknown, + isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [newField], + }) + ); + + const { result, waitForNextUpdate, rerender } = renderHook(useGroupedFields, { + initialProps: props, + }); + + await waitForNextUpdate(); + + let fieldListGroupedProps = result.current.fieldListGroupedProps; + const fieldGroups = fieldListGroupedProps.fieldGroups; + const scrollToTopResetCounter1 = fieldListGroupedProps.scrollToTopResetCounter; + + expect( + Object.keys(fieldGroups!).map( + (key) => `${key}-${fieldGroups![key as FieldsGroupNames]?.fields.length}` + ) + ).toStrictEqual([ + 'SpecialFields-0', + 'SelectedFields-0', + 'PopularFields-0', + 'AvailableFields-25', + 'UnmappedFields-1', + 'EmptyFields-0', + 'MetaFields-3', + ]); + + expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); + expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.allFieldsModified).toStrictEqual([ + ...allFields, + new DataViewField(newField), + ]); + expect(result.current.hasNewFields).toBe(true); + + rerender({ + ...props, + dataViewId: null, // for text-based queries + allFields, + }); + + fieldListGroupedProps = result.current.fieldListGroupedProps; + expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); + expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe( + scrollToTopResetCounter1 + ); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore(); }); @@ -438,6 +518,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== knownDataViewId, + getNewFields: () => [], }) ); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 16758ea4ee7f2..7853c7e67800b 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -10,8 +10,9 @@ import { groupBy } from 'lodash'; import { useEffect, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; -import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { type DataViewsContract } from '@kbn/data-views-plugin/public'; +import { type UseNewFieldsParams, useNewFields } from './use_new_fields'; import { type FieldListGroups, type FieldsGroup, @@ -41,6 +42,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; + getNewFieldsBySpec?: UseNewFieldsParams['getNewFieldsBySpec']; } export interface GroupedFieldsResult { @@ -52,6 +54,8 @@ export interface GroupedFieldsResult { fieldsExistInIndex: boolean; screenReaderDescriptionId?: string; }; + allFieldsModified: T[] | null; // `null` is for loading indicator + hasNewFields: boolean; } export function useGroupedFields({ @@ -65,6 +69,7 @@ export function useGroupedFields({ onOverrideFieldGroupDetails, onSupportedFieldFilter, onSelectedFieldFilter, + getNewFieldsBySpec, }: GroupedFieldsParams): GroupedFieldsResult { const fieldsExistenceReader = useExistingFieldsReader(); const fieldListFilters = useFieldFilters({ @@ -73,6 +78,7 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); + const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName); @@ -101,6 +107,13 @@ export function useGroupedFields({ // if field existence information changed, reload the data view too }, [dataViewId, services.dataViews, setDataView, hasFieldDataHandler]); + const { allFieldsModified, hasNewFields } = useNewFields({ + dataView, + allFields, + getNewFieldsBySpec, + fieldsExistenceReader, + }); + // important when switching from a known dataViewId to no data view (like in text-based queries) useEffect(() => { if (dataView && !dataViewId) { @@ -120,13 +133,16 @@ export function useGroupedFields({ }; const selectedFields = sortedSelectedFields || []; - const sortedFields = [...(allFields || [])].sort(sortFields); + + const sortedFields = [...(allFieldsModified || [])].sort(sortFields); + const groupedFields = { ...getDefaultFieldGroups(), ...groupBy(sortedFields, (field) => { if (!sortedSelectedFields && onSelectedFieldFilter && onSelectedFieldFilter(field)) { selectedFields.push(field); } + if (onSupportedFieldFilter && !onSupportedFieldFilter(field)) { return 'skippedFields'; } @@ -311,7 +327,7 @@ export function useGroupedFields({ return fieldGroupDefinitions; }, [ - allFields, + allFieldsModified, onSupportedFieldFilter, onSelectedFieldFilter, onOverrideFieldGroupDetails, @@ -381,6 +397,8 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, + allFieldsModified, + hasNewFields, }; } diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx new file mode 100644 index 0000000000000..2be5d50764bc9 --- /dev/null +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { stubLogstashDataView as dataView } from '@kbn/data-views-plugin/common/data_view.stub'; +import { DataViewField } from '@kbn/data-views-plugin/common'; +import { useNewFields, type UseNewFieldsParams } from './use_new_fields'; +import { type ExistingFieldsReader } from './use_existing_fields'; +import { ExistenceFetchStatus } from '../types'; + +const fieldsExistenceReader: ExistingFieldsReader = { + hasFieldData: (dataViewId) => { + return dataViewId === dataView.id; + }, + getFieldsExistenceStatus: (dataViewId) => + dataViewId === dataView.id ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, + isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== dataView.id, + getNewFields: () => [], +}; + +describe('UnifiedFieldList useNewFields()', () => { + const allFields = dataView.fields; + + it('should work correctly in loading state', async () => { + const props: UseNewFieldsParams = { + dataView, + allFields: null, + fieldsExistenceReader, + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toBe(null); + expect(result.current.hasNewFields).toBe(false); + }); + + it('should work correctly with empty new fields', async () => { + const props: UseNewFieldsParams = { + dataView, + allFields, + fieldsExistenceReader, + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); + }); + + it('should work correctly with new fields', async () => { + const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true }; + const newField2 = { ...newField, name: 'test2' }; + const props: UseNewFieldsParams = { + dataView, + allFields, + fieldsExistenceReader: { + ...fieldsExistenceReader, + getNewFields: () => [newField, newField2], + }, + getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)), + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toStrictEqual([ + ...allFields, + new DataViewField(newField), + new DataViewField(newField2), + ]); + expect(result.current.hasNewFields).toBe(true); + }); +}); diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts new file mode 100644 index 0000000000000..51e143cc524c3 --- /dev/null +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { useMemo } from 'react'; +import type { FieldSpec } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; +import type { FieldListItem } from '../types'; +import type { ExistingFieldsReader } from './use_existing_fields'; + +export interface UseNewFieldsParams { + dataView?: DataView | null; + allFields: T[] | null; // `null` is for loading indicator + getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[]; + fieldsExistenceReader: ExistingFieldsReader; +} + +export interface UseNewFieldsResult { + allFieldsModified: T[] | null; + hasNewFields: boolean; +} + +/** + * This hook is used to get the new fields of previous fields for wildcards request, and merges those + * with the existing fields. + */ +export function useNewFields({ + dataView, + allFields, + getNewFieldsBySpec, + fieldsExistenceReader, +}: UseNewFieldsParams): UseNewFieldsResult { + const dataViewId = dataView?.id; + + const newFields = useMemo(() => { + const newLoadedFields = + allFields && dataView?.id && getNewFieldsBySpec + ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) + : null; + + return newLoadedFields?.length ? newLoadedFields : null; + }, [allFields, dataView, fieldsExistenceReader, getNewFieldsBySpec]); + + const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); + + const allFieldsModified = useMemo(() => { + if (!allFields || !newFields?.length || !dataViewId) return allFields; + // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request + // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way + const allFieldsExlNew = allFields.filter( + (field) => !newFields.some((newField) => newField.name === field.name) + ); + + return [...allFieldsExlNew, ...newFields]; + }, [newFields, allFields, dataViewId]); + + return { allFieldsModified, hasNewFields }; +} diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 2aa6137ffb1fb..41a1e53a33849 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -7,7 +7,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { RuntimeField } from '@kbn/data-views-plugin/common'; +import type { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; @@ -49,15 +49,25 @@ export async function fetchFieldExistence({ metaFields: string[]; dataViewsService: DataViewsContract; }) { - const allFields = buildFieldList(dataView, metaFields); const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, { // filled in by data views service pattern: '', indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery), }); + + // take care of fields of existingFieldList, that are not yet available + // in the given data view. Those fields we consider as new fields, + // that were ingested after the data view was loaded + const newFields = existingFieldList.filter((field) => !dataView.getFieldByName(field.name)); + // refresh the data view in case there are new fields + if (newFields.length) { + await dataViewsService.refreshFields(dataView, false); + } + const allFields = buildFieldList(dataView, metaFields); return { - indexPatternTitle: dataView.title, + indexPatternTitle: dataView.getIndexPattern(), existingFieldNames: existingFields(existingFieldList, allFields), + newFields, }; } @@ -66,19 +76,23 @@ export async function fetchFieldExistence({ */ export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] { return indexPattern.fields.map((field) => { - return { - name: field.name, - isScript: !!field.scripted, - lang: field.lang, - script: field.script, - // id is a special case - it doesn't show up in the meta field list, - // but as it's not part of source, it has to be handled separately. - isMeta: metaFields?.includes(field.name) || field.name === '_id', - runtimeField: !field.isMapped ? field.runtimeField : undefined, - }; + return buildField(field, metaFields); }); } +export function buildField(field: DataViewField, metaFields: string[]): Field { + return { + name: field.name, + isScript: !!field.scripted, + lang: field.lang, + script: field.script, + // id is a special case - it doesn't show up in the meta field list, + // but as it's not part of source, it has to be handled separately. + isMeta: metaFields?.includes(field.name) || field.name === '_id', + runtimeField: !field.isMapped ? field.runtimeField : undefined, + }; +} + function toQuery( timeFieldName: string | undefined, fromDate: string | undefined, diff --git a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts index 796062bc60559..590bc46eff573 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts @@ -9,7 +9,7 @@ import { IUiSettingsClient } from '@kbn/core/public'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import { UI_SETTINGS } from '@kbn/data-service/src/constants'; -import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common'; import { lastValueFrom } from 'rxjs'; import { fetchFieldExistence } from './field_existing_utils'; @@ -27,6 +27,7 @@ interface FetchFieldExistenceParams { export type LoadFieldExistingHandler = (params: FetchFieldExistenceParams) => Promise<{ existingFieldNames: string[]; indexPatternTitle: string; + newFields?: FieldSpec[]; }>; export const loadFieldExisting: LoadFieldExistingHandler = async ({ diff --git a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx index 9076bcb37c7df..2d857de608834 100644 --- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx +++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx @@ -307,12 +307,6 @@ export function ChangeDataView({ isTextBasedLangSelected={isTextBasedLangSelected} setPopoverIsOpen={setPopoverIsOpen} onChangeDataView={async (newId) => { - try { - // refreshing the field list - await dataViews.get(newId, false, true); - } catch (e) { - // - } setSelectedDataViewId(newId); setPopoverIsOpen(false); if (isTextBasedLangSelected && !isTextLangTransitionModalDismissed) { diff --git a/test/functional/apps/discover/group4/_field_list_new_fields.ts b/test/functional/apps/discover/group4/_field_list_new_fields.ts new file mode 100644 index 0000000000000..3c24bcf613ae4 --- /dev/null +++ b/test/functional/apps/discover/group4/_field_list_new_fields.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const security = getService('security'); + const es = getService('es'); + const retry = getService('retry'); + const queryBar = getService('queryBar'); + const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'unifiedFieldList']); + + describe('Field list new fields in background handling', function () { + before(async () => { + await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); + await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); + + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setCommonlyUsedTime('This_week'); + }); + + after(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); + await es.transport.request({ + path: '/my-index-000001', + method: 'DELETE', + }); + }); + + it('Check that new ingested fields are added to the available fields section', async function () { + const initialPattern = 'my-index-'; + await es.transport.request({ + path: '/my-index-000001/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + a: 'GET /search HTTP/1.1 200 1070000', + }, + }); + + await PageObjects.discover.createAdHocDataView(initialPattern, true); + + await retry.waitFor('current data view to get updated', async () => { + return (await PageObjects.discover.getCurrentlySelectedDataView()) === `${initialPattern}*`; + }); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + + expect(await PageObjects.discover.getHitCountInt()).to.be(1); + expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([ + '@timestamp', + 'a', + ]); + + await es.transport.request({ + path: '/my-index-000001/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + b: 'GET /search HTTP/1.1 200 1070000', + }, + }); + + await retry.waitFor('the new record was found', async () => { + await queryBar.submitQuery(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + return (await PageObjects.discover.getHitCountInt()) === 2; + }); + + expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([ + '@timestamp', + 'a', + 'b', + ]); + }); + }); +} diff --git a/test/functional/apps/discover/group4/index.ts b/test/functional/apps/discover/group4/index.ts index 1aab3db2bfc43..656a116551db8 100644 --- a/test/functional/apps/discover/group4/index.ts +++ b/test/functional/apps/discover/group4/index.ts @@ -33,5 +33,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_context_encoded_url_params')); loadTestFile(require.resolve('./_hide_announcements')); loadTestFile(require.resolve('./_data_view_edit')); + loadTestFile(require.resolve('./_field_list_new_fields')); }); } diff --git a/x-pack/plugins/lens/public/data_views_service/loader.test.ts b/x-pack/plugins/lens/public/data_views_service/loader.test.ts index f91d236986b11..4c648a7782896 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.test.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.test.ts @@ -5,8 +5,13 @@ * 2.0. */ -import { DataViewsContract } from '@kbn/data-views-plugin/public'; -import { ensureIndexPattern, loadIndexPatternRefs, loadIndexPatterns } from './loader'; +import { DataViewsContract, DataViewField } from '@kbn/data-views-plugin/public'; +import { + ensureIndexPattern, + loadIndexPatternRefs, + loadIndexPatterns, + buildIndexPatternField, +} from './loader'; import { sampleIndexPatterns, mockDataViewsService } from './mocks'; import { documentField } from '../datasources/form_based/document_field'; @@ -313,4 +318,32 @@ describe('loader', () => { expect(onError).not.toHaveBeenCalled(); }); }); + + describe('buildIndexPatternField', () => { + it('should return a field with the correct name and derived parameters', async () => { + const field = buildIndexPatternField({ + name: 'foo', + displayName: 'Foo', + type: 'string', + aggregatable: true, + searchable: true, + } as DataViewField); + expect(field.name).toEqual('foo'); + expect(field.meta).toEqual(false); + expect(field.runtime).toEqual(false); + }); + it('should return return the right meta field value', async () => { + const field = buildIndexPatternField( + { + name: 'meta', + displayName: 'Meta', + type: 'string', + aggregatable: true, + searchable: true, + } as DataViewField, + new Set(['meta']) + ); + expect(field.meta).toEqual(true); + }); + }); }); diff --git a/x-pack/plugins/lens/public/data_views_service/loader.ts b/x-pack/plugins/lens/public/data_views_service/loader.ts index 784c97d832e34..8a52146991b8d 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.ts @@ -6,7 +6,12 @@ */ import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; -import type { DataViewsContract, DataView, DataViewSpec } from '@kbn/data-views-plugin/public'; +import { + DataViewsContract, + DataView, + DataViewSpec, + DataViewField, +} from '@kbn/data-views-plugin/public'; import { keyBy } from 'lodash'; import { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef } from '../types'; import { documentField } from '../datasources/form_based/document_field'; @@ -32,46 +37,7 @@ export function convertDataViewIntoLensIndexPattern( const metaKeys = new Set(dataView.metaFields); const newFields = dataView.fields .filter(isFieldLensCompatible) - .map((field): IndexPatternField => { - // Convert the getters on the index pattern service into plain JSON - const base = { - name: field.name, - displayName: field.displayName, - type: field.type, - aggregatable: field.aggregatable, - filterable: field.filterable, - searchable: field.searchable, - meta: metaKeys.has(field.name), - esTypes: field.esTypes, - scripted: field.scripted, - isMapped: field.isMapped, - customLabel: field.customLabel, - runtimeField: field.runtimeField, - runtime: Boolean(field.runtimeField), - timeSeriesDimension: field.timeSeriesDimension, - timeSeriesMetric: field.timeSeriesMetric, - timeSeriesRollup: field.isRolledUpField, - partiallyApplicableFunctions: field.isRolledUpField - ? { - percentile: true, - percentile_rank: true, - median: true, - last_value: true, - unique_count: true, - standard_deviation: true, - } - : undefined, - }; - - // Simplifies tests by hiding optional properties instead of undefined - return base.scripted - ? { - ...base, - lang: field.lang, - script: field.script, - } - : base; - }) + .map((field) => buildIndexPatternField(field, metaKeys)) .concat(documentField); const { typeMeta, title, name, timeFieldName, fieldFormatMap } = dataView; @@ -113,6 +79,51 @@ export function convertDataViewIntoLensIndexPattern( }; } +export function buildIndexPatternField( + field: DataViewField, + metaKeys?: Set +): IndexPatternField { + const meta = metaKeys ? metaKeys.has(field.name) : false; + // Convert the getters on the index pattern service into plain JSON + const base = { + name: field.name, + displayName: field.displayName, + type: field.type, + aggregatable: field.aggregatable, + filterable: field.filterable, + searchable: field.searchable, + meta, + esTypes: field.esTypes, + scripted: field.scripted, + isMapped: field.isMapped, + customLabel: field.customLabel, + runtimeField: field.runtimeField, + runtime: Boolean(field.runtimeField), + timeSeriesDimension: field.timeSeriesDimension, + timeSeriesMetric: field.timeSeriesMetric, + timeSeriesRollup: field.isRolledUpField, + partiallyApplicableFunctions: field.isRolledUpField + ? { + percentile: true, + percentile_rank: true, + median: true, + last_value: true, + unique_count: true, + standard_deviation: true, + } + : undefined, + }; + + // Simplifies tests by hiding optional properties instead of undefined + return base.scripted + ? { + ...base, + lang: field.lang, + script: field.script, + } + : base; +} + export async function loadIndexPatternRefs( dataViews: MinimalDataViewsContract ): Promise { diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 53a26693b7a7a..5d7e928c95594 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -13,7 +13,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { CoreStart } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { type DataView } from '@kbn/data-plugin/common'; +import { type DataView, DataViewField, FieldSpec } from '@kbn/data-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public'; import { VISUALIZE_GEO_FIELD_TRIGGER } from '@kbn/ui-actions-plugin/public'; @@ -28,6 +28,9 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; +import useLatest from 'react-use/lib/useLatest'; +import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; +import { buildIndexPatternField } from '../../data_views_service/loader'; import type { DatasourceDataPanelProps, FramePublicAPI, @@ -249,18 +252,20 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ } }, []); - const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields({ - dataViewId: currentIndexPatternId, - allFields, - services: { - dataViews, - core, - }, - isAffectedByGlobalFilter: Boolean(filters.length), - onSupportedFieldFilter, - onSelectedFieldFilter, - onOverrideFieldGroupDetails, - }); + const { fieldListFiltersProps, fieldListGroupedProps, hasNewFields } = + useGroupedFields({ + dataViewId: currentIndexPatternId, + allFields, + services: { + dataViews, + core, + }, + isAffectedByGlobalFilter: Boolean(filters.length), + onSupportedFieldFilter, + onSelectedFieldFilter, + onOverrideFieldGroupDetails, + getNewFieldsBySpec, + }); const closeFieldEditor = useRef<() => void | undefined>(); @@ -273,7 +278,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ }; }, []); - const refreshFieldList = useCallback(async () => { + const refreshFieldList = useLatest(async () => { if (currentIndexPattern) { const newlyMappedIndexPattern = await indexPatternService.loadIndexPatterns({ patterns: [currentIndexPattern.id], @@ -289,13 +294,13 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ } // start a new session so all charts are refreshed data.search.session.start(); - }, [ - indexPatternService, - currentIndexPattern, - onIndexPatternRefresh, - frame.dataViews.indexPatterns, - data.search.session, - ]); + }); + + useEffect(() => { + if (hasNewFields) { + refreshFieldList.current(); + } + }, [hasNewFields, refreshFieldList]); const editField = useMemo( () => @@ -309,7 +314,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ fieldName, onSave: () => { if (indexPatternInstance.isPersisted()) { - refreshFieldList(); + refreshFieldList.current(); refetchFieldsExistenceInfo(indexPatternInstance.id); } else { indexPatternService.replaceDataViewId(indexPatternInstance); @@ -341,7 +346,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ fieldName, onDelete: () => { if (indexPatternInstance.isPersisted()) { - refreshFieldList(); + refreshFieldList.current(); refetchFieldsExistenceInfo(indexPatternInstance.id); } else { indexPatternService.replaceDataViewId(indexPatternInstance); @@ -408,4 +413,16 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ ); }; +function getNewFieldsBySpec(spec: FieldSpec[], dataView: DataView | null) { + const metaKeys = dataView ? new Set(dataView.metaFields) : undefined; + + return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { + const field = new DataViewField(fieldSpec); + if (isFieldLensCompatible(field)) { + result.push(buildIndexPatternField(field, metaKeys)); + } + return result; + }, []); +} + export const MemoizedDataPanel = memo(InnerFormBasedDataPanel); diff --git a/x-pack/test/functional/apps/lens/group2/fields_list.ts b/x-pack/test/functional/apps/lens/group2/fields_list.ts index 4e1f771d0b042..79baafe6100a6 100644 --- a/x-pack/test/functional/apps/lens/group2/fields_list.ts +++ b/x-pack/test/functional/apps/lens/group2/fields_list.ts @@ -9,13 +9,15 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header', 'timePicker']); const find = getService('find'); const log = getService('log'); const testSubjects = getService('testSubjects'); const filterBar = getService('filterBar'); const fieldEditor = getService('fieldEditor'); const retry = getService('retry'); + const es = getService('es'); + const queryBar = getService('queryBar'); describe('lens fields list tests', () => { for (const datasourceType of ['form-based', 'ad-hoc', 'ad-hoc-no-timefield']) { @@ -48,7 +50,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); }); }); - it('should show all fields as available', async () => { expect( await (await testSubjects.find('lnsIndexPatternAvailableFields-count')).getVisibleText() @@ -231,5 +232,50 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } }); } + + describe(`update field list test`, () => { + before(async () => { + await es.transport.request({ + path: '/field-update-test/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + oldField: 10, + }, + }); + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.timePicker.setCommonlyUsedTime('This_week'); + + await PageObjects.lens.createAdHocDataView('field-update-test', true); + await retry.try(async () => { + const selectedPattern = await PageObjects.lens.getDataPanelIndexPattern(); + expect(selectedPattern).to.eql('field-update-test*'); + }); + }); + after(async () => { + await es.transport.request({ + path: '/field-update-test', + method: 'DELETE', + }); + }); + + it('should show new fields Available fields', async () => { + await es.transport.request({ + path: '/field-update-test/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + oldField: 10, + newField: 20, + }, + }); + await PageObjects.lens.waitForField('oldField'); + await queryBar.setQuery('oldField: 10'); + await queryBar.submitQuery(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.waitForField('newField'); + }); + }); }); }