diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.test.tsx index 78d5ed68787bf..1dad2a79d9651 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.test.tsx @@ -12,6 +12,7 @@ import React from 'react'; import { SAME_FAMILY } from '../../data_quality_panel/same_family/translations'; import { eventCategory, + someField, eventCategoryWithUnallowedValues, } from '../../mock/enriched_field_metadata/mock_enriched_field_metadata'; import { TestProviders } from '../../mock/test_providers/test_providers'; @@ -261,15 +262,9 @@ describe('getCommonTableColumns', () => { const columns = getCommonTableColumns(); const descriptionolumnRender = columns[5].render; - const withDescription: EnrichedFieldMetadata = { - ...eventCategory, - description: undefined, - }; - render( - {descriptionolumnRender != null && - descriptionolumnRender(withDescription.description, withDescription)} + {descriptionolumnRender != null && descriptionolumnRender(undefined, someField)} ); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.tsx index cc3ccf395bc9e..2e0d2506d6e0f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.tsx @@ -44,19 +44,25 @@ export const getCommonTableColumns = (): Array< { field: 'indexFieldType', name: i18n.INDEX_MAPPING_TYPE_ACTUAL, - render: (_, x) => - x.type != null && x.indexFieldType !== x.type ? ( - getIsInSameFamily({ ecsExpectedType: x.type, type: x.indexFieldType }) ? ( + render: (_, x) => { + // if custom field or ecs based field with mapping match + if (!x.hasEcsMetadata || x.indexFieldType === x.type) { + return {x.indexFieldType}; + } + + // mapping mismatch due to same family + if (getIsInSameFamily({ ecsExpectedType: x.type, type: x.indexFieldType })) { + return (
{x.indexFieldType}
- ) : ( - {x.indexFieldType} - ) - ) : ( - {x.indexFieldType} - ), + ); + } + + // mapping mismatch + return {x.indexFieldType}; + }, sortable: true, truncateText: false, width: '15%', diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.test.tsx index bbc026ba0b364..367ff38fa1093 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.test.tsx @@ -12,8 +12,8 @@ import React from 'react'; import { SAME_FAMILY } from '../../data_quality_panel/same_family/translations'; import { TestProviders } from '../../mock/test_providers/test_providers'; import { eventCategory } from '../../mock/enriched_field_metadata/mock_enriched_field_metadata'; -import { EnrichedFieldMetadata } from '../../types'; -import { EMPTY_PLACEHOLDER, getIncompatibleMappingsTableColumns } from '.'; +import { EcsBasedFieldMetadata } from '../../types'; +import { getIncompatibleMappingsTableColumns } from '.'; describe('getIncompatibleMappingsTableColumns', () => { test('it returns the expected column configuration', () => { @@ -65,19 +65,6 @@ describe('getIncompatibleMappingsTableColumns', () => { expect(screen.getByTestId('codeSuccess')).toHaveTextContent(expected); }); - - test('it renders an empty placeholder when type is undefined', () => { - const columns = getIncompatibleMappingsTableColumns(); - const typeColumnRender = columns[1].render; - - render( - - {typeColumnRender != null && typeColumnRender(undefined, eventCategory)} - - ); - - expect(screen.getByTestId('codeSuccess')).toHaveTextContent(EMPTY_PLACEHOLDER); - }); }); describe('indexFieldType column render()', () => { @@ -88,7 +75,7 @@ describe('getIncompatibleMappingsTableColumns', () => { const columns = getIncompatibleMappingsTableColumns(); const indexFieldTypeColumnRender = columns[2].render; - const withTypeMismatchSameFamily: EnrichedFieldMetadata = { + const withTypeMismatchSameFamily: EcsBasedFieldMetadata = { ...eventCategory, // `event.category` is a `keyword` per the ECS spec indexFieldType, // this index has a mapping of `wildcard` instead of `keyword` isInSameFamily: true, // `wildcard` and `keyword` are in the same family @@ -121,7 +108,7 @@ describe('getIncompatibleMappingsTableColumns', () => { const columns = getIncompatibleMappingsTableColumns(); const indexFieldTypeColumnRender = columns[2].render; - const withTypeMismatchDifferentFamily: EnrichedFieldMetadata = { + const withTypeMismatchDifferentFamily: EcsBasedFieldMetadata = { ...eventCategory, // `event.category` is a `keyword` per the ECS spec indexFieldType, // this index has a mapping of `text` instead of `keyword` isInSameFamily: false, // `text` and `wildcard` are not in the same family diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.tsx index 4a6c042f67e96..70078ab7ccc16 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.tsx @@ -11,12 +11,12 @@ import React from 'react'; import { SameFamily } from '../../data_quality_panel/same_family'; import { CodeDanger, CodeSuccess } from '../../styles'; import * as i18n from '../translations'; -import type { EnrichedFieldMetadata } from '../../types'; +import type { EcsBasedFieldMetadata } from '../../types'; export const EMPTY_PLACEHOLDER = '--'; export const getIncompatibleMappingsTableColumns = (): Array< - EuiTableFieldDataColumnType + EuiTableFieldDataColumnType > => [ { field: 'indexFieldName', @@ -28,11 +28,7 @@ export const getIncompatibleMappingsTableColumns = (): Array< { field: 'type', name: i18n.ECS_MAPPING_TYPE_EXPECTED, - render: (type: string) => ( - - {type != null ? type : EMPTY_PLACEHOLDER} - - ), + render: (type: string) => {type}, sortable: true, truncateText: false, width: '25%', diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.test.tsx index 7c72289290942..d9897cfa3d399 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.test.tsx @@ -10,7 +10,6 @@ import { omit } from 'lodash/fp'; import React from 'react'; import { - EMPTY_PLACEHOLDER, getCustomTableColumns, getEcsCompliantTableColumns, getIncompatibleValuesTableColumns, @@ -117,31 +116,6 @@ describe('helpers', () => { expect(screen.queryByTestId('typePlaceholder')).not.toBeInTheDocument(); }); }); - - describe('when `type` is undefined', () => { - beforeEach(() => { - const withUndefinedType = { - ...eventCategory, - type: undefined, // <-- - }; - const columns = getEcsCompliantTableColumns(); - const typeRender = columns[1].render; - - render( - - <>{typeRender != null && typeRender(withUndefinedType.type, withUndefinedType)} - - ); - }); - - test('it does NOT render the `type`', () => { - expect(screen.queryByTestId('type')).not.toBeInTheDocument(); - }); - - test('it renders the placeholder', () => { - expect(screen.getByTestId('typePlaceholder')).toHaveTextContent(EMPTY_PLACEHOLDER); - }); - }); }); describe('allowed values render()', () => { @@ -230,35 +204,6 @@ describe('helpers', () => { expect(screen.queryByTestId('emptyPlaceholder')).not.toBeInTheDocument(); }); }); - - describe('when `description` is undefined', () => { - const withUndefinedDescription = { - ...eventCategory, - description: undefined, // <-- - }; - - beforeEach(() => { - const columns = getEcsCompliantTableColumns(); - const descriptionRender = columns[3].render; - - render( - - <> - {descriptionRender != null && - descriptionRender(withUndefinedDescription.description, withUndefinedDescription)} - - - ); - }); - - test('it does NOT render the `description`', () => { - expect(screen.queryByTestId('description')).not.toBeInTheDocument(); - }); - - test('it renders the placeholder', () => { - expect(screen.getByTestId('emptyPlaceholder')).toBeInTheDocument(); - }); - }); }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.tsx index 8153380c140c3..a9f5c17034833 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.tsx @@ -13,12 +13,17 @@ import { EcsAllowedValues } from './ecs_allowed_values'; import { IndexInvalidValues } from './index_invalid_values'; import { CodeSuccess } from '../styles'; import * as i18n from './translations'; -import type { AllowedValue, EnrichedFieldMetadata, UnallowedValueCount } from '../types'; +import type { + AllowedValue, + CustomFieldMetadata, + EcsBasedFieldMetadata, + UnallowedValueCount, +} from '../types'; export const EMPTY_PLACEHOLDER = '--'; export const getCustomTableColumns = (): Array< - EuiTableFieldDataColumnType + EuiTableFieldDataColumnType > => [ { field: 'indexFieldName', @@ -40,7 +45,7 @@ export const getCustomTableColumns = (): Array< ]; export const getEcsCompliantTableColumns = (): Array< - EuiTableFieldDataColumnType + EuiTableFieldDataColumnType > => [ { field: 'indexFieldName', @@ -52,12 +57,7 @@ export const getEcsCompliantTableColumns = (): Array< { field: 'type', name: i18n.ECS_MAPPING_TYPE, - render: (type: string | undefined) => - type != null ? ( - {type} - ) : ( - {EMPTY_PLACEHOLDER} - ), + render: (type: string) => {type}, sortable: true, truncateText: false, width: '25%', @@ -75,12 +75,7 @@ export const getEcsCompliantTableColumns = (): Array< { field: 'description', name: i18n.ECS_DESCRIPTION, - render: (description: string | undefined) => - description != null ? ( - {description} - ) : ( - {EMPTY_PLACEHOLDER} - ), + render: (description: string) => {description}, sortable: false, truncateText: false, width: '25%', @@ -88,7 +83,7 @@ export const getEcsCompliantTableColumns = (): Array< ]; export const getIncompatibleValuesTableColumns = (): Array< - EuiTableFieldDataColumnType + EuiTableFieldDataColumnType > => [ { field: 'indexFieldName', diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/index.tsx index 145686785cafa..460663fb28790 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/index.tsx @@ -20,17 +20,17 @@ const search: Search = { }, }; -interface Props { - enrichedFieldMetadata: EnrichedFieldMetadata[]; - getTableColumns: () => Array>; +interface Props { + enrichedFieldMetadata: T[]; + getTableColumns: () => Array>; title: string; } -const CompareFieldsTableComponent: React.FC = ({ +const CompareFieldsTableComponent = ({ enrichedFieldMetadata, getTableColumns, title, -}) => { +}: Props): React.ReactElement => { const columns = useMemo(() => getTableColumns(), [getTableColumns]); return ( @@ -53,4 +53,8 @@ const CompareFieldsTableComponent: React.FC = ({ CompareFieldsTableComponent.displayName = 'CompareFieldsTableComponent'; -export const CompareFieldsTable = React.memo(CompareFieldsTableComponent); +export const CompareFieldsTable = React.memo( + CompareFieldsTableComponent + // React.memo doesn't pass generics through so + // this is a cheap fix without sacrificing type safety +) as typeof CompareFieldsTableComponent; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/constants.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/constants.ts new file mode 100644 index 0000000000000..a53c50edc1084 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EcsFlat } from '@elastic/ecs'; +import { EcsFieldMetadata } from './types'; + +export const EcsFlatTyped = EcsFlat as unknown as Record; +export type EcsFlatTyped = typeof EcsFlatTyped; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.test.tsx index 43c05f5757dd0..7fd0a3f3b133d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.test.tsx @@ -5,20 +5,15 @@ * 2.0. */ -import { EcsFlat } from '@elastic/ecs'; -import { omit } from 'lodash/fp'; - +import { EcsFlatTyped } from '../../constants'; import { getUnallowedValueRequestItems, getValidValues, hasAllowedValues } from './helpers'; -import { AllowedValue, EcsMetadata } from '../../types'; - -const ecsMetadata: Record = EcsFlat as unknown as Record; describe('helpers', () => { describe('hasAllowedValues', () => { test('it returns true for a field that has `allowed_values`', () => { expect( hasAllowedValues({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldName: 'event.category', }) ).toBe(true); @@ -27,7 +22,7 @@ describe('helpers', () => { test('it returns false for a field that does NOT have `allowed_values`', () => { expect( hasAllowedValues({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldName: 'host.name', }) ).toBe(false); @@ -36,25 +31,16 @@ describe('helpers', () => { test('it returns false for a field that does NOT exist in `ecsMetadata`', () => { expect( hasAllowedValues({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldName: 'does.NOT.exist', }) ).toBe(false); }); - - test('it returns false when `ecsMetadata` is null', () => { - expect( - hasAllowedValues({ - ecsMetadata: null, // <-- - fieldName: 'event.category', - }) - ).toBe(false); - }); }); describe('getValidValues', () => { test('it returns the expected valid values', () => { - expect(getValidValues(ecsMetadata['event.category'])).toEqual( + expect(getValidValues(EcsFlatTyped['event.category'])).toEqual( expect.arrayContaining([ 'authentication', 'configuration', @@ -79,60 +65,19 @@ describe('helpers', () => { }); test('it returns an empty array when the `field` does NOT have `allowed_values`', () => { - expect(getValidValues(ecsMetadata['host.name'])).toEqual([]); + expect(getValidValues(EcsFlatTyped['host.name'])).toEqual([]); }); test('it returns an empty array when `field` is undefined', () => { expect(getValidValues(undefined)).toEqual([]); }); - - test('it skips `allowed_values` where `name` is undefined', () => { - // omit the `name` property from the `database` `AllowedValue`: - const missingDatabase = - ecsMetadata['event.category'].allowed_values?.map((x) => - x.name === 'database' ? omit('name', x) : x - ) ?? []; - - const field = { - ...ecsMetadata['event.category'], - allowed_values: missingDatabase, - }; - - expect(getValidValues(field)).toEqual( - expect.arrayContaining([ - 'authentication', - 'configuration', - 'driver', - 'email', - 'file', - 'host', - 'iam', - 'intrusion_detection', - 'malware', - 'network', - 'package', - 'process', - 'registry', - 'session', - 'threat', - 'vulnerability', - 'web', - ]) - ); - expect(getValidValues(field)).not.toEqual( - expect.arrayContaining([ - // there should be no entry for 'database' - 'database', - ]) - ); - }); }); describe('getUnallowedValueRequestItems', () => { test('it returns the expected request items', () => { expect( getUnallowedValueRequestItems({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, indexName: 'auditbeat-*', }) ).toEqual([ @@ -203,14 +148,5 @@ describe('helpers', () => { }, ]); }); - - test('it returns an empty array when `ecsMetadata` is null', () => { - expect( - getUnallowedValueRequestItems({ - ecsMetadata: null, // <-- - indexName: 'auditbeat-*', - }) - ).toEqual([]); - }); }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.tsx index 707e83fd0df52..fd356b9fe60d5 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.tsx @@ -5,40 +5,38 @@ * 2.0. */ -import type { EcsMetadata, UnallowedValueRequestItem } from '../../types'; +import type { EcsFlatTyped } from '../../constants'; +import type { EcsFieldMetadata, UnallowedValueRequestItem } from '../../types'; export const hasAllowedValues = ({ ecsMetadata, fieldName, }: { - ecsMetadata: Record | null; + ecsMetadata: EcsFlatTyped; fieldName: string; -}): boolean => - ecsMetadata != null ? (ecsMetadata[fieldName]?.allowed_values?.length ?? 0) > 0 : false; +}): boolean => (ecsMetadata[fieldName]?.allowed_values?.length ?? 0) > 0; -export const getValidValues = (field: EcsMetadata | undefined): string[] => - field?.allowed_values?.flatMap(({ name }) => (name != null ? name : [])) ?? []; +export const getValidValues = (field?: EcsFieldMetadata): string[] => + field?.allowed_values?.flatMap(({ name }) => name) ?? []; export const getUnallowedValueRequestItems = ({ ecsMetadata, indexName, }: { - ecsMetadata: Record | null; + ecsMetadata: EcsFlatTyped; indexName: string; }): UnallowedValueRequestItem[] => - ecsMetadata != null - ? Object.keys(ecsMetadata).reduce( - (acc, fieldName) => - hasAllowedValues({ ecsMetadata, fieldName }) - ? [ - ...acc, - { - indexName, - indexFieldName: fieldName, - allowedValues: getValidValues(ecsMetadata[fieldName]), - }, - ] - : acc, - [] - ) - : []; + Object.keys(ecsMetadata).reduce( + (acc, fieldName) => + hasAllowedValues({ ecsMetadata, fieldName }) + ? [ + ...acc, + { + indexName, + indexFieldName: fieldName, + allowedValues: getValidValues(ecsMetadata[fieldName]), + }, + ] + : acc, + [] + ); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.test.ts index c97a8f4fb9a50..70d522f18177a 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.test.ts @@ -5,15 +5,14 @@ * 2.0. */ -import { EcsFlat, EcsVersion } from '@elastic/ecs'; +import { EcsVersion } from '@elastic/ecs'; import { checkIndex, EMPTY_PARTITIONED_FIELD_METADATA } from './check_index'; import { EMPTY_STAT } from '../../../../helpers'; import { mockMappingsResponse } from '../../../../mock/mappings_response/mock_mappings_response'; import { mockUnallowedValuesResponse } from '../../../../mock/unallowed_values/mock_unallowed_values'; -import { EcsMetadata, UnallowedValueRequestItem } from '../../../../types'; - -const ecsMetadata = EcsFlat as unknown as Record; +import { UnallowedValueRequestItem } from '../../../../types'; +import { EcsFlatTyped } from '../../../../constants'; let mockFetchMappings = jest.fn( ({ @@ -99,7 +98,7 @@ describe('checkIndex', () => { abortController: new AbortController(), batchId: 'batch-id', checkAllStartTime: Date.now(), - ecsMetadata, + ecsMetadata: EcsFlatTyped, formatBytes, formatNumber, httpFetch, @@ -149,7 +148,7 @@ describe('checkIndex', () => { abortController, batchId: 'batch-id', checkAllStartTime: Date.now(), - ecsMetadata, + ecsMetadata: EcsFlatTyped, formatBytes, formatNumber, httpFetch, @@ -164,51 +163,6 @@ describe('checkIndex', () => { }); }); - describe('when `ecsMetadata` is null', () => { - const onCheckCompleted = jest.fn(); - - beforeEach(async () => { - jest.clearAllMocks(); - - await checkIndex({ - abortController: new AbortController(), - batchId: 'batch-id', - checkAllStartTime: Date.now(), - ecsMetadata: null, // <-- - formatBytes, - formatNumber, - httpFetch, - indexName, - isLastCheck: false, - onCheckCompleted, - pattern, - version: EcsVersion, - }); - }); - - test('it invokes onCheckCompleted with a null `error`', () => { - expect(onCheckCompleted.mock.calls[0][0].error).toBeNull(); - }); - - test('it invokes onCheckCompleted with the expected `indexName`', () => { - expect(onCheckCompleted.mock.calls[0][0].indexName).toEqual(indexName); - }); - - test('it invokes onCheckCompleted with the default `partitionedFieldMetadata`', () => { - expect(onCheckCompleted.mock.calls[0][0].partitionedFieldMetadata).toEqual( - EMPTY_PARTITIONED_FIELD_METADATA - ); - }); - - test('it invokes onCheckCompleted with the expected `pattern`', () => { - expect(onCheckCompleted.mock.calls[0][0].pattern).toEqual(pattern); - }); - - test('it invokes onCheckCompleted with the expected `version`', () => { - expect(onCheckCompleted.mock.calls[0][0].version).toEqual(EcsVersion); - }); - }); - describe('when an error occurs', () => { const onCheckCompleted = jest.fn(); const error = 'simulated fetch mappings error'; @@ -230,7 +184,7 @@ describe('checkIndex', () => { abortController: new AbortController(), batchId: 'batch-id', checkAllStartTime: Date.now(), - ecsMetadata, + ecsMetadata: EcsFlatTyped, formatBytes, formatNumber, httpFetch, @@ -284,7 +238,7 @@ describe('checkIndex', () => { abortController: new AbortController(), batchId: 'batch-id', checkAllStartTime: Date.now(), - ecsMetadata, + ecsMetadata: EcsFlatTyped, formatBytes, formatNumber, httpFetch, @@ -346,7 +300,7 @@ describe('checkIndex', () => { abortController, batchId: 'batch-id', checkAllStartTime: Date.now(), - ecsMetadata, + ecsMetadata: EcsFlatTyped, formatBytes, formatNumber, httpFetch, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.ts index 914f244ec39bc..a9216b9d09fdd 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.ts @@ -12,9 +12,10 @@ import { getSortedPartitionedFieldMetadata, } from '../../../index_properties/helpers'; import * as i18n from './translations'; -import type { EcsMetadata, OnCheckCompleted, PartitionedFieldMetadata } from '../../../../types'; +import type { OnCheckCompleted, PartitionedFieldMetadata } from '../../../../types'; import { fetchMappings } from '../../../../use_mappings/helpers'; import { fetchUnallowedValues, getUnallowedValues } from '../../../../use_unallowed_values/helpers'; +import { EcsFlatTyped } from '../../../../constants'; export const EMPTY_PARTITIONED_FIELD_METADATA: PartitionedFieldMetadata = { all: [], @@ -41,7 +42,7 @@ export async function checkIndex({ abortController: AbortController; batchId: string; checkAllStartTime: number; - ecsMetadata: Record | null; + ecsMetadata: EcsFlatTyped; formatBytes: (value: number | undefined) => string; formatNumber: (value: number | undefined) => string; httpFetch: HttpHandler; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/index.tsx index dafa39492ab6e..5155c2a07f9cf 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { EcsFlat, EcsVersion } from '@elastic/ecs'; +import { EcsVersion } from '@elastic/ecs'; import { EuiButton } from '@elastic/eui'; import React, { useCallback, useEffect, useRef, useState } from 'react'; @@ -16,7 +16,8 @@ import { checkIndex } from './check_index'; import { useDataQualityContext } from '../../../data_quality_context'; import { getAllIndicesToCheck } from './helpers'; import * as i18n from '../../../../translations'; -import type { EcsMetadata, IndexToCheck, OnCheckCompleted } from '../../../../types'; +import type { IndexToCheck, OnCheckCompleted } from '../../../../types'; +import { EcsFlatTyped } from '../../../../constants'; const CheckAllButton = styled(EuiButton)` width: 112px; @@ -97,7 +98,7 @@ const CheckAllComponent: React.FC = ({ abortController: abortController.current, batchId, checkAllStartTime: startTime, - ecsMetadata: EcsFlat as unknown as Record, + ecsMetadata: EcsFlatTyped, formatBytes, formatNumber, httpFetch, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.test.ts index a4c2bf01f3b41..2ef3eb9c1c190 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.test.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { EcsFlat } from '@elastic/ecs'; - import { getMappingsProperties, getSortedPartitionedFieldMetadata, @@ -14,16 +12,14 @@ import { } from './helpers'; import { mockIndicesGetMappingIndexMappingRecords } from '../../mock/indices_get_mapping_index_mapping_record/mock_indices_get_mapping_index_mapping_record'; import { mockMappingsProperties } from '../../mock/mappings_properties/mock_mappings_properties'; -import { EcsMetadata } from '../../types'; - -const ecsMetadata: Record = EcsFlat as unknown as Record; +import { EcsFlatTyped } from '../../constants'; describe('helpers', () => { describe('getSortedPartitionedFieldMetadata', () => { test('it returns null when mappings are loading', () => { expect( getSortedPartitionedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, loadingMappings: true, // <-- mappingsProperties: mockMappingsProperties, unallowedValues: {}, @@ -31,21 +27,10 @@ describe('helpers', () => { ).toBeNull(); }); - test('it returns null when `ecsMetadata` is null', () => { - expect( - getSortedPartitionedFieldMetadata({ - ecsMetadata: null, // <-- - loadingMappings: false, - mappingsProperties: mockMappingsProperties, - unallowedValues: {}, - }) - ).toBeNull(); - }); - test('it returns null when `unallowedValues` is null', () => { expect( getSortedPartitionedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, loadingMappings: false, mappingsProperties: mockMappingsProperties, unallowedValues: null, // <-- @@ -54,30 +39,27 @@ describe('helpers', () => { }); describe('when `mappingsProperties` is unknown', () => { + const incompatibleFieldMetadata = { + ...EcsFlatTyped['@timestamp'], + hasEcsMetadata: true, + indexFieldName: '@timestamp', + indexFieldType: '-', + indexInvalidValues: [], + isEcsCompliant: false, + isInSameFamily: false, + }; const expected = { - all: [], + all: [incompatibleFieldMetadata], custom: [], ecsCompliant: [], - incompatible: [ - { - description: - 'Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events.', - hasEcsMetadata: true, - indexFieldName: '@timestamp', - indexFieldType: '-', - indexInvalidValues: [], - isEcsCompliant: false, - isInSameFamily: false, - type: 'date', - }, - ], + incompatible: [incompatibleFieldMetadata], sameFamily: [], }; test('it returns a `PartitionedFieldMetadata` with an `incompatible` `@timestamp` when `mappingsProperties` is undefined', () => { expect( getSortedPartitionedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, loadingMappings: false, mappingsProperties: undefined, // <-- unallowedValues: {}, @@ -88,7 +70,7 @@ describe('helpers', () => { test('it returns a `PartitionedFieldMetadata` with an `incompatible` `@timestamp` when `mappingsProperties` is null', () => { expect( getSortedPartitionedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, loadingMappings: false, mappingsProperties: null, // <-- unallowedValues: {}, @@ -116,7 +98,7 @@ describe('helpers', () => { expect( getSortedPartitionedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, loadingMappings: false, mappingsProperties: mockMappingsProperties, unallowedValues, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.ts index 5c19b83dd0dfd..e47d18685615f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.ts @@ -10,6 +10,7 @@ import type { MappingProperty, } from '@elastic/elasticsearch/lib/api/types'; import { sortBy } from 'lodash/fp'; +import { EcsFlatTyped } from '../../constants'; import { getEnrichedFieldMetadata, @@ -17,7 +18,7 @@ import { getMissingTimestampFieldMetadata, getPartitionedFieldMetadata, } from '../../helpers'; -import type { EcsMetadata, PartitionedFieldMetadata, UnallowedValueCount } from '../../types'; +import type { PartitionedFieldMetadata, UnallowedValueCount } from '../../types'; export const ALL_TAB_ID = 'allTab'; export const ECS_COMPLIANT_TAB_ID = 'ecsCompliantTab'; @@ -40,19 +41,26 @@ export const getSortedPartitionedFieldMetadata = ({ mappingsProperties, unallowedValues, }: { - ecsMetadata: Record | null; + ecsMetadata: EcsFlatTyped; loadingMappings: boolean; mappingsProperties: Record | null | undefined; unallowedValues: Record | null; }): PartitionedFieldMetadata | null => { - if (loadingMappings || ecsMetadata == null || unallowedValues == null) { + if (loadingMappings || unallowedValues == null) { return null; } + // this covers scenario when we try to check an empty index + // or index without required @timestamp field in the mapping + // + // we create an artifical incompatible timestamp field metadata + // so that we can signal to user that the incompatibility is due to missing timestamp if (mappingsProperties == null) { + const missingTimestampFieldMetadata = getMissingTimestampFieldMetadata(); return { ...EMPTY_METADATA, - incompatible: [getMissingTimestampFieldMetadata()], + all: [missingTimestampFieldMetadata], + incompatible: [missingTimestampFieldMetadata], }; } diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx index 9a00f588a15d7..a9dce4dafc741 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { EcsFlat, EcsVersion } from '@elastic/ecs'; +import { EcsVersion } from '@elastic/ecs'; import type { FlameElementEvent, HeatmapElementEvent, @@ -39,12 +39,13 @@ import { getSameFamilyFields, } from '../tabs/incompatible_tab/helpers'; import * as i18n from './translations'; -import type { EcsMetadata, IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../types'; +import type { IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../types'; import { useAddToNewCase } from '../../use_add_to_new_case'; import { useMappings } from '../../use_mappings'; import { useUnallowedValues } from '../../use_unallowed_values'; import { useDataQualityContext } from '../data_quality_context'; import { formatStorageResult, postStorageResult, getSizeInBytes } from '../../helpers'; +import { EcsFlatTyped } from '../../constants'; const EMPTY_MARKDOWN_COMMENTS: string[] = []; @@ -109,7 +110,7 @@ const IndexPropertiesComponent: React.FC = ({ const requestItems = useMemo( () => getUnallowedValueRequestItems({ - ecsMetadata: EcsFlat as unknown as Record, + ecsMetadata: EcsFlatTyped, indexName, }), [indexName] @@ -134,7 +135,7 @@ const IndexPropertiesComponent: React.FC = ({ const partitionedFieldMetadata: PartitionedFieldMetadata | null = useMemo( () => getSortedPartitionedFieldMetadata({ - ecsMetadata: EcsFlat as unknown as Record, + ecsMetadata: EcsFlatTyped, loadingMappings, mappingsProperties, unallowedValues, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts index 098c66370f1c9..8c14a214cabf3 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts @@ -57,7 +57,7 @@ import { import { SAME_FAMILY } from '../../same_family/translations'; import { INCOMPATIBLE_FIELD_MAPPINGS_TABLE_TITLE } from '../../tabs/incompatible_tab/translations'; import { - EnrichedFieldMetadata, + EcsBasedFieldMetadata, ErrorSummary, PatternRollup, UnallowedValueCount, @@ -230,17 +230,6 @@ describe('helpers', () => { '| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |' ); }); - - test('it returns the expected table rows when some have allowed values', () => { - const withAllowedValues = [ - ...mockCustomFields, - eventCategory, // note: this is not a real-world use case, because custom fields don't have allowed values - ]; - - expect(getCustomMarkdownTableRows(withAllowedValues)).toEqual( - '| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |\n| event.category | `keyword` | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` |' - ); - }); }); describe('getSameFamilyBadge', () => { @@ -265,7 +254,7 @@ describe('helpers', () => { describe('getIncompatibleMappingsMarkdownTableRows', () => { test('it returns the expected table rows when the field is in the same family', () => { - const eventCategoryWithWildcard: EnrichedFieldMetadata = { + const eventCategoryWithWildcard: EcsBasedFieldMetadata = { ...eventCategory, // `event.category` is a `keyword` per the ECS spec indexFieldType: 'wildcard', // this index has a mapping of `wildcard` instead of `keyword` isInSameFamily: true, // `wildcard` and `keyword` are in the same family @@ -282,7 +271,7 @@ describe('helpers', () => { }); test('it returns the expected table rows when the field is NOT in the same family', () => { - const eventCategoryWithText: EnrichedFieldMetadata = { + const eventCategoryWithText: EcsBasedFieldMetadata = { ...eventCategory, // `event.category` is a `keyword` per the ECS spec indexFieldType: 'text', // this index has a mapping of `text` instead of `keyword` isInSameFamily: false, // `text` and `keyword` are NOT in the same family diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts index 850986e3959f1..bdbfdba4ddb35 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts @@ -25,6 +25,8 @@ import { HOT, WARM, COLD, FROZEN, UNMANAGED } from '../../../ilm_phases_empty_pr import * as i18n from '../translations'; import type { AllowedValue, + CustomFieldMetadata, + EcsBasedFieldMetadata, EnrichedFieldMetadata, ErrorSummary, IlmExplainPhaseCounts, @@ -85,23 +87,21 @@ export const getIndexInvalidValues = (indexInvalidValues: UnallowedValueCount[]) .map(({ fieldName, count }) => `${getCodeFormattedValue(escape(fieldName))} (${count})`) .join(', '); // newlines are instead joined with spaces -export const getCustomMarkdownTableRows = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] -): string => - enrichedFieldMetadata +export const getCustomMarkdownTableRows = (customFieldMetadata: CustomFieldMetadata[]): string => + customFieldMetadata .map( (x) => `| ${escape(x.indexFieldName)} | ${getCodeFormattedValue( x.indexFieldType - )} | ${getAllowedValues(x.allowed_values)} |` + )} | ${getAllowedValues(undefined)} |` ) .join('\n'); -export const getSameFamilyBadge = (enrichedFieldMetadata: EnrichedFieldMetadata): string => - enrichedFieldMetadata.isInSameFamily ? getCodeFormattedValue(SAME_FAMILY) : ''; +export const getSameFamilyBadge = (ecsBasedFieldMetadata: EcsBasedFieldMetadata): string => + ecsBasedFieldMetadata.isInSameFamily ? getCodeFormattedValue(SAME_FAMILY) : ''; export const getIncompatibleMappingsMarkdownTableRows = ( - incompatibleMappings: EnrichedFieldMetadata[] + incompatibleMappings: EcsBasedFieldMetadata[] ): string => incompatibleMappings .map( @@ -113,7 +113,7 @@ export const getIncompatibleMappingsMarkdownTableRows = ( .join('\n'); export const getIncompatibleValuesMarkdownTableRows = ( - incompatibleValues: EnrichedFieldMetadata[] + incompatibleValues: EcsBasedFieldMetadata[] ): string => incompatibleValues .map( @@ -173,14 +173,14 @@ ${getMarkdownTableRows(errorSummary)} ` : ''; -export const getMarkdownTable = ({ +export const getMarkdownTable = ({ enrichedFieldMetadata, getMarkdownTableRows, headerNames, title, }: { - enrichedFieldMetadata: EnrichedFieldMetadata[]; - getMarkdownTableRows: (enrichedFieldMetadata: EnrichedFieldMetadata[]) => string; + enrichedFieldMetadata: T; + getMarkdownTableRows: (enrichedFieldMetadata: T) => string; headerNames: string[]; title: string; }): string => diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.test.tsx index 41eb828b1be28..fc2ba0327062f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.test.tsx @@ -23,7 +23,7 @@ describe('CustomCallout', () => { beforeEach(() => { render( - +
{content}
 diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.tsx index 74cf0dd6ce068..8d1bc0c20a72c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.tsx @@ -9,27 +9,27 @@ import { EcsVersion } from '@elastic/ecs'; import { EuiCallOut, EuiSpacer } from '@elastic/eui'; import React, { useMemo } from 'react'; -import type { EnrichedFieldMetadata } from '../../../../types'; +import type { CustomFieldMetadata } from '../../../../types'; import * as i18n from '../../../index_properties/translations'; interface Props { children?: React.ReactNode; - enrichedFieldMetadata: EnrichedFieldMetadata[]; + customFieldMetadata: CustomFieldMetadata[]; } -const CustomCalloutComponent: React.FC = ({ children, enrichedFieldMetadata }) => { +const CustomCalloutComponent: React.FC = ({ children, customFieldMetadata }) => { const title = useMemo( () => ( - {i18n.CUSTOM_CALLOUT_TITLE(enrichedFieldMetadata.length)} + {i18n.CUSTOM_CALLOUT_TITLE(customFieldMetadata.length)} ), - [enrichedFieldMetadata.length] + [customFieldMetadata.length] ); return (
- {i18n.CUSTOM_CALLOUT({ fieldCount: enrichedFieldMetadata.length, version: EcsVersion })} + {i18n.CUSTOM_CALLOUT({ fieldCount: customFieldMetadata.length, version: EcsVersion })}
{i18n.ECS_IS_A_PERMISSIVE_SCHEMA}
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.test.ts deleted file mode 100644 index b059c444f4c60..0000000000000 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.test.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getIncompatiableFieldsInSameFamilyCount } from './helpers'; -import { - eventCategory, - eventCategoryWithUnallowedValues, - hostNameWithTextMapping, - someField, - sourceIpWithTextMapping, -} from '../../../../mock/enriched_field_metadata/mock_enriched_field_metadata'; -import { EnrichedFieldMetadata } from '../../../../types'; - -const sameFamily: EnrichedFieldMetadata = { - ...eventCategory, // `event.category` is a `keyword` per the ECS spec - indexFieldType: 'wildcard', // this index has a mapping of `wildcard` instead of `keyword` - isInSameFamily: true, // `wildcard` and `keyword` are in the same family - isEcsCompliant: false, // wildcard !== keyword -}; - -describe('helpers', () => { - describe('getFieldsInSameFamilyCount', () => { - test('it filters out fields that are ECS compliant', () => { - expect( - getIncompatiableFieldsInSameFamilyCount([ - eventCategory, // isEcsCompliant: true, indexInvalidValues.length: 0, isInSameFamily: true, `keyword` and `keyword` are in the same family - ]) - ).toEqual(0); - }); - - test('it filters out fields with unallowed values', () => { - expect( - getIncompatiableFieldsInSameFamilyCount([ - eventCategoryWithUnallowedValues, // isEcsCompliant: false, indexInvalidValues.length: 2, isInSameFamily: true, `keyword` and `keyword` are in the same family - ]) - ).toEqual(0); - }); - - test('it filters out fields that are not in the same family', () => { - expect( - getIncompatiableFieldsInSameFamilyCount([ - hostNameWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, `keyword` and `text` are not in the family - ]) - ).toEqual(0); - }); - - test('it returns 1 for an incompatible field in the same family', () => { - expect( - getIncompatiableFieldsInSameFamilyCount([ - sameFamily, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: true, `wildcard` and `keyword` are in the same family - ]) - ).toEqual(1); - }); - - test('it returns the expected count when some of the input should be counted', () => { - expect( - getIncompatiableFieldsInSameFamilyCount([ - sameFamily, - eventCategoryWithUnallowedValues, // isEcsCompliant: false, indexInvalidValues.length: 2, isInSameFamily: true, `keyword` and `keyword` are in the same family - hostNameWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length, isInSameFamily: false, `text` and `keyword` not in the same family - someField, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, custom fields are never in the same family - sourceIpWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, `ip` is not a member of any families - ]) - ).toEqual(1); - }); - - test('it returns zero when none of the input should be counted', () => { - expect( - getIncompatiableFieldsInSameFamilyCount([ - eventCategoryWithUnallowedValues, // isEcsCompliant: false, indexInvalidValues.length: 2, isInSameFamily: true, `keyword` and `keyword` are in the same family - hostNameWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length, isInSameFamily: false, `text` and `keyword` not in the same family - someField, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, custom fields are never in the same family - sourceIpWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, `ip` is not a member of any families - ]) - ).toEqual(0); - }); - }); -}); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.ts deleted file mode 100644 index ca0e6c42098b9..0000000000000 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EnrichedFieldMetadata } from '../../../../types'; - -export const getIncompatiableFieldsInSameFamilyCount = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] -): number => - enrichedFieldMetadata.filter( - (x) => !x.isEcsCompliant && x.indexInvalidValues.length === 0 && x.isInSameFamily - ).length; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/index.test.tsx index 522162fff699b..590dff6d64a0b 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/index.test.tsx @@ -21,12 +21,12 @@ import { sourceIpWithTextMapping, } from '../../../../mock/enriched_field_metadata/mock_enriched_field_metadata'; import { TestProviders } from '../../../../mock/test_providers/test_providers'; -import { EnrichedFieldMetadata } from '../../../../types'; +import { EcsBasedFieldMetadata } from '../../../../types'; import { IncompatibleCallout } from '.'; const content = 'Is your name Michael?'; -const eventCategoryWithWildcard: EnrichedFieldMetadata = { +const eventCategoryWithWildcard: EcsBasedFieldMetadata = { ...eventCategory, // `event.category` is a `keyword` per the ECS spec indexFieldType: 'wildcard', // this index has a mapping of `wildcard` instead of `keyword` isInSameFamily: true, // `wildcard` and `keyword` are in the same family @@ -38,7 +38,7 @@ describe('IncompatibleCallout', () => { render( = ({ children, enrichedFieldMetadata }) => { - const fieldCount = enrichedFieldMetadata.length; +const IncompatibleCalloutComponent: React.FC = ({ children, ecsBasedFieldMetadata }) => { + const fieldCount = ecsBasedFieldMetadata.length; const title = useMemo( () => {i18n.INCOMPATIBLE_CALLOUT_TITLE(fieldCount)}, [fieldCount] diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.test.tsx index 89494a2c91b03..774b0a04ec174 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.test.tsx @@ -21,7 +21,7 @@ describe('SameFamilyCallout', () => { render(
{content}
 diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.tsx index 961f9a43aa781..ffec880687c8c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.tsx @@ -10,28 +10,28 @@ import { EuiCallOut, EuiSpacer, EuiText } from '@elastic/eui'; import React, { useMemo } from 'react'; import * as i18n from '../../../index_properties/translations'; -import type { EnrichedFieldMetadata } from '../../../../types'; +import type { EcsBasedFieldMetadata } from '../../../../types'; interface Props { children?: React.ReactNode; - enrichedFieldMetadata: EnrichedFieldMetadata[]; + ecsBasedFieldMetadata: EcsBasedFieldMetadata[]; } -const SameFamilyCalloutComponent: React.FC = ({ children, enrichedFieldMetadata }) => { +const SameFamilyCalloutComponent: React.FC = ({ children, ecsBasedFieldMetadata }) => { const title = useMemo( () => ( - {i18n.SAME_FAMILY_CALLOUT_TITLE(enrichedFieldMetadata.length)} + {i18n.SAME_FAMILY_CALLOUT_TITLE(ecsBasedFieldMetadata.length)} ), - [enrichedFieldMetadata.length] + [ecsBasedFieldMetadata.length] ); return (
{i18n.SAME_FAMILY_CALLOUT({ - fieldCount: enrichedFieldMetadata.length, + fieldCount: ecsBasedFieldMetadata.length, version: EcsVersion, })}
diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts index 30aae990fc7dd..31dd9644bbc1d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts @@ -35,7 +35,7 @@ const formatNumber = (value: number | undefined) => describe('helpers', () => { describe('getCustomMarkdownComment', () => { test('it returns a comment for custom fields with the expected field counts and ECS version', () => { - expect(getCustomMarkdownComment({ enrichedFieldMetadata: [hostNameKeyword, someField] })) + expect(getCustomMarkdownComment({ customFieldMetadata: [hostNameKeyword, someField] })) .toEqual(`#### 2 Custom field mappings These fields are not defined by the Elastic Common Schema (ECS), version ${EcsVersion}. diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.ts index 0d659ac6570f5..f5b6ca7220809 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.ts @@ -19,26 +19,26 @@ import { } from '../../index_properties/markdown/helpers'; import * as i18n from '../../index_properties/translations'; import { getFillColor } from '../summary_tab/helpers'; -import type { EnrichedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types'; +import type { CustomFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types'; export const getCustomMarkdownComment = ({ - enrichedFieldMetadata, + customFieldMetadata, }: { - enrichedFieldMetadata: EnrichedFieldMetadata[]; + customFieldMetadata: CustomFieldMetadata[]; }): string => getMarkdownComment({ suggestedAction: `${i18n.CUSTOM_CALLOUT({ - fieldCount: enrichedFieldMetadata.length, + fieldCount: customFieldMetadata.length, version: EcsVersion, })} ${i18n.ECS_IS_A_PERMISSIVE_SCHEMA} `, - title: i18n.CUSTOM_CALLOUT_TITLE(enrichedFieldMetadata.length), + title: i18n.CUSTOM_CALLOUT_TITLE(customFieldMetadata.length), }); -export const showCustomCallout = (enrichedFieldMetadata: EnrichedFieldMetadata[]): boolean => - enrichedFieldMetadata.length > 0; +export const showCustomCallout = (customFieldMetadata: CustomFieldMetadata[]): boolean => + customFieldMetadata.length > 0; export const getCustomColor = (partitionedFieldMetadata: PartitionedFieldMetadata): string => showCustomCallout(partitionedFieldMetadata.custom) @@ -80,7 +80,7 @@ export const getAllCustomMarkdownComments = ({ }), getTabCountsMarkdownComment(partitionedFieldMetadata), getCustomMarkdownComment({ - enrichedFieldMetadata: partitionedFieldMetadata.custom, + customFieldMetadata: partitionedFieldMetadata.custom, }), getMarkdownTable({ enrichedFieldMetadata: partitionedFieldMetadata.custom, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/index.tsx index 019e65e891474..b976be6193087 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/index.tsx @@ -91,7 +91,7 @@ const CustomTabComponent: React.FC = ({ <> {showCustomCallout(partitionedFieldMetadata.custom) ? ( <> - + diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx index c6386d7710bfc..3c04a6473facf 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx @@ -11,7 +11,6 @@ import { omit } from 'lodash/fp'; import { eventCategory, - someField, timestamp, } from '../../mock/enriched_field_metadata/mock_enriched_field_metadata'; import { mockPartitionedFieldMetadata } from '../../mock/partitioned_field_metadata/mock_partitioned_field_metadata'; @@ -38,11 +37,11 @@ describe('helpers', () => { }); test('it returns false when `enrichedFieldMetadata` contains an @timestamp field', () => { - expect(showMissingTimestampCallout([timestamp, eventCategory, someField])).toBe(false); + expect(showMissingTimestampCallout([timestamp, eventCategory])).toBe(false); }); test('it returns true when `enrichedFieldMetadata` does NOT contain an @timestamp field', () => { - expect(showMissingTimestampCallout([eventCategory, someField])).toBe(true); + expect(showMissingTimestampCallout([eventCategory])).toBe(true); }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx index 0f59258949b2d..670357c3730f7 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx @@ -39,7 +39,7 @@ import { SameFamilyTab } from './same_family_tab'; import { SummaryTab } from './summary_tab'; import { getFillColor } from './summary_tab/helpers'; import type { - EnrichedFieldMetadata, + EcsBasedFieldMetadata, IlmPhase, MeteringStatsIndex, PartitionedFieldMetadata, @@ -56,8 +56,8 @@ ${i18n.PAGES_MAY_NOT_DISPLAY_EVENTS} }); export const showMissingTimestampCallout = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] -): boolean => !enrichedFieldMetadata.some((x) => x.name === '@timestamp'); + ecsBasedFieldMetadata: EcsBasedFieldMetadata[] +): boolean => !ecsBasedFieldMetadata.some((x) => x.name === '@timestamp'); export const getEcsCompliantColor = (partitionedFieldMetadata: PartitionedFieldMetadata): string => showMissingTimestampCallout(partitionedFieldMetadata.ecsCompliant) diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.ts index b2f5a5c430175..857b53589f163 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.ts @@ -19,7 +19,7 @@ import { } from '../../index_properties/markdown/helpers'; import { getFillColor } from '../summary_tab/helpers'; import * as i18n from '../../index_properties/translations'; -import type { EnrichedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types'; +import type { EcsBasedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types'; import { INCOMPATIBLE_FIELD_MAPPINGS_TABLE_TITLE, INCOMPATIBLE_FIELD_VALUES_TABLE_TITLE, @@ -44,17 +44,17 @@ ${i18n.MAPPINGS_THAT_CONFLICT_WITH_ECS} title: i18n.INCOMPATIBLE_CALLOUT_TITLE(incompatible), }); -export const showInvalidCallout = (enrichedFieldMetadata: EnrichedFieldMetadata[]): boolean => - enrichedFieldMetadata.length > 0; +export const showInvalidCallout = (ecsBasedFieldMetadata: EcsBasedFieldMetadata[]): boolean => + ecsBasedFieldMetadata.length > 0; export const getIncompatibleColor = (): string => getFillColor('incompatible'); export const getSameFamilyColor = (): string => getFillColor('same-family'); export const getIncompatibleMappings = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] -): EnrichedFieldMetadata[] => - enrichedFieldMetadata.filter( + ecsBasedFieldMetadata: EcsBasedFieldMetadata[] +): EcsBasedFieldMetadata[] => + ecsBasedFieldMetadata.filter( (x) => !x.isEcsCompliant && x.type !== x.indexFieldType && @@ -62,9 +62,9 @@ export const getIncompatibleMappings = ( ); export const getIncompatibleMappingsFields = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] + ecsBasedFieldMetadata: EcsBasedFieldMetadata[] ): string[] => - enrichedFieldMetadata.reduce((acc, x) => { + ecsBasedFieldMetadata.reduce((acc, x) => { if ( !x.isEcsCompliant && x.type !== x.indexFieldType && @@ -78,8 +78,8 @@ export const getIncompatibleMappingsFields = ( return acc; }, []); -export const getSameFamilyFields = (enrichedFieldMetadata: EnrichedFieldMetadata[]): string[] => - enrichedFieldMetadata.reduce((acc, x) => { +export const getSameFamilyFields = (ecsBasedFieldMetadata: EcsBasedFieldMetadata[]): string[] => + ecsBasedFieldMetadata.reduce((acc, x) => { if (!x.isEcsCompliant && x.type !== x.indexFieldType && x.isInSameFamily) { const field = escape(x.indexFieldName); if (field != null) { @@ -90,14 +90,14 @@ export const getSameFamilyFields = (enrichedFieldMetadata: EnrichedFieldMetadata }, []); export const getIncompatibleValues = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] -): EnrichedFieldMetadata[] => - enrichedFieldMetadata.filter((x) => !x.isEcsCompliant && x.indexInvalidValues.length > 0); + ecsBasedFieldMetadata: EcsBasedFieldMetadata[] +): EcsBasedFieldMetadata[] => + ecsBasedFieldMetadata.filter((x) => !x.isEcsCompliant && x.indexInvalidValues.length > 0); export const getIncompatibleValuesFields = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] + ecsBasedFieldMetadata: EcsBasedFieldMetadata[] ): string[] => - enrichedFieldMetadata.reduce((acc, x) => { + ecsBasedFieldMetadata.reduce((acc, x) => { if (!x.isEcsCompliant && x.indexInvalidValues.length > 0) { const field = escape(x.indexFieldName); if (field != null) { @@ -112,8 +112,8 @@ export const getIncompatibleFieldsMarkdownTablesComment = ({ incompatibleValues, indexName, }: { - incompatibleMappings: EnrichedFieldMetadata[]; - incompatibleValues: EnrichedFieldMetadata[]; + incompatibleMappings: EcsBasedFieldMetadata[]; + incompatibleValues: EcsBasedFieldMetadata[]; indexName: string; }): string => ` ${ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/index.tsx index aa6866555a3b3..94333d57da0b2 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/index.tsx @@ -127,7 +127,7 @@ const IncompatibleTabComponent: React.FC = ({
{showInvalidCallout(partitionedFieldMetadata.incompatible) ? ( <> - + getMarkdownComment({ @@ -37,14 +37,14 @@ ${i18n.FIELDS_WITH_MAPPINGS_SAME_FAMILY} }); export const getSameFamilyMappings = ( - enrichedFieldMetadata: EnrichedFieldMetadata[] -): EnrichedFieldMetadata[] => enrichedFieldMetadata.filter((x) => x.isInSameFamily); + enrichedFieldMetadata: EcsBasedFieldMetadata[] +): EcsBasedFieldMetadata[] => enrichedFieldMetadata.filter((x) => x.isInSameFamily); export const getSameFamilyMarkdownTablesComment = ({ sameFamilyMappings, indexName, }: { - sameFamilyMappings: EnrichedFieldMetadata[]; + sameFamilyMappings: EcsBasedFieldMetadata[]; indexName: string; }): string => ` ${ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/same_family_tab/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/same_family_tab/index.tsx index a02ff03d24d4d..26295bbbc2ee8 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/same_family_tab/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/same_family_tab/index.tsx @@ -83,7 +83,7 @@ const SameFamilyTabComponent: React.FC = ({ return (
- + {i18n.COPY_TO_CLIPBOARD} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/index.tsx index aa32baafecf19..96b110a05807e 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/index.tsx @@ -109,7 +109,7 @@ const CalloutSummaryComponent: React.FC = ({ <> {showInvalidCallout(partitionedFieldMetadata.incompatible) && ( <> - + )} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.test.ts index 3838a4dc1c168..c89cf1d5158ec 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.test.ts @@ -35,6 +35,7 @@ import { getSummaryData, getTabId, } from './helpers'; +import { EcsFlatTyped } from '../../../constants'; describe('helpers', () => { describe('getSummaryData', () => { @@ -216,15 +217,13 @@ describe('helpers', () => { custom: [], incompatible: [ { - description: - 'Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events.', + ...EcsFlatTyped['@timestamp'], hasEcsMetadata: true, indexFieldName: '@timestamp', indexFieldType: '-', indexInvalidValues: [], isEcsCompliant: false, isInSameFamily: false, - type: 'date', }, ], sameFamily: [], diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts index f4d44e65309f0..bfe3ffc0c5e55 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts @@ -7,7 +7,6 @@ import { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types'; import { euiThemeVars } from '@kbn/ui-theme'; -import { EcsFlat } from '@elastic/ecs'; import { omit } from 'lodash/fp'; import { @@ -33,11 +32,11 @@ import { getTotalPatternIndicesChecked, getTotalPatternSameFamily, getTotalSizeInBytes, - hasValidTimestampMapping, isMappingCompatible, postStorageResult, getStorageResults, StorageResult, + formatStorageResult, } from './helpers'; import { hostNameWithTextMapping, @@ -68,13 +67,11 @@ import { COLD_DESCRIPTION, FROZEN_DESCRIPTION, HOT_DESCRIPTION, - TIMESTAMP_DESCRIPTION, UNMANAGED_DESCRIPTION, WARM_DESCRIPTION, } from './translations'; import { DataQualityCheckResult, - EcsMetadata, EnrichedFieldMetadata, PartitionedFieldMetadata, PatternRollup, @@ -82,8 +79,8 @@ import { } from './types'; import { httpServiceMock } from '@kbn/core-http-browser-mocks'; import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; - -const ecsMetadata: Record = EcsFlat as unknown as Record; +import { EcsFlatTyped } from './constants'; +import { mockPartitionedFieldMetadataWithSameFamily } from './mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family'; describe('helpers', () => { describe('getTotalPatternSameFamily', () => { @@ -485,7 +482,7 @@ describe('helpers', () => { test('it returns the happy path result when the index has no mapping conflicts, and no unallowed values', () => { expect( getEnrichedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index }) @@ -501,7 +498,7 @@ describe('helpers', () => { expect( getEnrichedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index unallowedValues: noEntryForEventCategory, // a lookup in this map for the `event.category` field will return undefined }) @@ -511,7 +508,7 @@ describe('helpers', () => { test('it returns a result with the expected `indexInvalidValues` and `isEcsCompliant` when the index has no mapping conflict, but it has unallowed values', () => { expect( getEnrichedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index unallowedValues, // this index has unallowed values for the event.category field }) @@ -537,7 +534,7 @@ describe('helpers', () => { expect( getEnrichedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldMetadata: { field: 'event.category', // `event.category` is a `keyword`, per the ECS spec type: indexFieldType, // this index has a mapping of `text` instead @@ -558,7 +555,7 @@ describe('helpers', () => { expect( getEnrichedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldMetadata: { field: 'event.category', // `event.category` is a `keyword` per the ECS spec type: indexFieldType, // this index has a mapping of `wildcard` instead @@ -579,7 +576,7 @@ describe('helpers', () => { expect( getEnrichedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldMetadata: { field: 'event.category', // `event.category` is a `keyword` per the ECS spec type: indexFieldType, // this index has a mapping of `text` instead @@ -611,7 +608,7 @@ describe('helpers', () => { expect( getEnrichedFieldMetadata({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, fieldMetadata: { field, type: indexFieldType, // no mapping conflict, because ECS doesn't define this field @@ -632,14 +629,13 @@ describe('helpers', () => { describe('getMissingTimestampFieldMetadata', () => { test('it returns the expected `EnrichedFieldMetadata`', () => { expect(getMissingTimestampFieldMetadata()).toEqual({ - description: TIMESTAMP_DESCRIPTION, + ...EcsFlatTyped['@timestamp'], hasEcsMetadata: true, indexFieldName: '@timestamp', indexFieldType: '-', // the index did NOT define a mapping for @timestamp indexInvalidValues: [], isEcsCompliant: false, // an index must define the @timestamp mapping isInSameFamily: false, // `date` is not a member of any families - type: 'date', }); }); }); @@ -717,37 +713,6 @@ describe('helpers', () => { }); }); - describe('hasValidTimestampMapping', () => { - test('it returns true when the `enrichedFieldMetadata` has a valid @timestamp', () => { - const enrichedFieldMetadata: EnrichedFieldMetadata[] = [timestamp, sourcePort]; - - expect(hasValidTimestampMapping(enrichedFieldMetadata)).toBe(true); - }); - - test('it returns false when the `enrichedFieldMetadata` collection does NOT include a valid @timestamp', () => { - const enrichedFieldMetadata: EnrichedFieldMetadata[] = [sourcePort]; - - expect(hasValidTimestampMapping(enrichedFieldMetadata)).toBe(false); - }); - - test('it returns false when the `enrichedFieldMetadata` has an @timestamp with an invalid mapping', () => { - const timestampWithInvalidMapping: EnrichedFieldMetadata = { - ...timestamp, - indexFieldType: 'text', // invalid mapping, should be "date" - }; - const enrichedFieldMetadata: EnrichedFieldMetadata[] = [ - timestampWithInvalidMapping, - sourcePort, - ]; - - expect(hasValidTimestampMapping(enrichedFieldMetadata)).toBe(false); - }); - - test('it returns false when `enrichedFieldMetadata` is empty', () => { - expect(hasValidTimestampMapping([])).toBe(false); - }); - }); - describe('getDocsCount', () => { test('it returns the expected docs count when `stats` contains the `indexName`', () => { const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; @@ -1429,6 +1394,120 @@ describe('helpers', () => { }); }); + describe('formatStorageResult', () => { + it('should correctly format the input data into a StorageResult object', () => { + const inputData: Parameters[number] = { + result: { + indexName: 'testIndex', + pattern: 'testPattern', + checkedAt: 1627545600000, + docsCount: 100, + incompatible: 3, + sameFamily: 1, + ilmPhase: 'hot', + markdownComments: ['test comments'], + error: null, + }, + report: { + batchId: 'testBatch', + isCheckAll: true, + sameFamilyFields: ['agent.type'], + unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], + unallowedValueFields: ['event.category'], + sizeInBytes: 5000, + ecsVersion: '1.0.0', + indexName: 'testIndex', + indexId: 'testIndexId', + }, + partitionedFieldMetadata: mockPartitionedFieldMetadataWithSameFamily, + }; + + const expectedResult: StorageResult = { + batchId: 'testBatch', + indexName: 'testIndex', + indexPattern: 'testPattern', + isCheckAll: true, + checkedAt: 1627545600000, + docsCount: 100, + totalFieldCount: 10, + ecsFieldCount: 2, + customFieldCount: 4, + incompatibleFieldCount: 3, + incompatibleFieldMappingItems: [ + { + fieldName: 'event.category', + expectedValue: 'keyword', + actualValue: 'constant_keyword', + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + }, + { + fieldName: 'host.name', + expectedValue: 'keyword', + actualValue: 'text', + description: + 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', + }, + { + fieldName: 'source.ip', + expectedValue: 'ip', + actualValue: 'text', + description: 'IP address of the source (IPv4 or IPv6).', + }, + ], + incompatibleFieldValueItems: [ + { + fieldName: 'event.category', + expectedValues: [ + 'authentication', + 'configuration', + 'database', + 'driver', + 'email', + 'file', + 'host', + 'iam', + 'intrusion_detection', + 'malware', + 'network', + 'package', + 'process', + 'registry', + 'session', + 'threat', + 'vulnerability', + 'web', + ], + actualValues: [{ name: 'an_invalid_category', count: 2 }], + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + }, + ], + sameFamilyFieldCount: 1, + sameFamilyFields: ['agent.type'], + sameFamilyFieldItems: [ + { + fieldName: 'agent.type', + expectedValue: 'keyword', + actualValue: 'constant_keyword', + description: + 'Type of the agent.\nThe agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + }, + ], + unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], + unallowedValueFields: ['event.category'], + sizeInBytes: 5000, + ilmPhase: 'hot', + markdownComments: ['test comments'], + ecsVersion: '1.0.0', + indexId: 'testIndexId', + error: null, + }; + + expect(formatStorageResult(inputData)).toEqual(expectedResult); + }); + }); + describe('postStorageResult', () => { const { fetch } = httpServiceMock.createStartContract(); const { toasts } = notificationServiceMock.createStartContract(); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts index 48dd12688838c..0980ac3763347 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts @@ -17,16 +17,20 @@ import * as i18n from './translations'; import type { DataQualityCheckResult, DataQualityIndexCheckedParams, - EcsMetadata, + EcsBasedFieldMetadata, EnrichedFieldMetadata, ErrorSummary, IlmPhase, + IncompatibleFieldMappingItem, + IncompatibleFieldValueItem, MeteringStatsIndex, PartitionedFieldMetadata, PartitionedFieldMetadataStats, PatternRollup, + SameFamilyFieldItem, UnallowedValueCount, } from './types'; +import { EcsFlatTyped } from './constants'; const EMPTY_INDEX_NAMES: string[] = []; export const INTERNAL_API_VERSION = '1'; @@ -172,7 +176,7 @@ export const getEnrichedFieldMetadata = ({ fieldMetadata, unallowedValues, }: { - ecsMetadata: Record; + ecsMetadata: EcsFlatTyped; fieldMetadata: FieldType; unallowedValues: Record; }): EnrichedFieldMetadata => { @@ -202,7 +206,7 @@ export const getEnrichedFieldMetadata = ({ return { indexFieldName: field, indexFieldType: type, - indexInvalidValues, + indexInvalidValues: [], hasEcsMetadata: false, isEcsCompliant: false, isInSameFamily: false, // custom fields are never in the same family @@ -210,15 +214,14 @@ export const getEnrichedFieldMetadata = ({ } }; -export const getMissingTimestampFieldMetadata = (): EnrichedFieldMetadata => ({ - description: i18n.TIMESTAMP_DESCRIPTION, +export const getMissingTimestampFieldMetadata = (): EcsBasedFieldMetadata => ({ + ...EcsFlatTyped['@timestamp'], hasEcsMetadata: true, indexFieldName: '@timestamp', indexFieldType: '-', indexInvalidValues: [], isEcsCompliant: false, isInSameFamily: false, // `date` is not a member of any families - type: 'date', }); export const getPartitionedFieldMetadata = ( @@ -258,11 +261,6 @@ export const getPartitionedFieldMetadataStats = ( }; }; -export const hasValidTimestampMapping = (enrichedFieldMetadata: EnrichedFieldMetadata[]): boolean => - enrichedFieldMetadata.some( - (x) => x.indexFieldName === '@timestamp' && x.indexFieldType === 'date' - ); - export const getDocsCount = ({ indexName, stats, @@ -471,8 +469,11 @@ export interface StorageResult { ecsFieldCount: number; customFieldCount: number; incompatibleFieldCount: number; + incompatibleFieldMappingItems: IncompatibleFieldMappingItem[]; + incompatibleFieldValueItems: IncompatibleFieldValueItem[]; sameFamilyFieldCount: number; sameFamilyFields: string[]; + sameFamilyFieldItems: SameFamilyFieldItem[]; unallowedMappingFields: string[]; unallowedValueFields: string[]; sizeInBytes: number; @@ -491,28 +492,66 @@ export const formatStorageResult = ({ result: DataQualityCheckResult; report: DataQualityIndexCheckedParams; partitionedFieldMetadata: PartitionedFieldMetadata; -}): StorageResult => ({ - batchId: report.batchId, - indexName: result.indexName, - indexPattern: result.pattern, - isCheckAll: report.isCheckAll, - checkedAt: result.checkedAt ?? Date.now(), - docsCount: result.docsCount ?? 0, - totalFieldCount: partitionedFieldMetadata.all.length, - ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, - customFieldCount: partitionedFieldMetadata.custom.length, - incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, - sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, - sameFamilyFields: report.sameFamilyFields ?? [], - unallowedMappingFields: report.unallowedMappingFields ?? [], - unallowedValueFields: report.unallowedValueFields ?? [], - sizeInBytes: report.sizeInBytes ?? 0, - ilmPhase: result.ilmPhase, - markdownComments: result.markdownComments, - ecsVersion: report.ecsVersion, - indexId: report.indexId ?? '', // ---> we don't have this field when isILMAvailable is false - error: result.error, -}); +}): StorageResult => { + const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = []; + const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = []; + const sameFamilyFieldItems: SameFamilyFieldItem[] = []; + + partitionedFieldMetadata.incompatible.forEach((field) => { + if (field.type !== field.indexFieldType) { + incompatibleFieldMappingItems.push({ + fieldName: field.indexFieldName, + expectedValue: field.type, + actualValue: field.indexFieldType, + description: field.description, + }); + } + + if (field.indexInvalidValues.length > 0) { + incompatibleFieldValueItems.push({ + fieldName: field.indexFieldName, + expectedValues: field.allowed_values?.map((x) => x.name) ?? [], + actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })), + description: field.description, + }); + } + }); + + partitionedFieldMetadata.sameFamily.forEach((field) => { + sameFamilyFieldItems.push({ + fieldName: field.indexFieldName, + expectedValue: field.type, + actualValue: field.indexFieldType, + description: field.description, + }); + }); + + return { + batchId: report.batchId, + indexName: result.indexName, + indexPattern: result.pattern, + isCheckAll: report.isCheckAll, + checkedAt: result.checkedAt ?? Date.now(), + docsCount: result.docsCount ?? 0, + totalFieldCount: partitionedFieldMetadata.all.length, + ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, + customFieldCount: partitionedFieldMetadata.custom.length, + incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, + incompatibleFieldMappingItems, + incompatibleFieldValueItems, + sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, + sameFamilyFields: report.sameFamilyFields ?? [], + sameFamilyFieldItems, + unallowedMappingFields: report.unallowedMappingFields ?? [], + unallowedValueFields: report.unallowedValueFields ?? [], + sizeInBytes: report.sizeInBytes ?? 0, + ilmPhase: result.ilmPhase, + markdownComments: result.markdownComments, + ecsVersion: report.ecsVersion, + indexId: report.indexId ?? '', + error: result.error, + }; +}; export const formatResultFromStorage = ({ storageResult, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/enriched_field_metadata/mock_enriched_field_metadata.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/enriched_field_metadata/mock_enriched_field_metadata.ts index 1f5ea0ce4d4d4..351d0bb06310f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/enriched_field_metadata/mock_enriched_field_metadata.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/enriched_field_metadata/mock_enriched_field_metadata.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { EnrichedFieldMetadata } from '../../types'; +import { CustomFieldMetadata, EcsBasedFieldMetadata } from '../../types'; -export const timestamp: EnrichedFieldMetadata = { +export const timestamp: EcsBasedFieldMetadata = { dashed_name: 'timestamp', description: 'Date/time when the event originated.\nThis is the date/time extracted from the event, typically representing when the event was generated by the source.\nIf the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline.\nRequired field for all events.', @@ -27,7 +27,7 @@ export const timestamp: EnrichedFieldMetadata = { isInSameFamily: false, // `date` is not a member of any families }; -export const eventCategory: EnrichedFieldMetadata = { +export const eventCategory: EcsBasedFieldMetadata = { allowed_values: [ { description: @@ -166,7 +166,7 @@ export const eventCategory: EnrichedFieldMetadata = { isInSameFamily: false, }; -export const eventCategoryWithUnallowedValues: EnrichedFieldMetadata = { +export const eventCategoryWithUnallowedValues: EcsBasedFieldMetadata = { ...eventCategory, indexInvalidValues: [ { @@ -181,7 +181,7 @@ export const eventCategoryWithUnallowedValues: EnrichedFieldMetadata = { isEcsCompliant: false, // because this index has unallowed values }; -export const hostNameWithTextMapping: EnrichedFieldMetadata = { +export const hostNameWithTextMapping: EcsBasedFieldMetadata = { dashed_name: 'host-name', description: 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', @@ -200,7 +200,7 @@ export const hostNameWithTextMapping: EnrichedFieldMetadata = { isInSameFamily: false, // `keyword` and `text` are not in the family }; -export const hostNameKeyword: EnrichedFieldMetadata = { +export const hostNameKeyword: CustomFieldMetadata = { indexFieldName: 'host.name.keyword', indexFieldType: 'keyword', indexInvalidValues: [], @@ -209,7 +209,7 @@ export const hostNameKeyword: EnrichedFieldMetadata = { isInSameFamily: false, // custom fields are never in the same family }; -export const someField: EnrichedFieldMetadata = { +export const someField: CustomFieldMetadata = { indexFieldName: 'some.field', indexFieldType: 'text', indexInvalidValues: [], @@ -218,7 +218,7 @@ export const someField: EnrichedFieldMetadata = { isInSameFamily: false, // custom fields are never in the same family }; -export const someFieldKeyword: EnrichedFieldMetadata = { +export const someFieldKeyword: CustomFieldMetadata = { indexFieldName: 'some.field.keyword', indexFieldType: 'keyword', indexInvalidValues: [], @@ -227,7 +227,7 @@ export const someFieldKeyword: EnrichedFieldMetadata = { isInSameFamily: false, // custom fields are never in the same family }; -export const sourceIpWithTextMapping: EnrichedFieldMetadata = { +export const sourceIpWithTextMapping: EcsBasedFieldMetadata = { dashed_name: 'source-ip', description: 'IP address of the source (IPv4 or IPv6).', flat_name: 'source.ip', @@ -244,7 +244,7 @@ export const sourceIpWithTextMapping: EnrichedFieldMetadata = { isInSameFamily: false, // `ip` is not a member of any families }; -export const sourceIpKeyword: EnrichedFieldMetadata = { +export const sourceIpKeyword: CustomFieldMetadata = { indexFieldName: 'source.ip.keyword', indexFieldType: 'keyword', indexInvalidValues: [], @@ -253,7 +253,7 @@ export const sourceIpKeyword: EnrichedFieldMetadata = { isInSameFamily: false, // custom fields are never in the same family }; -export const sourcePort: EnrichedFieldMetadata = { +export const sourcePort: EcsBasedFieldMetadata = { dashed_name: 'source-port', description: 'Port of the source.', flat_name: 'source.port', @@ -271,7 +271,7 @@ export const sourcePort: EnrichedFieldMetadata = { isInSameFamily: false, // `long` is not a member of any families }; -export const mockCustomFields: EnrichedFieldMetadata[] = [ +export const mockCustomFields: CustomFieldMetadata[] = [ { indexFieldName: 'host.name.keyword', indexFieldType: 'keyword', @@ -306,7 +306,7 @@ export const mockCustomFields: EnrichedFieldMetadata[] = [ }, ]; -export const mockIncompatibleMappings: EnrichedFieldMetadata[] = [ +export const mockIncompatibleMappings: EcsBasedFieldMetadata[] = [ { dashed_name: 'host-name', description: diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts index bd551d80f531b..ed20efc3fd959 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts @@ -16,43 +16,73 @@ export interface Mappings { indexes: Record; } -export interface AllowedValue { - description?: string; - expected_event_types?: string[]; - name?: string; -} +export interface EcsFieldMetadata { + dashed_name: string; + description: string; + flat_name: string; + level: string; + name: string; + normalize: string[]; + short: string; + type: string; -export interface EcsMetadata { allowed_values?: AllowedValue[]; - dashed_name?: string; - description?: string; - example?: string; - flat_name?: string; + beta?: string; + doc_values?: boolean; + example?: string | number | boolean; + expected_values?: string[]; format?: string; ignore_above?: number; - level?: string; - name?: string; - normalize?: string[]; + index?: boolean; + input_format?: string; + multi_fields?: MultiField[]; + object_type?: string; + original_fieldset?: string; + output_format?: string; + output_precision?: number; + pattern?: string; required?: boolean; - short?: string; - type?: string; + scaling_factor?: number; +} + +export interface AllowedValue { + description: string; + name: string; + expected_event_types?: string[]; + beta?: string; +} + +export interface MultiField { + flat_name: string; + name: string; + type: string; } -export type EnrichedFieldMetadata = EcsMetadata & { - hasEcsMetadata: boolean; +export interface CustomFieldMetadata { + hasEcsMetadata: false; + indexFieldName: string; + indexFieldType: string; + indexInvalidValues: []; + isEcsCompliant: false; + isInSameFamily: false; +} +export interface EcsBasedFieldMetadata extends EcsFieldMetadata { + hasEcsMetadata: true; indexFieldName: string; indexFieldType: string; indexInvalidValues: UnallowedValueCount[]; isEcsCompliant: boolean; isInSameFamily: boolean; -}; +} + +export type EnrichedFieldMetadata = EcsBasedFieldMetadata | CustomFieldMetadata; export interface PartitionedFieldMetadata { all: EnrichedFieldMetadata[]; - custom: EnrichedFieldMetadata[]; - ecsCompliant: EnrichedFieldMetadata[]; - incompatible: EnrichedFieldMetadata[]; - sameFamily: EnrichedFieldMetadata[]; + custom: CustomFieldMetadata[]; + ecsCompliant: EcsBasedFieldMetadata[]; + incompatible: EcsBasedFieldMetadata[]; + sameFamily: EcsBasedFieldMetadata[]; } export interface PartitionedFieldMetadataStats { @@ -89,6 +119,32 @@ export interface UnallowedValueSearchResult { export type IlmPhase = 'hot' | 'warm' | 'cold' | 'frozen' | 'unmanaged'; +export interface IncompatibleFieldMappingItem { + fieldName: string; + expectedValue: string; + actualValue: string; + description: string; +} + +export interface ActualIncompatibleValue { + name: string; + count: number; +} + +export interface IncompatibleFieldValueItem { + fieldName: string; + expectedValues: string[]; + actualValues: ActualIncompatibleValue[]; + description: string; +} + +export interface SameFamilyFieldItem { + fieldName: string; + expectedValue: string; + actualValue: string; + description: string; +} + export interface IlmExplainPhaseCounts { hot: number; warm: number; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx index 0983308815464..fcbb8aae9337f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx @@ -5,7 +5,6 @@ * 2.0. */ -import { EcsFlat } from '@elastic/ecs'; import { renderHook } from '@testing-library/react-hooks'; import React, { FC, PropsWithChildren } from 'react'; @@ -13,9 +12,10 @@ import { getUnallowedValueRequestItems } from '../data_quality_panel/allowed_val import { DataQualityProvider } from '../data_quality_panel/data_quality_context'; import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unallowed_values'; import { ERROR_LOADING_UNALLOWED_VALUES } from '../translations'; -import { EcsMetadata, UnallowedValueRequestItem } from '../types'; +import { UnallowedValueRequestItem } from '../types'; import { useUnallowedValues, UseUnallowedValues } from '.'; import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; +import { EcsFlatTyped } from '../constants'; const mockHttpFetch = jest.fn(); const mockReportDataQualityIndexChecked = jest.fn(); @@ -37,10 +37,9 @@ const ContextWrapper: FC> = ({ children }) => ( ); -const ecsMetadata = EcsFlat as unknown as Record; const indexName = 'auditbeat-custom-index-1'; const requestItems = getUnallowedValueRequestItems({ - ecsMetadata, + ecsMetadata: EcsFlatTyped, indexName, }); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts index 8fbb3c354df8d..c82c70ad6bcc1 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts @@ -18,12 +18,18 @@ export const resultsFieldMap: FieldMap = { totalFieldCount: { type: 'long', required: true }, ecsFieldCount: { type: 'long', required: true }, customFieldCount: { type: 'long', required: true }, - incompatibleFieldItems: { type: 'nested', required: true, array: true }, - 'incompatibleFieldItems.fieldName': { type: 'keyword', required: true }, - 'incompatibleFieldItems.expectedValue': { type: 'keyword', required: true }, - 'incompatibleFieldItems.actualValue': { type: 'keyword', required: true }, - 'incompatibleFieldItems.description': { type: 'keyword', required: true }, - 'incompatibleFieldItems.reason': { type: 'keyword', required: true }, + incompatibleFieldMappingItems: { type: 'nested', required: true, array: true }, + 'incompatibleFieldMappingItems.fieldName': { type: 'keyword', required: true }, + 'incompatibleFieldMappingItems.expectedValue': { type: 'keyword', required: true }, + 'incompatibleFieldMappingItems.actualValue': { type: 'keyword', required: true }, + 'incompatibleFieldMappingItems.description': { type: 'keyword', required: true }, + incompatibleFieldValueItems: { type: 'nested', required: true, array: true }, + 'incompatibleFieldValueItems.fieldName': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.expectedValues': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.actualValues': { type: 'nested', required: true, array: true }, + 'incompatibleFieldValueItems.actualValues.name': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.actualValues.count': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.description': { type: 'keyword', required: true }, incompatibleFieldCount: { type: 'long', required: true }, sameFamilyFieldCount: { type: 'long', required: true }, sameFamilyFieldItems: { type: 'nested', required: true, array: true }, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts index 0b53c94b9b8e5..ad51233c5d203 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts @@ -19,8 +19,38 @@ export const resultDocument: ResultDocument = { ecsFieldCount: 677, customFieldCount: 904, incompatibleFieldCount: 1, + incompatibleFieldMappingItems: [], + incompatibleFieldValueItems: [ + { + fieldName: 'event.category', + expectedValues: [ + `authentication`, + `configuration`, + `database`, + `driver`, + `email`, + `file`, + `host`, + `iam`, + `intrusion_detection`, + `malware`, + `network`, + `package`, + `process`, + `registry`, + `session`, + `threat`, + 'vulnerability', + 'web', + ], + actualValues: [{ name: 'behavior', count: 6 }], + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + }, + ], sameFamilyFieldCount: 0, sameFamilyFields: [], + sameFamilyFieldItems: [], unallowedMappingFields: [], unallowedValueFields: ['event.category'], sizeInBytes: 173796, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts index c16a3d806b1bd..b242b6f80b1b2 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts @@ -19,8 +19,32 @@ const ResultDocumentInterface = t.interface({ ecsFieldCount: t.number, customFieldCount: t.number, incompatibleFieldCount: t.number, + incompatibleFieldMappingItems: t.array( + t.type({ + fieldName: t.string, + expectedValue: t.string, + actualValue: t.string, + description: t.string, + }) + ), + incompatibleFieldValueItems: t.array( + t.type({ + fieldName: t.string, + expectedValues: t.array(t.string), + actualValues: t.array(t.type({ name: t.string, count: t.number })), + description: t.string, + }) + ), sameFamilyFieldCount: t.number, sameFamilyFields: t.array(t.string), + sameFamilyFieldItems: t.array( + t.type({ + fieldName: t.string, + expectedValue: t.string, + actualValue: t.string, + description: t.string, + }) + ), unallowedMappingFields: t.array(t.string), unallowedValueFields: t.array(t.string), sizeInBytes: t.number,