diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts index 6dc7b9e41d88d..bfe3ffc0c5e55 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts @@ -1433,18 +1433,32 @@ describe('helpers', () => { ecsFieldCount: 2, customFieldCount: 4, incompatibleFieldCount: 3, - incompatibleFieldItems: [ + incompatibleFieldMappingItems: [ { fieldName: 'event.category', expectedValue: 'keyword', actualValue: 'constant_keyword', description: 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', - reason: 'mapping', }, + { + fieldName: 'host.name', + expectedValue: 'keyword', + actualValue: 'text', + description: + 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', + }, + { + fieldName: 'source.ip', + expectedValue: 'ip', + actualValue: 'text', + description: 'IP address of the source (IPv4 or IPv6).', + }, + ], + incompatibleFieldValueItems: [ { fieldName: 'event.category', - expectedValue: [ + expectedValues: [ 'authentication', 'configuration', 'database', @@ -1464,25 +1478,9 @@ describe('helpers', () => { 'vulnerability', 'web', ], - actualValue: ['an_invalid_category'], + actualValues: [{ name: 'an_invalid_category', count: 2 }], description: 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', - reason: 'value', - }, - { - fieldName: 'host.name', - expectedValue: 'keyword', - actualValue: 'text', - description: - 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', - reason: 'mapping', - }, - { - fieldName: 'source.ip', - expectedValue: 'ip', - actualValue: 'text', - description: 'IP address of the source (IPv4 or IPv6).', - reason: 'mapping', }, ], sameFamilyFieldCount: 1, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts index b09b19a96d341..0980ac3763347 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts @@ -21,7 +21,8 @@ import type { EnrichedFieldMetadata, ErrorSummary, IlmPhase, - IncompatibleFieldItem, + IncompatibleFieldMappingItem, + IncompatibleFieldValueItem, MeteringStatsIndex, PartitionedFieldMetadata, PartitionedFieldMetadataStats, @@ -468,7 +469,8 @@ export interface StorageResult { ecsFieldCount: number; customFieldCount: number; incompatibleFieldCount: number; - incompatibleFieldItems: IncompatibleFieldItem[]; + incompatibleFieldMappingItems: IncompatibleFieldMappingItem[]; + incompatibleFieldValueItems: IncompatibleFieldValueItem[]; sameFamilyFieldCount: number; sameFamilyFields: string[]; sameFamilyFieldItems: SameFamilyFieldItem[]; @@ -491,29 +493,26 @@ export const formatStorageResult = ({ report: DataQualityIndexCheckedParams; partitionedFieldMetadata: PartitionedFieldMetadata; }): StorageResult => { - const incompatibleFieldItems: IncompatibleFieldItem[] = []; + const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = []; + const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = []; const sameFamilyFieldItems: SameFamilyFieldItem[] = []; partitionedFieldMetadata.incompatible.forEach((field) => { if (field.type !== field.indexFieldType) { - // Mapping incompatibility - incompatibleFieldItems.push({ + incompatibleFieldMappingItems.push({ fieldName: field.indexFieldName, expectedValue: field.type, actualValue: field.indexFieldType, description: field.description, - reason: 'mapping', }); } if (field.indexInvalidValues.length > 0) { - // Value incompatibility - incompatibleFieldItems.push({ + incompatibleFieldValueItems.push({ fieldName: field.indexFieldName, - expectedValue: field.allowed_values?.map((x) => x.name) ?? [], - actualValue: field.indexInvalidValues.map((v) => v.fieldName), + expectedValues: field.allowed_values?.map((x) => x.name) ?? [], + actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })), description: field.description, - reason: 'value', }); } }); @@ -538,7 +537,8 @@ export const formatStorageResult = ({ ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, customFieldCount: partitionedFieldMetadata.custom.length, incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, - incompatibleFieldItems, + incompatibleFieldMappingItems, + incompatibleFieldValueItems, sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, sameFamilyFields: report.sameFamilyFields ?? [], sameFamilyFieldItems, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts index 2fc9d399cbb90..ed20efc3fd959 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts @@ -119,24 +119,25 @@ export interface UnallowedValueSearchResult { export type IlmPhase = 'hot' | 'warm' | 'cold' | 'frozen' | 'unmanaged'; -export interface IncompatibleMappingItem { +export interface IncompatibleFieldMappingItem { fieldName: string; expectedValue: string; actualValue: string; description: string; - reason: 'mapping'; } -export interface IncompatibleValueItem { +export interface ActualIncompatibleValue { + name: string; + count: number; +} + +export interface IncompatibleFieldValueItem { fieldName: string; - expectedValue: string[]; - actualValue: string[]; + expectedValues: string[]; + actualValues: ActualIncompatibleValue[]; description: string; - reason: 'value'; } -export type IncompatibleFieldItem = IncompatibleMappingItem | IncompatibleValueItem; - export interface SameFamilyFieldItem { fieldName: string; expectedValue: string; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts index 8fbb3c354df8d..c82c70ad6bcc1 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts @@ -18,12 +18,18 @@ export const resultsFieldMap: FieldMap = { totalFieldCount: { type: 'long', required: true }, ecsFieldCount: { type: 'long', required: true }, customFieldCount: { type: 'long', required: true }, - incompatibleFieldItems: { type: 'nested', required: true, array: true }, - 'incompatibleFieldItems.fieldName': { type: 'keyword', required: true }, - 'incompatibleFieldItems.expectedValue': { type: 'keyword', required: true }, - 'incompatibleFieldItems.actualValue': { type: 'keyword', required: true }, - 'incompatibleFieldItems.description': { type: 'keyword', required: true }, - 'incompatibleFieldItems.reason': { type: 'keyword', required: true }, + incompatibleFieldMappingItems: { type: 'nested', required: true, array: true }, + 'incompatibleFieldMappingItems.fieldName': { type: 'keyword', required: true }, + 'incompatibleFieldMappingItems.expectedValue': { type: 'keyword', required: true }, + 'incompatibleFieldMappingItems.actualValue': { type: 'keyword', required: true }, + 'incompatibleFieldMappingItems.description': { type: 'keyword', required: true }, + incompatibleFieldValueItems: { type: 'nested', required: true, array: true }, + 'incompatibleFieldValueItems.fieldName': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.expectedValues': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.actualValues': { type: 'nested', required: true, array: true }, + 'incompatibleFieldValueItems.actualValues.name': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.actualValues.count': { type: 'keyword', required: true }, + 'incompatibleFieldValueItems.description': { type: 'keyword', required: true }, incompatibleFieldCount: { type: 'long', required: true }, sameFamilyFieldCount: { type: 'long', required: true }, sameFamilyFieldItems: { type: 'nested', required: true, array: true }, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts index 40ce01478ce1f..ad51233c5d203 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts @@ -19,10 +19,11 @@ export const resultDocument: ResultDocument = { ecsFieldCount: 677, customFieldCount: 904, incompatibleFieldCount: 1, - incompatibleFieldItems: [ + incompatibleFieldMappingItems: [], + incompatibleFieldValueItems: [ { fieldName: 'event.category', - expectedValue: [ + expectedValues: [ `authentication`, `configuration`, `database`, @@ -42,10 +43,9 @@ export const resultDocument: ResultDocument = { 'vulnerability', 'web', ], - actualValue: ['behavior'], + actualValues: [{ name: 'behavior', count: 6 }], description: 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', - reason: 'value', }, ], sameFamilyFieldCount: 0, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts index 46bed0c9e6577..b242b6f80b1b2 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts @@ -19,13 +19,20 @@ const ResultDocumentInterface = t.interface({ ecsFieldCount: t.number, customFieldCount: t.number, incompatibleFieldCount: t.number, - incompatibleFieldItems: t.array( + incompatibleFieldMappingItems: t.array( t.type({ fieldName: t.string, - expectedValue: t.union([t.string, t.array(t.string)]), - actualValue: t.union([t.string, t.array(t.string)]), + expectedValue: t.string, + actualValue: t.string, + description: t.string, + }) + ), + incompatibleFieldValueItems: t.array( + t.type({ + fieldName: t.string, + expectedValues: t.array(t.string), + actualValues: t.array(t.type({ name: t.string, count: t.number })), description: t.string, - reason: t.string, }) ), sameFamilyFieldCount: t.number,