From cd8e3af3b37780272718bf327bb17500d4854ab5 Mon Sep 17 00:00:00 2001 From: Karen Grigoryan Date: Fri, 7 Jun 2024 14:31:02 +0200 Subject: [PATCH] feat(ecs_data_quality_dashboard): enhance StorageResult with detailed field items - Added `incompatibleFieldItems` and `sameFamilyFieldItems` to `StorageResult` for detailed field information. - Updated tests to validate the new functionality. - Adjusted type definitions and mock data to support the changes. Addresses #184751 --- .../impl/data_quality/helpers.test.ts | 118 ++++++++++++++++++ .../impl/data_quality/helpers.ts | 88 +++++++++---- .../impl/data_quality/types.ts | 25 ++++ .../server/routes/results/results.mock.ts | 30 +++++ .../server/schemas/result.ts | 17 +++ 5 files changed, 256 insertions(+), 22 deletions(-) diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts index 2cd6c3bec2aa2..6dc7b9e41d88d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts @@ -36,6 +36,7 @@ import { postStorageResult, getStorageResults, StorageResult, + formatStorageResult, } from './helpers'; import { hostNameWithTextMapping, @@ -79,6 +80,7 @@ import { import { httpServiceMock } from '@kbn/core-http-browser-mocks'; import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; import { EcsFlatTyped } from './constants'; +import { mockPartitionedFieldMetadataWithSameFamily } from './mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family'; describe('helpers', () => { describe('getTotalPatternSameFamily', () => { @@ -1392,6 +1394,122 @@ describe('helpers', () => { }); }); + describe('formatStorageResult', () => { + it('should correctly format the input data into a StorageResult object', () => { + const inputData: Parameters[number] = { + result: { + indexName: 'testIndex', + pattern: 'testPattern', + checkedAt: 1627545600000, + docsCount: 100, + incompatible: 3, + sameFamily: 1, + ilmPhase: 'hot', + markdownComments: ['test comments'], + error: null, + }, + report: { + batchId: 'testBatch', + isCheckAll: true, + sameFamilyFields: ['agent.type'], + unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], + unallowedValueFields: ['event.category'], + sizeInBytes: 5000, + ecsVersion: '1.0.0', + indexName: 'testIndex', + indexId: 'testIndexId', + }, + partitionedFieldMetadata: mockPartitionedFieldMetadataWithSameFamily, + }; + + const expectedResult: StorageResult = { + batchId: 'testBatch', + indexName: 'testIndex', + indexPattern: 'testPattern', + isCheckAll: true, + checkedAt: 1627545600000, + docsCount: 100, + totalFieldCount: 10, + ecsFieldCount: 2, + customFieldCount: 4, + incompatibleFieldCount: 3, + incompatibleFieldItems: [ + { + fieldName: 'event.category', + expectedValue: 'keyword', + actualValue: 'constant_keyword', + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + reason: 'mapping', + }, + { + fieldName: 'event.category', + expectedValue: [ + 'authentication', + 'configuration', + 'database', + 'driver', + 'email', + 'file', + 'host', + 'iam', + 'intrusion_detection', + 'malware', + 'network', + 'package', + 'process', + 'registry', + 'session', + 'threat', + 'vulnerability', + 'web', + ], + actualValue: ['an_invalid_category'], + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + reason: 'value', + }, + { + fieldName: 'host.name', + expectedValue: 'keyword', + actualValue: 'text', + description: + 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', + reason: 'mapping', + }, + { + fieldName: 'source.ip', + expectedValue: 'ip', + actualValue: 'text', + description: 'IP address of the source (IPv4 or IPv6).', + reason: 'mapping', + }, + ], + sameFamilyFieldCount: 1, + sameFamilyFields: ['agent.type'], + sameFamilyFieldItems: [ + { + fieldName: 'agent.type', + expectedValue: 'keyword', + actualValue: 'constant_keyword', + description: + 'Type of the agent.\nThe agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + }, + ], + unallowedMappingFields: ['event.category', 'host.name', 'source.ip'], + unallowedValueFields: ['event.category'], + sizeInBytes: 5000, + ilmPhase: 'hot', + markdownComments: ['test comments'], + ecsVersion: '1.0.0', + indexId: 'testIndexId', + error: null, + }; + + expect(formatStorageResult(inputData)).toEqual(expectedResult); + }); + }); + describe('postStorageResult', () => { const { fetch } = httpServiceMock.createStartContract(); const { toasts } = notificationServiceMock.createStartContract(); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts index 0580dd7e80788..b09b19a96d341 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts @@ -21,10 +21,12 @@ import type { EnrichedFieldMetadata, ErrorSummary, IlmPhase, + IncompatibleFieldItem, MeteringStatsIndex, PartitionedFieldMetadata, PartitionedFieldMetadataStats, PatternRollup, + SameFamilyFieldItem, UnallowedValueCount, } from './types'; import { EcsFlatTyped } from './constants'; @@ -466,8 +468,10 @@ export interface StorageResult { ecsFieldCount: number; customFieldCount: number; incompatibleFieldCount: number; + incompatibleFieldItems: IncompatibleFieldItem[]; sameFamilyFieldCount: number; sameFamilyFields: string[]; + sameFamilyFieldItems: SameFamilyFieldItem[]; unallowedMappingFields: string[]; unallowedValueFields: string[]; sizeInBytes: number; @@ -486,28 +490,68 @@ export const formatStorageResult = ({ result: DataQualityCheckResult; report: DataQualityIndexCheckedParams; partitionedFieldMetadata: PartitionedFieldMetadata; -}): StorageResult => ({ - batchId: report.batchId, - indexName: result.indexName, - indexPattern: result.pattern, - isCheckAll: report.isCheckAll, - checkedAt: result.checkedAt ?? Date.now(), - docsCount: result.docsCount ?? 0, - totalFieldCount: partitionedFieldMetadata.all.length, - ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, - customFieldCount: partitionedFieldMetadata.custom.length, - incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, - sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, - sameFamilyFields: report.sameFamilyFields ?? [], - unallowedMappingFields: report.unallowedMappingFields ?? [], - unallowedValueFields: report.unallowedValueFields ?? [], - sizeInBytes: report.sizeInBytes ?? 0, - ilmPhase: result.ilmPhase, - markdownComments: result.markdownComments, - ecsVersion: report.ecsVersion, - indexId: report.indexId ?? '', // ---> we don't have this field when isILMAvailable is false - error: result.error, -}); +}): StorageResult => { + const incompatibleFieldItems: IncompatibleFieldItem[] = []; + const sameFamilyFieldItems: SameFamilyFieldItem[] = []; + + partitionedFieldMetadata.incompatible.forEach((field) => { + if (field.type !== field.indexFieldType) { + // Mapping incompatibility + incompatibleFieldItems.push({ + fieldName: field.indexFieldName, + expectedValue: field.type, + actualValue: field.indexFieldType, + description: field.description, + reason: 'mapping', + }); + } + + if (field.indexInvalidValues.length > 0) { + // Value incompatibility + incompatibleFieldItems.push({ + fieldName: field.indexFieldName, + expectedValue: field.allowed_values?.map((x) => x.name) ?? [], + actualValue: field.indexInvalidValues.map((v) => v.fieldName), + description: field.description, + reason: 'value', + }); + } + }); + + partitionedFieldMetadata.sameFamily.forEach((field) => { + sameFamilyFieldItems.push({ + fieldName: field.indexFieldName, + expectedValue: field.type, + actualValue: field.indexFieldType, + description: field.description, + }); + }); + + return { + batchId: report.batchId, + indexName: result.indexName, + indexPattern: result.pattern, + isCheckAll: report.isCheckAll, + checkedAt: result.checkedAt ?? Date.now(), + docsCount: result.docsCount ?? 0, + totalFieldCount: partitionedFieldMetadata.all.length, + ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, + customFieldCount: partitionedFieldMetadata.custom.length, + incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, + incompatibleFieldItems, + sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, + sameFamilyFields: report.sameFamilyFields ?? [], + sameFamilyFieldItems, + unallowedMappingFields: report.unallowedMappingFields ?? [], + unallowedValueFields: report.unallowedValueFields ?? [], + sizeInBytes: report.sizeInBytes ?? 0, + ilmPhase: result.ilmPhase, + markdownComments: result.markdownComments, + ecsVersion: report.ecsVersion, + indexId: report.indexId ?? '', + error: result.error, + }; +}; export const formatResultFromStorage = ({ storageResult, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts index cdd4265615b40..2fc9d399cbb90 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts @@ -119,6 +119,31 @@ export interface UnallowedValueSearchResult { export type IlmPhase = 'hot' | 'warm' | 'cold' | 'frozen' | 'unmanaged'; +export interface IncompatibleMappingItem { + fieldName: string; + expectedValue: string; + actualValue: string; + description: string; + reason: 'mapping'; +} + +export interface IncompatibleValueItem { + fieldName: string; + expectedValue: string[]; + actualValue: string[]; + description: string; + reason: 'value'; +} + +export type IncompatibleFieldItem = IncompatibleMappingItem | IncompatibleValueItem; + +export interface SameFamilyFieldItem { + fieldName: string; + expectedValue: string; + actualValue: string; + description: string; +} + export interface IlmExplainPhaseCounts { hot: number; warm: number; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts index 0b53c94b9b8e5..40ce01478ce1f 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts @@ -19,8 +19,38 @@ export const resultDocument: ResultDocument = { ecsFieldCount: 677, customFieldCount: 904, incompatibleFieldCount: 1, + incompatibleFieldItems: [ + { + fieldName: 'event.category', + expectedValue: [ + `authentication`, + `configuration`, + `database`, + `driver`, + `email`, + `file`, + `host`, + `iam`, + `intrusion_detection`, + `malware`, + `network`, + `package`, + `process`, + `registry`, + `session`, + `threat`, + 'vulnerability', + 'web', + ], + actualValue: ['behavior'], + description: + 'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.', + reason: 'value', + }, + ], sameFamilyFieldCount: 0, sameFamilyFields: [], + sameFamilyFieldItems: [], unallowedMappingFields: [], unallowedValueFields: ['event.category'], sizeInBytes: 173796, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts index c16a3d806b1bd..46bed0c9e6577 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts @@ -19,8 +19,25 @@ const ResultDocumentInterface = t.interface({ ecsFieldCount: t.number, customFieldCount: t.number, incompatibleFieldCount: t.number, + incompatibleFieldItems: t.array( + t.type({ + fieldName: t.string, + expectedValue: t.union([t.string, t.array(t.string)]), + actualValue: t.union([t.string, t.array(t.string)]), + description: t.string, + reason: t.string, + }) + ), sameFamilyFieldCount: t.number, sameFamilyFields: t.array(t.string), + sameFamilyFieldItems: t.array( + t.type({ + fieldName: t.string, + expectedValue: t.string, + actualValue: t.string, + description: t.string, + }) + ), unallowedMappingFields: t.array(t.string), unallowedValueFields: t.array(t.string), sizeInBytes: t.number,