feat(ecs_data_quality_dashboard): enhance StorageResult with detailed…

… field items

- Added `incompatibleFieldItems` and `sameFamilyFieldItems` to `StorageResult` for detailed field information.
- Updated tests to validate the new functionality.
- Adjusted type definitions and mock data to support the changes.

Addresses elastic#184751

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts

@@ -36,6 +36,7 @@ import {
   postStorageResult,
   getStorageResults,
   StorageResult,
+  formatStorageResult,
 } from './helpers';
 import {
   hostNameWithTextMapping,
@@ -79,6 +80,7 @@ import {
 import { httpServiceMock } from '@kbn/core-http-browser-mocks';
 import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
 import { EcsFlatTyped } from './constants';
+import { mockPartitionedFieldMetadataWithSameFamily } from './mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family';
 
 describe('helpers', () => {
   describe('getTotalPatternSameFamily', () => {
@@ -1392,6 +1394,122 @@ describe('helpers', () => {
     });
   });
 
+  describe('formatStorageResult', () => {
+    it('should correctly format the input data into a StorageResult object', () => {
+      const inputData: Parameters<typeof formatStorageResult>[number] = {
+        result: {
+          indexName: 'testIndex',
+          pattern: 'testPattern',
+          checkedAt: 1627545600000,
+          docsCount: 100,
+          incompatible: 3,
+          sameFamily: 1,
+          ilmPhase: 'hot',
+          markdownComments: ['test comments'],
+          error: null,
+        },
+        report: {
+          batchId: 'testBatch',
+          isCheckAll: true,
+          sameFamilyFields: ['agent.type'],
+          unallowedMappingFields: ['event.category', 'host.name', 'source.ip'],
+          unallowedValueFields: ['event.category'],
+          sizeInBytes: 5000,
+          ecsVersion: '1.0.0',
+          indexName: 'testIndex',
+          indexId: 'testIndexId',
+        },
+        partitionedFieldMetadata: mockPartitionedFieldMetadataWithSameFamily,
+      };
+
+      const expectedResult: StorageResult = {
+        batchId: 'testBatch',
+        indexName: 'testIndex',
+        indexPattern: 'testPattern',
+        isCheckAll: true,
+        checkedAt: 1627545600000,
+        docsCount: 100,
+        totalFieldCount: 10,
+        ecsFieldCount: 2,
+        customFieldCount: 4,
+        incompatibleFieldCount: 3,
+        incompatibleFieldItems: [
+          {
+            fieldName: 'event.category',
+            expectedValue: 'keyword',
+            actualValue: 'constant_keyword',
+            description:
+              'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
+            reason: 'mapping',
+          },
+          {
+            fieldName: 'event.category',
+            expectedValue: [
+              'authentication',
+              'configuration',
+              'database',
+              'driver',
+              'email',
+              'file',
+              'host',
+              'iam',
+              'intrusion_detection',
+              'malware',
+              'network',
+              'package',
+              'process',
+              'registry',
+              'session',
+              'threat',
+              'vulnerability',
+              'web',
+            ],
+            actualValue: ['an_invalid_category'],
+            description:
+              'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
+            reason: 'value',
+          },
+          {
+            fieldName: 'host.name',
+            expectedValue: 'keyword',
+            actualValue: 'text',
+            description:
+              'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
+            reason: 'mapping',
+          },
+          {
+            fieldName: 'source.ip',
+            expectedValue: 'ip',
+            actualValue: 'text',
+            description: 'IP address of the source (IPv4 or IPv6).',
+            reason: 'mapping',
+          },
+        ],
+        sameFamilyFieldCount: 1,
+        sameFamilyFields: ['agent.type'],
+        sameFamilyFieldItems: [
+          {
+            fieldName: 'agent.type',
+            expectedValue: 'keyword',
+            actualValue: 'constant_keyword',
+            description:
+              'Type of the agent.\nThe agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
+          },
+        ],
+        unallowedMappingFields: ['event.category', 'host.name', 'source.ip'],
+        unallowedValueFields: ['event.category'],
+        sizeInBytes: 5000,
+        ilmPhase: 'hot',
+        markdownComments: ['test comments'],
+        ecsVersion: '1.0.0',
+        indexId: 'testIndexId',
+        error: null,
+      };
+
+      expect(formatStorageResult(inputData)).toEqual(expectedResult);
+    });
+  });
+
   describe('postStorageResult', () => {
     const { fetch } = httpServiceMock.createStartContract();
     const { toasts } = notificationServiceMock.createStartContract();

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts

@@ -21,10 +21,12 @@ import type {
   EnrichedFieldMetadata,
   ErrorSummary,
   IlmPhase,
+  IncompatibleFieldItem,
   MeteringStatsIndex,
   PartitionedFieldMetadata,
   PartitionedFieldMetadataStats,
   PatternRollup,
+  SameFamilyFieldItem,
   UnallowedValueCount,
 } from './types';
 import { EcsFlatTyped } from './constants';
@@ -466,8 +468,10 @@ export interface StorageResult {
   ecsFieldCount: number;
   customFieldCount: number;
   incompatibleFieldCount: number;
+  incompatibleFieldItems: IncompatibleFieldItem[];
   sameFamilyFieldCount: number;
   sameFamilyFields: string[];
+  sameFamilyFieldItems: SameFamilyFieldItem[];
   unallowedMappingFields: string[];
   unallowedValueFields: string[];
   sizeInBytes: number;
@@ -486,28 +490,68 @@ export const formatStorageResult = ({
   result: DataQualityCheckResult;
   report: DataQualityIndexCheckedParams;
   partitionedFieldMetadata: PartitionedFieldMetadata;
-}): StorageResult => ({
-  batchId: report.batchId,
-  indexName: result.indexName,
-  indexPattern: result.pattern,
-  isCheckAll: report.isCheckAll,
-  checkedAt: result.checkedAt ?? Date.now(),
-  docsCount: result.docsCount ?? 0,
-  totalFieldCount: partitionedFieldMetadata.all.length,
-  ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
-  customFieldCount: partitionedFieldMetadata.custom.length,
-  incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
-  sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
-  sameFamilyFields: report.sameFamilyFields ?? [],
-  unallowedMappingFields: report.unallowedMappingFields ?? [],
-  unallowedValueFields: report.unallowedValueFields ?? [],
-  sizeInBytes: report.sizeInBytes ?? 0,
-  ilmPhase: result.ilmPhase,
-  markdownComments: result.markdownComments,
-  ecsVersion: report.ecsVersion,
-  indexId: report.indexId ?? '', // ---> we don't have this field when isILMAvailable is false
-  error: result.error,
-});
+}): StorageResult => {
+  const incompatibleFieldItems: IncompatibleFieldItem[] = [];
+  const sameFamilyFieldItems: SameFamilyFieldItem[] = [];
+
+  partitionedFieldMetadata.incompatible.forEach((field) => {
+    if (field.type !== field.indexFieldType) {
+      // Mapping incompatibility
+      incompatibleFieldItems.push({
+        fieldName: field.indexFieldName,
+        expectedValue: field.type,
+        actualValue: field.indexFieldType,
+        description: field.description,
+        reason: 'mapping',
+      });
+    }
+
+    if (field.indexInvalidValues.length > 0) {
+      // Value incompatibility
+      incompatibleFieldItems.push({
+        fieldName: field.indexFieldName,
+        expectedValue: field.allowed_values?.map((x) => x.name) ?? [],
+        actualValue: field.indexInvalidValues.map((v) => v.fieldName),
+        description: field.description,
+        reason: 'value',
+      });
+    }
+  });
+
+  partitionedFieldMetadata.sameFamily.forEach((field) => {
+    sameFamilyFieldItems.push({
+      fieldName: field.indexFieldName,
+      expectedValue: field.type,
+      actualValue: field.indexFieldType,
+      description: field.description,
+    });
+  });
+
+  return {
+    batchId: report.batchId,
+    indexName: result.indexName,
+    indexPattern: result.pattern,
+    isCheckAll: report.isCheckAll,
+    checkedAt: result.checkedAt ?? Date.now(),
+    docsCount: result.docsCount ?? 0,
+    totalFieldCount: partitionedFieldMetadata.all.length,
+    ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
+    customFieldCount: partitionedFieldMetadata.custom.length,
+    incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
+    incompatibleFieldItems,
+    sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
+    sameFamilyFields: report.sameFamilyFields ?? [],
+    sameFamilyFieldItems,
+    unallowedMappingFields: report.unallowedMappingFields ?? [],
+    unallowedValueFields: report.unallowedValueFields ?? [],
+    sizeInBytes: report.sizeInBytes ?? 0,
+    ilmPhase: result.ilmPhase,
+    markdownComments: result.markdownComments,
+    ecsVersion: report.ecsVersion,
+    indexId: report.indexId ?? '',
+    error: result.error,
+  };
+};
 
 export const formatResultFromStorage = ({
   storageResult,

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts

@@ -119,6 +119,31 @@ export interface UnallowedValueSearchResult {
 
 export type IlmPhase = 'hot' | 'warm' | 'cold' | 'frozen' | 'unmanaged';
 
+export interface IncompatibleMappingItem {
+  fieldName: string;
+  expectedValue: string;
+  actualValue: string;
+  description: string;
+  reason: 'mapping';
+}
+
+export interface IncompatibleValueItem {
+  fieldName: string;
+  expectedValue: string[];
+  actualValue: string[];
+  description: string;
+  reason: 'value';
+}
+
+export type IncompatibleFieldItem = IncompatibleMappingItem | IncompatibleValueItem;
+
+export interface SameFamilyFieldItem {
+  fieldName: string;
+  expectedValue: string;
+  actualValue: string;
+  description: string;
+}
+
 export interface IlmExplainPhaseCounts {
   hot: number;
   warm: number;

x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts

@@ -19,8 +19,38 @@ export const resultDocument: ResultDocument = {
   ecsFieldCount: 677,
   customFieldCount: 904,
   incompatibleFieldCount: 1,
+  incompatibleFieldItems: [
+    {
+      fieldName: 'event.category',
+      expectedValue: [
+        `authentication`,
+        `configuration`,
+        `database`,
+        `driver`,
+        `email`,
+        `file`,
+        `host`,
+        `iam`,
+        `intrusion_detection`,
+        `malware`,
+        `network`,
+        `package`,
+        `process`,
+        `registry`,
+        `session`,
+        `threat`,
+        'vulnerability',
+        'web',
+      ],
+      actualValue: ['behavior'],
+      description:
+        'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
+      reason: 'value',
+    },
+  ],
   sameFamilyFieldCount: 0,
   sameFamilyFields: [],
+  sameFamilyFieldItems: [],
   unallowedMappingFields: [],
   unallowedValueFields: ['event.category'],
   sizeInBytes: 173796,

x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts

@@ -19,8 +19,25 @@ const ResultDocumentInterface = t.interface({
   ecsFieldCount: t.number,
   customFieldCount: t.number,
   incompatibleFieldCount: t.number,
+  incompatibleFieldItems: t.array(
+    t.type({
+      fieldName: t.string,
+      expectedValue: t.union([t.string, t.array(t.string)]),
+      actualValue: t.union([t.string, t.array(t.string)]),
+      description: t.string,
+      reason: t.string,
+    })
+  ),
   sameFamilyFieldCount: t.number,
   sameFamilyFields: t.array(t.string),
+  sameFamilyFieldItems: t.array(
+    t.type({
+      fieldName: t.string,
+      expectedValue: t.string,
+      actualValue: t.string,
+      description: t.string,
+    })
+  ),
   unallowedMappingFields: t.array(t.string),
   unallowedValueFields: t.array(t.string),
   sizeInBytes: t.number,