refactor: split incompatibleFieldItems into mapping and value items

- Refactored `incompatibleFieldItems` into `incompatibleFieldMappingItems` and `incompatibleFieldValueItems`. - Updated related test cases in `helpers.test.ts`. - Modified `helpers.ts` to handle the new structure. - Updated type definitions in `types.ts`. - Adjusted `results_field_map.ts` to reflect the new structure. - Updated mock data in `results.mock.ts`. - Modified schema definitions in `result.ts`.
elastic · Jun 12, 2024 · 00aa6c7 · 00aa6c7
1 parent cd8e3af
commit 00aa6c7
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 54 deletions.
diff --git a/...k/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts b/...k/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts
@@ -1433,18 +1433,32 @@ describe('helpers', () => {
         ecsFieldCount: 2,
         customFieldCount: 4,
         incompatibleFieldCount: 3,
-        incompatibleFieldItems: [
+        incompatibleFieldMappingItems: [
           {
             fieldName: 'event.category',
             expectedValue: 'keyword',
             actualValue: 'constant_keyword',
             description:
               'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
-            reason: 'mapping',
           },
+          {
+            fieldName: 'host.name',
+            expectedValue: 'keyword',
+            actualValue: 'text',
+            description:
+              'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
+          },
+          {
+            fieldName: 'source.ip',
+            expectedValue: 'ip',
+            actualValue: 'text',
+            description: 'IP address of the source (IPv4 or IPv6).',
+          },
+        ],
+        incompatibleFieldValueItems: [
           {
             fieldName: 'event.category',
-            expectedValue: [
+            expectedValues: [
               'authentication',
               'configuration',
               'database',
@@ -1464,25 +1478,9 @@ describe('helpers', () => {
               'vulnerability',
               'web',
             ],
-            actualValue: ['an_invalid_category'],
+            actualValues: [{ name: 'an_invalid_category', count: 2 }],
             description:
               'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
-            reason: 'value',
-          },
-          {
-            fieldName: 'host.name',
-            expectedValue: 'keyword',
-            actualValue: 'text',
-            description:
-              'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
-            reason: 'mapping',
-          },
-          {
-            fieldName: 'source.ip',
-            expectedValue: 'ip',
-            actualValue: 'text',
-            description: 'IP address of the source (IPv4 or IPv6).',
-            reason: 'mapping',
           },
         ],
         sameFamilyFieldCount: 1,

diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts
@@ -21,7 +21,8 @@ import type {
   EnrichedFieldMetadata,
   ErrorSummary,
   IlmPhase,
-  IncompatibleFieldItem,
+  IncompatibleFieldMappingItem,
+  IncompatibleFieldValueItem,
   MeteringStatsIndex,
   PartitionedFieldMetadata,
   PartitionedFieldMetadataStats,
@@ -468,7 +469,8 @@ export interface StorageResult {
   ecsFieldCount: number;
   customFieldCount: number;
   incompatibleFieldCount: number;
-  incompatibleFieldItems: IncompatibleFieldItem[];
+  incompatibleFieldMappingItems: IncompatibleFieldMappingItem[];
+  incompatibleFieldValueItems: IncompatibleFieldValueItem[];
   sameFamilyFieldCount: number;
   sameFamilyFields: string[];
   sameFamilyFieldItems: SameFamilyFieldItem[];
@@ -491,29 +493,26 @@ export const formatStorageResult = ({
   report: DataQualityIndexCheckedParams;
   partitionedFieldMetadata: PartitionedFieldMetadata;
 }): StorageResult => {
-  const incompatibleFieldItems: IncompatibleFieldItem[] = [];
+  const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = [];
+  const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = [];
   const sameFamilyFieldItems: SameFamilyFieldItem[] = [];
 
   partitionedFieldMetadata.incompatible.forEach((field) => {
     if (field.type !== field.indexFieldType) {
-      // Mapping incompatibility
-      incompatibleFieldItems.push({
+      incompatibleFieldMappingItems.push({
         fieldName: field.indexFieldName,
         expectedValue: field.type,
         actualValue: field.indexFieldType,
         description: field.description,
-        reason: 'mapping',
       });
     }
 
     if (field.indexInvalidValues.length > 0) {
-      // Value incompatibility
-      incompatibleFieldItems.push({
+      incompatibleFieldValueItems.push({
         fieldName: field.indexFieldName,
-        expectedValue: field.allowed_values?.map((x) => x.name) ?? [],
-        actualValue: field.indexInvalidValues.map((v) => v.fieldName),
+        expectedValues: field.allowed_values?.map((x) => x.name) ?? [],
+        actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })),
         description: field.description,
-        reason: 'value',
       });
     }
   });
@@ -538,7 +537,8 @@ export const formatStorageResult = ({
     ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
     customFieldCount: partitionedFieldMetadata.custom.length,
     incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
-    incompatibleFieldItems,
+    incompatibleFieldMappingItems,
+    incompatibleFieldValueItems,
     sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
     sameFamilyFields: report.sameFamilyFields ?? [],
     sameFamilyFieldItems,

diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts
@@ -119,24 +119,25 @@ export interface UnallowedValueSearchResult {
 
 export type IlmPhase = 'hot' | 'warm' | 'cold' | 'frozen' | 'unmanaged';
 
-export interface IncompatibleMappingItem {
+export interface IncompatibleFieldMappingItem {
   fieldName: string;
   expectedValue: string;
   actualValue: string;
   description: string;
-  reason: 'mapping';
 }
 
-export interface IncompatibleValueItem {
+export interface ActualIncompatibleValue {
+  name: string;
+  count: number;
+}
+
+export interface IncompatibleFieldValueItem {
   fieldName: string;
-  expectedValue: string[];
-  actualValue: string[];
+  expectedValues: string[];
+  actualValues: ActualIncompatibleValue[];
   description: string;
-  reason: 'value';
 }
 
-export type IncompatibleFieldItem = IncompatibleMappingItem | IncompatibleValueItem;
-
 export interface SameFamilyFieldItem {
   fieldName: string;
   expectedValue: string;

diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts
@@ -18,12 +18,18 @@ export const resultsFieldMap: FieldMap = {
   totalFieldCount: { type: 'long', required: true },
   ecsFieldCount: { type: 'long', required: true },
   customFieldCount: { type: 'long', required: true },
-  incompatibleFieldItems: { type: 'nested', required: true, array: true },
-  'incompatibleFieldItems.fieldName': { type: 'keyword', required: true },
-  'incompatibleFieldItems.expectedValue': { type: 'keyword', required: true },
-  'incompatibleFieldItems.actualValue': { type: 'keyword', required: true },
-  'incompatibleFieldItems.description': { type: 'keyword', required: true },
-  'incompatibleFieldItems.reason': { type: 'keyword', required: true },
+  incompatibleFieldMappingItems: { type: 'nested', required: true, array: true },
+  'incompatibleFieldMappingItems.fieldName': { type: 'keyword', required: true },
+  'incompatibleFieldMappingItems.expectedValue': { type: 'keyword', required: true },
+  'incompatibleFieldMappingItems.actualValue': { type: 'keyword', required: true },
+  'incompatibleFieldMappingItems.description': { type: 'keyword', required: true },
+  incompatibleFieldValueItems: { type: 'nested', required: true, array: true },
+  'incompatibleFieldValueItems.fieldName': { type: 'keyword', required: true },
+  'incompatibleFieldValueItems.expectedValues': { type: 'keyword', required: true },
+  'incompatibleFieldValueItems.actualValues': { type: 'nested', required: true, array: true },
+  'incompatibleFieldValueItems.actualValues.name': { type: 'keyword', required: true },
+  'incompatibleFieldValueItems.actualValues.count': { type: 'keyword', required: true },
+  'incompatibleFieldValueItems.description': { type: 'keyword', required: true },
   incompatibleFieldCount: { type: 'long', required: true },
   sameFamilyFieldCount: { type: 'long', required: true },
   sameFamilyFieldItems: { type: 'nested', required: true, array: true },

diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts
@@ -19,10 +19,11 @@ export const resultDocument: ResultDocument = {
   ecsFieldCount: 677,
   customFieldCount: 904,
   incompatibleFieldCount: 1,
-  incompatibleFieldItems: [
+  incompatibleFieldMappingItems: [],
+  incompatibleFieldValueItems: [
     {
       fieldName: 'event.category',
-      expectedValue: [
+      expectedValues: [
         `authentication`,
         `configuration`,
         `database`,
@@ -42,10 +43,9 @@ export const resultDocument: ResultDocument = {
         'vulnerability',
         'web',
       ],
-      actualValue: ['behavior'],
+      actualValues: [{ name: 'behavior', count: 6 }],
       description:
         'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
-      reason: 'value',
     },
   ],
   sameFamilyFieldCount: 0,

diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts
@@ -19,13 +19,20 @@ const ResultDocumentInterface = t.interface({
   ecsFieldCount: t.number,
   customFieldCount: t.number,
   incompatibleFieldCount: t.number,
-  incompatibleFieldItems: t.array(
+  incompatibleFieldMappingItems: t.array(
     t.type({
       fieldName: t.string,
-      expectedValue: t.union([t.string, t.array(t.string)]),
-      actualValue: t.union([t.string, t.array(t.string)]),
+      expectedValue: t.string,
+      actualValue: t.string,
+      description: t.string,
+    })
+  ),
+  incompatibleFieldValueItems: t.array(
+    t.type({
+      fieldName: t.string,
+      expectedValues: t.array(t.string),
+      actualValues: t.array(t.type({ name: t.string, count: t.number })),
       description: t.string,
-      reason: t.string,
     })
   ),
   sameFamilyFieldCount: t.number,