[8.15] [integration automatic-import] Adding ECS 'reserved'…

… fields list and adding check in the invalid ecs check (elastic#189006) (elastic#189215)

# Backport

This will backport the following commits from `main` to `8.15`:
- [[integration automatic-import] Adding ECS 'reserved' fields
list and adding check in the invalid ecs check
(elastic#189006)](elastic#189006)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kylie
Meli","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-07-25T18:05:30Z","message":"[integration
automatic-import] Adding ECS 'reserved' fields list and adding check in
the invalid ecs check (elastic#189006)\n\nThis PR updates the invalid ecs check
to include a check for 'reserved'\r\nECS fields. Reserved ECS fields are
valid ecs fields, but ones we do not\r\nwant to add mappings for as they
are reserved for agent operations or\r\nutilized in
categorization.\r\n\r\nECS reserved:\r\n- ecs.version\r\n-
error.message\r\n- event.category\r\n- event.created\r\n-
event.dataset\r\n- event.ingested\r\n- event.original\r\n-
event.type","sha":"45f63a3b97084ac1fb7e5297b2b534e3d97c935d","branchLabelMapping":{"^v8.16.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v8.15.0","v8.16.0","Team:Security-Scalability"],"title":"[integration
automatic-import] Adding ECS 'reserved' fields list and adding check in
the invalid ecs
check","number":189006,"url":"https://github.com/elastic/kibana/pull/189006","mergeCommit":{"message":"[integration
automatic-import] Adding ECS 'reserved' fields list and adding check in
the invalid ecs check (elastic#189006)\n\nThis PR updates the invalid ecs check
to include a check for 'reserved'\r\nECS fields. Reserved ECS fields are
valid ecs fields, but ones we do not\r\nwant to add mappings for as they
are reserved for agent operations or\r\nutilized in
categorization.\r\n\r\nECS reserved:\r\n- ecs.version\r\n-
error.message\r\n- event.category\r\n- event.created\r\n-
event.dataset\r\n- event.ingested\r\n- event.original\r\n-
event.type","sha":"45f63a3b97084ac1fb7e5297b2b534e3d97c935d"}},"sourceBranch":"main","suggestedTargetBranches":["8.15"],"targetPullRequestStates":[{"branch":"8.15","label":"v8.15.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/189006","number":189006,"mergeCommit":{"message":"[integration
automatic-import] Adding ECS 'reserved' fields list and adding check in
the invalid ecs check (elastic#189006)\n\nThis PR updates the invalid ecs check
to include a check for 'reserved'\r\nECS fields. Reserved ECS fields are
valid ecs fields, but ones we do not\r\nwant to add mappings for as they
are reserved for agent operations or\r\nutilized in
categorization.\r\n\r\nECS reserved:\r\n- ecs.version\r\n-
error.message\r\n- event.category\r\n- event.created\r\n-
event.dataset\r\n- event.ingested\r\n- event.original\r\n-
event.type","sha":"45f63a3b97084ac1fb7e5297b2b534e3d97c935d"}}]}]
BACKPORT-->

Co-authored-by: Kylie Meli <[email protected]>

x-pack/plugins/integration_assistant/server/graphs/ecs/constants.ts

@@ -2248,3 +2248,14 @@ export const ECS_EXAMPLE_ANSWER = {
     },
   },
 };
+
+export const ECS_RESERVED = [
+  'ecs.version',
+  'error.message',
+  'event.category',
+  'event.created',
+  'event.dataset',
+  'event.ingested',
+  'event.original',
+  'event.type',
+];

x-pack/plugins/integration_assistant/server/graphs/ecs/validate.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { processMapping } from './validate';
+import { findInvalidEcsFields, processMapping } from './validate';
 
 describe('Testing ecs handler', () => {
   it('processMapping()', async () => {
@@ -50,3 +50,43 @@ describe('Testing ecs handler', () => {
     });
   });
 });
+
+describe('findInvalidEcsFields', () => {
+  it('invalid: invalid ecs mapping', async () => {
+    const ecsMappingInvalid = {
+      mysql_enterprise: {
+        audit: {
+          test_array: null,
+          bytes: {
+            target: 'myField.bytes',
+            confidence: 0.99,
+            type: 'number',
+            date_formats: [],
+          },
+        },
+      },
+    };
+
+    const invalid = findInvalidEcsFields(ecsMappingInvalid);
+    expect(invalid.length).toBe(1);
+  });
+
+  it('invalid: reserved ecs field', async () => {
+    const ecsMappingReserved = {
+      mysql_enterprise: {
+        audit: {
+          test_array: null,
+          type: {
+            target: 'event.type',
+            confidence: 'error',
+            type: 'string',
+            date_formats: [],
+          },
+        },
+      },
+    };
+
+    const invalid = findInvalidEcsFields(ecsMappingReserved);
+    expect(invalid.length).toBe(1);
+  });
+});

x-pack/plugins/integration_assistant/server/graphs/ecs/validate.ts

@@ -7,6 +7,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { ECS_FULL } from '../../../common/ecs';
 import type { EcsMappingState } from '../../types';
+import { ECS_RESERVED } from './constants';
 
 const valueFieldKeys = new Set(['target', 'confidence', 'date_formats', 'type']);
 type AnyObject = Record<string, any>;
@@ -127,10 +128,11 @@ function findDuplicateFields(samples: string[], ecsMapping: AnyObject): string[]
 }
 
 // Function to find invalid ECS fields
-function findInvalidEcsFields(ecsMapping: AnyObject): string[] {
+export function findInvalidEcsFields(ecsMapping: AnyObject): string[] {
   const results: string[] = [];
   const output: Record<string, string[][]> = {};
   const ecsDict = ECS_FULL;
+  const ecsReserved = ECS_RESERVED;
 
   processMapping([], ecsMapping, output);
   const filteredOutput = Object.fromEntries(
@@ -142,6 +144,11 @@ function findInvalidEcsFields(ecsMapping: AnyObject): string[] {
       const field = paths.map((p) => p.join('.'));
       results.push(`Invalid ECS field mapping identified for ${ecsValue} : ${field.join(', ')}`);
     }
+
+    if (ecsReserved.includes(ecsValue)) {
+      const field = paths.map((p) => p.join('.'));
+      results.push(`Reserved ECS field mapping identified for ${ecsValue} : ${field.join(', ')}`);
+    }
   }
 
   return results;