From 618cc48bc5250f4d5553e70779eb4d8b02645a8e Mon Sep 17 00:00:00 2001
From: Ying Mao <ying.mao@elastic.co>
Date: Mon, 27 Nov 2023 08:54:56 -0500
Subject: [PATCH] [Reponse Ops][Alerting] Excluding ECS experimental fields
 from ECS component template (#170571)

Resolves https://github.com/elastic/kibana/issues/168959

## Summary

Hard-coding a list of experimental (RFC stage 2) ECS fields to exclude
from the ECS component template. These are only the fields that are not
currently defined in `ecs_flat.yml`. The only existing field that is
excluded is `faas.trigger` which, if included, will cause a mapping
conflict exception because of an ECS mapping change from `nested` to
`object`.

## To Verify

Compare the mappings for the `.alerts-ecs-mappings` component template
between `main` and this branch and notice that the `faas.trigger` field
is excluded from the component template on this branch.

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../src/field_maps/ecs_field_map.ts           | 57 ++++++++++++++++++-
 .../src/schemas/generated/ecs_schema.ts       |  1 -
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts
index 3704edee72701..8aea9ca56e029 100644
--- a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts
+++ b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts
@@ -11,9 +11,64 @@ import { EcsMetadata, FieldMap } from './types';
 
 const EXCLUDED_TYPES = ['constant_keyword'];
 
+// ECS fields that have reached Stage 2 in the RFC process
+// are included in the generated Yaml but are still considered
+// experimental. Some are correctly marked as beta but most are
+// not.
+
+// More about the RFC stages here: https://elastic.github.io/ecs/stages.html
+
+// The following RFCS are currently in stage 2:
+// https://github.com/elastic/ecs/blob/main/rfcs/text/0027-faas-fields.md
+// https://github.com/elastic/ecs/blob/main/rfcs/text/0035-tty-output.md
+// https://github.com/elastic/ecs/blob/main/rfcs/text/0037-host-metrics.md
+// https://github.com/elastic/ecs/blob/main/rfcs/text/0040-volume-device.md
+
+// Fields from these RFCs that are not already in the ECS component template
+// as of 8.11 are manually identified as experimental below.
+// The next time this list is updated, we should check the above list of RFCs to
+// see if any have moved to Stage 3 and remove them from the list and check if
+// there are any new stage 2 RFCs with fields we should exclude as experimental.
+
+const EXPERIMENTAL_FIELDS = [
+  'faas.trigger', // this was previously mapped as nested but changed to object
+  'faas.trigger.request_id',
+  'faas.trigger.type',
+  'host.cpu.system.norm.pct',
+  'host.cpu.user.norm.pct',
+  'host.fsstats.total_size.total',
+  'host.fsstats.total_size.used',
+  'host.fsstats.total_size.used.pct',
+  'host.load.norm.1',
+  'host.load.norm.5',
+  'host.load.norm.15',
+  'host.memory.actual.used.bytes',
+  'host.memory.actual.used.pct',
+  'host.memory.total',
+  'process.io.bytes',
+  'volume.bus_type',
+  'volume.default_access',
+  'volume.device_name',
+  'volume.device_type',
+  'volume.dos_name',
+  'volume.file_system_type',
+  'volume.mount_name',
+  'volume.nt_name',
+  'volume.product_id',
+  'volume.product_name',
+  'volume.removable',
+  'volume.serial_number',
+  'volume.size',
+  'volume.vendor_id',
+  'volume.vendor_name',
+  'volume.writable',
+];
+
 export const ecsFieldMap: FieldMap = Object.fromEntries(
   Object.entries(EcsFlat)
-    .filter(([_, value]) => !EXCLUDED_TYPES.includes(value.type))
+    .filter(
+      ([key, value]) => !EXCLUDED_TYPES.includes(value.type) && !EXPERIMENTAL_FIELDS.includes(key)
+    )
     .map(([key, _]) => {
       const value: EcsMetadata = EcsFlat[key as keyof typeof EcsFlat];
       return [
diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts
index a3d3ef6f0a8a7..b3bc0eb161720 100644
--- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts
+++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts
@@ -308,7 +308,6 @@ const EcsOptional = rt.partial({
   'faas.execution': schemaString,
   'faas.id': schemaString,
   'faas.name': schemaString,
-  'faas.trigger': schemaUnknown,
   'faas.version': schemaString,
   'file.accessed': schemaDate,
   'file.attributes': schemaStringArray,