From d8aebc86cab9167b4d3a19b0c913754d1b81a82f Mon Sep 17 00:00:00 2001 From: Ying Date: Fri, 3 Nov 2023 14:41:14 -0400 Subject: [PATCH 1/4] Adding list of experimental ECS fields to exclude --- .../src/field_maps/ecs_field_map.ts | 162 +++++++++++++++++- 1 file changed, 161 insertions(+), 1 deletion(-) diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts index 3704edee72701..5659c71196922 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts @@ -11,9 +11,169 @@ import { EcsMetadata, FieldMap } from './types'; const EXCLUDED_TYPES = ['constant_keyword']; +// ECS fields that have reached Stage 2 in the RFC process +// are included in the generated Yaml but are still considered +// experimental. Some are correctly marked as beta but most are +// not. + +// More about the RFC stages here: https://elastic.github.io/ecs/stages.html + +// The following RFCS are currently in stage 2: +// https://github.com/elastic/ecs/blob/main/rfcs/text/0002-rfc-environment.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0009-data_stream-fields.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0015-create-file-elf.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0027-faas-fields.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0028-cgroups.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0034-device-fields.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0035-tty-output.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0037-host-metrics.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0040-volume-device.md + +// Fields from these RFCs are manually identified as experimental below. +// The next time this list is updated, we should check the above list of RFCs to +// see if any have moved to Stage 3 and remove them from the list and check if +// there are any new stage 2 RFCs with fields we should identify as experimental. + +const EXPERIMENTAL_FIELDS = [ + 'data_stream.type', + 'data_stream.dataset', + 'data_stream.namespace', + 'device.id', + 'device.manufacturer', + 'device.model.identifier', + 'device.model.name', + 'faas.coldstart', + 'faas.execution', + 'faas.id', + 'faas.name', + 'faas.trigger', + 'faas.trigger.request_id', + 'faas.trigger.type', + 'faas.version', + 'file.elf.architecture', + 'file.elf.byte_order', + 'file.elf.cpu_type', + 'file.elf.creation_date', + 'file.elf.exports', + 'file.elf.segments', + 'file.elf.segments.sections', + 'file.elf.segments.type', + 'file.elf.header.abi_version', + 'file.elf.header.class', + 'file.elf.header.data', + 'file.elf.header.entrypoint', + 'file.elf.header.object_version', + 'file.elf.header.os_abi', + 'file.elf.header.type', + 'file.elf.header.version', + 'file.elf.imports', + 'file.elf.sections', + 'file.elf.sections.chi2', + 'file.elf.sections.entropy', + 'file.elf.sections.flags', + 'file.elf.sections.name', + 'file.elf.sections.physical_offset', + 'file.elf.sections.physical_size', + 'file.elf.sections.type', + 'file.elf.sections.virtual_address', + 'file.elf.sections.virtual_size', + 'file.elf.shared_libraries', + 'file.elf.telfhash', + 'host.network.egress.bytes', + 'host.network.ingress.bytes', + 'process.io', + 'process.io.bytes_skipped', + 'process.io.bytes_skipped.length', + 'process.io.bytes_skipped.offset', + 'process.io.max_bytes_per_process_exceeded', + 'process.io.text', + 'process.io.total_bytes_captured', + 'process.io.total_bytes_skipped', + 'process.io.type', + 'process.tty.columns', + 'process.tty.rows', + 'service.environment', + 'threat.enrichments.indicator.file.elf.architecture', + 'threat.enrichments.indicator.file.elf.byte_order', + 'threat.enrichments.indicator.file.elf.cpu_type', + 'threat.enrichments.indicator.file.elf.creation_date', + 'threat.enrichments.indicator.file.elf.exports', + 'threat.enrichments.indicator.file.elf.segments', + 'threat.enrichments.indicator.file.elf.segments.sections', + 'threat.enrichments.indicator.file.elf.segments.type', + 'threat.enrichments.indicator.file.elf.header.abi_version', + 'threat.enrichments.indicator.file.elf.header.class', + 'threat.enrichments.indicator.file.elf.header.data', + 'threat.enrichments.indicator.file.elf.header.entrypoint', + 'threat.enrichments.indicator.file.elf.header.object_version', + 'threat.enrichments.indicator.file.elf.header.os_abi', + 'threat.enrichments.indicator.file.elf.header.type', + 'threat.enrichments.indicator.file.elf.header.version', + 'threat.enrichments.indicator.file.elf.imports', + 'threat.enrichments.indicator.file.elf.sections', + 'threat.enrichments.indicator.file.elf.sections.chi2', + 'threat.enrichments.indicator.file.elf.sections.entropy', + 'threat.enrichments.indicator.file.elf.sections.flags', + 'threat.enrichments.indicator.file.elf.sections.name', + 'threat.enrichments.indicator.file.elf.sections.physical_offset', + 'threat.enrichments.indicator.file.elf.sections.physical_size', + 'threat.enrichments.indicator.file.elf.sections.type', + 'threat.enrichments.indicator.file.elf.sections.virtual_address', + 'threat.enrichments.indicator.file.elf.sections.virtual_size', + 'threat.enrichments.indicator.file.elf.shared_libraries', + 'threat.enrichments.indicator.file.elf.telfhash', + 'threat.indicator.file.elf.architecture', + 'threat.indicator.file.elf.byte_order', + 'threat.indicator.file.elf.cpu_type', + 'threat.indicator.file.elf.creation_date', + 'threat.indicator.file.elf.exports', + 'threat.indicator.file.elf.segments', + 'threat.indicator.file.elf.segments.sections', + 'threat.indicator.file.elf.segments.type', + 'threat.indicator.file.elf.header.abi_version', + 'threat.indicator.file.elf.header.class', + 'threat.indicator.file.elf.header.data', + 'threat.indicator.file.elf.header.entrypoint', + 'threat.indicator.file.elf.header.object_version', + 'threat.indicator.file.elf.header.os_abi', + 'threat.indicator.file.elf.header.type', + 'threat.indicator.file.elf.header.version', + 'threat.indicator.file.elf.imports', + 'threat.indicator.file.elf.sections', + 'threat.indicator.file.elf.sections.chi2', + 'threat.indicator.file.elf.sections.entropy', + 'threat.indicator.file.elf.sections.flags', + 'threat.indicator.file.elf.sections.name', + 'threat.indicator.file.elf.sections.physical_offset', + 'threat.indicator.file.elf.sections.physical_size', + 'threat.indicator.file.elf.sections.type', + 'threat.indicator.file.elf.sections.virtual_address', + 'threat.indicator.file.elf.sections.virtual_size', + 'threat.indicator.file.elf.shared_libraries', + 'threat.indicator.file.elf.telfhash', + 'volume.bus_type', + 'volume.default_access', + 'volume.device_name', + 'volume.device_type', + 'volume.dos_name', + 'volume.file_system_type', + 'volume.mount_name', + 'volume.nt_name', + 'volume.product_id', + 'volume.product_name', + 'volume.removable', + 'volume.serial_number', + 'volume.size', + 'volume.vendor_id', + 'volume.vendor_name', + 'volume.writable', +]; + export const ecsFieldMap: FieldMap = Object.fromEntries( Object.entries(EcsFlat) - .filter(([_, value]) => !EXCLUDED_TYPES.includes(value.type)) + .filter( + ([key, value]) => !EXCLUDED_TYPES.includes(value.type) && !EXPERIMENTAL_FIELDS.includes(key) + ) .map(([key, _]) => { const value: EcsMetadata = EcsFlat[key as keyof typeof EcsFlat]; return [ From 066ba53b91502a852213d3e092b3fef61432f375 Mon Sep 17 00:00:00 2001 From: Ying Date: Fri, 3 Nov 2023 17:41:44 -0400 Subject: [PATCH 2/4] Fixing tests --- .../src/schemas/generated/ecs_schema.ts | 84 ------------------- .../utils/strip_non_ecs_fields.test.ts | 21 ----- 2 files changed, 105 deletions(-) diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts index a3d3ef6f0a8a7..6e936a56effe1 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts @@ -193,10 +193,6 @@ const EcsOptional = rt.partial({ 'destination.user.id': schemaString, 'destination.user.name': schemaString, 'destination.user.roles': schemaStringArray, - 'device.id': schemaString, - 'device.manufacturer': schemaString, - 'device.model.identifier': schemaString, - 'device.model.name': schemaString, 'dll.code_signature.digest_algorithm': schemaString, 'dll.code_signature.exists': schemaBoolean, 'dll.code_signature.signing_id': schemaString, @@ -304,12 +300,6 @@ const EcsOptional = rt.partial({ 'event.timezone': schemaString, 'event.type': schemaStringArray, 'event.url': schemaString, - 'faas.coldstart': schemaBoolean, - 'faas.execution': schemaString, - 'faas.id': schemaString, - 'faas.name': schemaString, - 'faas.trigger': schemaUnknown, - 'faas.version': schemaString, 'file.accessed': schemaDate, 'file.attributes': schemaStringArray, 'file.code_signature.digest_algorithm': schemaString, @@ -326,41 +316,6 @@ const EcsOptional = rt.partial({ 'file.device': schemaString, 'file.directory': schemaString, 'file.drive_letter': schemaString, - 'file.elf.architecture': schemaString, - 'file.elf.byte_order': schemaString, - 'file.elf.cpu_type': schemaString, - 'file.elf.creation_date': schemaDate, - 'file.elf.exports': schemaUnknownArray, - 'file.elf.header.abi_version': schemaString, - 'file.elf.header.class': schemaString, - 'file.elf.header.data': schemaString, - 'file.elf.header.entrypoint': schemaStringOrNumber, - 'file.elf.header.object_version': schemaString, - 'file.elf.header.os_abi': schemaString, - 'file.elf.header.type': schemaString, - 'file.elf.header.version': schemaString, - 'file.elf.imports': schemaUnknownArray, - 'file.elf.sections': rt.array( - rt.partial({ - chi2: schemaStringOrNumber, - entropy: schemaStringOrNumber, - flags: schemaString, - name: schemaString, - physical_offset: schemaString, - physical_size: schemaStringOrNumber, - type: schemaString, - virtual_address: schemaStringOrNumber, - virtual_size: schemaStringOrNumber, - }) - ), - 'file.elf.segments': rt.array( - rt.partial({ - sections: schemaString, - type: schemaString, - }) - ), - 'file.elf.shared_libraries': schemaStringArray, - 'file.elf.telfhash': schemaString, 'file.extension': schemaString, 'file.fork_name': schemaString, 'file.gid': schemaString, @@ -440,9 +395,7 @@ const EcsOptional = rt.partial({ 'host.ip': schemaStringArray, 'host.mac': schemaStringArray, 'host.name': schemaString, - 'host.network.egress.bytes': schemaStringOrNumber, 'host.network.egress.packets': schemaStringOrNumber, - 'host.network.ingress.bytes': schemaStringOrNumber, 'host.network.ingress.packets': schemaStringOrNumber, 'host.os.family': schemaString, 'host.os.full': schemaString, @@ -676,7 +629,6 @@ const EcsOptional = rt.partial({ 'process.hash.ssdeep': schemaString, 'process.hash.tlsh': schemaString, 'process.interactive': schemaBoolean, - 'process.io': schemaUnknown, 'process.name': schemaString, 'process.parent.args': schemaStringArray, 'process.parent.args_count': schemaStringOrNumber, @@ -893,7 +845,6 @@ const EcsOptional = rt.partial({ 'server.user.name': schemaString, 'server.user.roles': schemaStringArray, 'service.address': schemaString, - 'service.environment': schemaString, 'service.ephemeral_id': schemaString, 'service.id': schemaString, 'service.name': schemaString, @@ -1003,41 +954,6 @@ const EcsOptional = rt.partial({ 'threat.indicator.file.device': schemaString, 'threat.indicator.file.directory': schemaString, 'threat.indicator.file.drive_letter': schemaString, - 'threat.indicator.file.elf.architecture': schemaString, - 'threat.indicator.file.elf.byte_order': schemaString, - 'threat.indicator.file.elf.cpu_type': schemaString, - 'threat.indicator.file.elf.creation_date': schemaDate, - 'threat.indicator.file.elf.exports': schemaUnknownArray, - 'threat.indicator.file.elf.header.abi_version': schemaString, - 'threat.indicator.file.elf.header.class': schemaString, - 'threat.indicator.file.elf.header.data': schemaString, - 'threat.indicator.file.elf.header.entrypoint': schemaStringOrNumber, - 'threat.indicator.file.elf.header.object_version': schemaString, - 'threat.indicator.file.elf.header.os_abi': schemaString, - 'threat.indicator.file.elf.header.type': schemaString, - 'threat.indicator.file.elf.header.version': schemaString, - 'threat.indicator.file.elf.imports': schemaUnknownArray, - 'threat.indicator.file.elf.sections': rt.array( - rt.partial({ - chi2: schemaStringOrNumber, - entropy: schemaStringOrNumber, - flags: schemaString, - name: schemaString, - physical_offset: schemaString, - physical_size: schemaStringOrNumber, - type: schemaString, - virtual_address: schemaStringOrNumber, - virtual_size: schemaStringOrNumber, - }) - ), - 'threat.indicator.file.elf.segments': rt.array( - rt.partial({ - sections: schemaString, - type: schemaString, - }) - ), - 'threat.indicator.file.elf.shared_libraries': schemaStringArray, - 'threat.indicator.file.elf.telfhash': schemaString, 'threat.indicator.file.extension': schemaString, 'threat.indicator.file.fork_name': schemaString, 'threat.indicator.file.gid': schemaString, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts index 21f9adc96bd60..9c10a317ee17c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts @@ -71,27 +71,6 @@ describe('stripNonEcsFields', () => { ]); }); - // https://github.com/elastic/sdh-security-team/issues/736 - describe('fields that exists in the alerts mapping but not in local ECS(ruleRegistry) definition', () => { - it('should strip object type "device" field if it is supplied as a keyword', () => { - const { result, removed } = stripNonEcsFields({ - device: 'test', - message: 'test message', - }); - - expect(result).toEqual({ - message: 'test message', - }); - - expect(removed).toEqual([ - { - key: 'device', - value: 'test', - }, - ]); - }); - }); - describe('array fields', () => { it('should not strip arrays of objects when an object is expected', () => { const { result, removed } = stripNonEcsFields({ From 7714668177d04f8382f0ed7eca372921080fa186 Mon Sep 17 00:00:00 2001 From: Ying Date: Mon, 13 Nov 2023 10:54:23 -0500 Subject: [PATCH 3/4] Only excluding new to us experimental fields --- .../src/field_maps/ecs_field_map.ts | 139 ++---------------- .../src/schemas/generated/ecs_schema.ts | 85 +++++++++++ .../utils/strip_non_ecs_fields.test.ts | 21 +++ 3 files changed, 122 insertions(+), 123 deletions(-) diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts index 5659c71196922..b9af904d16254 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts @@ -19,138 +19,31 @@ const EXCLUDED_TYPES = ['constant_keyword']; // More about the RFC stages here: https://elastic.github.io/ecs/stages.html // The following RFCS are currently in stage 2: -// https://github.com/elastic/ecs/blob/main/rfcs/text/0002-rfc-environment.md -// https://github.com/elastic/ecs/blob/main/rfcs/text/0009-data_stream-fields.md -// https://github.com/elastic/ecs/blob/main/rfcs/text/0015-create-file-elf.md // https://github.com/elastic/ecs/blob/main/rfcs/text/0027-faas-fields.md -// https://github.com/elastic/ecs/blob/main/rfcs/text/0028-cgroups.md -// https://github.com/elastic/ecs/blob/main/rfcs/text/0034-device-fields.md // https://github.com/elastic/ecs/blob/main/rfcs/text/0035-tty-output.md // https://github.com/elastic/ecs/blob/main/rfcs/text/0037-host-metrics.md // https://github.com/elastic/ecs/blob/main/rfcs/text/0040-volume-device.md -// Fields from these RFCs are manually identified as experimental below. +// Fields from these RFCs that are not already in the ECS component template +// as of 8.11 are manually identified as experimental below. // The next time this list is updated, we should check the above list of RFCs to // see if any have moved to Stage 3 and remove them from the list and check if -// there are any new stage 2 RFCs with fields we should identify as experimental. +// there are any new stage 2 RFCs with fields we should exclude as experimental. const EXPERIMENTAL_FIELDS = [ - 'data_stream.type', - 'data_stream.dataset', - 'data_stream.namespace', - 'device.id', - 'device.manufacturer', - 'device.model.identifier', - 'device.model.name', - 'faas.coldstart', - 'faas.execution', - 'faas.id', - 'faas.name', - 'faas.trigger', - 'faas.trigger.request_id', - 'faas.trigger.type', - 'faas.version', - 'file.elf.architecture', - 'file.elf.byte_order', - 'file.elf.cpu_type', - 'file.elf.creation_date', - 'file.elf.exports', - 'file.elf.segments', - 'file.elf.segments.sections', - 'file.elf.segments.type', - 'file.elf.header.abi_version', - 'file.elf.header.class', - 'file.elf.header.data', - 'file.elf.header.entrypoint', - 'file.elf.header.object_version', - 'file.elf.header.os_abi', - 'file.elf.header.type', - 'file.elf.header.version', - 'file.elf.imports', - 'file.elf.sections', - 'file.elf.sections.chi2', - 'file.elf.sections.entropy', - 'file.elf.sections.flags', - 'file.elf.sections.name', - 'file.elf.sections.physical_offset', - 'file.elf.sections.physical_size', - 'file.elf.sections.type', - 'file.elf.sections.virtual_address', - 'file.elf.sections.virtual_size', - 'file.elf.shared_libraries', - 'file.elf.telfhash', - 'host.network.egress.bytes', - 'host.network.ingress.bytes', - 'process.io', - 'process.io.bytes_skipped', - 'process.io.bytes_skipped.length', - 'process.io.bytes_skipped.offset', - 'process.io.max_bytes_per_process_exceeded', - 'process.io.text', - 'process.io.total_bytes_captured', - 'process.io.total_bytes_skipped', - 'process.io.type', - 'process.tty.columns', - 'process.tty.rows', - 'service.environment', - 'threat.enrichments.indicator.file.elf.architecture', - 'threat.enrichments.indicator.file.elf.byte_order', - 'threat.enrichments.indicator.file.elf.cpu_type', - 'threat.enrichments.indicator.file.elf.creation_date', - 'threat.enrichments.indicator.file.elf.exports', - 'threat.enrichments.indicator.file.elf.segments', - 'threat.enrichments.indicator.file.elf.segments.sections', - 'threat.enrichments.indicator.file.elf.segments.type', - 'threat.enrichments.indicator.file.elf.header.abi_version', - 'threat.enrichments.indicator.file.elf.header.class', - 'threat.enrichments.indicator.file.elf.header.data', - 'threat.enrichments.indicator.file.elf.header.entrypoint', - 'threat.enrichments.indicator.file.elf.header.object_version', - 'threat.enrichments.indicator.file.elf.header.os_abi', - 'threat.enrichments.indicator.file.elf.header.type', - 'threat.enrichments.indicator.file.elf.header.version', - 'threat.enrichments.indicator.file.elf.imports', - 'threat.enrichments.indicator.file.elf.sections', - 'threat.enrichments.indicator.file.elf.sections.chi2', - 'threat.enrichments.indicator.file.elf.sections.entropy', - 'threat.enrichments.indicator.file.elf.sections.flags', - 'threat.enrichments.indicator.file.elf.sections.name', - 'threat.enrichments.indicator.file.elf.sections.physical_offset', - 'threat.enrichments.indicator.file.elf.sections.physical_size', - 'threat.enrichments.indicator.file.elf.sections.type', - 'threat.enrichments.indicator.file.elf.sections.virtual_address', - 'threat.enrichments.indicator.file.elf.sections.virtual_size', - 'threat.enrichments.indicator.file.elf.shared_libraries', - 'threat.enrichments.indicator.file.elf.telfhash', - 'threat.indicator.file.elf.architecture', - 'threat.indicator.file.elf.byte_order', - 'threat.indicator.file.elf.cpu_type', - 'threat.indicator.file.elf.creation_date', - 'threat.indicator.file.elf.exports', - 'threat.indicator.file.elf.segments', - 'threat.indicator.file.elf.segments.sections', - 'threat.indicator.file.elf.segments.type', - 'threat.indicator.file.elf.header.abi_version', - 'threat.indicator.file.elf.header.class', - 'threat.indicator.file.elf.header.data', - 'threat.indicator.file.elf.header.entrypoint', - 'threat.indicator.file.elf.header.object_version', - 'threat.indicator.file.elf.header.os_abi', - 'threat.indicator.file.elf.header.type', - 'threat.indicator.file.elf.header.version', - 'threat.indicator.file.elf.imports', - 'threat.indicator.file.elf.sections', - 'threat.indicator.file.elf.sections.chi2', - 'threat.indicator.file.elf.sections.entropy', - 'threat.indicator.file.elf.sections.flags', - 'threat.indicator.file.elf.sections.name', - 'threat.indicator.file.elf.sections.physical_offset', - 'threat.indicator.file.elf.sections.physical_size', - 'threat.indicator.file.elf.sections.type', - 'threat.indicator.file.elf.sections.virtual_address', - 'threat.indicator.file.elf.sections.virtual_size', - 'threat.indicator.file.elf.shared_libraries', - 'threat.indicator.file.elf.telfhash', + 'faas.trigger', // this was previously mapped as nested but changed to object + 'host.cpu.system.norm.pct', + 'host.cpu.user.norm.pct', + 'host.fsstats.total_size.total', + 'host.fsstats.total_size.used', + 'host.fsstats.total_size.used.pct', + 'host.load.norm.1', + 'host.load.norm.5', + 'host.load.norm.15', + 'host.memory.actual.used.bytes', + 'host.memory.actual.used.pct', + 'host.memory.total', + 'process.io.bytes', 'volume.bus_type', 'volume.default_access', 'volume.device_name', diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts index 6e936a56effe1..50e2b47dbbedd 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts @@ -193,6 +193,10 @@ const EcsOptional = rt.partial({ 'destination.user.id': schemaString, 'destination.user.name': schemaString, 'destination.user.roles': schemaStringArray, + 'device.id': schemaString, + 'device.manufacturer': schemaString, + 'device.model.identifier': schemaString, + 'device.model.name': schemaString, 'dll.code_signature.digest_algorithm': schemaString, 'dll.code_signature.exists': schemaBoolean, 'dll.code_signature.signing_id': schemaString, @@ -300,6 +304,13 @@ const EcsOptional = rt.partial({ 'event.timezone': schemaString, 'event.type': schemaStringArray, 'event.url': schemaString, + 'faas.coldstart': schemaBoolean, + 'faas.execution': schemaString, + 'faas.id': schemaString, + 'faas.name': schemaString, + 'faas.trigger.request_id': schemaString, + 'faas.trigger.type': schemaString, + 'faas.version': schemaString, 'file.accessed': schemaDate, 'file.attributes': schemaStringArray, 'file.code_signature.digest_algorithm': schemaString, @@ -316,6 +327,41 @@ const EcsOptional = rt.partial({ 'file.device': schemaString, 'file.directory': schemaString, 'file.drive_letter': schemaString, + 'file.elf.architecture': schemaString, + 'file.elf.byte_order': schemaString, + 'file.elf.cpu_type': schemaString, + 'file.elf.creation_date': schemaDate, + 'file.elf.exports': schemaUnknownArray, + 'file.elf.header.abi_version': schemaString, + 'file.elf.header.class': schemaString, + 'file.elf.header.data': schemaString, + 'file.elf.header.entrypoint': schemaStringOrNumber, + 'file.elf.header.object_version': schemaString, + 'file.elf.header.os_abi': schemaString, + 'file.elf.header.type': schemaString, + 'file.elf.header.version': schemaString, + 'file.elf.imports': schemaUnknownArray, + 'file.elf.sections': rt.array( + rt.partial({ + chi2: schemaStringOrNumber, + entropy: schemaStringOrNumber, + flags: schemaString, + name: schemaString, + physical_offset: schemaString, + physical_size: schemaStringOrNumber, + type: schemaString, + virtual_address: schemaStringOrNumber, + virtual_size: schemaStringOrNumber, + }) + ), + 'file.elf.segments': rt.array( + rt.partial({ + sections: schemaString, + type: schemaString, + }) + ), + 'file.elf.shared_libraries': schemaStringArray, + 'file.elf.telfhash': schemaString, 'file.extension': schemaString, 'file.fork_name': schemaString, 'file.gid': schemaString, @@ -395,7 +441,9 @@ const EcsOptional = rt.partial({ 'host.ip': schemaStringArray, 'host.mac': schemaStringArray, 'host.name': schemaString, + 'host.network.egress.bytes': schemaStringOrNumber, 'host.network.egress.packets': schemaStringOrNumber, + 'host.network.ingress.bytes': schemaStringOrNumber, 'host.network.ingress.packets': schemaStringOrNumber, 'host.os.family': schemaString, 'host.os.full': schemaString, @@ -629,6 +677,7 @@ const EcsOptional = rt.partial({ 'process.hash.ssdeep': schemaString, 'process.hash.tlsh': schemaString, 'process.interactive': schemaBoolean, + 'process.io': schemaUnknown, 'process.name': schemaString, 'process.parent.args': schemaStringArray, 'process.parent.args_count': schemaStringOrNumber, @@ -845,6 +894,7 @@ const EcsOptional = rt.partial({ 'server.user.name': schemaString, 'server.user.roles': schemaStringArray, 'service.address': schemaString, + 'service.environment': schemaString, 'service.ephemeral_id': schemaString, 'service.id': schemaString, 'service.name': schemaString, @@ -954,6 +1004,41 @@ const EcsOptional = rt.partial({ 'threat.indicator.file.device': schemaString, 'threat.indicator.file.directory': schemaString, 'threat.indicator.file.drive_letter': schemaString, + 'threat.indicator.file.elf.architecture': schemaString, + 'threat.indicator.file.elf.byte_order': schemaString, + 'threat.indicator.file.elf.cpu_type': schemaString, + 'threat.indicator.file.elf.creation_date': schemaDate, + 'threat.indicator.file.elf.exports': schemaUnknownArray, + 'threat.indicator.file.elf.header.abi_version': schemaString, + 'threat.indicator.file.elf.header.class': schemaString, + 'threat.indicator.file.elf.header.data': schemaString, + 'threat.indicator.file.elf.header.entrypoint': schemaStringOrNumber, + 'threat.indicator.file.elf.header.object_version': schemaString, + 'threat.indicator.file.elf.header.os_abi': schemaString, + 'threat.indicator.file.elf.header.type': schemaString, + 'threat.indicator.file.elf.header.version': schemaString, + 'threat.indicator.file.elf.imports': schemaUnknownArray, + 'threat.indicator.file.elf.sections': rt.array( + rt.partial({ + chi2: schemaStringOrNumber, + entropy: schemaStringOrNumber, + flags: schemaString, + name: schemaString, + physical_offset: schemaString, + physical_size: schemaStringOrNumber, + type: schemaString, + virtual_address: schemaStringOrNumber, + virtual_size: schemaStringOrNumber, + }) + ), + 'threat.indicator.file.elf.segments': rt.array( + rt.partial({ + sections: schemaString, + type: schemaString, + }) + ), + 'threat.indicator.file.elf.shared_libraries': schemaStringArray, + 'threat.indicator.file.elf.telfhash': schemaString, 'threat.indicator.file.extension': schemaString, 'threat.indicator.file.fork_name': schemaString, 'threat.indicator.file.gid': schemaString, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts index 9c10a317ee17c..21f9adc96bd60 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts @@ -71,6 +71,27 @@ describe('stripNonEcsFields', () => { ]); }); + // https://github.com/elastic/sdh-security-team/issues/736 + describe('fields that exists in the alerts mapping but not in local ECS(ruleRegistry) definition', () => { + it('should strip object type "device" field if it is supplied as a keyword', () => { + const { result, removed } = stripNonEcsFields({ + device: 'test', + message: 'test message', + }); + + expect(result).toEqual({ + message: 'test message', + }); + + expect(removed).toEqual([ + { + key: 'device', + value: 'test', + }, + ]); + }); + }); + describe('array fields', () => { it('should not strip arrays of objects when an object is expected', () => { const { result, removed } = stripNonEcsFields({ From 2e89046d5678ea2af5545042ba116ef5dbec2c6f Mon Sep 17 00:00:00 2001 From: Ying Date: Mon, 20 Nov 2023 11:20:22 -0500 Subject: [PATCH 4/4] Exclude all the faas.trigger fields --- .../kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts | 2 ++ .../src/schemas/generated/ecs_schema.ts | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts index b9af904d16254..8aea9ca56e029 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts @@ -32,6 +32,8 @@ const EXCLUDED_TYPES = ['constant_keyword']; const EXPERIMENTAL_FIELDS = [ 'faas.trigger', // this was previously mapped as nested but changed to object + 'faas.trigger.request_id', + 'faas.trigger.type', 'host.cpu.system.norm.pct', 'host.cpu.user.norm.pct', 'host.fsstats.total_size.total', diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts index 50e2b47dbbedd..b3bc0eb161720 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ecs_schema.ts @@ -308,8 +308,6 @@ const EcsOptional = rt.partial({ 'faas.execution': schemaString, 'faas.id': schemaString, 'faas.name': schemaString, - 'faas.trigger.request_id': schemaString, - 'faas.trigger.type': schemaString, 'faas.version': schemaString, 'file.accessed': schemaDate, 'file.attributes': schemaStringArray,