From 56ec535d7c273e56f7c2f0ee7d969070303f78b6 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 12 Jul 2024 17:24:23 +0200 Subject: [PATCH 01/10] top_metrics metadata agg --- .../kbn-entities-schema/src/schema/common.ts | 20 +++- .../server/lib/entities/built_in/services.ts | 92 +++++++++++++++---- .../generate_history_processors.ts | 35 ++++--- .../generate_metadata_aggregations.ts | 34 +++++-- 4 files changed, 142 insertions(+), 39 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index bb9d07b1957f4..a46eba1567cc9 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -82,13 +82,29 @@ export const keyMetricSchema = z.object({ export type KeyMetric = z.infer; +export const metadataAggregation = z.union([ + z.object({ type: z.literal('terms'), limit: z.number().default(1000) }), + z.object({ + type: z.literal('top_metrics'), + sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), + }), +]); + export const metadataSchema = z .object({ source: z.string(), destination: z.optional(z.string()), - limit: z.optional(z.number().default(1000)), + aggregation: z.optional(metadataAggregation).default({ type: 'terms', limit: 1000 }), }) - .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))); + .or( + z.string().transform((value) => ({ + source: value, + destination: value, + aggregation: { type: z.literal('terms'), limit: 1000 }, + })) + ); + +export type MetadataField = z.infer; export const identityFieldsSchema = z .object({ diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts index 29d501de8312f..9e015a47050ba 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts @@ -8,16 +8,33 @@ import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; import { BUILT_IN_ID_PREFIX } from './constants'; +const serviceTransactionFilter = (additionalFilters: string[] = []) => { + const baseFilters = [ + 'processor.event: "metric"', + 'metricset.name: "service_transaction"', + 'metricset.interval: "1m"', + ]; + + return [...baseFilters, ...additionalFilters].join(' AND '); +}; + export const builtInServicesEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ - id: `${BUILT_IN_ID_PREFIX}services`, version: '0.1.0', - name: 'Services from logs', + id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`, + name: 'Services from ECS data', + description: + 'This definition extracts service entities from common data streams by looking for the ECS field service.name', type: 'service', managed: true, - indexPatterns: ['logs-*', 'filebeat*'], + filter: '@timestamp >= now-10m', + indexPatterns: ['logs-*', 'filebeat*', 'metrics-apm.service_transaction.1m*'], history: { timestampField: '@timestamp', interval: '1m', + settings: { + frequency: '2m', + syncDelay: '2m', + }, }, latest: { lookback: '5m', @@ -26,43 +43,86 @@ export const builtInServicesEntityDefinition: EntityDefinition = entityDefinitio displayNameTemplate: '{{service.name}}{{#service.environment}}:{{.}}{{/service.environment}}', metadata: [ { source: '_index', destination: 'sourceIndex' }, + { + source: 'agent.name', + aggregation: { type: 'top_metrics', sort: { 'transaction.duration.summary': 'desc' } }, + }, 'data_stream.type', - 'service.instance.id', + 'service.environment', + 'service.name', 'service.namespace', 'service.version', 'service.runtime.name', 'service.runtime.version', - 'service.node.name', 'service.language.name', - 'agent.name', 'cloud.provider', - 'cloud.instance.id', 'cloud.availability_zone', - 'cloud.instance.name', 'cloud.machine.type', - 'host.name', - 'container.id', ], metrics: [ { - name: 'logRate', - equation: 'A / 5', + name: 'latency', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'avg', + filter: serviceTransactionFilter(), + field: 'transaction.duration.histogram', + }, + ], + }, + { + name: 'throughput', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: serviceTransactionFilter(), + }, + ], + }, + { + name: 'failedTransactionRate', + equation: 'A / B', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: serviceTransactionFilter(['event.outcome: "failure"']), + }, + { + name: 'B', + aggregation: 'doc_count', + filter: serviceTransactionFilter(['event.outcome: *']), + }, + ], + }, + { + name: 'logErrorRate', + equation: 'A / B', metrics: [ { name: 'A', aggregation: 'doc_count', - filter: 'log.level: *', + filter: 'log.level: "error" OR error.log.level: "error"', + }, + { + name: 'B', + aggregation: 'doc_count', + filter: 'log.level: * OR error.log.level: *', }, ], }, { - name: 'errorRate', - equation: 'A / 5', + name: 'logRate', + equation: 'A', metrics: [ { name: 'A', aggregation: 'doc_count', - filter: 'log.level: "ERROR"', + filter: 'log.level: * OR error.log.level: *', }, ], }, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index b273be780910b..e86890bdca7e8 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/entities-schema'; +import { EntityDefinition, MetadataField } from '@kbn/entities-schema'; import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities'; import { generateHistoryIndexName } from '../helpers/generate_component_id'; @@ -15,13 +15,20 @@ function createIdTemplate(definition: EntityDefinition) { }, definition.displayNameTemplate); } -function mapDestinationToPainless(destination: string) { +function mapDestinationToPainless(metadata: MetadataField) { + const destination = metadata.destination || metadata.source; const fieldParts = destination.split('.'); return fieldParts.reduce((acc, _part, currentIndex, parts) => { if (currentIndex + 1 === parts.length) { - return `${acc}\n ctx${parts - .map((s) => `["${s}"]`) - .join('')} = ctx.entity.metadata.${destination}.keySet();`; + if (metadata.aggregation.type === 'terms') { + return `${acc}\n ctx${parts + .map((s) => `["${s}"]`) + .join('')} = ctx.entity.metadata.${destination}.keySet();`; + } else if (metadata.aggregation.type === 'top_metrics') { + return `${acc}\n ctx${parts + .map((s) => `["${s}"]`) + .join('')} = ctx.entity.metadata.${destination}["${metadata.source}"];`; + } } return `${acc}\n if(ctx.${parts.slice(0, currentIndex + 1).join('.')} == null) ctx${parts .slice(0, currentIndex + 1) @@ -34,12 +41,18 @@ function createMetadataPainlessScript(definition: EntityDefinition) { if (!definition.metadata) { return ''; } - return definition.metadata.reduce((script, def) => { - const destination = def.destination || def.source; - return `${script}if (ctx.entity?.metadata?.${destination.replaceAll( - '.', - '?.' - )} != null) {${mapDestinationToPainless(destination)}\n}\n`; + return definition.metadata.reduce((script, metadata) => { + const destination = metadata.destination || metadata.source; + if (metadata.aggregation.type === 'terms') { + return `${script}if (ctx.entity?.metadata?.${destination.replaceAll( + '.', + '?.' + )} != null) {${mapDestinationToPainless(metadata)}\n}\n`; + } + + return `${script}if (ctx.entity?.metadata?.${destination.replaceAll('.', '?.')}["${ + metadata.source + }"] != null) {${mapDestinationToPainless(metadata)}\n}\n`; }, ''); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 31ba3e9add0dc..216b49c2a2224 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -12,18 +12,29 @@ export function generateHistoryMetadataAggregations(definition: EntityDefinition if (!definition.metadata) { return {}; } - return definition.metadata.reduce( - (aggs, metadata) => ({ - ...aggs, - [`entity.metadata.${metadata.destination ?? metadata.source}`]: { + return definition.metadata.reduce((aggs, metadata) => { + let agg; + if (metadata.aggregation.type === 'terms') { + agg = { terms: { field: metadata.source, - size: metadata.limit ?? ENTITY_DEFAULT_METADATA_LIMIT, + size: metadata.aggregation.limit, }, - }, - }), - {} - ); + }; + } else if (metadata.aggregation.type === 'top_metrics') { + agg = { + top_metrics: { + metrics: [{ field: metadata.source }], + sort: metadata.aggregation.sort, + }, + }; + } + + return { + ...aggs, + [`entity.metadata.${metadata.destination ?? metadata.source}`]: agg, + }; + }, {}); } export function generateLatestMetadataAggregations(definition: EntityDefinition) { @@ -46,7 +57,10 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) data: { terms: { field: metadata.destination ?? metadata.source, - size: metadata.limit ?? ENTITY_DEFAULT_METADATA_LIMIT, + size: + metadata.aggregation?.type === 'terms' + ? metadata.aggregation?.limit ?? ENTITY_DEFAULT_METADATA_LIMIT + : 1, }, }, }, From 12f692536a25ae122286ff247891a106941540a9 Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 11 Sep 2024 11:16:36 +0200 Subject: [PATCH 02/10] rename to last_value --- .../kbn-entities-schema/src/schema/common.ts | 13 +++++++------ .../server/lib/entities/built_in/services.ts | 2 +- .../ingest_pipeline/generate_history_processors.ts | 10 ++++------ .../transform/generate_metadata_aggregations.ts | 5 +++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index 204e74e9c73d9..bae3ce44bd8b6 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -87,9 +87,8 @@ export type KeyMetric = z.infer; export const metadataAggregation = z.union([ z.object({ type: z.literal('terms'), limit: z.number().default(1000) }), z.object({ - type: z.literal('top_metrics'), + type: z.literal('last_value'), sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), - limit: z.number(), }), ]); @@ -97,13 +96,15 @@ export const metadataSchema = z .object({ source: z.string(), destination: z.optional(z.string()), - aggregation: z.optional(metadataAggregation).default({ type: 'terms', limit: 1000 }), + aggregation: z + .optional(metadataAggregation) + .default({ type: z.literal('terms').value, limit: 1000 }), }) .or( z.string().transform((value) => ({ source: value, destination: value, - aggregation: { type: z.literal('terms'), limit: 1000 }, + aggregation: { type: z.literal('terms').value, limit: 1000 }, })) ) .transform((metadata) => ({ @@ -111,11 +112,11 @@ export const metadataSchema = z destination: metadata.destination ?? metadata.source, })) .superRefine((value, ctx) => { - if (value.aggregation.limit < 1) { + if (value.aggregation.type === 'terms' && value.aggregation.limit < 1) { ctx.addIssue({ path: ['limit'], code: z.ZodIssueCode.custom, - message: 'limit should be greater than 1', + message: 'limit for terms aggregation should be greater than 1', }); } if (value.source.length === 0) { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts index e52abf4142fb4..daf5d1ccc1b3f 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts @@ -46,7 +46,7 @@ export const builtInServicesFromLogsEntityDefinition: EntityDefinition = displayNameTemplate: '{{service.name}}{{#service.environment}}:{{.}}{{/service.environment}}', metadata: [ { source: '_index', destination: 'sourceIndex' }, - { source: 'agent.name', limit: 100 }, + { source: 'agent.name', aggregation: { type: 'terms', limit: 100 } }, 'data_stream.type', 'service.environment', 'service.name', diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index d33f87f14fb61..964a64562560c 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -15,11 +15,9 @@ import { generateHistoryIndexName } from '../helpers/generate_component_id'; function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { if (aggregation.type === 'terms') { return `ctx.entity.metadata.${destination}.keySet()`; - } else if (aggregation.type === 'top_metrics') { + } else if (aggregation.type === 'last_value') { return `ctx.entity.metadata.${destination}["${source}"]`; } - - throw new Error(`unsupported metadata aggregation type [${aggregation.type}]`); } function mapDestinationToPainless(metadata: MetadataField) { @@ -46,16 +44,16 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } `; return `${acc}\n${next}`; - } else if (metadata.aggregation.type === 'top_metrics') { + } else if (metadata.aggregation.type === 'last_value') { const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}["${metadata.source}"] != null) { + if (ctx.entity?.metadata?.${optionalFieldPath}["${metadata.source}"] != "null") { ${mapDestinationToPainless(metadata)} } `; return `${acc}\n${next}`; } - throw new Error(`unsupported metadata aggregation [${metadata.aggregation.type}]`); + return acc; }, ''); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 1e2e5e89b6d72..09e403e4932d1 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -21,11 +21,12 @@ export function generateHistoryMetadataAggregations(definition: EntityDefinition size: metadata.aggregation.limit, }, }; - } else if (metadata.aggregation.type === 'top_metrics') { + } else if (metadata.aggregation.type === 'last_value') { agg = { top_metrics: { metrics: [{ field: metadata.source }], sort: metadata.aggregation.sort, + size: 1, }, }; } @@ -59,7 +60,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) data: { terms: { field: metadata.destination, - size: metadata.aggregation.limit, + size: metadata.aggregation.type === 'terms' ? metadata.aggregation.limit : 1, }, }, }, From ef6d3424139113bd5f3929b8a8714c3296215099 Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 11 Sep 2024 11:25:27 +0200 Subject: [PATCH 03/10] rename top_value --- x-pack/packages/kbn-entities-schema/src/schema/common.ts | 2 +- .../entities/ingest_pipeline/generate_history_processors.ts | 4 ++-- .../lib/entities/transform/generate_metadata_aggregations.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index bae3ce44bd8b6..d252dff99eda5 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -87,7 +87,7 @@ export type KeyMetric = z.infer; export const metadataAggregation = z.union([ z.object({ type: z.literal('terms'), limit: z.number().default(1000) }), z.object({ - type: z.literal('last_value'), + type: z.literal('top_value'), sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), }), ]); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index 964a64562560c..d413b060b6ba1 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -15,7 +15,7 @@ import { generateHistoryIndexName } from '../helpers/generate_component_id'; function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { if (aggregation.type === 'terms') { return `ctx.entity.metadata.${destination}.keySet()`; - } else if (aggregation.type === 'last_value') { + } else if (aggregation.type === 'top_value') { return `ctx.entity.metadata.${destination}["${source}"]`; } } @@ -44,7 +44,7 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } `; return `${acc}\n${next}`; - } else if (metadata.aggregation.type === 'last_value') { + } else if (metadata.aggregation.type === 'top_value') { const next = ` if (ctx.entity?.metadata?.${optionalFieldPath}["${metadata.source}"] != "null") { ${mapDestinationToPainless(metadata)} diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 09e403e4932d1..ac52009500db9 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -21,7 +21,7 @@ export function generateHistoryMetadataAggregations(definition: EntityDefinition size: metadata.aggregation.limit, }, }; - } else if (metadata.aggregation.type === 'last_value') { + } else if (metadata.aggregation.type === 'top_value') { agg = { top_metrics: { metrics: [{ field: metadata.source }], From 362d036159666da2981c0478ce10427b27c80de7 Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 11 Sep 2024 11:40:22 +0200 Subject: [PATCH 04/10] fix tests --- .../generate_metadata_aggregations.test.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts index 69f6d4f071696..19f5675c5285d 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts @@ -47,7 +47,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { it('should generate metadata aggregations for object format with source and limit', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10 }], + metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], }); expect(generateHistoryMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { @@ -62,13 +62,19 @@ describe('Generate Metadata Aggregations for history and latest', () => { it('should generate metadata aggregations for object format with source, limit, and destination', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }], + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 20 }, + destination: 'hostName', + }, + ], }); expect(generateHistoryMetadataAggregations(definition)).toEqual({ 'entity.metadata.hostName': { terms: { field: 'host.name', - size: 10, + size: 20, }, }, }); @@ -131,7 +137,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { it('should generate metadata aggregations for object format with source and limit', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10 }], + metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], }); expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { @@ -157,7 +163,13 @@ describe('Generate Metadata Aggregations for history and latest', () => { it('should generate metadata aggregations for object format with source, limit, and destination', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }], + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 10 }, + destination: 'hostName', + }, + ], }); expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.hostName': { From 56dc01f94ac1103361f7a3fb94f87aba5701100b Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 11 Sep 2024 12:46:12 +0200 Subject: [PATCH 05/10] implement top_value for latest transform --- .../generate_history_processors.ts | 6 +- .../generate_latest_processors.ts | 43 +++++-- .../generate_metadata_aggregations.test.ts | 121 +++++++++++++++++- .../generate_metadata_aggregations.ts | 73 +++++++++-- 4 files changed, 212 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index d413b060b6ba1..031e9107d0025 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -16,7 +16,7 @@ function getMetadataSourceField({ aggregation, destination, source }: MetadataFi if (aggregation.type === 'terms') { return `ctx.entity.metadata.${destination}.keySet()`; } else if (aggregation.type === 'top_value') { - return `ctx.entity.metadata.${destination}["${source}"]`; + return `ctx.entity.metadata.${destination}.top_value["${source}"]`; } } @@ -34,7 +34,7 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } return definition.metadata.reduce((acc, metadata) => { - const destination = metadata.destination; + const { destination, source } = metadata; const optionalFieldPath = destination.replaceAll('.', '?.'); if (metadata.aggregation.type === 'terms') { @@ -46,7 +46,7 @@ function createMetadataPainlessScript(definition: EntityDefinition) { return `${acc}\n${next}`; } else if (metadata.aggregation.type === 'top_value') { const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}["${metadata.source}"] != "null") { + if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) { ${mapDestinationToPainless(metadata)} } `; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index f1a45d297554e..75ce312f786d3 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -5,17 +5,26 @@ * 2.0. */ -import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1 } from '@kbn/entities-schema'; +import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema'; import { initializePathScript, cleanScript, } from '../helpers/ingest_pipeline_script_processor_helpers'; import { generateLatestIndexName } from '../helpers/generate_component_id'; -function mapDestinationToPainless(field: string) { +function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { + if (aggregation.type === 'terms') { + return `ctx.entity.metadata.${destination}.data.keySet()`; + } else if (aggregation.type === 'top_value') { + return `ctx.entity.metadata.${destination}.top_value["${destination}"]`; + } +} + +function mapDestinationToPainless(metadata: MetadataField) { + const field = metadata.destination; return ` ${initializePathScript(field)} - ctx.${field} = ctx.entity.metadata.${field}.data.keySet(); + ctx.${field} = ${getMetadataSourceField(metadata)}; `; } @@ -24,15 +33,27 @@ function createMetadataPainlessScript(definition: EntityDefinition) { return ''; } - return definition.metadata.reduce((acc, def) => { - const destination = def.destination || def.source; + return definition.metadata.reduce((acc, metadata) => { + const destination = metadata.destination; const optionalFieldPath = destination.replaceAll('.', '?.'); - const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) { - ${mapDestinationToPainless(destination)} - } - `; - return `${acc}\n${next}`; + + if (metadata.aggregation.type === 'terms') { + const next = ` + if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) { + ${mapDestinationToPainless(metadata)} + } + `; + return `${acc}\n${next}`; + } else if (metadata.aggregation.type === 'top_value') { + const next = ` + if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${destination}"] != null) { + ${mapDestinationToPainless(metadata)} + } + `; + return `${acc}\n${next}`; + } + + return acc; }, ''); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts index 19f5675c5285d..0508aaa4465fb 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts @@ -44,7 +44,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source and limit', () => { + it('should generate metadata aggregations for object format with source and aggregation', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], @@ -59,7 +59,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source, limit, and destination', () => { + it('should generate metadata aggregations for object format with source, aggregation, and destination', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, metadata: [ @@ -79,6 +79,49 @@ describe('Generate Metadata Aggregations for history and latest', () => { }, }); }); + + it('should generate metadata aggregations for terms and top_value', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 10 }, + destination: 'hostName', + }, + { + source: 'agent.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + destination: 'agentName', + }, + ], + }); + + expect(generateHistoryMetadataAggregations(definition)).toEqual({ + 'entity.metadata.hostName': { + terms: { + field: 'host.name', + size: 10, + }, + }, + 'entity.metadata.agentName': { + filter: { + exists: { + field: 'agent.name', + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { field: 'agent.name' }, + sort: { '@timestamp': 'desc' }, + size: 1, + }, + }, + }, + }, + }); + }); }); describe('generateLatestMetadataAggregations()', () => { @@ -134,7 +177,7 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source and limit', () => { + it('should generate metadata aggregations for object format with source and aggregation', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], @@ -160,7 +203,39 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source, limit, and destination', () => { + it('should generate metadata aggregations for object format with source, aggregation, and destination', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 10 }, + destination: 'hostName', + }, + ], + }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ + 'entity.metadata.hostName': { + filter: { + range: { + '@timestamp': { + gte: 'now-360s', + }, + }, + }, + aggs: { + data: { + terms: { + field: 'hostName', + size: 10, + }, + }, + }, + }, + }); + }); + + it('should generate metadata aggregations for terms and top_value', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, metadata: [ @@ -169,6 +244,11 @@ describe('Generate Metadata Aggregations for history and latest', () => { aggregation: { type: 'terms', limit: 10 }, destination: 'hostName', }, + { + source: 'agent.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + destination: 'agentName', + }, ], }); expect(generateLatestMetadataAggregations(definition)).toEqual({ @@ -189,6 +269,39 @@ describe('Generate Metadata Aggregations for history and latest', () => { }, }, }, + 'entity.metadata.agentName': { + filter: { + bool: { + must: [ + { + range: { + '@timestamp': { + gte: 'now-360s', + }, + }, + }, + { + exists: { + field: 'agentName', + }, + }, + ], + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { + field: 'agentName', + }, + sort: { + '@timestamp': 'desc', + }, + size: 1, + }, + }, + }, + }, }); }); }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index ac52009500db9..746c456a8817a 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -23,10 +23,21 @@ export function generateHistoryMetadataAggregations(definition: EntityDefinition }; } else if (metadata.aggregation.type === 'top_value') { agg = { - top_metrics: { - metrics: [{ field: metadata.source }], - sort: metadata.aggregation.sort, - size: 1, + filter: { + exists: { + field: metadata.source, + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { + field: metadata.source, + }, + sort: metadata.aggregation.sort, + size: 1, + }, + }, }, }; } @@ -45,10 +56,10 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) const offsetInSeconds = calculateOffset(definition); - return definition.metadata.reduce( - (aggs, metadata) => ({ - ...aggs, - [`entity.metadata.${metadata.destination}`]: { + return definition.metadata.reduce((aggs, metadata) => { + let agg; + if (metadata.aggregation.type === 'terms') { + agg = { filter: { range: { '@timestamp': { @@ -60,12 +71,48 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) data: { terms: { field: metadata.destination, - size: metadata.aggregation.type === 'terms' ? metadata.aggregation.limit : 1, + size: metadata.aggregation.limit, }, }, }, - }, - }), - {} - ); + }; + } else if (metadata.aggregation.type === 'top_value') { + agg = { + filter: { + bool: { + must: [ + { + range: { + '@timestamp': { + gte: `now-${offsetInSeconds}s`, + }, + }, + }, + { + exists: { + field: metadata.destination, + }, + }, + ], + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { + field: metadata.destination, + }, + sort: metadata.aggregation.sort, + size: 1, + }, + }, + }, + }; + } + + return { + ...aggs, + [`entity.metadata.${metadata.destination}`]: agg, + }; + }, {}); } From 14291a817f8655d241702ec6d2f60749866c617c Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 11 Sep 2024 15:29:58 +0200 Subject: [PATCH 06/10] fix tests --- .../schema/__snapshots__/common.test.ts.snap | 22 ++++++++++++++----- .../src/schema/common.test.ts | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap index 4067947f7ddcf..9210d3b9991cf 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap +++ b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap @@ -67,7 +67,7 @@ Object { "limit" ], "code": "custom", - "message": "limit should be greater than 1" + "message": "limit for terms aggregation should be greater than 1" } ]], "success": false, @@ -77,8 +77,11 @@ Object { exports[`schemas metadataSchema should parse successfully with a source and desitination 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "hostName", - "limit": 1000, "source": "host.name", }, "success": true, @@ -88,8 +91,11 @@ Object { exports[`schemas metadataSchema should parse successfully with an valid string 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "host.name", - "limit": 1000, "source": "host.name", }, "success": true, @@ -99,8 +105,11 @@ Object { exports[`schemas metadataSchema should parse successfully with just a source 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "host.name", - "limit": 1000, "source": "host.name", }, "success": true, @@ -110,8 +119,11 @@ Object { exports[`schemas metadataSchema should parse successfully with valid object 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "hostName", - "limit": 1000, "source": "host.name", }, "success": true, diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts index f59363866c37a..88b1fdef64ac7 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts @@ -28,7 +28,7 @@ describe('schemas', () => { const result = metadataSchema.safeParse({ source: 'host.name', destination: 'host.name', - limit: 0, + aggregation: { type: 'terms', limit: 0 }, }); expect(result.success).toBeFalsy(); expect(result).toMatchSnapshot(); From a102abe6be4f375f0c290de6e09f4a1a37527711 Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 11 Sep 2024 15:37:08 +0200 Subject: [PATCH 07/10] fix tests --- .../src/schema/common.test.ts | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts index 88b1fdef64ac7..1a737ac3f4d9b 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts @@ -52,11 +52,41 @@ describe('schemas', () => { const result = metadataSchema.safeParse({ source: 'host.name', destination: 'hostName', - size: 1, }); expect(result.success).toBeTruthy(); expect(result).toMatchSnapshot(); }); + + it('should default to terms aggregation when none provided', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'hostName', + }); + expect(result.success).toBeTruthy(); + expect(result.data).toEqual({ + source: 'host.name', + destination: 'hostName', + aggregation: { type: 'terms', limit: 1000 }, + }); + }); + + it('should parse supported aggregations', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'hostName', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }); + expect(result.success).toBeTruthy(); + }); + + it('should reject unsupported aggregation', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'hostName', + aggregation: { type: 'unknown_agg', limit: 10 }, + }); + expect(result.success).toBeFalsy(); + }); }); describe('durationSchema', () => { From 517f2216391c0f51d9195436014528ff205bdd17 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 13 Sep 2024 06:50:39 +0200 Subject: [PATCH 08/10] add lookbackPeriod to top_value agg --- x-pack/packages/kbn-entities-schema/src/schema/common.ts | 1 + .../lib/entities/transform/generate_metadata_aggregations.ts | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index d252dff99eda5..8f4efca2badff 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -89,6 +89,7 @@ export const metadataAggregation = z.union([ z.object({ type: z.literal('top_value'), sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), + lookbackPeriod: durationSchema, }), ]); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 746c456a8817a..853ea4dcb5722 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -35,7 +35,6 @@ export function generateHistoryMetadataAggregations(definition: EntityDefinition field: metadata.source, }, sort: metadata.aggregation.sort, - size: 1, }, }, }, @@ -84,7 +83,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) { range: { '@timestamp': { - gte: `now-${offsetInSeconds}s`, + gte: `now-${metadata.aggregation.lookbackPeriod}`, }, }, }, @@ -103,7 +102,6 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) field: metadata.destination, }, sort: metadata.aggregation.sort, - size: 1, }, }, }, From a330f53ae0be75af82631e711fcc0ea82c4bcaee Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 13 Sep 2024 09:26:49 +0200 Subject: [PATCH 09/10] optional lookbackPeriod --- x-pack/packages/kbn-entities-schema/src/schema/common.ts | 2 +- .../entities/transform/generate_metadata_aggregations.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index 8f4efca2badff..aa54dbd16c9aa 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -89,7 +89,7 @@ export const metadataAggregation = z.union([ z.object({ type: z.literal('top_value'), sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), - lookbackPeriod: durationSchema, + lookbackPeriod: z.optional(durationSchema), }), ]); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 853ea4dcb5722..0fc4464672219 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -53,7 +53,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) return {}; } - const offsetInSeconds = calculateOffset(definition); + const offsetInSeconds = `${calculateOffset(definition)}s`; return definition.metadata.reduce((aggs, metadata) => { let agg; @@ -62,7 +62,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) filter: { range: { '@timestamp': { - gte: `now-${offsetInSeconds}s`, + gte: `now-${offsetInSeconds}`, }, }, }, @@ -83,7 +83,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) { range: { '@timestamp': { - gte: `now-${metadata.aggregation.lookbackPeriod}`, + gte: `now-${metadata.aggregation.lookbackPeriod ?? offsetInSeconds}`, }, }, }, From b4b2d73b7c450cc77e5fe8603ff72eee2a5b2299 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 13 Sep 2024 10:57:11 +0200 Subject: [PATCH 10/10] fix test --- .../entities/transform/generate_metadata_aggregations.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts index 0508aaa4465fb..7746be66f5033 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts @@ -115,7 +115,6 @@ describe('Generate Metadata Aggregations for history and latest', () => { top_metrics: { metrics: { field: 'agent.name' }, sort: { '@timestamp': 'desc' }, - size: 1, }, }, }, @@ -297,7 +296,6 @@ describe('Generate Metadata Aggregations for history and latest', () => { sort: { '@timestamp': 'desc', }, - size: 1, }, }, },