diff --git a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap index 4067947f7ddcf..9210d3b9991cf 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap +++ b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap @@ -67,7 +67,7 @@ Object { "limit" ], "code": "custom", - "message": "limit should be greater than 1" + "message": "limit for terms aggregation should be greater than 1" } ]], "success": false, @@ -77,8 +77,11 @@ Object { exports[`schemas metadataSchema should parse successfully with a source and desitination 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "hostName", - "limit": 1000, "source": "host.name", }, "success": true, @@ -88,8 +91,11 @@ Object { exports[`schemas metadataSchema should parse successfully with an valid string 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "host.name", - "limit": 1000, "source": "host.name", }, "success": true, @@ -99,8 +105,11 @@ Object { exports[`schemas metadataSchema should parse successfully with just a source 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "host.name", - "limit": 1000, "source": "host.name", }, "success": true, @@ -110,8 +119,11 @@ Object { exports[`schemas metadataSchema should parse successfully with valid object 1`] = ` Object { "data": Object { + "aggregation": Object { + "limit": 1000, + "type": "terms", + }, "destination": "hostName", - "limit": 1000, "source": "host.name", }, "success": true, diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts index f59363866c37a..1a737ac3f4d9b 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts @@ -28,7 +28,7 @@ describe('schemas', () => { const result = metadataSchema.safeParse({ source: 'host.name', destination: 'host.name', - limit: 0, + aggregation: { type: 'terms', limit: 0 }, }); expect(result.success).toBeFalsy(); expect(result).toMatchSnapshot(); @@ -52,11 +52,41 @@ describe('schemas', () => { const result = metadataSchema.safeParse({ source: 'host.name', destination: 'hostName', - size: 1, }); expect(result.success).toBeTruthy(); expect(result).toMatchSnapshot(); }); + + it('should default to terms aggregation when none provided', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'hostName', + }); + expect(result.success).toBeTruthy(); + expect(result.data).toEqual({ + source: 'host.name', + destination: 'hostName', + aggregation: { type: 'terms', limit: 1000 }, + }); + }); + + it('should parse supported aggregations', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'hostName', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + }); + expect(result.success).toBeTruthy(); + }); + + it('should reject unsupported aggregation', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'hostName', + aggregation: { type: 'unknown_agg', limit: 10 }, + }); + expect(result.success).toBeFalsy(); + }); }); describe('durationSchema', () => { diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index 6e46a335dadde..aa54dbd16c9aa 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -84,24 +84,40 @@ export const keyMetricSchema = z.object({ export type KeyMetric = z.infer; +export const metadataAggregation = z.union([ + z.object({ type: z.literal('terms'), limit: z.number().default(1000) }), + z.object({ + type: z.literal('top_value'), + sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])), + lookbackPeriod: z.optional(durationSchema), + }), +]); + export const metadataSchema = z .object({ source: z.string(), destination: z.optional(z.string()), - limit: z.optional(z.number().default(1000)), + aggregation: z + .optional(metadataAggregation) + .default({ type: z.literal('terms').value, limit: 1000 }), }) + .or( + z.string().transform((value) => ({ + source: value, + destination: value, + aggregation: { type: z.literal('terms').value, limit: 1000 }, + })) + ) .transform((metadata) => ({ ...metadata, destination: metadata.destination ?? metadata.source, - limit: metadata.limit ?? 1000, })) - .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))) .superRefine((value, ctx) => { - if (value.limit < 1) { + if (value.aggregation.type === 'terms' && value.aggregation.limit < 1) { ctx.addIssue({ path: ['limit'], code: z.ZodIssueCode.custom, - message: 'limit should be greater than 1', + message: 'limit for terms aggregation should be greater than 1', }); } if (value.source.length === 0) { @@ -120,6 +136,8 @@ export const metadataSchema = z } }); +export type MetadataField = z.infer; + export const identityFieldsSchema = z .object({ field: z.string(), diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts index 1c04fe8a924ac..aa1d86ee25adf 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts @@ -46,7 +46,7 @@ export const builtInServicesFromLogsEntityDefinition: EntityDefinition = displayNameTemplate: '{{service.name}}{{#service.environment}}:{{.}}{{/service.environment}}', metadata: [ { source: '_index', destination: 'sourceIndex' }, - { source: 'agent.name', limit: 100 }, + { source: 'agent.name', aggregation: { type: 'terms', limit: 100 } }, 'data_stream.type', 'service.environment', 'service.name', diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index 91a03479a3547..d51ab0be75db1 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1 } from '@kbn/entities-schema'; +import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema'; import { initializePathScript, cleanScript, @@ -13,10 +13,19 @@ import { import { generateHistoryIndexName } from '../helpers/generate_component_id'; import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; -function mapDestinationToPainless(field: string) { +function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { + if (aggregation.type === 'terms') { + return `ctx.entity.metadata.${destination}.keySet()`; + } else if (aggregation.type === 'top_value') { + return `ctx.entity.metadata.${destination}.top_value["${source}"]`; + } +} + +function mapDestinationToPainless(metadata: MetadataField) { + const field = metadata.destination; return ` ${initializePathScript(field)} - ctx.${field} = ctx.entity.metadata.${field}.keySet(); + ctx.${field} = ${getMetadataSourceField(metadata)}; `; } @@ -25,15 +34,27 @@ function createMetadataPainlessScript(definition: EntityDefinition) { return ''; } - return definition.metadata.reduce((acc, def) => { - const destination = def.destination; + return definition.metadata.reduce((acc, metadata) => { + const { destination, source } = metadata; const optionalFieldPath = destination.replaceAll('.', '?.'); - const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath} != null) { - ${mapDestinationToPainless(destination)} - } - `; - return `${acc}\n${next}`; + + if (metadata.aggregation.type === 'terms') { + const next = ` + if (ctx.entity?.metadata?.${optionalFieldPath} != null) { + ${mapDestinationToPainless(metadata)} + } + `; + return `${acc}\n${next}`; + } else if (metadata.aggregation.type === 'top_value') { + const next = ` + if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) { + ${mapDestinationToPainless(metadata)} + } + `; + return `${acc}\n${next}`; + } + + return acc; }, ''); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts index 8efefe8ae33d9..16823221fffb3 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1 } from '@kbn/entities-schema'; +import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema'; import { initializePathScript, cleanScript, @@ -13,10 +13,19 @@ import { import { generateLatestIndexName } from '../helpers/generate_component_id'; import { isBuiltinDefinition } from '../helpers/is_builtin_definition'; -function mapDestinationToPainless(field: string) { +function getMetadataSourceField({ aggregation, destination, source }: MetadataField) { + if (aggregation.type === 'terms') { + return `ctx.entity.metadata.${destination}.data.keySet()`; + } else if (aggregation.type === 'top_value') { + return `ctx.entity.metadata.${destination}.top_value["${destination}"]`; + } +} + +function mapDestinationToPainless(metadata: MetadataField) { + const field = metadata.destination; return ` ${initializePathScript(field)} - ctx.${field} = ctx.entity.metadata.${field}.data.keySet(); + ctx.${field} = ${getMetadataSourceField(metadata)}; `; } @@ -25,15 +34,27 @@ function createMetadataPainlessScript(definition: EntityDefinition) { return ''; } - return definition.metadata.reduce((acc, def) => { - const destination = def.destination || def.source; + return definition.metadata.reduce((acc, metadata) => { + const destination = metadata.destination; const optionalFieldPath = destination.replaceAll('.', '?.'); - const next = ` - if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) { - ${mapDestinationToPainless(destination)} - } - `; - return `${acc}\n${next}`; + + if (metadata.aggregation.type === 'terms') { + const next = ` + if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) { + ${mapDestinationToPainless(metadata)} + } + `; + return `${acc}\n${next}`; + } else if (metadata.aggregation.type === 'top_value') { + const next = ` + if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${destination}"] != null) { + ${mapDestinationToPainless(metadata)} + } + `; + return `${acc}\n${next}`; + } + + return acc; }, ''); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts index 69f6d4f071696..7746be66f5033 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts @@ -44,10 +44,10 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source and limit', () => { + it('should generate metadata aggregations for object format with source and aggregation', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10 }], + metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], }); expect(generateHistoryMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { @@ -59,11 +59,44 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source, limit, and destination', () => { + it('should generate metadata aggregations for object format with source, aggregation, and destination', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }], + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 20 }, + destination: 'hostName', + }, + ], + }); + expect(generateHistoryMetadataAggregations(definition)).toEqual({ + 'entity.metadata.hostName': { + terms: { + field: 'host.name', + size: 20, + }, + }, + }); + }); + + it('should generate metadata aggregations for terms and top_value', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 10 }, + destination: 'hostName', + }, + { + source: 'agent.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + destination: 'agentName', + }, + ], }); + expect(generateHistoryMetadataAggregations(definition)).toEqual({ 'entity.metadata.hostName': { terms: { @@ -71,6 +104,21 @@ describe('Generate Metadata Aggregations for history and latest', () => { size: 10, }, }, + 'entity.metadata.agentName': { + filter: { + exists: { + field: 'agent.name', + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { field: 'agent.name' }, + sort: { '@timestamp': 'desc' }, + }, + }, + }, + }, }); }); }); @@ -128,10 +176,10 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source and limit', () => { + it('should generate metadata aggregations for object format with source and aggregation', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10 }], + metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }], }); expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.host.name': { @@ -154,10 +202,16 @@ describe('Generate Metadata Aggregations for history and latest', () => { }); }); - it('should generate metadata aggregations for object format with source, limit, and destination', () => { + it('should generate metadata aggregations for object format with source, aggregation, and destination', () => { const definition = entityDefinitionSchema.parse({ ...rawEntityDefinition, - metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }], + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 10 }, + destination: 'hostName', + }, + ], }); expect(generateLatestMetadataAggregations(definition)).toEqual({ 'entity.metadata.hostName': { @@ -179,5 +233,74 @@ describe('Generate Metadata Aggregations for history and latest', () => { }, }); }); + + it('should generate metadata aggregations for terms and top_value', () => { + const definition = entityDefinitionSchema.parse({ + ...rawEntityDefinition, + metadata: [ + { + source: 'host.name', + aggregation: { type: 'terms', limit: 10 }, + destination: 'hostName', + }, + { + source: 'agent.name', + aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } }, + destination: 'agentName', + }, + ], + }); + expect(generateLatestMetadataAggregations(definition)).toEqual({ + 'entity.metadata.hostName': { + filter: { + range: { + '@timestamp': { + gte: 'now-360s', + }, + }, + }, + aggs: { + data: { + terms: { + field: 'hostName', + size: 10, + }, + }, + }, + }, + 'entity.metadata.agentName': { + filter: { + bool: { + must: [ + { + range: { + '@timestamp': { + gte: 'now-360s', + }, + }, + }, + { + exists: { + field: 'agentName', + }, + }, + ], + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { + field: 'agentName', + }, + sort: { + '@timestamp': 'desc', + }, + }, + }, + }, + }, + }); + }); }); }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 1da8988209db6..0fc4464672219 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -6,25 +6,46 @@ */ import { EntityDefinition } from '@kbn/entities-schema'; -import { ENTITY_DEFAULT_METADATA_LIMIT } from '../../../../common/constants_entities'; import { calculateOffset } from '../helpers/calculate_offset'; export function generateHistoryMetadataAggregations(definition: EntityDefinition) { if (!definition.metadata) { return {}; } - return definition.metadata.reduce( - (aggs, metadata) => ({ - ...aggs, - [`entity.metadata.${metadata.destination ?? metadata.source}`]: { + return definition.metadata.reduce((aggs, metadata) => { + let agg; + if (metadata.aggregation.type === 'terms') { + agg = { terms: { field: metadata.source, - size: metadata.limit ?? ENTITY_DEFAULT_METADATA_LIMIT, + size: metadata.aggregation.limit, + }, + }; + } else if (metadata.aggregation.type === 'top_value') { + agg = { + filter: { + exists: { + field: metadata.source, + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { + field: metadata.source, + }, + sort: metadata.aggregation.sort, + }, + }, }, - }, - }), - {} - ); + }; + } + + return { + ...aggs, + [`entity.metadata.${metadata.destination}`]: agg, + }; + }, {}); } export function generateLatestMetadataAggregations(definition: EntityDefinition) { @@ -32,29 +53,64 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) return {}; } - const offsetInSeconds = calculateOffset(definition); + const offsetInSeconds = `${calculateOffset(definition)}s`; - return definition.metadata.reduce( - (aggs, metadata) => ({ - ...aggs, - [`entity.metadata.${metadata.destination}`]: { + return definition.metadata.reduce((aggs, metadata) => { + let agg; + if (metadata.aggregation.type === 'terms') { + agg = { filter: { range: { '@timestamp': { - gte: `now-${offsetInSeconds}s`, + gte: `now-${offsetInSeconds}`, }, }, }, aggs: { data: { terms: { - field: metadata.destination ?? metadata.source, - size: metadata.limit ?? ENTITY_DEFAULT_METADATA_LIMIT, + field: metadata.destination, + size: metadata.aggregation.limit, + }, + }, + }, + }; + } else if (metadata.aggregation.type === 'top_value') { + agg = { + filter: { + bool: { + must: [ + { + range: { + '@timestamp': { + gte: `now-${metadata.aggregation.lookbackPeriod ?? offsetInSeconds}`, + }, + }, + }, + { + exists: { + field: metadata.destination, + }, + }, + ], + }, + }, + aggs: { + top_value: { + top_metrics: { + metrics: { + field: metadata.destination, + }, + sort: metadata.aggregation.sort, }, }, }, - }, - }), - {} - ); + }; + } + + return { + ...aggs, + [`entity.metadata.${metadata.destination}`]: agg, + }; + }, {}); }