diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts index b9119143ec37e..6ce76e127c8e8 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts @@ -12,7 +12,7 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}containers_from_ecs_data`, managed: true, - version: '1.0.0', + version: '0.1.0', name: 'Containers from ECS data', description: 'This definition extracts container entities from common data streams by looking for the ECS field container.id', @@ -65,94 +65,4 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition = 'agent.type', 'agent.ephemeral_id', ], - metrics: [ - { - name: 'log_rate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: 'log.level: * OR error.log.level: *', - }, - ], - }, - { - name: 'error_log_rate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")', - }, - ], - }, - { - name: 'cpu_usage_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'docker.cpu.total.pct', - }, - ], - }, - { - name: 'memory_usage_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'docker.memory.usage.pct', - }, - ], - }, - { - name: 'network_in_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'docker.network.in.bytes', - }, - ], - }, - { - name: 'network_out_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'docker.network.out.bytes', - }, - ], - }, - { - name: 'disk_read_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'docker.diskio.read.ops', - }, - ], - }, - { - name: 'disk_write_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'docker.diskio.write.ops', - }, - ], - }, - ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts index 5fead32f5c0e8..56f83d5fbaed6 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts @@ -11,7 +11,7 @@ import { BUILT_IN_ID_PREFIX } from './constants'; export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ id: `${BUILT_IN_ID_PREFIX}hosts_from_ecs_data`, managed: true, - version: '1.0.0', + version: '0.1.0', name: 'Hosts from ECS data', description: 'This definition extracts host entities from common data streams by looking for the ECS field host.name', @@ -65,115 +65,4 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin 'agent.type', 'agent.version', ], - metrics: [ - { - name: 'log_rate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: 'log.level: * OR error.log.level: *', - }, - ], - }, - { - name: 'error_log_rate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")', - }, - ], - }, - { - name: 'cpu_usage_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'system.cpu.total.norm.pct', - }, - ], - }, - { - name: 'normalized_load_avg', - equation: 'A / B', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'system.load.1', - }, - { - name: 'B', - aggregation: 'max', - field: 'system.load.cores', - }, - ], - }, - { - name: 'memory_usage_avg', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - field: 'system.memory.actual.used.pct', - }, - ], - }, - { - name: 'memory_free_avg', - equation: 'A - B', - metrics: [ - { - name: 'A', - aggregation: 'max', - field: 'system.memory.total', - }, - { - name: 'B', - aggregation: 'avg', - field: 'system.memory.actual.used.bytes', - }, - ], - }, - { - name: 'disk_usage_max', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'max', - field: 'system.filesystem.used.pct', - }, - ], - }, - { - name: 'rx_avg', - equation: 'A * 8', - metrics: [ - { - name: 'A', - aggregation: 'sum', - field: 'host.network.ingress.bytes', - }, - ], - }, - { - name: 'tx_avg', - equation: 'A * 8', - metrics: [ - { - name: 'A', - aggregation: 'sum', - field: 'host.network.egress.bytes', - }, - ], - }, - ], }); diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts index c1496f424d393..6caa209da02ca 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts @@ -8,31 +8,16 @@ import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema'; import { BUILT_IN_ID_PREFIX } from './constants'; -const serviceTransactionFilter = (additionalFilters: string[] = []) => { - const baseFilters = [ - 'processor.event: "metric"', - 'metricset.name: "service_transaction"', - 'metricset.interval: "1m"', - ]; - - return [...baseFilters, ...additionalFilters].join(' AND '); -}; - export const builtInServicesFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({ - version: '0.3.0', + version: '0.4.0', id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`, name: 'Services from ECS data', description: 'This definition extracts service entities from common data streams by looking for the ECS field service.name', type: 'service', managed: true, - indexPatterns: [ - 'logs-*', - 'filebeat*', - 'metrics-apm.service_transaction.1m*', - 'metrics-apm.service_summary.1m*', - ], + indexPatterns: ['logs-*', 'filebeat*', 'traces-apm*'], history: { timestampField: '@timestamp', interval: '1m', @@ -65,72 +50,9 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition = 'cloud.provider', 'cloud.availability_zone', 'cloud.machine.type', - ], - metrics: [ - { - name: 'latency', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'avg', - filter: serviceTransactionFilter(), - field: 'transaction.duration.histogram', - }, - ], - }, - { - name: 'throughput', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'value_count', - filter: serviceTransactionFilter(), - field: 'transaction.duration.summary', - }, - ], - }, - { - name: 'failedTransactionRate', - equation: '1 - (A / B)', - metrics: [ - { - name: 'A', - aggregation: 'sum', - filter: serviceTransactionFilter(), - field: 'event.success_count', - }, - { - name: 'B', - aggregation: 'value_count', - filter: serviceTransactionFilter(), - field: 'event.success_count', - }, - ], - }, - { - name: 'logErrorRate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: - 'log.level: "error" OR log.level: "ERROR" OR error.log.level: "error" OR error.log.level: "ERROR"', - }, - ], - }, - { - name: 'logRate', - equation: 'A', - metrics: [ - { - name: 'A', - aggregation: 'doc_count', - filter: 'data_stream.type: logs', - }, - ], - }, + 'kubernetes.namespace', + 'orchestrator.cluster.name', + 'k8s.namespace.name', + 'k8s.cluster.name', ], }); diff --git a/x-pack/plugins/observability_solution/apm/common/entities/types.ts b/x-pack/plugins/observability_solution/apm/common/entities/types.ts index 1e13d7c1d3634..9775b1e32eae6 100644 --- a/x-pack/plugins/observability_solution/apm/common/entities/types.ts +++ b/x-pack/plugins/observability_solution/apm/common/entities/types.ts @@ -10,16 +10,3 @@ export enum EntityDataStreamType { TRACES = 'traces', LOGS = 'logs', } - -interface TraceMetrics { - latency?: number | null; - throughput?: number | null; - failedTransactionRate?: number | null; -} - -interface LogsMetrics { - logRate?: number | null; - logErrorRate?: number | null; -} - -export type EntityMetrics = TraceMetrics & LogsMetrics; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts index 7395f639bb6e9..6cedb09efa7c2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts @@ -12,7 +12,6 @@ import { import type { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { getEntityLatestServices } from './get_entity_latest_services'; import type { EntityLatestServiceRaw } from './types'; -import { getEntityHistoryServicesMetrics } from './get_entity_history_services_metrics'; export function entitiesRangeQuery(start?: number, end?: number): QueryDslQueryContainer[] { if (!start || !end) { @@ -64,30 +63,5 @@ export async function getEntities({ serviceName, }); - const serviceEntitiesHistoryMetricsMap = entityLatestServices.length - ? await getEntityHistoryServicesMetrics({ - start, - end, - entitiesESClient, - entityIds: entityLatestServices.map((latestEntity) => latestEntity.entity.id), - size, - }) - : undefined; - - return entityLatestServices.map((latestEntity) => { - const historyEntityMetrics = serviceEntitiesHistoryMetricsMap?.[latestEntity.entity.id]; - return { - ...latestEntity, - entity: { - ...latestEntity.entity, - metrics: historyEntityMetrics || { - latency: undefined, - logErrorRate: undefined, - failedTransactionRate: undefined, - logRate: undefined, - throughput: undefined, - }, - }, - }; - }); + return entityLatestServices; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_metrics.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_metrics.ts deleted file mode 100644 index 009f384af901b..0000000000000 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_metrics.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { rangeQuery, termsQuery } from '@kbn/observability-plugin/server'; -import { - ENTITY_ID, - ENTITY_LAST_SEEN, -} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch'; -import { EntityMetrics } from '../../../common/entities/types'; -import { - ENTITY_METRICS_FAILED_TRANSACTION_RATE, - ENTITY_METRICS_LATENCY, - ENTITY_METRICS_LOG_ERROR_RATE, - ENTITY_METRICS_LOG_RATE, - ENTITY_METRICS_THROUGHPUT, -} from '../../../common/es_fields/entities'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; - -interface Params { - entitiesESClient: EntitiesESClient; - start: number; - end: number; - entityIds: string[]; - size: number; -} - -export async function getEntityHistoryServicesMetrics({ - end, - entityIds, - start, - entitiesESClient, - size, -}: Params) { - const response = await entitiesESClient.searchHistory('get_entities_history', { - body: { - size: 0, - track_total_hits: false, - query: { - bool: { - filter: [ - ...rangeQuery(start, end, ENTITY_LAST_SEEN), - ...termsQuery(ENTITY_ID, ...entityIds), - ], - }, - }, - aggs: { - entityIds: { - terms: { field: ENTITY_ID, size }, - aggs: { - latency: { avg: { field: ENTITY_METRICS_LATENCY } }, - logErrorRate: { avg: { field: ENTITY_METRICS_LOG_ERROR_RATE } }, - logRate: { avg: { field: ENTITY_METRICS_LOG_RATE } }, - throughput: { avg: { field: ENTITY_METRICS_THROUGHPUT } }, - failedTransactionRate: { avg: { field: ENTITY_METRICS_FAILED_TRANSACTION_RATE } }, - }, - }, - }, - }, - }); - - if (!response.aggregations) { - return {}; - } - - return response.aggregations.entityIds.buckets.reduce>( - (acc, currBucket) => { - return { - ...acc, - [currBucket.key]: { - latency: currBucket.latency.value, - logErrorRate: currBucket.logErrorRate.value, - logRate: currBucket.logRate.value, - throughput: currBucket.throughput.value, - failedTransactionRate: currBucket.failedTransactionRate.value, - }, - }; - }, - {} - ); -} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_timeseries.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_timeseries.ts deleted file mode 100644 index 4ee23966f282c..0000000000000 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_timeseries.ts +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getBucketSize } from '@kbn/apm-data-access-plugin/common'; -import { rangeQuery, termsQuery } from '@kbn/observability-plugin/server'; -import { ENTITY_LAST_SEEN } from '@kbn/observability-shared-plugin/common/field_names/elasticsearch'; -import { keyBy } from 'lodash'; -import { SERVICE_NAME } from '../../../common/es_fields/apm'; -import { - ENTITY_METRICS_FAILED_TRANSACTION_RATE, - ENTITY_METRICS_LATENCY, - ENTITY_METRICS_LOG_ERROR_RATE, - ENTITY_METRICS_LOG_RATE, - ENTITY_METRICS_THROUGHPUT, -} from '../../../common/es_fields/entities'; -import { environmentQuery } from '../../../common/utils/environment_query'; -import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; - -interface Params { - entitiesESClient: EntitiesESClient; - start: number; - end: number; - serviceNames: string[]; - environment: string; -} - -export async function getEntityHistoryServicesTimeseries({ - start, - end, - serviceNames, - entitiesESClient, - environment, -}: Params) { - const { intervalString } = getBucketSize({ - start, - end, - minBucketSize: 60, - }); - - const response = await entitiesESClient.searchHistory('get_entities_history_timeseries', { - body: { - size: 0, - track_total_hits: false, - query: { - bool: { - filter: [ - ...rangeQuery(start, end, ENTITY_LAST_SEEN), - ...termsQuery(SERVICE_NAME, ...serviceNames), - ...environmentQuery(environment), - ], - }, - }, - aggs: { - serviceNames: { - terms: { field: SERVICE_NAME, size: serviceNames.length }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { min: start, max: end }, - }, - aggs: { - latency: { avg: { field: ENTITY_METRICS_LATENCY } }, - logErrorRate: { avg: { field: ENTITY_METRICS_LOG_ERROR_RATE } }, - logRate: { avg: { field: ENTITY_METRICS_LOG_RATE } }, - throughput: { avg: { field: ENTITY_METRICS_THROUGHPUT } }, - failedTransactionRate: { avg: { field: ENTITY_METRICS_FAILED_TRANSACTION_RATE } }, - }, - }, - }, - }, - }, - }, - }); - - if (!response.aggregations) { - return {}; - } - - return keyBy( - response.aggregations.serviceNames.buckets.map((serviceBucket) => { - const serviceName = serviceBucket.key as string; - - return { - serviceName, - latency: serviceBucket.timeseries.buckets.map((bucket) => ({ - x: bucket.key, - y: bucket.latency.value ?? null, - })), - logErrorRate: serviceBucket.timeseries.buckets.map((bucket) => ({ - x: bucket.key, - y: bucket.logErrorRate.value ?? null, - })), - logRate: serviceBucket.timeseries.buckets.map((bucket) => ({ - x: bucket.key, - y: bucket.logRate.value ?? null, - })), - throughput: serviceBucket.timeseries.buckets.map((bucket) => ({ - x: bucket.key, - y: bucket.throughput.value ?? null, - })), - failedTransactionRate: serviceBucket.timeseries.buckets.map((bucket) => ({ - x: bucket.key, - y: bucket.failedTransactionRate.value ?? null, - })), - }; - }), - 'serviceName' - ); -} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts index 084fbbe438952..9e6bb34bceafe 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts @@ -10,7 +10,6 @@ import { WrappedElasticsearchClientError } from '@kbn/observability-plugin/serve import { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { withApmSpan } from '../../../utils/with_apm_span'; import { getEntities } from '../get_entities'; -import { calculateAvgMetrics } from '../utils/calculate_avg_metrics'; import { mergeEntities } from '../utils/merge_entities'; export const MAX_NUMBER_OF_SERVICES = 1_000; @@ -41,7 +40,7 @@ export async function getServiceEntities({ size: MAX_NUMBER_OF_SERVICES, }); - return calculateAvgMetrics(mergeEntities({ entities })); + return mergeEntities({ entities }); } catch (error) { // If the index does not exist, handle it gracefully if ( diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts index e99ecc0217ed9..3ab3b907f5be2 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts @@ -8,7 +8,6 @@ import type { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; import { withApmSpan } from '../../../utils/with_apm_span'; import { getEntityLatestServices } from '../get_entity_latest_services'; -import { calculateAvgMetrics } from '../utils/calculate_avg_metrics'; import { mergeEntities } from '../utils/merge_entities'; import { MAX_NUMBER_OF_SERVICES } from './get_service_entities'; @@ -27,7 +26,7 @@ export function getServiceEntitySummary({ entitiesESClient, environment, service serviceName, }); - const serviceEntity = calculateAvgMetrics(mergeEntities({ entities: entityLatestServices })); + const serviceEntity = mergeEntities({ entities: entityLatestServices }); return serviceEntity[0]; }); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts index c3f36a6d86c52..218de180cbc00 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts @@ -4,8 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import Boom from '@hapi/boom'; -import { jsonRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { createEntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client'; @@ -13,7 +11,6 @@ import { createApmServerRoute } from '../../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../../default_api_types'; import { getServiceEntities } from './get_service_entities'; import { getServiceEntitySummary } from './get_service_entity_summary'; -import { getEntityHistoryServicesTimeseries } from '../get_entity_history_services_timeseries'; const serviceEntitiesSummaryRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/entities/services/{serviceName}/summary', @@ -72,46 +69,6 @@ const servicesEntitiesRoute = createApmServerRoute({ }, }); -const servicesEntitiesDetailedStatisticsRoute = createApmServerRoute({ - endpoint: 'POST /internal/apm/entities/services/detailed_statistics', - params: t.type({ - query: t.intersection([environmentRt, kueryRt, rangeRt]), - body: t.type({ serviceNames: jsonRt.pipe(t.array(t.string)) }), - }), - options: { tags: ['access:apm'] }, - handler: async (resources) => { - const { context, params, request } = resources; - const coreContext = await context.core; - - const entitiesESClient = await createEntitiesESClient({ - request, - esClient: coreContext.elasticsearch.client.asCurrentUser, - }); - - const { environment, start, end } = params.query; - - const { serviceNames } = params.body; - - if (!serviceNames.length) { - throw Boom.badRequest(`serviceNames cannot be empty`); - } - - const serviceEntitiesTimeseries = await getEntityHistoryServicesTimeseries({ - start, - end, - serviceNames, - environment, - entitiesESClient, - }); - - return { - currentPeriod: { - ...serviceEntitiesTimeseries, - }, - }; - }, -}); - const serviceLogRateTimeseriesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/entities/services/{serviceName}/logs_rate_timeseries', params: t.type({ @@ -183,6 +140,5 @@ export const servicesEntitiesRoutesRepository = { ...servicesEntitiesRoute, ...serviceLogRateTimeseriesRoute, ...serviceLogErrorRateTimeseriesRoute, - ...servicesEntitiesDetailedStatisticsRoute, ...serviceEntitiesSummaryRoute, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts index 00f1b0580a669..5713c0a2b67fb 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts @@ -5,7 +5,6 @@ * 2.0. */ import { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; -import { EntityMetrics } from '../../../common/entities/types'; export enum EntityType { SERVICE = 'service', @@ -30,5 +29,4 @@ interface Entity { lastSeenTimestamp: string; firstSeenTimestamp: string; identityFields: string[]; - metrics: EntityMetrics; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.test.ts deleted file mode 100644 index 4d0a562295224..0000000000000 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.test.ts +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EntityMetrics, EntityDataStreamType } from '../../../../common/entities/types'; -import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import { calculateAvgMetrics, mergeMetrics } from './calculate_avg_metrics'; - -describe('calculateAverageMetrics', () => { - it('calculates average metrics', () => { - const entities = [ - { - agentName: 'nodejs' as AgentName, - dataStreamTypes: [EntityDataStreamType.METRICS, EntityDataStreamType.LOGS], - environments: [], - latestTimestamp: '2024-03-05T10:34:40.810Z', - metrics: [ - { - failedTransactionRate: 5, - latency: 5, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - { - failedTransactionRate: 10, - latency: 10, - logErrorRate: 10, - logRate: 10, - throughput: 10, - }, - ], - serviceName: 'service-1', - hasLogMetrics: true, - }, - { - agentName: 'java' as AgentName, - dataStreamTypes: [EntityDataStreamType.METRICS], - environments: [], - latestTimestamp: '2024-06-05T10:34:40.810Z', - metrics: [ - { - failedTransactionRate: 15, - latency: 15, - logErrorRate: 15, - logRate: 15, - throughput: 15, - }, - { - failedTransactionRate: 5, - latency: 5, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - ], - serviceName: 'service-2', - hasLogMetrics: true, - }, - ]; - - const result = calculateAvgMetrics(entities); - - expect(result).toEqual([ - { - agentName: 'nodejs', - dataStreamTypes: [EntityDataStreamType.METRICS, EntityDataStreamType.LOGS], - environments: [], - latestTimestamp: '2024-03-05T10:34:40.810Z', - metrics: { - failedTransactionRate: 7.5, - latency: 7.5, - logErrorRate: 7.5, - logRate: 7.5, - throughput: 7.5, - }, - serviceName: 'service-1', - hasLogMetrics: true, - }, - { - agentName: 'java' as AgentName, - dataStreamTypes: [EntityDataStreamType.METRICS], - environments: [], - latestTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - failedTransactionRate: 10, - latency: 10, - logErrorRate: 10, - logRate: 10, - throughput: 10, - }, - serviceName: 'service-2', - hasLogMetrics: true, - }, - ]); - }); - it('calculates average metrics with null', () => { - const entities = [ - { - agentName: 'nodejs' as AgentName, - dataStreamTypes: [EntityDataStreamType.METRICS], - environments: ['env-service-1', 'env-service-2'], - latestTimestamp: '2024-03-05T10:34:40.810Z', - metrics: [ - { - failedTransactionRate: 5, - latency: null, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - { - failedTransactionRate: 10, - latency: null, - logErrorRate: 10, - logRate: 10, - throughput: 10, - }, - ], - serviceName: 'service-1', - hasLogMetrics: true, - }, - ]; - - const result = calculateAvgMetrics(entities); - - expect(result).toEqual([ - { - agentName: 'nodejs', - dataStreamTypes: [EntityDataStreamType.METRICS], - environments: ['env-service-1', 'env-service-2'], - latestTimestamp: '2024-03-05T10:34:40.810Z', - metrics: { - failedTransactionRate: 7.5, - logErrorRate: 7.5, - logRate: 7.5, - throughput: 7.5, - }, - serviceName: 'service-1', - hasLogMetrics: true, - }, - ]); - }); -}); - -describe('mergeMetrics', () => { - it('merges metrics correctly', () => { - const metrics = [ - { - failedTransactionRate: 5, - latency: 5, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - { - failedTransactionRate: 10, - latency: 10, - logErrorRate: 10, - logRate: 10, - throughput: 10, - }, - ]; - - const result = mergeMetrics(metrics); - - expect(result).toEqual({ - failedTransactionRate: [5, 10], - latency: [5, 10], - logErrorRate: [5, 10], - logRate: [5, 10], - throughput: [5, 10], - }); - }); - - it('handles empty metrics array', () => { - const metrics: EntityMetrics[] = []; - - const result = mergeMetrics(metrics); - - expect(result).toEqual({}); - }); - - it('returns metrics with zero value', () => { - const metrics = [ - { - failedTransactionRate: 0, - latency: 4, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - ]; - - const result = mergeMetrics(metrics); - - expect(result).toEqual({ - failedTransactionRate: [0], - latency: [4], - logErrorRate: [5], - logRate: [5], - throughput: [5], - }); - }); - - it('does not return metrics with null', () => { - const metrics = [ - { - failedTransactionRate: null, - latency: null, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - { - failedTransactionRate: 5, - latency: null, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - ]; - - const result = mergeMetrics(metrics); - - expect(result).toEqual({ - failedTransactionRate: [5], - logErrorRate: [5, 5], - logRate: [5, 5], - throughput: [5, 5], - }); - }); -}); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.ts deleted file mode 100644 index bee5f7ce9cc2b..0000000000000 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { mapValues, isNumber } from 'lodash'; -import { EntityMetrics } from '../../../../common/entities/types'; -import type { MergedServiceEntity } from './merge_entities'; - -export function calculateAvgMetrics(entities: MergedServiceEntity[]) { - return entities.map((entity) => { - const transformedMetrics = mergeMetrics(entity.metrics); - const averages = mapValues(transformedMetrics, (values: number[]) => { - const sum = values.reduce((acc: number, val: number) => acc + (val !== null ? val : 0), 0); - return sum / values.length; - }); - - return { - ...entity, - metrics: averages, - }; - }); -} -type MetricsKey = keyof EntityMetrics; - -export function mergeMetrics(metrics: EntityMetrics[]) { - return metrics.reduce((acc, metric) => { - for (const key in metric) { - if (Object.hasOwn(metric, key)) { - const metricsKey = key as MetricsKey; - - const value = metric[metricsKey]; - if (isNumber(value)) { - if (!acc[metricsKey]) { - acc[metricsKey] = []; - } - acc[metricsKey].push(value); - } - } - } - return acc; - }, {} as { [key in MetricsKey]: number[] }); -} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts index 587cb03975105..bb5c4f48b4125 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts @@ -22,13 +22,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name', 'service.environment'], id: 'service-1:test', }, @@ -41,16 +34,6 @@ describe('mergeEntities', () => { dataStreamTypes: ['metrics', 'logs'], environments: ['test'], lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - logErrorRate: null, - logRate: 1, - throughput: 0, - }, - ], serviceName: 'service-1', }, ]); @@ -68,13 +51,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-03-05T10:34:40.810Z', lastSeenTimestamp: '2024-03-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name', 'service.environment'], id: 'service-1:env-service-1', }, @@ -89,13 +65,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-03-05T10:34:40.810Z', lastSeenTimestamp: '2024-03-05T10:34:40.810Z', - metrics: { - logRate: 10, - logErrorRate: 10, - throughput: 10, - failedTransactionRate: 10, - latency: 10, - }, identityFields: ['service.name', 'service.environment'], id: 'apm-only-1:synthtrace-env-2', }, @@ -110,13 +79,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 15, - logErrorRate: 15, - throughput: 15, - failedTransactionRate: 15, - latency: 15, - }, identityFields: ['service.name', 'service.environment'], id: 'service-2:env-service-3', }, @@ -131,13 +93,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 5, - logErrorRate: 5, - throughput: 5, - failedTransactionRate: 5, - latency: 5, - }, identityFields: ['service.name', 'service.environment'], id: 'service-2:env-service-3', }, @@ -151,23 +106,6 @@ describe('mergeEntities', () => { dataStreamTypes: ['foo', 'bar'], environments: ['env-service-1', 'env-service-2'], lastSeenTimestamp: '2024-03-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - logErrorRate: null, - logRate: 1, - throughput: 0, - }, - { - failedTransactionRate: 10, - latency: 10, - logErrorRate: 10, - logRate: 10, - throughput: 10, - }, - ], serviceName: 'service-1', }, { @@ -175,23 +113,6 @@ describe('mergeEntities', () => { dataStreamTypes: ['baz'], environments: ['env-service-3', 'env-service-4'], lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 15, - latency: 15, - logErrorRate: 15, - logRate: 15, - throughput: 15, - }, - { - failedTransactionRate: 5, - latency: 5, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - ], serviceName: 'service-2', }, ]); @@ -208,13 +129,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 5, - logErrorRate: 5, - throughput: 5, - failedTransactionRate: 5, - latency: 5, - }, identityFields: ['service.name', 'service.environment'], id: 'service-1:test', }, @@ -229,13 +143,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 10, - logErrorRate: 10, - throughput: 10, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name', 'service.environment'], id: 'service-1:test', }, @@ -250,13 +157,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-23-05T10:34:40.810Z', lastSeenTimestamp: '2024-23-05T10:34:40.810Z', - metrics: { - logRate: 0.333, - logErrorRate: 0.333, - throughput: 0.333, - failedTransactionRate: 0.333, - latency: 0.333, - }, identityFields: ['service.name', 'service.environment'], id: 'service-1:prod', }, @@ -269,30 +169,6 @@ describe('mergeEntities', () => { dataStreamTypes: ['metrics', 'logs', 'foo'], environments: ['test', 'prod'], lastSeenTimestamp: '2024-23-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 5, - latency: 5, - logErrorRate: 5, - logRate: 5, - throughput: 5, - }, - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - logErrorRate: 10, - logRate: 10, - throughput: 10, - }, - { - failedTransactionRate: 0.333, - latency: 0.333, - logErrorRate: 0.333, - logRate: 0.333, - throughput: 0.333, - }, - ], serviceName: 'service-1', }, ]); @@ -309,13 +185,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name'], id: 'service-1:test', }, @@ -328,16 +197,6 @@ describe('mergeEntities', () => { dataStreamTypes: [], environments: [], lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - logErrorRate: null, - logRate: 1, - throughput: 0, - }, - ], serviceName: 'service-1', }, ]); @@ -352,13 +211,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name'], id: 'service-1:test', }, @@ -372,13 +224,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name'], id: 'service-1:test', }, @@ -391,23 +236,6 @@ describe('mergeEntities', () => { dataStreamTypes: [], environments: [], lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - logErrorRate: null, - logRate: 1, - throughput: 0, - }, - { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, - ], serviceName: 'service-1', }, ]); @@ -424,13 +252,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name'], id: 'service-1:test', }, @@ -443,16 +264,6 @@ describe('mergeEntities', () => { dataStreamTypes: [], environments: [], lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - logErrorRate: null, - logRate: 1, - throughput: 0, - }, - ], serviceName: 'service-1', }, ]); @@ -467,13 +278,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name'], id: 'service-1:test', }, @@ -487,13 +291,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name'], id: 'service-1:test', }, @@ -506,23 +303,6 @@ describe('mergeEntities', () => { dataStreamTypes: [], environments: [], lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - hasLogMetrics: true, - metrics: [ - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - logErrorRate: null, - logRate: 1, - throughput: 0, - }, - { - logRate: 1, - logErrorRate: null, - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, - ], serviceName: 'service-1', }, ]); @@ -540,11 +320,6 @@ describe('mergeEntities', () => { entity: { firstSeenTimestamp: '2024-06-05T10:34:40.810Z', lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - metrics: { - throughput: 0, - failedTransactionRate: 0.3333333333333333, - latency: 10, - }, identityFields: ['service.name', 'service.environment'], id: 'service-1:test', }, @@ -557,14 +332,6 @@ describe('mergeEntities', () => { dataStreamTypes: ['metrics'], environments: ['test'], lastSeenTimestamp: '2024-06-05T10:34:40.810Z', - hasLogMetrics: false, - metrics: [ - { - failedTransactionRate: 0.3333333333333333, - latency: 10, - throughput: 0, - }, - ], serviceName: 'service-1', }, ]); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts index 4017d922d63c5..c7269989a3564 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts @@ -7,17 +7,14 @@ import { compact, uniq } from 'lodash'; import type { EntityLatestServiceRaw } from '../types'; -import { isFiniteNumber } from '../../../../common/utils/is_finite_number'; import type { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import type { EntityDataStreamType, EntityMetrics } from '../../../../common/entities/types'; +import type { EntityDataStreamType } from '../../../../common/entities/types'; export interface MergedServiceEntity { serviceName: string; agentName: AgentName; dataStreamTypes: EntityDataStreamType[]; environments: string[]; - metrics: EntityMetrics[]; - hasLogMetrics: boolean; } export function mergeEntities({ @@ -40,10 +37,6 @@ export function mergeEntities({ } function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServiceEntity) { - const hasLogMetrics = isFiniteNumber(entity.entity.metrics.logRate) - ? entity.entity.metrics.logRate > 0 - : false; - const commonEntityFields = { serviceName: entity.service.name, agentName: entity.agent.name[0], @@ -55,8 +48,6 @@ function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServic ...commonEntityFields, dataStreamTypes: entity.source_data_stream.type, environments: compact([entity?.service.environment]), - metrics: [entity.entity.metrics], - hasLogMetrics, }; } return { @@ -65,7 +56,5 @@ function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServic compact([...(existingEntity?.dataStreamTypes ?? []), ...entity.source_data_stream.type]) ), environments: uniq(compact([...existingEntity?.environments, entity?.service.environment])), - metrics: [...existingEntity?.metrics, entity.entity.metrics], - hasLogMetrics: hasLogMetrics || existingEntity.hasLogMetrics, }; } diff --git a/x-pack/test/apm_api_integration/tests/entities/services/services_entities_detailed_statistics.spec.ts b/x-pack/test/apm_api_integration/tests/entities/services/services_entities_detailed_statistics.spec.ts deleted file mode 100644 index e1bf212c6e9c0..0000000000000 --- a/x-pack/test/apm_api_integration/tests/entities/services/services_entities_detailed_statistics.spec.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import expect from '@kbn/expect'; -import { APIClientRequestParamsOf } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - - const apmApiClient = getService('apmApiClient'); - - const start = '2024-01-01T00:00:00.000Z'; - const end = '2024-01-01T00:59:59.999Z'; - - const serviceNames = ['my-service', 'synth-go']; - - async function getServiceEntitiesDetailedStats( - overrides?: Partial< - APIClientRequestParamsOf<'POST /internal/apm/entities/services/detailed_statistics'>['params']['query'] - > - ) { - const response = await apmApiClient.readUser({ - endpoint: `POST /internal/apm/entities/services/detailed_statistics`, - params: { - query: { - start, - end, - environment: 'ENVIRONMENT_ALL', - kuery: '', - ...overrides, - }, - body: { - serviceNames: JSON.stringify(serviceNames), - }, - }, - }); - - return response; - } - - registry.when( - 'Services entities detailed statistics when no data is generated', - { config: 'basic', archives: [] }, - () => { - describe('Service entities detailed', () => { - it('handles the empty state', async () => { - const response = await getServiceEntitiesDetailedStats(); - expect(response.status).to.be(200); - expect(response.body.currentPeriod).to.empty(); - }); - }); - } - ); -}