diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 6b0a68379f5d4..5d50833161979 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -7,7 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export const TIMESTAMP = 'timestamp.us'; +export const TIMESTAMP_US = 'timestamp.us'; +export const AT_TIMESTAMP = '@timestamp'; export const AGENT = 'agent'; export const AGENT_NAME = 'agent.name'; export const AGENT_VERSION = 'agent.version'; @@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider'; export const CLOUD_REGION = 'cloud.region'; export const CLOUD_MACHINE_TYPE = 'cloud.machine.type'; export const CLOUD_ACCOUNT_ID = 'cloud.account.id'; +export const CLOUD_ACCOUNT_NAME = 'cloud.account.name'; export const CLOUD_INSTANCE_ID = 'cloud.instance.id'; export const CLOUD_INSTANCE_NAME = 'cloud.instance.name'; export const CLOUD_SERVICE_NAME = 'cloud.service.name'; +export const CLOUD_PROJECT_NAME = 'cloud.project.name'; export const EVENT_SUCCESS_COUNT = 'event.success_count'; @@ -48,10 +51,14 @@ export const USER_ID = 'user.id'; export const USER_AGENT_ORIGINAL = 'user_agent.original'; export const USER_AGENT_NAME = 'user_agent.name'; +export const OBSERVER_VERSION = 'observer.version'; +export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const OBSERVER_HOSTNAME = 'observer.hostname'; export const OBSERVER_LISTENING = 'observer.listening'; export const PROCESSOR_EVENT = 'processor.event'; +export const PROCESSOR_NAME = 'processor.name'; +export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary'; @@ -95,6 +102,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us'; export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy'; export const SPAN_SYNC = 'span.sync'; +export const SPAN_STACKTRACE = 'span.stacktrace'; // Parent ID for a transaction or span export const PARENT_ID = 'parent.id'; @@ -110,6 +118,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array export const ERROR_EXC_TYPE = 'error.exception.type'; export const ERROR_PAGE_URL = 'error.page.url'; +export const ERROR_STACK_TRACE = 'error.stack_trace'; export const ERROR_TYPE = 'error.type'; // METRICS @@ -153,6 +162,12 @@ export const CONTAINER_IMAGE = 'container.image.name'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; export const KUBERNETES_POD_UID = 'kubernetes.pod.uid'; +export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; +export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; +export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; +export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; +export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; +export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name'; export const FAAS_ID = 'faas.id'; export const FAAS_NAME = 'faas.name'; @@ -198,3 +213,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name'; export const CHILD_ID = 'child.id'; export const LOG_LEVEL = 'log.level'; + +// Process +export const PROCESS_ARGS = 'process.args'; +export const PROCESS_PID = 'process.pid'; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts b/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts index b3a6066631346..14d26354e44ed 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts @@ -14,10 +14,10 @@ export interface APMBaseDoc { '@timestamp': string; agent: { name: string; - version: string; + version?: string; }; - parent?: { id: string }; // parent ID is not available on root transactions - trace?: { id: string }; + parent?: { id?: string }; // parent ID is not available on root transactions + trace?: { id?: string }; labels?: { [key: string]: string | number | boolean; }; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts index 7ee972faf7680..290be75091e18 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts @@ -10,26 +10,26 @@ export interface Cloud { availability_zone?: string; instance?: { - name: string; - id: string; + name?: string; + id?: string; }; machine?: { - type: string; + type?: string; }; project?: { - id: string; - name: string; + id?: string; + name?: string; }; provider?: string; region?: string; account?: { - id: string; - name: string; + id?: string; + name?: string; }; image?: { - id: string; + id?: string; }; service?: { - name: string; + name?: string; }; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts index 64dd497710b97..4c8d1ed4e52b4 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts @@ -9,5 +9,7 @@ export interface Container { id?: string | null; - image?: string | null; + image?: { + name?: string; + }; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts index 458731f690838..f3c62298ca8cb 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts @@ -8,7 +8,7 @@ */ export interface Http { - request?: { method: string; [key: string]: unknown }; - response?: { status_code: number; [key: string]: unknown }; + request?: { method?: string }; + response?: { status_code?: number }; version?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts index 704d77f19f858..2a4f1465db9a5 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts @@ -8,7 +8,7 @@ */ export interface Kubernetes { - pod?: { uid?: string | null; [key: string]: unknown }; + pod?: { uid?: string | null; name?: string }; namespace?: string; replicaset?: { name?: string; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts index 067ecb9436ff9..7d286d4c3581e 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts @@ -13,6 +13,6 @@ export interface Observer { id?: string; name?: string; type?: string; - version: string; - version_major: number; + version?: string; + version_major?: number; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts index 6cc058ef75642..a18f3c5578eb5 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts @@ -9,5 +9,5 @@ // only for RUM agent: shared by error and transaction export interface Page { - url: string; + url?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts index bcd9af08706ec..bd52784576dce 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts @@ -11,18 +11,18 @@ export interface Service { name: string; environment?: string; framework?: { - name: string; + name?: string; version?: string; }; node?: { name?: string; }; runtime?: { - name: string; - version: string; + name?: string; + version?: string; }; language?: { - name: string; + name?: string; version?: string; }; version?: string; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts index 3703763724f38..0f8cd3c814315 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts @@ -9,6 +9,6 @@ export interface Url { domain?: string; - full: string; + full?: string; original?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts index 1c2235288a661..962ed1060b826 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts @@ -8,5 +8,5 @@ */ export interface User { - id: string; + id?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts b/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts index ea3ebf39555d2..e8734de141e83 100644 --- a/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts +++ b/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts @@ -14,5 +14,5 @@ export type { ElasticAgentName, OpenTelemetryAgentName, AgentName } from '@kbn/e export interface Agent { ephemeral_id?: string; name: AgentName; - version: string; + version?: string; } diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts index deb7ed998c478..13a8174f4f1cf 100644 --- a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts +++ b/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts @@ -21,6 +21,18 @@ describe('flattenObject', () => { }); }); + it('flattens arrays', () => { + expect( + flattenObject({ + child: { + id: [1, 2], + }, + }) + ).toEqual({ + 'child.id': [1, 2], + }); + }); + it('does not flatten arrays', () => { expect( flattenObject({ diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts new file mode 100644 index 0000000000000..22cee17bb1a64 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { unflattenObject } from './unflatten_object'; + +describe('unflattenObject', () => { + it('unflattens deeply nested objects', () => { + expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({ + first: { + second: { + third: 'third', + }, + }, + }); + }); + + it('does not unflatten arrays', () => { + expect( + unflattenObject({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + 'nested.array': [0, 1, 2], + 'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }], + }) + ).toEqual({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + nested: { + array: [0, 1, 2], + }, + complex: { + nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }], + }, + }); + }); +}); diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts new file mode 100644 index 0000000000000..142ea2eea6461 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@kbn/safer-lodash-set'; + +export function unflattenObject(source: Record, target: Record = {}) { + // eslint-disable-next-line guard-for-in + for (const key in source) { + const val = source[key as keyof typeof source]; + + if (Array.isArray(val)) { + const unflattenedArray = val.map((item) => { + if (item && typeof item === 'object' && !Array.isArray(item)) { + return unflattenObject(item); + } + return item; + }); + set(target, key, unflattenedArray); + } else { + set(target, key, val); + } + } + return target; +} diff --git a/x-pack/packages/observability/observability_utils/tsconfig.json b/x-pack/packages/observability/observability_utils/tsconfig.json index 2ed47d10cfad9..b3f1a4a21c4e7 100644 --- a/x-pack/packages/observability/observability_utils/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/tsconfig.json @@ -21,5 +21,6 @@ "@kbn/es-types", "@kbn/apm-utils", "@kbn/es-query", + "@kbn/safer-lodash-set", ] } diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap index 6fa3e146a423d..88d00196e074b 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap @@ -37,6 +37,8 @@ Object { exports[`Error CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Error CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Error CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Error CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -45,6 +47,8 @@ exports[`Error CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Error CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Error CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Error CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Error CLOUD_REGION 1`] = `"europe-west1"`; @@ -94,6 +98,8 @@ exports[`Error ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Error ERROR_PAGE_URL 1`] = `undefined`; +exports[`Error ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Error ERROR_TYPE 1`] = `undefined`; exports[`Error EVENT_NAME 1`] = `undefined`; @@ -140,6 +146,8 @@ exports[`Error INDEX 1`] = `undefined`; exports[`Error KUBERNETES 1`] = `undefined`; +exports[`Error KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Error KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Error KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -150,6 +158,8 @@ exports[`Error KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Error KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Error KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Error KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Error KUBERNETES_POD_UID 1`] = `undefined`; @@ -228,10 +238,20 @@ exports[`Error OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Error OBSERVER_LISTENING 1`] = `undefined`; +exports[`Error OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Error PARENT_ID 1`] = `"parentId"`; +exports[`Error PROCESS_ARGS 1`] = `undefined`; + +exports[`Error PROCESS_PID 1`] = `undefined`; + exports[`Error PROCESSOR_EVENT 1`] = `"error"`; +exports[`Error PROCESSOR_NAME 1`] = `"error"`; + exports[`Error SERVICE 1`] = ` Object { "language": Object { @@ -296,6 +316,8 @@ exports[`Error SPAN_NAME 1`] = `undefined`; exports[`Error SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Error SPAN_STACKTRACE 1`] = `undefined`; + exports[`Error SPAN_SUBTYPE 1`] = `undefined`; exports[`Error SPAN_SYNC 1`] = `undefined`; @@ -304,10 +326,12 @@ exports[`Error SPAN_TYPE 1`] = `undefined`; exports[`Error TIER 1`] = `undefined`; -exports[`Error TIMESTAMP 1`] = `1337`; +exports[`Error TIMESTAMP_US 1`] = `1337`; exports[`Error TRACE_ID 1`] = `"trace id"`; +exports[`Error TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Error TRANSACTION_DURATION 1`] = `undefined`; exports[`Error TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; @@ -385,6 +409,8 @@ Object { exports[`Span CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Span CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Span CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Span CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -393,6 +419,8 @@ exports[`Span CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Span CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Span CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Span CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Span CLOUD_REGION 1`] = `"europe-west1"`; @@ -433,6 +461,8 @@ exports[`Span ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Span ERROR_PAGE_URL 1`] = `undefined`; +exports[`Span ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Span ERROR_TYPE 1`] = `undefined`; exports[`Span EVENT_NAME 1`] = `undefined`; @@ -475,6 +505,8 @@ exports[`Span INDEX 1`] = `undefined`; exports[`Span KUBERNETES 1`] = `undefined`; +exports[`Span KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Span KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Span KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -485,6 +517,8 @@ exports[`Span KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Span KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Span KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Span KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Span KUBERNETES_POD_UID 1`] = `undefined`; @@ -563,10 +597,20 @@ exports[`Span OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Span OBSERVER_LISTENING 1`] = `undefined`; +exports[`Span OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Span PARENT_ID 1`] = `"parentId"`; +exports[`Span PROCESS_ARGS 1`] = `undefined`; + +exports[`Span PROCESS_PID 1`] = `undefined`; + exports[`Span PROCESSOR_EVENT 1`] = `"span"`; +exports[`Span PROCESSOR_NAME 1`] = `"transaction"`; + exports[`Span SERVICE 1`] = ` Object { "name": "service name", @@ -627,6 +671,8 @@ exports[`Span SPAN_NAME 1`] = `"span name"`; exports[`Span SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Span SPAN_STACKTRACE 1`] = `undefined`; + exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`; exports[`Span SPAN_SYNC 1`] = `false`; @@ -635,10 +681,12 @@ exports[`Span SPAN_TYPE 1`] = `"span type"`; exports[`Span TIER 1`] = `undefined`; -exports[`Span TIMESTAMP 1`] = `1337`; +exports[`Span TIMESTAMP_US 1`] = `1337`; exports[`Span TRACE_ID 1`] = `"trace id"`; +exports[`Span TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Span TRANSACTION_DURATION 1`] = `undefined`; exports[`Span TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; @@ -716,6 +764,8 @@ Object { exports[`Transaction CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Transaction CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Transaction CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Transaction CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -724,6 +774,8 @@ exports[`Transaction CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Transaction CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Transaction CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Transaction CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Transaction CLOUD_REGION 1`] = `"europe-west1"`; @@ -768,6 +820,8 @@ exports[`Transaction ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Transaction ERROR_PAGE_URL 1`] = `undefined`; +exports[`Transaction ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Transaction ERROR_TYPE 1`] = `undefined`; exports[`Transaction EVENT_NAME 1`] = `undefined`; @@ -820,6 +874,8 @@ Object { } `; +exports[`Transaction KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Transaction KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Transaction KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -830,6 +886,8 @@ exports[`Transaction KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Transaction KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Transaction KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Transaction KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Transaction KUBERNETES_POD_UID 1`] = `"pod1234567890abcdef"`; @@ -908,10 +966,20 @@ exports[`Transaction OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; +exports[`Transaction OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Transaction PARENT_ID 1`] = `"parentId"`; +exports[`Transaction PROCESS_ARGS 1`] = `undefined`; + +exports[`Transaction PROCESS_PID 1`] = `undefined`; + exports[`Transaction PROCESSOR_EVENT 1`] = `"transaction"`; +exports[`Transaction PROCESSOR_NAME 1`] = `"transaction"`; + exports[`Transaction SERVICE 1`] = ` Object { "language": Object { @@ -976,6 +1044,8 @@ exports[`Transaction SPAN_NAME 1`] = `undefined`; exports[`Transaction SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Transaction SPAN_STACKTRACE 1`] = `undefined`; + exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`; exports[`Transaction SPAN_SYNC 1`] = `undefined`; @@ -984,10 +1054,12 @@ exports[`Transaction SPAN_TYPE 1`] = `undefined`; exports[`Transaction TIER 1`] = `undefined`; -exports[`Transaction TIMESTAMP 1`] = `1337`; +exports[`Transaction TIMESTAMP_US 1`] = `1337`; exports[`Transaction TRACE_ID 1`] = `"trace id"`; +exports[`Transaction TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Transaction TRANSACTION_DURATION 1`] = `1337`; exports[`Transaction TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts index f33fddd430e8d..12537d35afefe 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts @@ -10,12 +10,13 @@ import { AllowUnknownProperties } from '../../typings/common'; import { APMError } from '../../typings/es_schemas/ui/apm_error'; import { Span } from '../../typings/es_schemas/ui/span'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; -import * as apmFieldnames from './apm'; -import * as infraMetricsFieldnames from './infra_metrics'; +import * as allApmFieldNames from './apm'; +import * as infraMetricsFieldNames from './infra_metrics'; +const { AT_TIMESTAMP, ...apmFieldNames } = allApmFieldNames; const fieldnames = { - ...apmFieldnames, - ...infraMetricsFieldnames, + ...apmFieldNames, + ...infraMetricsFieldNames, }; describe('Transaction', () => { diff --git a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts index 0ccede67741b7..4136ea361392e 100644 --- a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts @@ -4,5 +4,63 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_INSTANCE_NAME, + CLOUD_MACHINE_TYPE, + CLOUD_PROVIDER, + CONTAINER_ID, + HOST_NAME, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_REPLICASET_NAME, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_VERSION, +} from './es_fields/apm'; +import { asMutableArray } from './utils/as_mutable_array'; + +export const SERVICE_METADATA_SERVICE_KEYS = asMutableArray([ + SERVICE_NODE_NAME, + SERVICE_VERSION, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, +] as const); + +export const SERVICE_METADATA_CONTAINER_KEYS = asMutableArray([ + CONTAINER_ID, + HOST_NAME, + KUBERNETES_POD_UID, + KUBERNETES_POD_NAME, +] as const); + +export const SERVICE_METADATA_INFRA_METRICS_KEYS = asMutableArray([ + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, +] as const); + +export const SERVICE_METADATA_CLOUD_KEYS = asMutableArray([ + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_INSTANCE_NAME, + CLOUD_MACHINE_TYPE, + CLOUD_PROVIDER, +] as const); + +export const SERVICE_METADATA_KUBERNETES_KEYS = asMutableArray([ + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_REPLICASET_NAME, +] as const); export type ContainerType = 'Kubernetes' | 'Docker' | undefined; diff --git a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts index 49f282473e9dd..2fd0be94a5c5f 100644 --- a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts +++ b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts @@ -64,16 +64,17 @@ export interface WaterfallSpan { links?: SpanLink[]; }; transaction?: { - id: string; + id?: string; }; child?: { id: string[] }; } export interface WaterfallError { timestamp: TimestampUs; - trace?: { id: string }; - transaction?: { id: string }; - parent?: { id: string }; + trace?: { id?: string }; + transaction?: { id?: string }; + parent?: { id?: string }; + span?: { id?: string }; error: { id: string; log?: { diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx index 2e91865083b8c..20d5521b43ebf 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx @@ -8,9 +8,9 @@ import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import type { Message } from '@kbn/observability-ai-assistant-plugin/public'; import React, { useMemo, useState } from 'react'; +import { AT_TIMESTAMP } from '@kbn/apm-types'; import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context'; import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; -import { Transaction } from '../../../../../typings/es_schemas/ui/transaction'; import { ErrorSampleDetailTabContent } from './error_sample_detail'; import { exceptionStacktraceTab, logStacktraceTab } from './error_tabs'; @@ -18,8 +18,26 @@ export function ErrorSampleContextualInsight({ error, transaction, }: { - error: APMError; - transaction?: Transaction; + error: { + [AT_TIMESTAMP]: string; + error: Pick; + service: { + name: string; + environment?: string; + language?: { + name?: string; + }; + runtime?: { + name?: string; + version?: string; + }; + }; + }; + transaction?: { + transaction: { + name: string; + }; + }; }) { const { observabilityAIAssistant } = useApmPluginContext(); diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx index a38c4dfc96f63..2edb2c1a3fea6 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx @@ -29,14 +29,14 @@ import { first } from 'lodash'; import React, { useEffect, useState } from 'react'; import { useHistory } from 'react-router-dom'; import useAsync from 'react-use/lib/useAsync'; -import { ERROR_GROUP_ID } from '../../../../../common/es_fields/apm'; +import { AT_TIMESTAMP, ERROR_GROUP_ID } from '../../../../../common/es_fields/apm'; import { TraceSearchType } from '../../../../../common/trace_explorer'; import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context'; import { useLegacyUrlParams } from '../../../../context/url_params_context/use_url_params'; import { useAnyOfApmParams } from '../../../../hooks/use_apm_params'; import { useApmRouter } from '../../../../hooks/use_apm_router'; -import { FETCH_STATUS, isPending } from '../../../../hooks/use_fetcher'; +import { FETCH_STATUS, isPending, isSuccess } from '../../../../hooks/use_fetcher'; import { useTraceExplorerEnabledSetting } from '../../../../hooks/use_trace_explorer_enabled_setting'; import { APIReturnType } from '../../../../services/rest/create_call_apm_api'; import { TransactionDetailLink } from '../../../shared/links/apm/transaction_detail_link'; @@ -111,8 +111,7 @@ export function ErrorSampleDetails({ const loadingErrorData = isPending(errorFetchStatus); const isLoading = loadingErrorSamplesData || loadingErrorData; - const isSucceded = - errorSamplesFetchStatus === FETCH_STATUS.SUCCESS && errorFetchStatus === FETCH_STATUS.SUCCESS; + const isSucceeded = isSuccess(errorSamplesFetchStatus) && isSuccess(errorFetchStatus); useEffect(() => { setSampleActivePage(0); @@ -137,7 +136,7 @@ export function ErrorSampleDetails({ }); }, [error, transaction, uiActions]); - if (!error && errorSampleIds?.length === 0 && isSucceded) { + if (!error && errorSampleIds?.length === 0 && isSucceeded) { return ( ; + }; currentTab: ErrorTab; }) { const codeLanguage = error?.service.language?.name; const exceptions = error?.error.exception || []; const logStackframes = error?.error.log?.stacktrace; const isPlaintextException = - !!error?.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace; + !!error.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace; switch (currentTab.key) { case ErrorTabKey.LogStackTrace: return ; @@ -363,7 +370,7 @@ export function ErrorSampleDetailTabContent({ return isPlaintextException ? ( diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx index 893e842513c8f..86b69eb480b3f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx @@ -41,7 +41,7 @@ export const metadataTab: ErrorTab = { }), }; -export function getTabs(error: APMError) { +export function getTabs(error: { error: { log?: APMError['error']['log'] } }) { const hasLogStacktrace = !isEmpty(error?.error.log?.stacktrace); return [...(hasLogStacktrace ? [logStacktraceTab] : []), exceptionStacktraceTab, metadataTab]; } diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx index af05e8766994c..c7acbfee7e45e 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx @@ -18,7 +18,9 @@ const Label = euiStyled.div` `; interface Props { - error: APMError; + error: { + error: Pick; + }; } export function SampleSummary({ error }: Props) { const logMessage = error.error.log?.message; diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts index a3467d7272ff5..acd9e9445ad8f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts +++ b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts @@ -6,7 +6,7 @@ */ import { format } from 'url'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; +import type { TransactionDetailRedirectInfo } from '../../../../server/routes/transactions/get_transaction_by_trace'; export const getRedirectToTransactionDetailPageUrl = ({ transaction, @@ -14,7 +14,7 @@ export const getRedirectToTransactionDetailPageUrl = ({ rangeTo, waterfallItemId, }: { - transaction: Transaction; + transaction: TransactionDetailRedirectInfo; rangeFrom?: string; rangeTo?: string; waterfallItemId?: string; diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx index 2958d2af7d68f..a32c01f3b15e5 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx @@ -7,10 +7,18 @@ import React, { ReactNode } from 'react'; import { ERROR_GROUP_ID, SERVICE_NAME } from '../../../../../common/es_fields/apm'; -import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; import { DiscoverLink } from './discover_link'; -function getDiscoverQuery(error: APMError, kuery?: string) { +interface ErrorForDiscoverQuery { + service: { + name: string; + }; + error: { + grouping_key: string; + }; +} + +function getDiscoverQuery(error: ErrorForDiscoverQuery, kuery?: string) { const serviceName = error.service.name; const groupId = error.error.grouping_key; let query = `${SERVICE_NAME}:"${serviceName}" AND ${ERROR_GROUP_ID}:"${groupId}"`; @@ -36,7 +44,7 @@ function DiscoverErrorLink({ children, }: { children?: ReactNode; - readonly error: APMError; + readonly error: ErrorForDiscoverQuery; readonly kuery?: string; }) { return ; diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx index ae688f8917602..dab585180fce9 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx @@ -7,13 +7,16 @@ import React, { useMemo } from 'react'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; +import { APMError, AT_TIMESTAMP } from '@kbn/apm-types'; import { getSectionsFromFields } from '../helper'; import { MetadataTable } from '..'; import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; interface Props { - error: APMError; + error: { + [AT_TIMESTAMP]: string; + error: Pick; + }; } export function ErrorMetadata({ error }: Props) { @@ -26,8 +29,8 @@ export function ErrorMetadata({ error }: Props) { id: error.error.id, }, query: { - start: error['@timestamp'], - end: error['@timestamp'], + start: error[AT_TIMESTAMP], + end: error[AT_TIMESTAMP], }, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index 319582f61b664..a2b6809f855e7 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -8,7 +8,7 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; import { tasks } from './tasks'; -import { SERVICE_NAME, SERVICE_ENVIRONMENT } from '../../../../common/es_fields/apm'; +import { SERVICE_NAME, SERVICE_ENVIRONMENT, AT_TIMESTAMP } from '../../../../common/es_fields/apm'; import { IndicesStatsResponse } from '../telemetry_client'; describe('data telemetry collection tasks', () => { @@ -101,7 +101,7 @@ describe('data telemetry collection tasks', () => { // a fixed date range. .mockReturnValueOnce({ hits: { - hits: [{ _source: { '@timestamp': new Date().toISOString() } }], + hits: [{ fields: { [AT_TIMESTAMP]: [new Date().toISOString()] } }], }, total: { value: 1, @@ -314,7 +314,7 @@ describe('data telemetry collection tasks', () => { ? { hits: { total: { value: 1 } } } : { hits: { - hits: [{ _source: { '@timestamp': 1 } }], + hits: [{ fields: { [AT_TIMESTAMP]: [1] } }], }, } ); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index db6a6a918177a..1347cbb4e3641 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -11,11 +11,13 @@ import { createHash } from 'crypto'; import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash'; import { SavedObjectsClient } from '@kbn/core/server'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name'; import { AGENT_ACTIVATION_METHOD, AGENT_NAME, AGENT_VERSION, + AT_TIMESTAMP, CLIENT_GEO_COUNTRY_ISO_CODE, CLOUD_AVAILABILITY_ZONE, CLOUD_PROVIDER, @@ -29,6 +31,7 @@ import { METRICSET_INTERVAL, METRICSET_NAME, OBSERVER_HOSTNAME, + OBSERVER_VERSION, PARENT_ID, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, @@ -54,10 +57,7 @@ import { SavedServiceGroup, } from '../../../../common/service_groups'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; -import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import { Span } from '../../../../typings/es_schemas/ui/span'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMDataTelemetry, APMPerService, @@ -193,17 +193,19 @@ export const tasks: TelemetryTask[] = [ size: 1, track_total_hits: false, sort: { - '@timestamp': 'desc' as const, + [AT_TIMESTAMP]: 'desc' as const, }, + fields: [AT_TIMESTAMP], }, }) - ).hits.hits[0] as { _source: { '@timestamp': string } }; + ).hits.hits[0]; if (!lastTransaction) { return {}; } - const end = new Date(lastTransaction._source['@timestamp']).getTime() - 5 * 60 * 1000; + const end = + new Date(lastTransaction.fields[AT_TIMESTAMP]![0] as string).getTime() - 5 * 60 * 1000; const start = end - 60 * 1000; @@ -512,16 +514,16 @@ export const tasks: TelemetryTask[] = [ }, }, sort: { - '@timestamp': 'asc', + [AT_TIMESTAMP]: 'asc', }, - _source: ['@timestamp'], + fields: [AT_TIMESTAMP], }, }) : null; - const event = retainmentResponse?.hits.hits[0]?._source as + const event = retainmentResponse?.hits.hits[0]?.fields as | { - '@timestamp': number; + [AT_TIMESTAMP]: number[]; } | undefined; @@ -535,7 +537,7 @@ export const tasks: TelemetryTask[] = [ ? { retainment: { [processorEvent]: { - ms: new Date().getTime() - new Date(event['@timestamp']).getTime(), + ms: new Date().getTime() - new Date(event[AT_TIMESTAMP][0]).getTime(), }, }, } @@ -690,16 +692,16 @@ export const tasks: TelemetryTask[] = [ sort: { '@timestamp': 'desc', }, + fields: asMutableArray([OBSERVER_VERSION] as const), }, }); - const hit = response.hits.hits[0]?._source as Pick; - - if (!hit || !hit.observer?.version) { + const event = unflattenKnownApmEventFields(response.hits.hits[0]?.fields); + if (!event || !event.observer?.version) { return {}; } - const [major, minor, patch] = hit.observer.version.split('.').map((part) => Number(part)); + const [major, minor, patch] = event.observer.version.split('.').map((part) => Number(part)); return { version: { diff --git a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts index 4ab2e753832a4..cbcad6dea5baf 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts @@ -186,7 +186,6 @@ export const getDestinationMap = ({ }, size: destinationsBySpanId.size, fields: asMutableArray([SERVICE_NAME, SERVICE_ENVIRONMENT, AGENT_NAME, PARENT_ID] as const), - _source: false, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts index 88d0040f70fc9..5d4977a73b42f 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts @@ -9,6 +9,10 @@ import { NOT_AVAILABLE_LABEL } from '../../../common/i18n'; import { Maybe } from '../../../typings/common'; import { APMError } from '../../../typings/es_schemas/ui/apm_error'; -export function getErrorName({ error }: { error: Maybe }): string { +export function getErrorName({ + error, +}: { + error: Maybe> & { log?: { message?: string } }; +}): string { return error?.log?.message || error?.exception?.[0]?.message || NOT_AVAILABLE_LABEL; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts index 96ddbe15c4287..dfc32ec9eb54e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts @@ -184,6 +184,7 @@ export function registerTransactionDurationRuleType({ body: { track_total_hits: false, size: 0, + _source: false as const, query: { bool: { filter: [ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts index 5f36325031ccb..7072639f8526e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts @@ -8,6 +8,9 @@ import datemath from '@elastic/datemath'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { flattenObject, KeyValuePair } from '../../../../common/utils/flatten_object'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { PROCESSOR_EVENT, TRACE_ID } from '../../../../common/es_fields/apm'; @@ -86,6 +89,7 @@ export async function getLogCategories({ const rawSamplingProbability = Math.min(100_000 / totalDocCount, 1); const samplingProbability = rawSamplingProbability < 0.5 ? rawSamplingProbability : 1; + const fields = asMutableArray(['message', TRACE_ID] as const); const categorizedLogsRes = await search({ index, size: 1, @@ -108,7 +112,7 @@ export async function getLogCategories({ top_hits: { sort: { '@timestamp': 'desc' as const }, size: 1, - _source: ['message', TRACE_ID], + fields, }, }, }, @@ -120,9 +124,11 @@ export async function getLogCategories({ const promises = categorizedLogsRes.aggregations?.sampling.categories?.buckets.map( async ({ doc_count: docCount, key, sample }) => { - const hit = sample.hits.hits[0]._source as { message: string; trace?: { id: string } }; - const sampleMessage = hit?.message; - const sampleTraceId = hit?.trace?.id; + const hit = sample.hits.hits[0]; + const event = unflattenKnownApmEventFields(hit?.fields); + + const sampleMessage = event.message as string; + const sampleTraceId = event.trace?.id; const errorCategory = key as string; if (!sampleTraceId) { @@ -140,7 +146,9 @@ export async function getLogCategories({ } ); - const sampleDoc = categorizedLogsRes.hits.hits?.[0]?._source as Record; + const event = unflattenKnownApmEventFields(maybe(categorizedLogsRes.hits.hits[0])?.fields); + + const sampleDoc = event as Record; return { logCategories: await Promise.all(promises ?? []), diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts index e7f3ace07e2a1..93c55cf1a9a30 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts @@ -13,6 +13,10 @@ import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { CONTAINER_ID } from '@kbn/apm-types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -79,13 +83,17 @@ async function getContainerIdFromLogs({ esClient: ElasticsearchClient; logSourcesService: LogSourcesService; }) { + const requiredFields = asMutableArray([CONTAINER_ID] as const); const index = await logSourcesService.getFlattenedLogSources(); const res = await typedSearch<{ container: { id: string } }, any>(esClient, { index, ...params, + fields: requiredFields, }); - return res.hits.hits[0]?._source?.container?.id; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.container.id; } async function getContainerIdFromTraces({ @@ -95,6 +103,7 @@ async function getContainerIdFromTraces({ params: APMEventESSearchRequest['body']; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([CONTAINER_ID] as const); const res = await apmEventClient.search('get_container_id_from_traces', { apm: { sources: [ @@ -104,8 +113,10 @@ async function getContainerIdFromTraces({ }, ], }, - body: params, + body: { ...params, fields: requiredFields }, }); - return res.hits.hits[0]?._source.container?.id; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.container.id; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts index a5b75f76ff237..d957372285b02 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts @@ -6,6 +6,9 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { maybe } from '../../../../common/utils/maybe'; import { ApmDocumentType } from '../../../../common/document_type'; import { termQuery } from '../../../../common/utils/term_query'; import { @@ -27,6 +30,7 @@ export async function getDownstreamServiceResource({ end: number; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([SPAN_DESTINATION_SERVICE_RESOURCE] as const); const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { sources: [ @@ -50,9 +54,11 @@ export async function getDownstreamServiceResource({ ], }, }, + fields: requiredFields, }, }); - const hit = response.hits.hits[0]; - return hit?._source?.span.destination?.service.resource; + const event = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredFields); + + return event?.span.destination.service.resource; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts index 0168431d0ac4e..bd966c500d1bc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts @@ -12,6 +12,10 @@ import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { SERVICE_NAME } from '@kbn/apm-types'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -102,6 +106,7 @@ async function getServiceNameFromTraces({ params: APMEventESSearchRequest['body']; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([SERVICE_NAME] as const); const res = await apmEventClient.search('get_service_name_from_traces', { apm: { sources: [ @@ -111,8 +116,13 @@ async function getServiceNameFromTraces({ }, ], }, - body: params, + body: { + ...params, + fields: requiredFields, + }, }); - return res.hits.hits[0]?._source.service.name; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.service.name; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts index ebb3d3f57d18a..5b84743064142 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts @@ -7,8 +7,14 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { maybe } from '../../../common/utils/maybe'; -import { SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm'; +import { + SPAN_DESTINATION_SERVICE_RESOURCE, + SPAN_SUBTYPE, + SPAN_TYPE, +} from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; export interface MetadataForDependencyResponse { @@ -27,6 +33,7 @@ export async function getMetadataForDependency({ start: number; end: number; }): Promise { + const fields = asMutableArray([SPAN_TYPE, SPAN_SUBTYPE] as const); const sampleResponse = await apmEventClient.search('get_metadata_for_dependency', { apm: { events: [ProcessorEvent.span], @@ -46,16 +53,17 @@ export async function getMetadataForDependency({ ], }, }, + fields, sort: { '@timestamp': 'desc', }, }, }); - const sample = maybe(sampleResponse.hits.hits[0])?._source; + const sample = unflattenKnownApmEventFields(maybe(sampleResponse.hits.hits[0])?.fields); return { - spanType: sample?.span.type, - spanSubtype: sample?.span.subtype, + spanType: sample?.span?.type, + spanSubtype: sample?.span?.subtype, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts index 1c8448579806f..2a5a804d57f04 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts @@ -8,8 +8,11 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server'; import { keyBy } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, + AT_TIMESTAMP, EVENT_OUTCOME, SERVICE_ENVIRONMENT, SERVICE_NAME, @@ -66,6 +69,19 @@ export async function getTopDependencySpans({ sampleRangeFrom?: number; sampleRangeTo?: number; }): Promise { + const topDedsRequiredFields = asMutableArray([ + SPAN_ID, + TRACE_ID, + TRANSACTION_ID, + SPAN_NAME, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + AGENT_NAME, + SPAN_DURATION, + EVENT_OUTCOME, + AT_TIMESTAMP, + ] as const); + const spans = ( await apmEventClient.search('get_top_dependency_spans', { apm: { @@ -98,23 +114,18 @@ export async function getTopDependencySpans({ ], }, }, - _source: [ - SPAN_ID, - TRACE_ID, - TRANSACTION_ID, - SPAN_NAME, - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - SPAN_DURATION, - EVENT_OUTCOME, - '@timestamp', - ], + fields: topDedsRequiredFields, }, }) - ).hits.hits.map((hit) => hit._source); + ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, topDedsRequiredFields)); + + const transactionIds = spans.map((span) => span.transaction.id); - const transactionIds = spans.map((span) => span.transaction!.id); + const txRequiredFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_TYPE, + TRANSACTION_NAME, + ] as const); const transactions = ( await apmEventClient.search('get_transactions_for_dependency_spans', { @@ -129,13 +140,13 @@ export async function getTopDependencySpans({ filter: [...termsQuery(TRANSACTION_ID, ...transactionIds)], }, }, - _source: [TRANSACTION_ID, TRANSACTION_TYPE, TRANSACTION_NAME], + fields: txRequiredFields, sort: { '@timestamp': 'desc', }, }, }) - ).hits.hits.map((hit) => hit._source); + ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, txRequiredFields)); const transactionsById = keyBy(transactions, (transaction) => transaction.transaction.id); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts index 1e9576ea2f7e4..3d6fa0f5a5ef6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts @@ -7,13 +7,17 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery, wildcardQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, ERROR_CULPRIT, ERROR_EXC_HANDLED, ERROR_EXC_MESSAGE, ERROR_EXC_TYPE, ERROR_GROUP_ID, ERROR_GROUP_NAME, + ERROR_ID, ERROR_LOG_MESSAGE, SERVICE_NAME, TRACE_ID, @@ -93,6 +97,21 @@ export async function getErrorGroupMainStatistics({ ] : []; + const requiredFields = asMutableArray([ + TRACE_ID, + AT_TIMESTAMP, + ERROR_GROUP_ID, + ERROR_ID, + ] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { sources: [ @@ -129,16 +148,8 @@ export async function getErrorGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - TRACE_ID, - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { '@timestamp': 'desc', }, @@ -157,15 +168,33 @@ export async function getErrorGroupMainStatistics({ const errorGroups = response.aggregations?.error_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + }, + }; + return { groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]._source['@timestamp']).getTime(), + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]._source.error.culprit, - handled: bucket.sample.hits.hits[0]._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]._source.error.exception?.[0].type, - traceId: bucket.sample.hits.hits[0]._source.trace?.id, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, + traceId: mergedEvent.trace?.id, }; }) ?? []; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts index 0a154d3ad13fa..fc80c3f411651 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts @@ -6,6 +6,7 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ERROR_GROUP_ID, @@ -42,6 +43,7 @@ export async function getErrorGroupSampleIds({ start: number; end: number; }): Promise { + const requiredFields = asMutableArray([ERROR_ID] as const); const resp = await apmEventClient.search('get_error_group_sample_ids', { apm: { sources: [ @@ -66,7 +68,7 @@ export async function getErrorGroupSampleIds({ should: [{ term: { [TRANSACTION_SAMPLED]: true } }], // prefer error samples with related transactions }, }, - _source: [ERROR_ID, 'transaction'], + fields: requiredFields, sort: asMutableArray([ { _score: { order: 'desc' } }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error @@ -74,8 +76,8 @@ export async function getErrorGroupSampleIds({ }, }); const errorSampleIds = resp.hits.hits.map((item) => { - const source = item._source; - return source.error.id; + const event = unflattenKnownApmEventFields(item.fields, requiredFields); + return event.error?.id; }); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index 348949d3ecca5..91da19224d83c 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -6,7 +6,28 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; -import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { maybe } from '../../../../common/utils/maybe'; +import { + AGENT_NAME, + AGENT_VERSION, + AT_TIMESTAMP, + ERROR_EXCEPTION, + ERROR_GROUP_ID, + ERROR_ID, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + PROCESSOR_EVENT, + PROCESSOR_NAME, + SERVICE_NAME, + TIMESTAMP_US, + TRACE_ID, + TRANSACTION_ID, + ERROR_STACK_TRACE, + SPAN_ID, +} from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { ApmDocumentType } from '../../../../common/document_type'; import { RollupInterval } from '../../../../common/rollup'; @@ -17,7 +38,15 @@ import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; export interface ErrorSampleDetailsResponse { transaction: Transaction | undefined; - error: APMError; + error: Omit & { + transaction?: { id?: string; type?: string }; + error: { + id: string; + } & Omit & { + exception?: APMError['error']['exception']; + log?: APMError['error']['log']; + }; + }; } export async function getErrorSampleDetails({ @@ -36,7 +65,29 @@ export async function getErrorSampleDetails({ apmEventClient: APMEventClient; start: number; end: number; -}): Promise { +}): Promise> { + const requiredFields = asMutableArray([ + AGENT_NAME, + PROCESSOR_EVENT, + TRACE_ID, + TIMESTAMP_US, + AT_TIMESTAMP, + SERVICE_NAME, + ERROR_ID, + ERROR_GROUP_ID, + ] as const); + + const optionalFields = asMutableArray([ + TRANSACTION_ID, + SPAN_ID, + AGENT_VERSION, + PROCESSOR_NAME, + ERROR_STACK_TRACE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const params = { apm: { sources: [ @@ -60,15 +111,29 @@ export async function getErrorSampleDetails({ ], }, }, + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_EXCEPTION, 'error.log'], }, }; const resp = await apmEventClient.search('get_error_sample_details', params); - const error = resp.hits.hits[0]?._source; - const transactionId = error?.transaction?.id; - const traceId = error?.trace?.id; + const hit = maybe(resp.hits.hits[0]); + + if (!hit) { + return { + transaction: undefined, + error: undefined, + }; + } + + const source = 'error' in hit._source ? hit._source : undefined; - let transaction; + const errorFromFields = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const transactionId = errorFromFields.transaction?.id ?? errorFromFields.span?.id; + const traceId = errorFromFields.trace.id; + + let transaction: Transaction | undefined; if (transactionId && traceId) { transaction = await getTransaction({ transactionId, @@ -81,6 +146,20 @@ export async function getErrorSampleDetails({ return { transaction, - error, + error: { + ...errorFromFields, + processor: { + name: errorFromFields.processor.name as 'error', + event: errorFromFields.processor.event as 'error', + }, + error: { + ...errorFromFields.error, + exception: + (source?.error.exception?.length ?? 0) > 1 + ? source?.error.exception + : errorFromFields?.error.exception && [errorFromFields.error.exception], + log: source?.error?.log, + }, + }, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts index dd262246a80b7..62d9d883ba896 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts @@ -7,6 +7,7 @@ import { jsonRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; +import { notFound } from '@hapi/boom'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { ErrorDistributionResponse, getErrorDistribution } from './distribution/get_distribution'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; @@ -205,7 +206,7 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({ const { serviceName, errorId } = params.path; const { environment, kuery, start, end } = params.query; - return getErrorSampleDetails({ + const { transaction, error } = await getErrorSampleDetails({ environment, errorId, kuery, @@ -214,6 +215,12 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({ start, end, }); + + if (!error) { + throw notFound(); + } + + return { error, transaction }; }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts index b0d3eabe0bab2..c606a6b045a93 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts @@ -8,6 +8,8 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../../common/utils/as_mutable_array'; import { ERROR_CULPRIT, ERROR_TYPE, @@ -19,6 +21,7 @@ import { SERVICE_NAME, TRANSACTION_NAME, TRANSACTION_TYPE, + AT_TIMESTAMP, } from '../../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../../common/utils/environment_query'; import { getErrorName } from '../../../../lib/helpers/get_error_name'; @@ -68,6 +71,16 @@ export async function getMobileCrashGroupMainStatistics({ ? { [maxTimestampAggKey]: sortDirection } : { _count: sortDirection }; + const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_crash_group_main_statistics', { apm: { events: [ProcessorEvent.error], @@ -99,22 +112,15 @@ export async function getMobileCrashGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { - '@timestamp': 'desc', + [AT_TIMESTAMP]: 'desc', }, }, }, ...(sortByLatestOccurrence - ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } } + ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } } : {}), }, }, @@ -123,14 +129,34 @@ export async function getMobileCrashGroupMainStatistics({ }); return ( - response.aggregations?.crash_groups.buckets.map((bucket) => ({ - groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(), - occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]?._source.error.culprit, - handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type, - })) ?? [] + response.aggregations?.crash_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + }, + }; + + return { + groupId: event.error?.grouping_key, + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), + occurrences: bucket.doc_count, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, + }; + }) ?? [] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts index f259e17d6154c..1181aa5b02870 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts @@ -8,7 +8,10 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, ERROR_CULPRIT, ERROR_EXC_HANDLED, ERROR_EXC_MESSAGE, @@ -67,6 +70,16 @@ export async function getMobileErrorGroupMainStatistics({ ? { [maxTimestampAggKey]: sortDirection } : { _count: sortDirection }; + const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { events: [ProcessorEvent.error], @@ -100,22 +113,15 @@ export async function getMobileErrorGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { - '@timestamp': 'desc', + [AT_TIMESTAMP]: 'desc', }, }, }, ...(sortByLatestOccurrence - ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } } + ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } } : {}), }, }, @@ -124,14 +130,34 @@ export async function getMobileErrorGroupMainStatistics({ }); return ( - response.aggregations?.error_groups.buckets.map((bucket) => ({ - groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(), - occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]?._source.error.culprit, - handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type, - })) ?? [] + response.aggregations?.error_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + }, + }; + + return { + groupId: event.error?.grouping_key, + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), + occurrences: bucket.doc_count, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, + }; + }) ?? [] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts index e766d56c44ae4..c683e308b73b8 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts @@ -7,9 +7,12 @@ import type { ESFilter } from '@kbn/es-types'; import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { isFiniteNumber } from '../../../../common/utils/is_finite_number'; import { Annotation, AnnotationType } from '../../../../common/annotations'; -import { SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm'; +import { AT_TIMESTAMP, SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { getBackwardCompatibleDocumentTypeFilter, @@ -66,6 +69,8 @@ export async function getDerivedServiceAnnotations({ if (versions.length <= 1) { return []; } + + const requiredFields = asMutableArray([AT_TIMESTAMP] as const); const annotations = await Promise.all( versions.map(async (version) => { const response = await apmEventClient.search('get_first_seen_of_version', { @@ -83,11 +88,21 @@ export async function getDerivedServiceAnnotations({ sort: { '@timestamp': 'asc', }, + fields: requiredFields, }, }); - const firstSeen = new Date(response.hits.hits[0]._source['@timestamp']).getTime(); + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields, + requiredFields + ); + + const timestamp = event?.[AT_TIMESTAMP]; + if (!timestamp) { + throw new Error('First seen for version was unexpectedly undefined or null.'); + } + const firstSeen = new Date(timestamp).getTime(); if (!isFiniteNumber(firstSeen)) { throw new Error('First seen for version was unexpectedly undefined or null.'); } @@ -99,7 +114,7 @@ export async function getDerivedServiceAnnotations({ return { type: AnnotationType.VERSION, id: version, - '@timestamp': firstSeen, + [AT_TIMESTAMP]: firstSeen, text: version, }; }) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts index 94c5bcbac4e66..dd272eadf57d6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts @@ -7,6 +7,8 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, SERVICE_NAME, @@ -16,23 +18,7 @@ import { } from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getServerlessTypeFromCloudData, ServerlessType } from '../../../common/serverless'; - -interface ServiceAgent { - agent?: { - name: string; - }; - service?: { - runtime?: { - name?: string; - }; - }; - cloud?: { - provider?: string; - service?: { - name?: string; - }; - }; -} +import { maybe } from '../../../common/utils/maybe'; export interface ServiceAgentResponse { agentName?: string; @@ -51,6 +37,13 @@ export async function getServiceAgent({ start: number; end: number; }): Promise { + const fields = asMutableArray([ + AGENT_NAME, + SERVICE_RUNTIME_NAME, + CLOUD_PROVIDER, + CLOUD_SERVICE_NAME, + ] as const); + const params = { terminate_after: 1, apm: { @@ -90,6 +83,7 @@ export async function getServiceAgent({ ], }, }, + fields, sort: { _score: { order: 'desc' as const }, }, @@ -97,11 +91,14 @@ export async function getServiceAgent({ }; const response = await apmEventClient.search('get_service_agent_name', params); - if (response.hits.total.value === 0) { + const hit = maybe(response.hits.hits[0]); + if (!hit) { return {}; } - const { agent, service, cloud } = response.hits.hits[0]._source as ServiceAgent; + const event = unflattenKnownApmEventFields(hit.fields); + + const { agent, service, cloud } = event; const serverlessType = getServerlessTypeFromCloudData(cloud?.provider, cloud?.service?.name); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts index 400429617d803..d16910f5984fc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts @@ -6,19 +6,20 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { CONTAINER_ID, CONTAINER_IMAGE, KUBERNETES, KUBERNETES_POD_NAME, KUBERNETES_POD_UID, -} from '../../../common/es_fields/apm'; -import { KUBERNETES_CONTAINER_NAME, - KUBERNETES_NAMESPACE, KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, -} from '../../../common/es_fields/infra_metrics'; + KUBERNETES_CONTAINER_ID, + KUBERNETES_NAMESPACE, +} from '../../../common/es_fields/apm'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { maybe } from '../../../common/utils/maybe'; import { InfraMetricsClient } from '../../lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client'; @@ -51,9 +52,21 @@ export const getServiceInstanceContainerMetadata = async ({ { exists: { field: KUBERNETES_DEPLOYMENT_NAME } }, ]; + const fields = asMutableArray([ + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + ] as const); + const response = await infraMetricsClient.search({ size: 1, track_total_hits: false, + fields, query: { bool: { filter: [ @@ -69,7 +82,7 @@ export const getServiceInstanceContainerMetadata = async ({ }, }); - const sample = maybe(response.hits.hits[0])?._source as ServiceInstanceContainerMetadataDetails; + const sample = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields); return { kubernetes: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts index daa49d2ed59c8..3c139f2aee0de 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts @@ -7,7 +7,16 @@ import { merge } from 'lodash'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; +import { + AGENT_NAME, + AT_TIMESTAMP, + METRICSET_NAME, + SERVICE_ENVIRONMENT, + SERVICE_NAME, + SERVICE_NODE_NAME, +} from '../../../common/es_fields/apm'; import { maybe } from '../../../common/utils/maybe'; import { getBackwardCompatibleDocumentTypeFilter, @@ -20,6 +29,13 @@ import { Container } from '../../../typings/es_schemas/raw/fields/container'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { Host } from '../../../typings/es_schemas/raw/fields/host'; import { Cloud } from '../../../typings/es_schemas/raw/fields/cloud'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; +import { + SERVICE_METADATA_CLOUD_KEYS, + SERVICE_METADATA_CONTAINER_KEYS, + SERVICE_METADATA_INFRA_METRICS_KEYS, + SERVICE_METADATA_SERVICE_KEYS, +} from '../../../common/service_metadata'; export interface ServiceInstanceMetadataDetailsResponse { '@timestamp': string; @@ -50,6 +66,18 @@ export async function getServiceInstanceMetadataDetails({ ...rangeQuery(start, end), ]; + const requiredKeys = asMutableArray([AT_TIMESTAMP, SERVICE_NAME, AGENT_NAME] as const); + + const optionalKeys = asMutableArray([ + SERVICE_ENVIRONMENT, + ...SERVICE_METADATA_SERVICE_KEYS, + ...SERVICE_METADATA_CLOUD_KEYS, + ...SERVICE_METADATA_CONTAINER_KEYS, + ...SERVICE_METADATA_INFRA_METRICS_KEYS, + ] as const); + + const fields = [...requiredKeys, ...optionalKeys]; + async function getApplicationMetricSample() { const response = await apmEventClient.search( 'get_service_instance_metadata_details_application_metric', @@ -66,11 +94,12 @@ export async function getServiceInstanceMetadataDetails({ filter: filter.concat({ term: { [METRICSET_NAME]: 'app' } }), }, }, + fields, }, } ); - return maybe(response.hits.hits[0]?._source); + return unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredKeys); } async function getTransactionEventSample() { @@ -85,11 +114,14 @@ export async function getServiceInstanceMetadataDetails({ terminate_after: 1, size: 1, query: { bool: { filter } }, + fields, }, } ); - return maybe(response.hits.hits[0]?._source); + return unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); } async function getTransactionMetricSample() { @@ -108,10 +140,14 @@ export async function getServiceInstanceMetadataDetails({ filter: filter.concat(getBackwardCompatibleDocumentTypeFilter(true)), }, }, + fields, }, } ); - return maybe(response.hits.hits[0]?._source); + + return unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); } // we can expect the most detail of application metrics, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index fb44638d8a6b0..0319ae66039e5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -7,37 +7,26 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; import { environmentQuery } from '../../../common/utils/environment_query'; import { - AGENT, - CONTAINER, - CLOUD, CLOUD_AVAILABILITY_ZONE, CLOUD_REGION, CLOUD_MACHINE_TYPE, CLOUD_SERVICE_NAME, CONTAINER_ID, - HOST, - KUBERNETES, - SERVICE, SERVICE_NAME, SERVICE_NODE_NAME, SERVICE_VERSION, FAAS_ID, FAAS_TRIGGER_TYPE, - LABEL_TELEMETRY_AUTO_VERSION, } from '../../../common/es_fields/apm'; - import { ContainerType } from '../../../common/service_metadata'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { should } from './get_service_metadata_icons'; import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name'; - -type ServiceMetadataDetailsRaw = Pick< - TransactionRaw, - 'service' | 'agent' | 'host' | 'container' | 'kubernetes' | 'cloud' | 'labels' ->; +import { maybe } from '../../../common/utils/maybe'; export interface ServiceMetadataDetails { service?: { @@ -112,7 +101,6 @@ export async function getServiceMetadataDetails({ body: { track_total_hits: 1, size: 1, - _source: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION], query: { bool: { filter, should } }, aggs: { serviceVersions: { @@ -166,13 +154,17 @@ export async function getServiceMetadataDetails({ }, totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, + fields: ['*'], }, }; const response = await apmEventClient.search('get_service_metadata_details', params); - const hit = response.hits.hits[0]?._source as ServiceMetadataDetailsRaw | undefined; - if (!hit) { + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); + + if (!event) { return { service: undefined, container: undefined, @@ -180,7 +172,7 @@ export async function getServiceMetadataDetails({ }; } - const { service, agent, host, kubernetes, container, cloud, labels } = hit; + const { service, agent, host, kubernetes, container, cloud, labels } = event; const serviceMetadataDetails = { versions: response.aggregations?.serviceVersions.buckets.map((bucket) => bucket.key as string), diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 30580ddbf0ac8..ee0a857c9b719 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -7,12 +7,15 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; +import { maybe } from '../../../common/utils/maybe'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, CLOUD_PROVIDER, CLOUD_SERVICE_NAME, CONTAINER_ID, - KUBERNETES, SERVICE_NAME, KUBERNETES_POD_NAME, HOST_OS_PLATFORM, @@ -20,14 +23,11 @@ import { AGENT_VERSION, SERVICE_FRAMEWORK_NAME, } from '../../../common/es_fields/apm'; -import { ContainerType } from '../../../common/service_metadata'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; +import { ContainerType, SERVICE_METADATA_KUBERNETES_KEYS } from '../../../common/service_metadata'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless'; -type ServiceMetadataIconsRaw = Pick; - export interface ServiceMetadataIcons { agentName?: string; containerType?: ContainerType; @@ -61,6 +61,14 @@ export async function getServiceMetadataIcons({ }): Promise { const filter = [{ term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end)]; + const fields = asMutableArray([ + CLOUD_PROVIDER, + CONTAINER_ID, + AGENT_NAME, + CLOUD_SERVICE_NAME, + ...SERVICE_METADATA_KUBERNETES_KEYS, + ] as const); + const params = { apm: { events: [ @@ -72,8 +80,8 @@ export async function getServiceMetadataIcons({ body: { track_total_hits: 1, size: 1, - _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME], query: { bool: { filter, should } }, + fields, }, }; @@ -88,9 +96,11 @@ export async function getServiceMetadataIcons({ }; } - const { kubernetes, cloud, container, agent } = response.hits.hits[0] - ._source as ServiceMetadataIconsRaw; + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); + const { kubernetes, cloud, container, agent } = event ?? {}; let containerType: ContainerType; if (!!kubernetes) { containerType = 'Kubernetes'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts index 3f9cf1275cace..2ff34698c20bc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts @@ -7,6 +7,8 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { isEmpty } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { PROCESSOR_EVENT, SPAN_ID, @@ -16,8 +18,6 @@ import { TRACE_ID, TRANSACTION_ID, } from '../../../common/es_fields/apm'; -import type { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; -import type { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getBufferedTimerange } from './utils'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -39,12 +39,16 @@ async function fetchLinkedChildrenOfSpan({ end, }); + const requiredFields = asMutableArray([TRACE_ID, PROCESSOR_EVENT] as const); + const optionalFields = asMutableArray([SPAN_ID, TRANSACTION_ID] as const); + const response = await apmEventClient.search('fetch_linked_children_of_span', { apm: { events: [ProcessorEvent.span, ProcessorEvent.transaction], }, - _source: [SPAN_LINKS, TRACE_ID, SPAN_ID, PROCESSOR_EVENT, TRANSACTION_ID], + _source: [SPAN_LINKS], body: { + fields: [...requiredFields, ...optionalFields], track_total_hits: false, size: 1000, query: { @@ -58,19 +62,32 @@ async function fetchLinkedChildrenOfSpan({ }, }, }); + + const linkedChildren = response.hits.hits.map((hit) => { + const source = 'span' in hit._source ? hit._source : undefined; + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + return { + ...event, + span: { + ...event.span, + links: source?.span?.links ?? [], + }, + }; + }); // Filter out documents that don't have any span.links that match the combination of traceId and spanId - return response.hits.hits.filter(({ _source: source }) => { - const spanLinks = source.span?.links?.filter((spanLink) => { + return linkedChildren.filter((linkedChild) => { + const spanLinks = linkedChild?.span?.links?.filter((spanLink) => { return spanLink.trace.id === traceId && (spanId ? spanLink.span.id === spanId : true); }); return !isEmpty(spanLinks); }); } -function getSpanId(source: TransactionRaw | SpanRaw) { - return source.processor.event === ProcessorEvent.span - ? (source as SpanRaw).span.id - : (source as TransactionRaw).transaction?.id; +function getSpanId( + linkedChild: Awaited>[number] +): string { + return (linkedChild.span.id ?? linkedChild.transaction?.id) as string; } export async function getSpanLinksCountById({ @@ -90,8 +107,9 @@ export async function getSpanLinksCountById({ start, end, }); - return linkedChildren.reduce>((acc, { _source: source }) => { - source.span?.links?.forEach((link) => { + + return linkedChildren.reduce>((acc, item) => { + item.span?.links?.forEach((link) => { // Ignores span links that don't belong to this trace if (link.trace.id === traceId) { acc[link.span.id] = (acc[link.span.id] || 0) + 1; @@ -122,10 +140,10 @@ export async function getLinkedChildrenOfSpan({ end, }); - return linkedChildren.map(({ _source: source }) => { + return linkedChildren.map((item) => { return { - trace: { id: source.trace.id }, - span: { id: getSpanId(source) }, + trace: { id: item.trace.id }, + span: { id: getSpanId(item) }, }; }); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts index 2010cd5e86f2f..59e91e0b17e6b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts @@ -56,7 +56,7 @@ export async function getLinkedParentsOfSpan({ }, }); - const source = response.hits.hits?.[0]?._source as TransactionRaw | SpanRaw; + const source = response.hits.hits?.[0]?._source as Pick; return source?.span?.links || []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts index 13f47764af375..669adb1008080 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts @@ -7,6 +7,8 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { chunk, compact, isEmpty, keyBy } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { SERVICE_NAME, SPAN_ID, @@ -25,8 +27,6 @@ import { import { Environment } from '../../../common/environment_rt'; import { SpanLinkDetails } from '../../../common/span_links'; import { SpanLink } from '../../../typings/es_schemas/raw/fields/span_links'; -import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getBufferedTimerange } from './utils'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -48,26 +48,35 @@ async function fetchSpanLinksDetails({ end, }); + const requiredFields = asMutableArray([ + TRACE_ID, + SERVICE_NAME, + AGENT_NAME, + PROCESSOR_EVENT, + ] as const); + + const requiredTxFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_DURATION, + ] as const); + + const requiredSpanFields = asMutableArray([ + SPAN_ID, + SPAN_NAME, + SPAN_DURATION, + SPAN_SUBTYPE, + SPAN_TYPE, + ] as const); + + const optionalFields = asMutableArray([SERVICE_ENVIRONMENT] as const); + const response = await apmEventClient.search('get_span_links_details', { apm: { events: [ProcessorEvent.span, ProcessorEvent.transaction], }, - _source: [ - TRACE_ID, - SPAN_ID, - TRANSACTION_ID, - SERVICE_NAME, - SPAN_NAME, - TRANSACTION_NAME, - TRANSACTION_DURATION, - SPAN_DURATION, - PROCESSOR_EVENT, - SPAN_SUBTYPE, - SPAN_TYPE, - AGENT_NAME, - SERVICE_ENVIRONMENT, - ], body: { + fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields], track_total_hits: false, size: 1000, query: { @@ -106,16 +115,67 @@ async function fetchSpanLinksDetails({ const spanIdsMap = keyBy(spanLinks, 'span.id'); - return response.hits.hits.filter(({ _source: source }) => { - // The above query might return other spans from the same transaction because siblings spans share the same transaction.id - // so, if it is a span we need to guarantee that the span.id is the same as the span links ids - if (source.processor.event === ProcessorEvent.span) { - const span = source as SpanRaw; - const hasSpanId = spanIdsMap[span.span.id] || false; - return hasSpanId; - } - return true; - }); + return response.hits.hits + .filter((hit) => { + // The above query might return other spans from the same transaction because siblings spans share the same transaction.id + // so, if it is a span we need to guarantee that the span.id is the same as the span links ids + if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) { + const spanLink = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + const hasSpanId = Boolean(spanIdsMap[spanLink.span.id] || false); + return hasSpanId; + } + return true; + }) + .map((hit) => { + const commonEvent = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const commonDetails = { + serviceName: commonEvent.service.name, + agentName: commonEvent.agent.name, + environment: commonEvent.service.environment as Environment, + transactionId: commonEvent.transaction?.id, + }; + + if (commonEvent.processor.event === ProcessorEvent.transaction) { + const event = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredTxFields, + ]); + return { + traceId: event.trace.id, + spanId: event.transaction.id, + processorEvent: commonEvent.processor.event, + transactionId: event.transaction.id, + details: { + ...commonDetails, + spanName: event.transaction.name, + duration: event.transaction.duration.us, + }, + }; + } else { + const event = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + return { + traceId: event.trace.id, + spanId: event.span.id, + processorEvent: commonEvent.processor.event, + details: { + ...commonDetails, + spanName: event.span.name, + duration: event.span.duration.us, + spanSubtype: event.span.subtype, + spanType: event.span.type, + }, + }; + } + }); } export async function getSpanLinksDetails({ @@ -153,39 +213,20 @@ export async function getSpanLinksDetails({ // Creates a map for all span links details found const spanLinksDetailsMap = linkedSpans.reduce>( - (acc, { _source: source }) => { - const commonDetails = { - serviceName: source.service.name, - agentName: source.agent.name, - environment: source.service.environment as Environment, - transactionId: source.transaction?.id, - }; - - if (source.processor.event === ProcessorEvent.transaction) { - const transaction = source as TransactionRaw; - const key = `${transaction.trace.id}:${transaction.transaction.id}`; + (acc, spanLink) => { + if (spanLink.processorEvent === ProcessorEvent.transaction) { + const key = `${spanLink.traceId}:${spanLink.transactionId}`; acc[key] = { - traceId: source.trace.id, - spanId: transaction.transaction.id, - details: { - ...commonDetails, - spanName: transaction.transaction.name, - duration: transaction.transaction.duration.us, - }, + traceId: spanLink.traceId, + spanId: spanLink.transactionId, + details: spanLink.details, }; } else { - const span = source as SpanRaw; - const key = `${span.trace.id}:${span.span.id}`; + const key = `${spanLink.traceId}:${spanLink.spanId}`; acc[key] = { - traceId: source.trace.id, - spanId: span.span.id, - details: { - ...commonDetails, - spanName: span.span.name, - duration: span.span.duration.us, - spanSubtype: span.span.subtype, - spanType: span.span.type, - }, + traceId: spanLink.traceId, + spanId: spanLink.spanId, + details: spanLink.details, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap index d64c33a421e19..58ec97112e8f6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap @@ -12,15 +12,26 @@ Object { }, "body": Object { "_source": Array [ + "error.log.message", + "error.exception.message", + "error.exception.handled", + "error.exception.type", + ], + "fields": Array [ "timestamp.us", "trace.id", - "transaction.id", - "parent.id", "service.name", "error.id", - "error.log.message", - "error.exception", "error.grouping_key", + "processor.event", + "parent.id", + "transaction.id", + "span.id", + "error.culprit", + "error.log.message", + "error.exception.message", + "error.exception.handled", + "error.exception.type", ], "query": Object { "bool": Object { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index d38a49745653a..55fb0aab47f38 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -10,12 +10,17 @@ import { SortResults } from '@elastic/elasticsearch/lib/api/types'; import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { last } from 'lodash'; +import { last, omit } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { APMConfig } from '../..'; import { AGENT_NAME, CHILD_ID, - ERROR_EXCEPTION, + ERROR_CULPRIT, + ERROR_EXC_HANDLED, + ERROR_EXC_MESSAGE, + ERROR_EXC_TYPE, ERROR_GROUP_ID, ERROR_ID, ERROR_LOG_LEVEL, @@ -37,7 +42,7 @@ import { SPAN_SUBTYPE, SPAN_SYNC, SPAN_TYPE, - TIMESTAMP, + TIMESTAMP_US, TRACE_ID, TRANSACTION_DURATION, TRANSACTION_ID, @@ -84,6 +89,26 @@ export async function getTraceItems({ const maxTraceItems = maxTraceItemsFromUrlParam ?? config.ui.maxTraceItems; const excludedLogLevels = ['debug', 'info', 'warning']; + const requiredFields = asMutableArray([ + TIMESTAMP_US, + TRACE_ID, + SERVICE_NAME, + ERROR_ID, + ERROR_GROUP_ID, + PROCESSOR_EVENT, + ] as const); + + const optionalFields = asMutableArray([ + PARENT_ID, + TRANSACTION_ID, + SPAN_ID, + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const errorResponsePromise = apmEventClient.search('get_errors_docs', { apm: { sources: [ @@ -96,23 +121,14 @@ export async function getTraceItems({ body: { track_total_hits: false, size: 1000, - _source: [ - TIMESTAMP, - TRACE_ID, - TRANSACTION_ID, - PARENT_ID, - SERVICE_NAME, - ERROR_ID, - ERROR_LOG_MESSAGE, - ERROR_EXCEPTION, - ERROR_GROUP_ID, - ], query: { bool: { filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)], must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, }, }, + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], }, }); @@ -133,8 +149,32 @@ export async function getTraceItems({ const traceDocsTotal = traceResponse.total; const exceedsMax = traceDocsTotal > maxTraceItems; - const traceDocs = traceResponse.hits.map((hit) => hit._source); - const errorDocs = errorResponse.hits.hits.map((hit) => hit._source); + + const traceDocs = traceResponse.hits.map(({ hit }) => hit); + + const errorDocs = errorResponse.hits.hits.map((hit) => { + const errorSource = 'error' in hit._source ? hit._source : undefined; + + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const waterfallErrorEvent: WaterfallError = { + ...event, + parent: { + ...event?.parent, + id: event?.parent?.id ?? event?.span?.id, + }, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + log: errorSource?.error.log, + }, + }; + + return waterfallErrorEvent; + }); return { exceedsMax, @@ -220,41 +260,54 @@ async function getTraceDocsPerPage({ start: number; end: number; searchAfter?: SortResults; -}) { +}): Promise<{ + hits: Array<{ hit: WaterfallTransaction | WaterfallSpan; sort: SortResults | undefined }>; + total: number; +}> { const size = Math.min(maxTraceItems, MAX_ITEMS_PER_PAGE); + const requiredFields = asMutableArray([ + AGENT_NAME, + TIMESTAMP_US, + TRACE_ID, + SERVICE_NAME, + PROCESSOR_EVENT, + ] as const); + + const requiredTxFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_TYPE, + ] as const); + + const requiredSpanFields = asMutableArray([ + SPAN_ID, + SPAN_TYPE, + SPAN_NAME, + SPAN_DURATION, + ] as const); + + const optionalFields = asMutableArray([ + PARENT_ID, + SERVICE_ENVIRONMENT, + EVENT_OUTCOME, + TRANSACTION_RESULT, + FAAS_COLDSTART, + SPAN_SUBTYPE, + SPAN_ACTION, + SPAN_COMPOSITE_COUNT, + SPAN_COMPOSITE_COMPRESSION_STRATEGY, + SPAN_COMPOSITE_SUM, + SPAN_SYNC, + CHILD_ID, + ] as const); + const body = { track_total_hits: true, size, search_after: searchAfter, - _source: [ - TIMESTAMP, - TRACE_ID, - PARENT_ID, - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - EVENT_OUTCOME, - PROCESSOR_EVENT, - TRANSACTION_DURATION, - TRANSACTION_ID, - TRANSACTION_NAME, - TRANSACTION_TYPE, - TRANSACTION_RESULT, - FAAS_COLDSTART, - SPAN_ID, - SPAN_TYPE, - SPAN_SUBTYPE, - SPAN_ACTION, - SPAN_NAME, - SPAN_DURATION, - SPAN_LINKS, - SPAN_COMPOSITE_COUNT, - SPAN_COMPOSITE_COMPRESSION_STRATEGY, - SPAN_COMPOSITE_SUM, - SPAN_SYNC, - CHILD_ID, - ], + _source: [SPAN_LINKS], query: { bool: { filter: [ @@ -266,6 +319,7 @@ async function getTraceDocsPerPage({ }, }, }, + fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields], sort: [ { _score: 'asc' }, { @@ -291,7 +345,51 @@ async function getTraceDocsPerPage({ }); return { - hits: res.hits.hits, + hits: res.hits.hits.map((hit) => { + const sort = hit.sort; + const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined; + + if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) { + const spanEvent = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + const spanWaterfallEvent: WaterfallSpan = { + ...omit(spanEvent, 'child'), + processor: { + event: 'span', + }, + span: { + ...spanEvent.span, + composite: spanEvent.span.composite + ? (spanEvent.span.composite as Required['composite']) + : undefined, + links: spanLinksSource, + }, + ...(spanEvent.child ? { child: spanEvent.child as WaterfallSpan['child'] } : {}), + }; + + return { sort, hit: spanWaterfallEvent }; + } + + const txEvent = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredTxFields, + ]); + const txWaterfallEvent: WaterfallTransaction = { + ...txEvent, + processor: { + event: 'transaction', + }, + span: { + ...txEvent.span, + links: spanLinksSource, + }, + }; + + return { hit: txWaterfallEvent, sort }; + }), total: res.hits.total.value, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts index 033b666d0371c..dd1330dea4e48 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts @@ -89,10 +89,10 @@ export async function getTraceSamplesByQuery({ }, event_category_field: PROCESSOR_EVENT, query, - filter_path: 'hits.sequences.events._source.trace.id', + fields: [TRACE_ID], }) ).hits?.sequences?.flatMap((sequence) => - sequence.events.map((event) => (event._source as { trace: { id: string } }).trace.id) + sequence.events.map((event) => (event.fields as { [TRACE_ID]: [string] })[TRACE_ID][0]) ) ?? []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts index 328201ec9d143..0814bcdc5738f 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts @@ -12,7 +12,10 @@ import { getSearchTransactionsEvents } from '../../lib/helpers/transactions'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, probabilityRt, rangeRt } from '../default_api_types'; import { getTransaction } from '../transactions/get_transaction'; -import { getRootTransactionByTraceId } from '../transactions/get_transaction_by_trace'; +import { + type TransactionDetailRedirectInfo, + getRootTransactionByTraceId, +} from '../transactions/get_transaction_by_trace'; import { getTopTracesPrimaryStats, TopTracesPrimaryStatsResponse, @@ -128,7 +131,7 @@ const rootTransactionByTraceIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction?: TransactionDetailRedirectInfo; }> => { const { params: { @@ -155,7 +158,7 @@ const transactionByIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction?: Transaction; }> => { const { params: { @@ -191,7 +194,7 @@ const transactionByNameRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction?: TransactionDetailRedirectInfo; }> => { const { params: { @@ -295,7 +298,7 @@ const transactionFromTraceByIdRoute = createApmServerRoute({ query: rangeRt, }), options: { tags: ['access:apm'] }, - handler: async (resources): Promise => { + handler: async (resources): Promise => { const { params } = resources; const { path: { transactionId, traceId }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap index deb1dec096f08..c7f832fe5ca65 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap @@ -11,6 +11,25 @@ Object { ], }, "body": Object { + "_source": Array [ + "span.links", + "transaction.agent.marks", + ], + "fields": Array [ + "trace.id", + "agent.name", + "processor.event", + "@timestamp", + "timestamp.us", + "service.name", + "transaction.id", + "transaction.duration.us", + "transaction.name", + "transaction.sampled", + "transaction.type", + "processor.name", + "service.language.name", + ], "query": Object { "bool": Object { "filter": Array [ @@ -311,6 +330,11 @@ Object { ], }, "body": Object { + "fields": Array [ + "transaction.id", + "trace.id", + "@timestamp", + ], "query": Object { "bool": Object { "filter": Array [ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts index 4e1b7020a98a6..dc8ab0a6aab19 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts @@ -7,7 +7,11 @@ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { SPAN_ID, TRACE_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; +import { merge, omit } from 'lodash'; +import { maybe } from '../../../../common/utils/maybe'; +import { SPAN_ID, SPAN_STACKTRACE, TRACE_ID } from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getTransaction } from '../get_transaction'; @@ -38,6 +42,8 @@ export async function getSpan({ track_total_hits: false, size: 1, terminate_after: 1, + fields: ['*'], + _source: [SPAN_STACKTRACE], query: { bool: { filter: asMutableArray([ @@ -60,5 +66,17 @@ export async function getSpan({ : undefined, ]); - return { span: spanResp.hits.hits[0]?._source, parentTransaction }; + const hit = maybe(spanResp.hits.hits[0]); + const spanFromSource = hit && 'span' in hit._source ? hit._source : undefined; + + const event = unflattenKnownApmEventFields(hit?.fields as undefined | FlattenedApmEvent); + + return { + span: event + ? merge({}, omit(event, 'span.links'), spanFromSource, { + processor: { event: 'span' as const, name: 'transaction' as const }, + }) + : undefined, + parentTransaction, + }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index 8854f3075e59b..8fc9d93ceff87 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -6,7 +6,26 @@ */ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; -import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { Transaction } from '@kbn/apm-types'; +import { maybe } from '../../../../common/utils/maybe'; +import { + AGENT_NAME, + PROCESSOR_EVENT, + SERVICE_NAME, + TIMESTAMP_US, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + AT_TIMESTAMP, + PROCESSOR_NAME, + SPAN_LINKS, + TRANSACTION_AGENT_MARKS, + SERVICE_LANGUAGE_NAME, +} from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../../common/document_type'; @@ -24,7 +43,23 @@ export async function getTransaction({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise { + const requiredFields = asMutableArray([ + TRACE_ID, + AGENT_NAME, + PROCESSOR_EVENT, + AT_TIMESTAMP, + TIMESTAMP_US, + SERVICE_NAME, + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + ] as const); + + const optionalFields = asMutableArray([PROCESSOR_NAME, SERVICE_LANGUAGE_NAME] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -47,8 +82,37 @@ export async function getTransaction({ ]), }, }, + fields: [...requiredFields, ...optionalFields], + _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS], }, }); - return resp.hits.hits[0]?._source; + const hit = maybe(resp.hits.hits[0]); + + if (!hit) { + return undefined; + } + + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const source = + 'span' in hit._source && 'transaction' in hit._source + ? (hit._source as { + transaction: Pick['transaction'], 'marks'>; + span?: Pick['span'], 'links'>; + }) + : undefined; + + return { + ...event, + transaction: { + ...event.transaction, + marks: source?.transaction.marks, + }, + processor: { + name: 'transaction', + event: 'transaction', + }, + span: source?.span, + }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts index 75b4655d70117..160e3f736580a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts @@ -6,11 +6,22 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; import { ApmDocumentType } from '../../../../common/document_type'; -import { SERVICE_NAME, TRANSACTION_NAME } from '../../../../common/es_fields/apm'; +import { + AT_TIMESTAMP, + SERVICE_NAME, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/es_fields/apm'; import { RollupInterval } from '../../../../common/rollup'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +import { TransactionDetailRedirectInfo } from '../get_transaction_by_trace'; export async function getTransactionByName({ transactionName, @@ -24,7 +35,17 @@ export async function getTransactionByName({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise { + const requiredFields = asMutableArray([ + AT_TIMESTAMP, + TRACE_ID, + TRANSACTION_ID, + TRANSACTION_TYPE, + TRANSACTION_NAME, + TRANSACTION_DURATION, + SERVICE_NAME, + ] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -47,8 +68,9 @@ export async function getTransactionByName({ ]), }, }, + fields: requiredFields, }, }); - return resp.hits.hits[0]?._source; + return unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts index d27be0489f8da..803ae19a2228e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts @@ -7,9 +7,40 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { TRACE_ID, PARENT_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { + TRACE_ID, + PARENT_ID, + AT_TIMESTAMP, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, + SERVICE_NAME, +} from '../../../../common/es_fields/apm'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +export interface TransactionDetailRedirectInfo { + [AT_TIMESTAMP]: string; + trace: { + id: string; + }; + transaction: { + id: string; + type: string; + name: string; + + duration: { + us: number; + }; + }; + service: { + name: string; + }; +} + export async function getRootTransactionByTraceId({ traceId, apmEventClient, @@ -20,7 +51,19 @@ export async function getRootTransactionByTraceId({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise<{ + transaction: TransactionDetailRedirectInfo | undefined; +}> { + const requiredFields = asMutableArray([ + TRACE_ID, + TRANSACTION_ID, + TRANSACTION_NAME, + AT_TIMESTAMP, + TRANSACTION_TYPE, + TRANSACTION_DURATION, + SERVICE_NAME, + ] as const); + const params = { apm: { events: [ProcessorEvent.transaction as const], @@ -45,11 +88,15 @@ export async function getRootTransactionByTraceId({ filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)], }, }, + fields: requiredFields, }, }; const resp = await apmEventClient.search('get_root_transaction_by_trace_id', params); + + const event = unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields); + return { - transaction: resp.hits.hits[0]?._source, + transaction: event, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts index 191250d3781ee..18dad19635333 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts @@ -7,7 +7,10 @@ import { Sort, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, SERVICE_NAME, TRACE_ID, TRANSACTION_ID, @@ -77,6 +80,8 @@ export async function getTraceSamples({ }); } + const requiredFields = asMutableArray([TRANSACTION_ID, TRACE_ID, AT_TIMESTAMP] as const); + const response = await apmEventClient.search('get_trace_samples_hits', { apm: { events: [ProcessorEvent.transaction], @@ -94,6 +99,7 @@ export async function getTraceSamples({ }, }, size: TRACE_SAMPLES_SIZE, + fields: requiredFields, sort: [ { _score: { @@ -101,7 +107,7 @@ export async function getTraceSamples({ }, }, { - '@timestamp': { + [AT_TIMESTAMP]: { order: 'desc', }, }, @@ -109,12 +115,15 @@ export async function getTraceSamples({ }, }); - const traceSamples = response.hits.hits.map((hit) => ({ - score: hit._score, - timestamp: hit._source['@timestamp'], - transactionId: hit._source.transaction.id, - traceId: hit._source.trace.id, - })); + const traceSamples = response.hits.hits.map((hit) => { + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + return { + score: hit._score, + timestamp: event[AT_TIMESTAMP], + transactionId: event.transaction.id, + traceId: event.trace.id, + }; + }); return { traceSamples }; }); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts index 2fac072a8cdb5..71ceac4002919 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts @@ -15,3 +15,4 @@ export { } from './lib/helpers'; export { withApmSpan } from './utils/with_apm_span'; +export { unflattenKnownApmEventFields } from './utils/unflatten_known_fields'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts new file mode 100644 index 0000000000000..9dc861a5292df --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { unflattenKnownApmEventFields } from './unflatten_known_fields'; + +describe('unflattenKnownApmEventFields', () => { + it('should return an empty object when input is empty', () => { + const input = {}; + const expectedOutput = {}; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten a simple flat input', () => { + const input = { + '@timestamp': '2024-10-10T10:10:10.000Z', + }; + const expectedOutput = { + '@timestamp': '2024-10-10T10:10:10.000Z', + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should override unknown fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.name.text': 'node-svc', + }; + const expectedOutput = { + service: { + name: 'node-svc', + }, + }; + + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten multiple nested fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.version': '1.0.0', + 'service.environment': 'production', + 'agent.name': 'nodejs', + }; + const expectedOutput = { + service: { + name: 'node-svc', + version: '1.0.0', + environment: 'production', + }, + agent: { + name: 'nodejs', + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should handle multiple values for multi-valued fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.tags': ['foo', 'bar'], + }; + const expectedOutput = { + service: { + name: 'node-svc', + tags: ['foo', 'bar'], + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten with empty multi-valued fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.tags': [], + }; + const expectedOutput = { + service: { + name: 'node-svc', + tags: [], + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should retain unknown fields in the output', () => { + const input = { + 'service.name': 'node-svc', + 'unknown.texts': ['foo', 'bar'], + 'unknown.field': 'foo', + unknonwField: 'bar', + }; + const expectedOutput = { + service: { + name: 'node-svc', + }, + unknown: { + field: 'foo', + texts: ['foo', 'bar'], + }, + unknonwField: 'bar', + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten nested fields with mandatory field', () => { + const input = { + 'service.name': 'node-svc', + 'service.environment': undefined, + }; + + const requiredFields: ['service.name'] = ['service.name']; + + const expectedOutput = { + service: { + name: 'node-svc', + }, + }; + expect(unflattenKnownApmEventFields(input, requiredFields)).toEqual(expectedOutput); + }); + + it('should throw an exception when mandatory field is not in the input', () => { + const input = { + 'service.environment': 'PROD', + }; + + const requiredFields: ['service.name'] = ['service.name']; + + // @ts-expect-error + expect(() => unflattenKnownApmEventFields(input, requiredFields)).toThrowError( + 'Missing required fields service.name in event' + ); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts new file mode 100644 index 0000000000000..b9a4322269828 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DedotObject } from '@kbn/utility-types'; +import * as APM_EVENT_FIELDS_MAP from '@kbn/apm-types/es_fields'; +import type { ValuesType } from 'utility-types'; +import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object'; +import { mergePlainObjects } from '@kbn/observability-utils/object/merge_plain_objects'; +import { castArray, isArray } from 'lodash'; +import { AgentName } from '@kbn/elastic-agent-utils'; +import { EventOutcome } from '@kbn/apm-types/src/es_schemas/raw/fields'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; + +const { + CLOUD, + AGENT, + SERVICE, + ERROR_EXCEPTION, + SPAN_LINKS, + HOST, + KUBERNETES, + CONTAINER, + TIER, + INDEX, + DATA_STEAM_TYPE, + VALUE_OTEL_JVM_PROCESS_MEMORY_HEAP, + VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP, + SPAN_LINKS_SPAN_ID, + SPAN_LINKS_TRACE_ID, + SPAN_STACKTRACE, + ...CONCRETE_FIELDS +} = APM_EVENT_FIELDS_MAP; + +const ALL_FIELDS = Object.values(CONCRETE_FIELDS); + +const KNOWN_MULTI_VALUED_FIELDS = [ + APM_EVENT_FIELDS_MAP.CHILD_ID, + APM_EVENT_FIELDS_MAP.PROCESS_ARGS, +] as const; + +type KnownField = ValuesType; + +type KnownSingleValuedField = Exclude; +type KnownMultiValuedField = ValuesType; + +const KNOWN_SINGLE_VALUED_FIELDS = ALL_FIELDS.filter( + (field): field is KnownSingleValuedField => !KNOWN_MULTI_VALUED_FIELDS.includes(field as any) +); + +interface TypeOverrideMap { + [APM_EVENT_FIELDS_MAP.SPAN_DURATION]: number; + [APM_EVENT_FIELDS_MAP.AGENT_NAME]: AgentName; + [APM_EVENT_FIELDS_MAP.EVENT_OUTCOME]: EventOutcome; + [APM_EVENT_FIELDS_MAP.FAAS_COLDSTART]: true; + [APM_EVENT_FIELDS_MAP.TRANSACTION_DURATION]: number; + [APM_EVENT_FIELDS_MAP.TIMESTAMP_US]: number; + [APM_EVENT_FIELDS_MAP.PROCESSOR_EVENT]: ProcessorEvent; + [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_COUNT]: number; + [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_SUM]: number; + [APM_EVENT_FIELDS_MAP.SPAN_SYNC]: boolean; + [APM_EVENT_FIELDS_MAP.TRANSACTION_SAMPLED]: boolean; + [APM_EVENT_FIELDS_MAP.PROCESSOR_NAME]: 'transaction' | 'metric' | 'error'; + [APM_EVENT_FIELDS_MAP.HTTP_RESPONSE_STATUS_CODE]: number; + [APM_EVENT_FIELDS_MAP.PROCESS_PID]: number; + [APM_EVENT_FIELDS_MAP.OBSERVER_VERSION_MAJOR]: number; + [APM_EVENT_FIELDS_MAP.ERROR_EXC_HANDLED]: boolean; +} + +type MaybeMultiValue = T extends KnownMultiValuedField ? U[] : U; + +type TypeOfKnownField = MaybeMultiValue< + T, + T extends keyof TypeOverrideMap ? TypeOverrideMap[T] : string +>; + +type MapToSingleOrMultiValue> = { + [TKey in keyof T]: TKey extends KnownField + ? T[TKey] extends undefined + ? TypeOfKnownField | undefined + : TypeOfKnownField + : unknown; +}; + +type UnflattenedKnownFields> = DedotObject< + MapToSingleOrMultiValue +>; + +export type FlattenedApmEvent = Record; + +export type UnflattenedApmEvent = UnflattenedKnownFields; + +export function unflattenKnownApmEventFields | undefined = undefined>( + fields: T +): T extends Record ? UnflattenedKnownFields : undefined; + +export function unflattenKnownApmEventFields< + T extends Record | undefined, + U extends Array> +>( + fields: T, + required: U +): T extends Record + ? UnflattenedKnownFields & + (U extends any[] + ? UnflattenedKnownFields<{ + [TKey in ValuesType]: keyof T extends TKey ? T[TKey] : unknown[]; + }> + : {}) + : undefined; + +export function unflattenKnownApmEventFields( + hitFields?: Record, + requiredFields?: string[] +) { + if (!hitFields) { + return undefined; + } + const missingRequiredFields = + requiredFields?.filter((key) => { + const value = hitFields?.[key]; + return value === null || value === undefined || (isArray(value) && value.length === 0); + }) ?? []; + + if (missingRequiredFields.length > 0) { + throw new Error(`Missing required fields ${missingRequiredFields.join(', ')} in event`); + } + + const copy: Record = mapToSingleOrMultiValue({ + ...hitFields, + }); + + const [knownFields, unknownFields] = Object.entries(copy).reduce( + (prev, [key, value]) => { + if (ALL_FIELDS.includes(key as KnownField)) { + prev[0][key as KnownField] = value; + } else { + prev[1][key] = value; + } + return prev; + }, + [{} as Record, {} as Record] + ); + + const unflattened = mergePlainObjects( + {}, + unflattenObject(unknownFields), + unflattenObject(knownFields) + ); + + return unflattened; +} + +export function mapToSingleOrMultiValue>( + fields: T +): MapToSingleOrMultiValue { + KNOWN_SINGLE_VALUED_FIELDS.forEach((field) => { + const value = fields[field]; + if (value !== null && value !== undefined) { + fields[field as keyof T] = castArray(value)[0]; + } + }); + + return fields; +} diff --git a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json index ea3ebf77b25be..aeeb73bee2857 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json @@ -20,6 +20,8 @@ "@kbn/apm-utils", "@kbn/core-http-server", "@kbn/security-plugin-types-server", - "@kbn/observability-utils" + "@kbn/observability-utils", + "@kbn/utility-types", + "@kbn/elastic-agent-utils" ] }