From 7235ed0425100bbf04ff157d0af7980875473c99 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 15 Oct 2024 11:38:44 +0200 Subject: [PATCH] [APM][Otel] Use `fields` instead of `_source` on APM queries (#195242) closes https://github.com/elastic/kibana/issues/192606 ## Summary v2 based on the work done in this PR https://github.com/elastic/kibana/pull/192608 and the suggestion from Dario https://github.com/elastic/kibana/pull/194424 This PR replaces the _source usage in APM queries with fields to support Otel data. The idea is to get rid of existing UI errors we have and make sure that otel data is shown correctly in the UI. One way to check it is using the [e2e PoC](https://github.com/elastic/otel-apm-e2e-poc/blob/main/README.md). --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Elastic Machine Co-authored-by: Jenny --- packages/kbn-apm-types/src/es_fields/apm.ts | 21 +- .../src/es_schemas/raw/apm_base_doc.ts | 6 +- .../src/es_schemas/raw/fields/cloud.ts | 18 +- .../src/es_schemas/raw/fields/container.ts | 4 +- .../src/es_schemas/raw/fields/http.ts | 4 +- .../src/es_schemas/raw/fields/kubernetes.ts | 2 +- .../src/es_schemas/raw/fields/observer.ts | 4 +- .../src/es_schemas/raw/fields/page.ts | 2 +- .../src/es_schemas/raw/fields/service.ts | 8 +- .../src/es_schemas/raw/fields/url.ts | 2 +- .../src/es_schemas/raw/fields/user.ts | 2 +- .../src/es_schemas/ui/fields/agent.ts | 2 +- .../object/flatten_object.test.ts | 12 ++ .../object/unflatten_object.test.ts | 40 ++++ .../object/unflatten_object.ts | 28 +++ .../observability_utils/tsconfig.json | 1 + .../__snapshots__/es_fields.test.ts.snap | 78 ++++++- .../apm/common/es_fields/es_fields.test.ts | 9 +- .../apm/common/service_metadata.ts | 58 ++++++ .../apm/common/waterfall/typings.ts | 9 +- .../error_sample_contextual_insight.tsx | 24 ++- .../error_sampler/error_sample_detail.tsx | 25 ++- .../error_sampler/error_tabs.tsx | 2 +- .../error_sampler/sample_summary.tsx | 4 +- ...redirect_to_transaction_detail_page_url.ts | 4 +- .../discover_links/discover_error_link.tsx | 14 +- .../metadata_table/error_metadata/index.tsx | 11 +- .../collect_data_telemetry/tasks.test.ts | 6 +- .../collect_data_telemetry/tasks.ts | 32 +-- .../get_destination_map.ts | 1 - .../apm/server/lib/helpers/get_error_name.ts | 6 +- ...register_transaction_duration_rule_type.ts | 1 + .../get_log_categories/index.ts | 18 +- .../get_container_id_from_signals.ts | 17 +- .../get_downstream_dependency_name.ts | 10 +- .../get_service_name_from_signals.ts | 14 +- .../get_metadata_for_dependency.ts | 16 +- .../dependencies/get_top_dependency_spans.ts | 43 ++-- .../get_error_group_main_statistics.ts | 61 ++++-- .../get_error_group_sample_ids.ts | 8 +- .../get_error_sample_details.ts | 95 ++++++++- .../apm/server/routes/errors/route.ts | 9 +- .../get_crash_group_main_statistics.ts | 66 ++++-- .../get_mobile_error_group_main_statistics.ts | 66 ++++-- .../get_derived_service_annotations.ts | 21 +- .../routes/services/get_service_agent.ts | 35 ++-- ...get_service_instance_container_metadata.ts | 23 ++- .../get_service_instance_metadata_details.ts | 44 +++- .../services/get_service_metadata_details.ts | 28 +-- .../services/get_service_metadata_icons.ts | 26 ++- .../routes/span_links/get_linked_children.ts | 46 +++-- .../routes/span_links/get_linked_parents.ts | 2 +- .../span_links/get_span_links_details.ts | 153 ++++++++------ .../traces/__snapshots__/queries.test.ts.snap | 19 +- .../server/routes/traces/get_trace_items.ts | 190 +++++++++++++----- .../traces/get_trace_samples_by_query.ts | 4 +- .../apm/server/routes/traces/route.ts | 13 +- .../__snapshots__/queries.test.ts.snap | 24 +++ .../routes/transactions/get_span/index.ts | 22 +- .../transactions/get_transaction/index.ts | 70 ++++++- .../get_transaction_by_name/index.ts | 28 ++- .../get_transaction_by_trace/index.ts | 53 ++++- .../transactions/trace_samples/index.ts | 23 ++- .../apm_data_access/server/utils.ts | 1 + .../utils/unflatten_known_fields.test.ts | 137 +++++++++++++ .../server/utils/unflatten_known_fields.ts | 168 ++++++++++++++++ .../apm_data_access/tsconfig.json | 4 +- 67 files changed, 1612 insertions(+), 385 deletions(-) create mode 100644 x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts create mode 100644 x-pack/packages/observability/observability_utils/object/unflatten_object.ts create mode 100644 x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts create mode 100644 x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index 6b0a68379f5d4..5d50833161979 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -7,7 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export const TIMESTAMP = 'timestamp.us'; +export const TIMESTAMP_US = 'timestamp.us'; +export const AT_TIMESTAMP = '@timestamp'; export const AGENT = 'agent'; export const AGENT_NAME = 'agent.name'; export const AGENT_VERSION = 'agent.version'; @@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider'; export const CLOUD_REGION = 'cloud.region'; export const CLOUD_MACHINE_TYPE = 'cloud.machine.type'; export const CLOUD_ACCOUNT_ID = 'cloud.account.id'; +export const CLOUD_ACCOUNT_NAME = 'cloud.account.name'; export const CLOUD_INSTANCE_ID = 'cloud.instance.id'; export const CLOUD_INSTANCE_NAME = 'cloud.instance.name'; export const CLOUD_SERVICE_NAME = 'cloud.service.name'; +export const CLOUD_PROJECT_NAME = 'cloud.project.name'; export const EVENT_SUCCESS_COUNT = 'event.success_count'; @@ -48,10 +51,14 @@ export const USER_ID = 'user.id'; export const USER_AGENT_ORIGINAL = 'user_agent.original'; export const USER_AGENT_NAME = 'user_agent.name'; +export const OBSERVER_VERSION = 'observer.version'; +export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const OBSERVER_HOSTNAME = 'observer.hostname'; export const OBSERVER_LISTENING = 'observer.listening'; export const PROCESSOR_EVENT = 'processor.event'; +export const PROCESSOR_NAME = 'processor.name'; +export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary'; @@ -95,6 +102,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us'; export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy'; export const SPAN_SYNC = 'span.sync'; +export const SPAN_STACKTRACE = 'span.stacktrace'; // Parent ID for a transaction or span export const PARENT_ID = 'parent.id'; @@ -110,6 +118,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array export const ERROR_EXC_TYPE = 'error.exception.type'; export const ERROR_PAGE_URL = 'error.page.url'; +export const ERROR_STACK_TRACE = 'error.stack_trace'; export const ERROR_TYPE = 'error.type'; // METRICS @@ -153,6 +162,12 @@ export const CONTAINER_IMAGE = 'container.image.name'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; export const KUBERNETES_POD_UID = 'kubernetes.pod.uid'; +export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; +export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; +export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; +export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; +export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; +export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name'; export const FAAS_ID = 'faas.id'; export const FAAS_NAME = 'faas.name'; @@ -198,3 +213,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name'; export const CHILD_ID = 'child.id'; export const LOG_LEVEL = 'log.level'; + +// Process +export const PROCESS_ARGS = 'process.args'; +export const PROCESS_PID = 'process.pid'; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts b/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts index b3a6066631346..14d26354e44ed 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts @@ -14,10 +14,10 @@ export interface APMBaseDoc { '@timestamp': string; agent: { name: string; - version: string; + version?: string; }; - parent?: { id: string }; // parent ID is not available on root transactions - trace?: { id: string }; + parent?: { id?: string }; // parent ID is not available on root transactions + trace?: { id?: string }; labels?: { [key: string]: string | number | boolean; }; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts index 7ee972faf7680..290be75091e18 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts @@ -10,26 +10,26 @@ export interface Cloud { availability_zone?: string; instance?: { - name: string; - id: string; + name?: string; + id?: string; }; machine?: { - type: string; + type?: string; }; project?: { - id: string; - name: string; + id?: string; + name?: string; }; provider?: string; region?: string; account?: { - id: string; - name: string; + id?: string; + name?: string; }; image?: { - id: string; + id?: string; }; service?: { - name: string; + name?: string; }; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts index 64dd497710b97..4c8d1ed4e52b4 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts @@ -9,5 +9,7 @@ export interface Container { id?: string | null; - image?: string | null; + image?: { + name?: string; + }; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts index 458731f690838..f3c62298ca8cb 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts @@ -8,7 +8,7 @@ */ export interface Http { - request?: { method: string; [key: string]: unknown }; - response?: { status_code: number; [key: string]: unknown }; + request?: { method?: string }; + response?: { status_code?: number }; version?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts index 704d77f19f858..2a4f1465db9a5 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts @@ -8,7 +8,7 @@ */ export interface Kubernetes { - pod?: { uid?: string | null; [key: string]: unknown }; + pod?: { uid?: string | null; name?: string }; namespace?: string; replicaset?: { name?: string; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts index 067ecb9436ff9..7d286d4c3581e 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts @@ -13,6 +13,6 @@ export interface Observer { id?: string; name?: string; type?: string; - version: string; - version_major: number; + version?: string; + version_major?: number; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts index 6cc058ef75642..a18f3c5578eb5 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts @@ -9,5 +9,5 @@ // only for RUM agent: shared by error and transaction export interface Page { - url: string; + url?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts index bcd9af08706ec..bd52784576dce 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts @@ -11,18 +11,18 @@ export interface Service { name: string; environment?: string; framework?: { - name: string; + name?: string; version?: string; }; node?: { name?: string; }; runtime?: { - name: string; - version: string; + name?: string; + version?: string; }; language?: { - name: string; + name?: string; version?: string; }; version?: string; diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts index 3703763724f38..0f8cd3c814315 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts @@ -9,6 +9,6 @@ export interface Url { domain?: string; - full: string; + full?: string; original?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts index 1c2235288a661..962ed1060b826 100644 --- a/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts +++ b/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts @@ -8,5 +8,5 @@ */ export interface User { - id: string; + id?: string; } diff --git a/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts b/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts index ea3ebf39555d2..e8734de141e83 100644 --- a/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts +++ b/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts @@ -14,5 +14,5 @@ export type { ElasticAgentName, OpenTelemetryAgentName, AgentName } from '@kbn/e export interface Agent { ephemeral_id?: string; name: AgentName; - version: string; + version?: string; } diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts index deb7ed998c478..13a8174f4f1cf 100644 --- a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts +++ b/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts @@ -21,6 +21,18 @@ describe('flattenObject', () => { }); }); + it('flattens arrays', () => { + expect( + flattenObject({ + child: { + id: [1, 2], + }, + }) + ).toEqual({ + 'child.id': [1, 2], + }); + }); + it('does not flatten arrays', () => { expect( flattenObject({ diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts new file mode 100644 index 0000000000000..22cee17bb1a64 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { unflattenObject } from './unflatten_object'; + +describe('unflattenObject', () => { + it('unflattens deeply nested objects', () => { + expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({ + first: { + second: { + third: 'third', + }, + }, + }); + }); + + it('does not unflatten arrays', () => { + expect( + unflattenObject({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + 'nested.array': [0, 1, 2], + 'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }], + }) + ).toEqual({ + simpleArray: ['0', '1', '2'], + complexArray: [{ one: 'one', two: 'two', three: 'three' }], + nested: { + array: [0, 1, 2], + }, + complex: { + nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }], + }, + }); + }); +}); diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts new file mode 100644 index 0000000000000..142ea2eea6461 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@kbn/safer-lodash-set'; + +export function unflattenObject(source: Record, target: Record = {}) { + // eslint-disable-next-line guard-for-in + for (const key in source) { + const val = source[key as keyof typeof source]; + + if (Array.isArray(val)) { + const unflattenedArray = val.map((item) => { + if (item && typeof item === 'object' && !Array.isArray(item)) { + return unflattenObject(item); + } + return item; + }); + set(target, key, unflattenedArray); + } else { + set(target, key, val); + } + } + return target; +} diff --git a/x-pack/packages/observability/observability_utils/tsconfig.json b/x-pack/packages/observability/observability_utils/tsconfig.json index 2ed47d10cfad9..b3f1a4a21c4e7 100644 --- a/x-pack/packages/observability/observability_utils/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/tsconfig.json @@ -21,5 +21,6 @@ "@kbn/es-types", "@kbn/apm-utils", "@kbn/es-query", + "@kbn/safer-lodash-set", ] } diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap index 6fa3e146a423d..88d00196e074b 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap @@ -37,6 +37,8 @@ Object { exports[`Error CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Error CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Error CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Error CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -45,6 +47,8 @@ exports[`Error CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Error CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Error CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Error CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Error CLOUD_REGION 1`] = `"europe-west1"`; @@ -94,6 +98,8 @@ exports[`Error ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Error ERROR_PAGE_URL 1`] = `undefined`; +exports[`Error ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Error ERROR_TYPE 1`] = `undefined`; exports[`Error EVENT_NAME 1`] = `undefined`; @@ -140,6 +146,8 @@ exports[`Error INDEX 1`] = `undefined`; exports[`Error KUBERNETES 1`] = `undefined`; +exports[`Error KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Error KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Error KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -150,6 +158,8 @@ exports[`Error KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Error KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Error KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Error KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Error KUBERNETES_POD_UID 1`] = `undefined`; @@ -228,10 +238,20 @@ exports[`Error OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Error OBSERVER_LISTENING 1`] = `undefined`; +exports[`Error OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Error PARENT_ID 1`] = `"parentId"`; +exports[`Error PROCESS_ARGS 1`] = `undefined`; + +exports[`Error PROCESS_PID 1`] = `undefined`; + exports[`Error PROCESSOR_EVENT 1`] = `"error"`; +exports[`Error PROCESSOR_NAME 1`] = `"error"`; + exports[`Error SERVICE 1`] = ` Object { "language": Object { @@ -296,6 +316,8 @@ exports[`Error SPAN_NAME 1`] = `undefined`; exports[`Error SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Error SPAN_STACKTRACE 1`] = `undefined`; + exports[`Error SPAN_SUBTYPE 1`] = `undefined`; exports[`Error SPAN_SYNC 1`] = `undefined`; @@ -304,10 +326,12 @@ exports[`Error SPAN_TYPE 1`] = `undefined`; exports[`Error TIER 1`] = `undefined`; -exports[`Error TIMESTAMP 1`] = `1337`; +exports[`Error TIMESTAMP_US 1`] = `1337`; exports[`Error TRACE_ID 1`] = `"trace id"`; +exports[`Error TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Error TRANSACTION_DURATION 1`] = `undefined`; exports[`Error TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; @@ -385,6 +409,8 @@ Object { exports[`Span CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Span CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Span CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Span CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -393,6 +419,8 @@ exports[`Span CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Span CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Span CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Span CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Span CLOUD_REGION 1`] = `"europe-west1"`; @@ -433,6 +461,8 @@ exports[`Span ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Span ERROR_PAGE_URL 1`] = `undefined`; +exports[`Span ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Span ERROR_TYPE 1`] = `undefined`; exports[`Span EVENT_NAME 1`] = `undefined`; @@ -475,6 +505,8 @@ exports[`Span INDEX 1`] = `undefined`; exports[`Span KUBERNETES 1`] = `undefined`; +exports[`Span KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Span KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Span KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -485,6 +517,8 @@ exports[`Span KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Span KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Span KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Span KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Span KUBERNETES_POD_UID 1`] = `undefined`; @@ -563,10 +597,20 @@ exports[`Span OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Span OBSERVER_LISTENING 1`] = `undefined`; +exports[`Span OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Span PARENT_ID 1`] = `"parentId"`; +exports[`Span PROCESS_ARGS 1`] = `undefined`; + +exports[`Span PROCESS_PID 1`] = `undefined`; + exports[`Span PROCESSOR_EVENT 1`] = `"span"`; +exports[`Span PROCESSOR_NAME 1`] = `"transaction"`; + exports[`Span SERVICE 1`] = ` Object { "name": "service name", @@ -627,6 +671,8 @@ exports[`Span SPAN_NAME 1`] = `"span name"`; exports[`Span SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Span SPAN_STACKTRACE 1`] = `undefined`; + exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`; exports[`Span SPAN_SYNC 1`] = `false`; @@ -635,10 +681,12 @@ exports[`Span SPAN_TYPE 1`] = `"span type"`; exports[`Span TIER 1`] = `undefined`; -exports[`Span TIMESTAMP 1`] = `1337`; +exports[`Span TIMESTAMP_US 1`] = `1337`; exports[`Span TRACE_ID 1`] = `"trace id"`; +exports[`Span TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Span TRANSACTION_DURATION 1`] = `undefined`; exports[`Span TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; @@ -716,6 +764,8 @@ Object { exports[`Transaction CLOUD_ACCOUNT_ID 1`] = `undefined`; +exports[`Transaction CLOUD_ACCOUNT_NAME 1`] = `undefined`; + exports[`Transaction CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; exports[`Transaction CLOUD_INSTANCE_ID 1`] = `undefined`; @@ -724,6 +774,8 @@ exports[`Transaction CLOUD_INSTANCE_NAME 1`] = `undefined`; exports[`Transaction CLOUD_MACHINE_TYPE 1`] = `undefined`; +exports[`Transaction CLOUD_PROJECT_NAME 1`] = `undefined`; + exports[`Transaction CLOUD_PROVIDER 1`] = `"gcp"`; exports[`Transaction CLOUD_REGION 1`] = `"europe-west1"`; @@ -768,6 +820,8 @@ exports[`Transaction ERROR_LOG_MESSAGE 1`] = `undefined`; exports[`Transaction ERROR_PAGE_URL 1`] = `undefined`; +exports[`Transaction ERROR_STACK_TRACE 1`] = `undefined`; + exports[`Transaction ERROR_TYPE 1`] = `undefined`; exports[`Transaction EVENT_NAME 1`] = `undefined`; @@ -820,6 +874,8 @@ Object { } `; +exports[`Transaction KUBERNETES_CONTAINER_ID 1`] = `undefined`; + exports[`Transaction KUBERNETES_CONTAINER_NAME 1`] = `undefined`; exports[`Transaction KUBERNETES_DEPLOYMENT 1`] = `undefined`; @@ -830,6 +886,8 @@ exports[`Transaction KUBERNETES_NAMESPACE 1`] = `undefined`; exports[`Transaction KUBERNETES_NAMESPACE_NAME 1`] = `undefined`; +exports[`Transaction KUBERNETES_NODE_NAME 1`] = `undefined`; + exports[`Transaction KUBERNETES_POD_NAME 1`] = `undefined`; exports[`Transaction KUBERNETES_POD_UID 1`] = `"pod1234567890abcdef"`; @@ -908,10 +966,20 @@ exports[`Transaction OBSERVER_HOSTNAME 1`] = `undefined`; exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; +exports[`Transaction OBSERVER_VERSION 1`] = `"whatever"`; + +exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`; + exports[`Transaction PARENT_ID 1`] = `"parentId"`; +exports[`Transaction PROCESS_ARGS 1`] = `undefined`; + +exports[`Transaction PROCESS_PID 1`] = `undefined`; + exports[`Transaction PROCESSOR_EVENT 1`] = `"transaction"`; +exports[`Transaction PROCESSOR_NAME 1`] = `"transaction"`; + exports[`Transaction SERVICE 1`] = ` Object { "language": Object { @@ -976,6 +1044,8 @@ exports[`Transaction SPAN_NAME 1`] = `undefined`; exports[`Transaction SPAN_SELF_TIME_SUM 1`] = `undefined`; +exports[`Transaction SPAN_STACKTRACE 1`] = `undefined`; + exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`; exports[`Transaction SPAN_SYNC 1`] = `undefined`; @@ -984,10 +1054,12 @@ exports[`Transaction SPAN_TYPE 1`] = `undefined`; exports[`Transaction TIER 1`] = `undefined`; -exports[`Transaction TIMESTAMP 1`] = `1337`; +exports[`Transaction TIMESTAMP_US 1`] = `1337`; exports[`Transaction TRACE_ID 1`] = `"trace id"`; +exports[`Transaction TRANSACTION_AGENT_MARKS 1`] = `undefined`; + exports[`Transaction TRANSACTION_DURATION 1`] = `1337`; exports[`Transaction TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts index f33fddd430e8d..12537d35afefe 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts @@ -10,12 +10,13 @@ import { AllowUnknownProperties } from '../../typings/common'; import { APMError } from '../../typings/es_schemas/ui/apm_error'; import { Span } from '../../typings/es_schemas/ui/span'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; -import * as apmFieldnames from './apm'; -import * as infraMetricsFieldnames from './infra_metrics'; +import * as allApmFieldNames from './apm'; +import * as infraMetricsFieldNames from './infra_metrics'; +const { AT_TIMESTAMP, ...apmFieldNames } = allApmFieldNames; const fieldnames = { - ...apmFieldnames, - ...infraMetricsFieldnames, + ...apmFieldNames, + ...infraMetricsFieldNames, }; describe('Transaction', () => { diff --git a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts index 0ccede67741b7..4136ea361392e 100644 --- a/x-pack/plugins/observability_solution/apm/common/service_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/common/service_metadata.ts @@ -4,5 +4,63 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_INSTANCE_NAME, + CLOUD_MACHINE_TYPE, + CLOUD_PROVIDER, + CONTAINER_ID, + HOST_NAME, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_REPLICASET_NAME, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_VERSION, +} from './es_fields/apm'; +import { asMutableArray } from './utils/as_mutable_array'; + +export const SERVICE_METADATA_SERVICE_KEYS = asMutableArray([ + SERVICE_NODE_NAME, + SERVICE_VERSION, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, +] as const); + +export const SERVICE_METADATA_CONTAINER_KEYS = asMutableArray([ + CONTAINER_ID, + HOST_NAME, + KUBERNETES_POD_UID, + KUBERNETES_POD_NAME, +] as const); + +export const SERVICE_METADATA_INFRA_METRICS_KEYS = asMutableArray([ + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, +] as const); + +export const SERVICE_METADATA_CLOUD_KEYS = asMutableArray([ + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_INSTANCE_NAME, + CLOUD_MACHINE_TYPE, + CLOUD_PROVIDER, +] as const); + +export const SERVICE_METADATA_KUBERNETES_KEYS = asMutableArray([ + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_REPLICASET_NAME, +] as const); export type ContainerType = 'Kubernetes' | 'Docker' | undefined; diff --git a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts index 49f282473e9dd..2fd0be94a5c5f 100644 --- a/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts +++ b/x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts @@ -64,16 +64,17 @@ export interface WaterfallSpan { links?: SpanLink[]; }; transaction?: { - id: string; + id?: string; }; child?: { id: string[] }; } export interface WaterfallError { timestamp: TimestampUs; - trace?: { id: string }; - transaction?: { id: string }; - parent?: { id: string }; + trace?: { id?: string }; + transaction?: { id?: string }; + parent?: { id?: string }; + span?: { id?: string }; error: { id: string; log?: { diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx index 2e91865083b8c..20d5521b43ebf 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx @@ -8,9 +8,9 @@ import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import type { Message } from '@kbn/observability-ai-assistant-plugin/public'; import React, { useMemo, useState } from 'react'; +import { AT_TIMESTAMP } from '@kbn/apm-types'; import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context'; import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; -import { Transaction } from '../../../../../typings/es_schemas/ui/transaction'; import { ErrorSampleDetailTabContent } from './error_sample_detail'; import { exceptionStacktraceTab, logStacktraceTab } from './error_tabs'; @@ -18,8 +18,26 @@ export function ErrorSampleContextualInsight({ error, transaction, }: { - error: APMError; - transaction?: Transaction; + error: { + [AT_TIMESTAMP]: string; + error: Pick; + service: { + name: string; + environment?: string; + language?: { + name?: string; + }; + runtime?: { + name?: string; + version?: string; + }; + }; + }; + transaction?: { + transaction: { + name: string; + }; + }; }) { const { observabilityAIAssistant } = useApmPluginContext(); diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx index a38c4dfc96f63..2edb2c1a3fea6 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx @@ -29,14 +29,14 @@ import { first } from 'lodash'; import React, { useEffect, useState } from 'react'; import { useHistory } from 'react-router-dom'; import useAsync from 'react-use/lib/useAsync'; -import { ERROR_GROUP_ID } from '../../../../../common/es_fields/apm'; +import { AT_TIMESTAMP, ERROR_GROUP_ID } from '../../../../../common/es_fields/apm'; import { TraceSearchType } from '../../../../../common/trace_explorer'; import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context'; import { useLegacyUrlParams } from '../../../../context/url_params_context/use_url_params'; import { useAnyOfApmParams } from '../../../../hooks/use_apm_params'; import { useApmRouter } from '../../../../hooks/use_apm_router'; -import { FETCH_STATUS, isPending } from '../../../../hooks/use_fetcher'; +import { FETCH_STATUS, isPending, isSuccess } from '../../../../hooks/use_fetcher'; import { useTraceExplorerEnabledSetting } from '../../../../hooks/use_trace_explorer_enabled_setting'; import { APIReturnType } from '../../../../services/rest/create_call_apm_api'; import { TransactionDetailLink } from '../../../shared/links/apm/transaction_detail_link'; @@ -111,8 +111,7 @@ export function ErrorSampleDetails({ const loadingErrorData = isPending(errorFetchStatus); const isLoading = loadingErrorSamplesData || loadingErrorData; - const isSucceded = - errorSamplesFetchStatus === FETCH_STATUS.SUCCESS && errorFetchStatus === FETCH_STATUS.SUCCESS; + const isSucceeded = isSuccess(errorSamplesFetchStatus) && isSuccess(errorFetchStatus); useEffect(() => { setSampleActivePage(0); @@ -137,7 +136,7 @@ export function ErrorSampleDetails({ }); }, [error, transaction, uiActions]); - if (!error && errorSampleIds?.length === 0 && isSucceded) { + if (!error && errorSampleIds?.length === 0 && isSucceeded) { return ( ; + }; currentTab: ErrorTab; }) { const codeLanguage = error?.service.language?.name; const exceptions = error?.error.exception || []; const logStackframes = error?.error.log?.stacktrace; const isPlaintextException = - !!error?.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace; + !!error.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace; switch (currentTab.key) { case ErrorTabKey.LogStackTrace: return ; @@ -363,7 +370,7 @@ export function ErrorSampleDetailTabContent({ return isPlaintextException ? ( diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx index 893e842513c8f..86b69eb480b3f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx @@ -41,7 +41,7 @@ export const metadataTab: ErrorTab = { }), }; -export function getTabs(error: APMError) { +export function getTabs(error: { error: { log?: APMError['error']['log'] } }) { const hasLogStacktrace = !isEmpty(error?.error.log?.stacktrace); return [...(hasLogStacktrace ? [logStacktraceTab] : []), exceptionStacktraceTab, metadataTab]; } diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx index af05e8766994c..c7acbfee7e45e 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx @@ -18,7 +18,9 @@ const Label = euiStyled.div` `; interface Props { - error: APMError; + error: { + error: Pick; + }; } export function SampleSummary({ error }: Props) { const logMessage = error.error.log?.message; diff --git a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts index a3467d7272ff5..acd9e9445ad8f 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts +++ b/x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts @@ -6,7 +6,7 @@ */ import { format } from 'url'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; +import type { TransactionDetailRedirectInfo } from '../../../../server/routes/transactions/get_transaction_by_trace'; export const getRedirectToTransactionDetailPageUrl = ({ transaction, @@ -14,7 +14,7 @@ export const getRedirectToTransactionDetailPageUrl = ({ rangeTo, waterfallItemId, }: { - transaction: Transaction; + transaction: TransactionDetailRedirectInfo; rangeFrom?: string; rangeTo?: string; waterfallItemId?: string; diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx index 2958d2af7d68f..a32c01f3b15e5 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx @@ -7,10 +7,18 @@ import React, { ReactNode } from 'react'; import { ERROR_GROUP_ID, SERVICE_NAME } from '../../../../../common/es_fields/apm'; -import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; import { DiscoverLink } from './discover_link'; -function getDiscoverQuery(error: APMError, kuery?: string) { +interface ErrorForDiscoverQuery { + service: { + name: string; + }; + error: { + grouping_key: string; + }; +} + +function getDiscoverQuery(error: ErrorForDiscoverQuery, kuery?: string) { const serviceName = error.service.name; const groupId = error.error.grouping_key; let query = `${SERVICE_NAME}:"${serviceName}" AND ${ERROR_GROUP_ID}:"${groupId}"`; @@ -36,7 +44,7 @@ function DiscoverErrorLink({ children, }: { children?: ReactNode; - readonly error: APMError; + readonly error: ErrorForDiscoverQuery; readonly kuery?: string; }) { return ; diff --git a/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx b/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx index ae688f8917602..dab585180fce9 100644 --- a/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx +++ b/x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx @@ -7,13 +7,16 @@ import React, { useMemo } from 'react'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { APMError } from '../../../../../typings/es_schemas/ui/apm_error'; +import { APMError, AT_TIMESTAMP } from '@kbn/apm-types'; import { getSectionsFromFields } from '../helper'; import { MetadataTable } from '..'; import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; interface Props { - error: APMError; + error: { + [AT_TIMESTAMP]: string; + error: Pick; + }; } export function ErrorMetadata({ error }: Props) { @@ -26,8 +29,8 @@ export function ErrorMetadata({ error }: Props) { id: error.error.id, }, query: { - start: error['@timestamp'], - end: error['@timestamp'], + start: error[AT_TIMESTAMP], + end: error[AT_TIMESTAMP], }, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index 319582f61b664..a2b6809f855e7 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -8,7 +8,7 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; import { tasks } from './tasks'; -import { SERVICE_NAME, SERVICE_ENVIRONMENT } from '../../../../common/es_fields/apm'; +import { SERVICE_NAME, SERVICE_ENVIRONMENT, AT_TIMESTAMP } from '../../../../common/es_fields/apm'; import { IndicesStatsResponse } from '../telemetry_client'; describe('data telemetry collection tasks', () => { @@ -101,7 +101,7 @@ describe('data telemetry collection tasks', () => { // a fixed date range. .mockReturnValueOnce({ hits: { - hits: [{ _source: { '@timestamp': new Date().toISOString() } }], + hits: [{ fields: { [AT_TIMESTAMP]: [new Date().toISOString()] } }], }, total: { value: 1, @@ -314,7 +314,7 @@ describe('data telemetry collection tasks', () => { ? { hits: { total: { value: 1 } } } : { hits: { - hits: [{ _source: { '@timestamp': 1 } }], + hits: [{ fields: { [AT_TIMESTAMP]: [1] } }], }, } ); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index db6a6a918177a..1347cbb4e3641 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -11,11 +11,13 @@ import { createHash } from 'crypto'; import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash'; import { SavedObjectsClient } from '@kbn/core/server'; import type { APMIndices } from '@kbn/apm-data-access-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name'; import { AGENT_ACTIVATION_METHOD, AGENT_NAME, AGENT_VERSION, + AT_TIMESTAMP, CLIENT_GEO_COUNTRY_ISO_CODE, CLOUD_AVAILABILITY_ZONE, CLOUD_PROVIDER, @@ -29,6 +31,7 @@ import { METRICSET_INTERVAL, METRICSET_NAME, OBSERVER_HOSTNAME, + OBSERVER_VERSION, PARENT_ID, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, @@ -54,10 +57,7 @@ import { SavedServiceGroup, } from '../../../../common/service_groups'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; -import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import { Span } from '../../../../typings/es_schemas/ui/span'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMDataTelemetry, APMPerService, @@ -193,17 +193,19 @@ export const tasks: TelemetryTask[] = [ size: 1, track_total_hits: false, sort: { - '@timestamp': 'desc' as const, + [AT_TIMESTAMP]: 'desc' as const, }, + fields: [AT_TIMESTAMP], }, }) - ).hits.hits[0] as { _source: { '@timestamp': string } }; + ).hits.hits[0]; if (!lastTransaction) { return {}; } - const end = new Date(lastTransaction._source['@timestamp']).getTime() - 5 * 60 * 1000; + const end = + new Date(lastTransaction.fields[AT_TIMESTAMP]![0] as string).getTime() - 5 * 60 * 1000; const start = end - 60 * 1000; @@ -512,16 +514,16 @@ export const tasks: TelemetryTask[] = [ }, }, sort: { - '@timestamp': 'asc', + [AT_TIMESTAMP]: 'asc', }, - _source: ['@timestamp'], + fields: [AT_TIMESTAMP], }, }) : null; - const event = retainmentResponse?.hits.hits[0]?._source as + const event = retainmentResponse?.hits.hits[0]?.fields as | { - '@timestamp': number; + [AT_TIMESTAMP]: number[]; } | undefined; @@ -535,7 +537,7 @@ export const tasks: TelemetryTask[] = [ ? { retainment: { [processorEvent]: { - ms: new Date().getTime() - new Date(event['@timestamp']).getTime(), + ms: new Date().getTime() - new Date(event[AT_TIMESTAMP][0]).getTime(), }, }, } @@ -690,16 +692,16 @@ export const tasks: TelemetryTask[] = [ sort: { '@timestamp': 'desc', }, + fields: asMutableArray([OBSERVER_VERSION] as const), }, }); - const hit = response.hits.hits[0]?._source as Pick; - - if (!hit || !hit.observer?.version) { + const event = unflattenKnownApmEventFields(response.hits.hits[0]?.fields); + if (!event || !event.observer?.version) { return {}; } - const [major, minor, patch] = hit.observer.version.split('.').map((part) => Number(part)); + const [major, minor, patch] = event.observer.version.split('.').map((part) => Number(part)); return { version: { diff --git a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts index 4ab2e753832a4..cbcad6dea5baf 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts @@ -186,7 +186,6 @@ export const getDestinationMap = ({ }, size: destinationsBySpanId.size, fields: asMutableArray([SERVICE_NAME, SERVICE_ENVIRONMENT, AGENT_NAME, PARENT_ID] as const), - _source: false, }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts index 88d0040f70fc9..5d4977a73b42f 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts @@ -9,6 +9,10 @@ import { NOT_AVAILABLE_LABEL } from '../../../common/i18n'; import { Maybe } from '../../../typings/common'; import { APMError } from '../../../typings/es_schemas/ui/apm_error'; -export function getErrorName({ error }: { error: Maybe }): string { +export function getErrorName({ + error, +}: { + error: Maybe> & { log?: { message?: string } }; +}): string { return error?.log?.message || error?.exception?.[0]?.message || NOT_AVAILABLE_LABEL; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts index 96ddbe15c4287..dfc32ec9eb54e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts @@ -184,6 +184,7 @@ export function registerTransactionDurationRuleType({ body: { track_total_hits: false, size: 0, + _source: false as const, query: { bool: { filter: [ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts index 5f36325031ccb..7072639f8526e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts @@ -8,6 +8,9 @@ import datemath from '@elastic/datemath'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { flattenObject, KeyValuePair } from '../../../../common/utils/flatten_object'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { PROCESSOR_EVENT, TRACE_ID } from '../../../../common/es_fields/apm'; @@ -86,6 +89,7 @@ export async function getLogCategories({ const rawSamplingProbability = Math.min(100_000 / totalDocCount, 1); const samplingProbability = rawSamplingProbability < 0.5 ? rawSamplingProbability : 1; + const fields = asMutableArray(['message', TRACE_ID] as const); const categorizedLogsRes = await search({ index, size: 1, @@ -108,7 +112,7 @@ export async function getLogCategories({ top_hits: { sort: { '@timestamp': 'desc' as const }, size: 1, - _source: ['message', TRACE_ID], + fields, }, }, }, @@ -120,9 +124,11 @@ export async function getLogCategories({ const promises = categorizedLogsRes.aggregations?.sampling.categories?.buckets.map( async ({ doc_count: docCount, key, sample }) => { - const hit = sample.hits.hits[0]._source as { message: string; trace?: { id: string } }; - const sampleMessage = hit?.message; - const sampleTraceId = hit?.trace?.id; + const hit = sample.hits.hits[0]; + const event = unflattenKnownApmEventFields(hit?.fields); + + const sampleMessage = event.message as string; + const sampleTraceId = event.trace?.id; const errorCategory = key as string; if (!sampleTraceId) { @@ -140,7 +146,9 @@ export async function getLogCategories({ } ); - const sampleDoc = categorizedLogsRes.hits.hits?.[0]?._source as Record; + const event = unflattenKnownApmEventFields(maybe(categorizedLogsRes.hits.hits[0])?.fields); + + const sampleDoc = event as Record; return { logCategories: await Promise.all(promises ?? []), diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts index e7f3ace07e2a1..93c55cf1a9a30 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts @@ -13,6 +13,10 @@ import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { CONTAINER_ID } from '@kbn/apm-types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -79,13 +83,17 @@ async function getContainerIdFromLogs({ esClient: ElasticsearchClient; logSourcesService: LogSourcesService; }) { + const requiredFields = asMutableArray([CONTAINER_ID] as const); const index = await logSourcesService.getFlattenedLogSources(); const res = await typedSearch<{ container: { id: string } }, any>(esClient, { index, ...params, + fields: requiredFields, }); - return res.hits.hits[0]?._source?.container?.id; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.container.id; } async function getContainerIdFromTraces({ @@ -95,6 +103,7 @@ async function getContainerIdFromTraces({ params: APMEventESSearchRequest['body']; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([CONTAINER_ID] as const); const res = await apmEventClient.search('get_container_id_from_traces', { apm: { sources: [ @@ -104,8 +113,10 @@ async function getContainerIdFromTraces({ }, ], }, - body: params, + body: { ...params, fields: requiredFields }, }); - return res.hits.hits[0]?._source.container?.id; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.container.id; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts index a5b75f76ff237..d957372285b02 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts @@ -6,6 +6,9 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { maybe } from '../../../../common/utils/maybe'; import { ApmDocumentType } from '../../../../common/document_type'; import { termQuery } from '../../../../common/utils/term_query'; import { @@ -27,6 +30,7 @@ export async function getDownstreamServiceResource({ end: number; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([SPAN_DESTINATION_SERVICE_RESOURCE] as const); const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { sources: [ @@ -50,9 +54,11 @@ export async function getDownstreamServiceResource({ ], }, }, + fields: requiredFields, }, }); - const hit = response.hits.hits[0]; - return hit?._source?.span.destination?.service.resource; + const event = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredFields); + + return event?.span.destination.service.resource; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts index 0168431d0ac4e..bd966c500d1bc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts @@ -12,6 +12,10 @@ import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { SERVICE_NAME } from '@kbn/apm-types'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -102,6 +106,7 @@ async function getServiceNameFromTraces({ params: APMEventESSearchRequest['body']; apmEventClient: APMEventClient; }) { + const requiredFields = asMutableArray([SERVICE_NAME] as const); const res = await apmEventClient.search('get_service_name_from_traces', { apm: { sources: [ @@ -111,8 +116,13 @@ async function getServiceNameFromTraces({ }, ], }, - body: params, + body: { + ...params, + fields: requiredFields, + }, }); - return res.hits.hits[0]?._source.service.name; + const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields); + + return event?.service.name; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts index ebb3d3f57d18a..5b84743064142 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts @@ -7,8 +7,14 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { maybe } from '../../../common/utils/maybe'; -import { SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm'; +import { + SPAN_DESTINATION_SERVICE_RESOURCE, + SPAN_SUBTYPE, + SPAN_TYPE, +} from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; export interface MetadataForDependencyResponse { @@ -27,6 +33,7 @@ export async function getMetadataForDependency({ start: number; end: number; }): Promise { + const fields = asMutableArray([SPAN_TYPE, SPAN_SUBTYPE] as const); const sampleResponse = await apmEventClient.search('get_metadata_for_dependency', { apm: { events: [ProcessorEvent.span], @@ -46,16 +53,17 @@ export async function getMetadataForDependency({ ], }, }, + fields, sort: { '@timestamp': 'desc', }, }, }); - const sample = maybe(sampleResponse.hits.hits[0])?._source; + const sample = unflattenKnownApmEventFields(maybe(sampleResponse.hits.hits[0])?.fields); return { - spanType: sample?.span.type, - spanSubtype: sample?.span.subtype, + spanType: sample?.span?.type, + spanSubtype: sample?.span?.subtype, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts index 1c8448579806f..2a5a804d57f04 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts @@ -8,8 +8,11 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server'; import { keyBy } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, + AT_TIMESTAMP, EVENT_OUTCOME, SERVICE_ENVIRONMENT, SERVICE_NAME, @@ -66,6 +69,19 @@ export async function getTopDependencySpans({ sampleRangeFrom?: number; sampleRangeTo?: number; }): Promise { + const topDedsRequiredFields = asMutableArray([ + SPAN_ID, + TRACE_ID, + TRANSACTION_ID, + SPAN_NAME, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + AGENT_NAME, + SPAN_DURATION, + EVENT_OUTCOME, + AT_TIMESTAMP, + ] as const); + const spans = ( await apmEventClient.search('get_top_dependency_spans', { apm: { @@ -98,23 +114,18 @@ export async function getTopDependencySpans({ ], }, }, - _source: [ - SPAN_ID, - TRACE_ID, - TRANSACTION_ID, - SPAN_NAME, - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - SPAN_DURATION, - EVENT_OUTCOME, - '@timestamp', - ], + fields: topDedsRequiredFields, }, }) - ).hits.hits.map((hit) => hit._source); + ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, topDedsRequiredFields)); + + const transactionIds = spans.map((span) => span.transaction.id); - const transactionIds = spans.map((span) => span.transaction!.id); + const txRequiredFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_TYPE, + TRANSACTION_NAME, + ] as const); const transactions = ( await apmEventClient.search('get_transactions_for_dependency_spans', { @@ -129,13 +140,13 @@ export async function getTopDependencySpans({ filter: [...termsQuery(TRANSACTION_ID, ...transactionIds)], }, }, - _source: [TRANSACTION_ID, TRANSACTION_TYPE, TRANSACTION_NAME], + fields: txRequiredFields, sort: { '@timestamp': 'desc', }, }, }) - ).hits.hits.map((hit) => hit._source); + ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, txRequiredFields)); const transactionsById = keyBy(transactions, (transaction) => transaction.transaction.id); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts index 1e9576ea2f7e4..3d6fa0f5a5ef6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts @@ -7,13 +7,17 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery, wildcardQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, ERROR_CULPRIT, ERROR_EXC_HANDLED, ERROR_EXC_MESSAGE, ERROR_EXC_TYPE, ERROR_GROUP_ID, ERROR_GROUP_NAME, + ERROR_ID, ERROR_LOG_MESSAGE, SERVICE_NAME, TRACE_ID, @@ -93,6 +97,21 @@ export async function getErrorGroupMainStatistics({ ] : []; + const requiredFields = asMutableArray([ + TRACE_ID, + AT_TIMESTAMP, + ERROR_GROUP_ID, + ERROR_ID, + ] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { sources: [ @@ -129,16 +148,8 @@ export async function getErrorGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - TRACE_ID, - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { '@timestamp': 'desc', }, @@ -157,15 +168,33 @@ export async function getErrorGroupMainStatistics({ const errorGroups = response.aggregations?.error_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + }, + }; + return { groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]._source['@timestamp']).getTime(), + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]._source.error.culprit, - handled: bucket.sample.hits.hits[0]._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]._source.error.exception?.[0].type, - traceId: bucket.sample.hits.hits[0]._source.trace?.id, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, + traceId: mergedEvent.trace?.id, }; }) ?? []; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts index 0a154d3ad13fa..fc80c3f411651 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts @@ -6,6 +6,7 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { ERROR_GROUP_ID, @@ -42,6 +43,7 @@ export async function getErrorGroupSampleIds({ start: number; end: number; }): Promise { + const requiredFields = asMutableArray([ERROR_ID] as const); const resp = await apmEventClient.search('get_error_group_sample_ids', { apm: { sources: [ @@ -66,7 +68,7 @@ export async function getErrorGroupSampleIds({ should: [{ term: { [TRANSACTION_SAMPLED]: true } }], // prefer error samples with related transactions }, }, - _source: [ERROR_ID, 'transaction'], + fields: requiredFields, sort: asMutableArray([ { _score: { order: 'desc' } }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error @@ -74,8 +76,8 @@ export async function getErrorGroupSampleIds({ }, }); const errorSampleIds = resp.hits.hits.map((item) => { - const source = item._source; - return source.error.id; + const event = unflattenKnownApmEventFields(item.fields, requiredFields); + return event.error?.id; }); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index 348949d3ecca5..91da19224d83c 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -6,7 +6,28 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; -import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { maybe } from '../../../../common/utils/maybe'; +import { + AGENT_NAME, + AGENT_VERSION, + AT_TIMESTAMP, + ERROR_EXCEPTION, + ERROR_GROUP_ID, + ERROR_ID, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + PROCESSOR_EVENT, + PROCESSOR_NAME, + SERVICE_NAME, + TIMESTAMP_US, + TRACE_ID, + TRANSACTION_ID, + ERROR_STACK_TRACE, + SPAN_ID, +} from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { ApmDocumentType } from '../../../../common/document_type'; import { RollupInterval } from '../../../../common/rollup'; @@ -17,7 +38,15 @@ import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; export interface ErrorSampleDetailsResponse { transaction: Transaction | undefined; - error: APMError; + error: Omit & { + transaction?: { id?: string; type?: string }; + error: { + id: string; + } & Omit & { + exception?: APMError['error']['exception']; + log?: APMError['error']['log']; + }; + }; } export async function getErrorSampleDetails({ @@ -36,7 +65,29 @@ export async function getErrorSampleDetails({ apmEventClient: APMEventClient; start: number; end: number; -}): Promise { +}): Promise> { + const requiredFields = asMutableArray([ + AGENT_NAME, + PROCESSOR_EVENT, + TRACE_ID, + TIMESTAMP_US, + AT_TIMESTAMP, + SERVICE_NAME, + ERROR_ID, + ERROR_GROUP_ID, + ] as const); + + const optionalFields = asMutableArray([ + TRANSACTION_ID, + SPAN_ID, + AGENT_VERSION, + PROCESSOR_NAME, + ERROR_STACK_TRACE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const params = { apm: { sources: [ @@ -60,15 +111,29 @@ export async function getErrorSampleDetails({ ], }, }, + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_EXCEPTION, 'error.log'], }, }; const resp = await apmEventClient.search('get_error_sample_details', params); - const error = resp.hits.hits[0]?._source; - const transactionId = error?.transaction?.id; - const traceId = error?.trace?.id; + const hit = maybe(resp.hits.hits[0]); + + if (!hit) { + return { + transaction: undefined, + error: undefined, + }; + } + + const source = 'error' in hit._source ? hit._source : undefined; - let transaction; + const errorFromFields = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const transactionId = errorFromFields.transaction?.id ?? errorFromFields.span?.id; + const traceId = errorFromFields.trace.id; + + let transaction: Transaction | undefined; if (transactionId && traceId) { transaction = await getTransaction({ transactionId, @@ -81,6 +146,20 @@ export async function getErrorSampleDetails({ return { transaction, - error, + error: { + ...errorFromFields, + processor: { + name: errorFromFields.processor.name as 'error', + event: errorFromFields.processor.event as 'error', + }, + error: { + ...errorFromFields.error, + exception: + (source?.error.exception?.length ?? 0) > 1 + ? source?.error.exception + : errorFromFields?.error.exception && [errorFromFields.error.exception], + log: source?.error?.log, + }, + }, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts index dd262246a80b7..62d9d883ba896 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts @@ -7,6 +7,7 @@ import { jsonRt, toNumberRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; +import { notFound } from '@hapi/boom'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { ErrorDistributionResponse, getErrorDistribution } from './distribution/get_distribution'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; @@ -205,7 +206,7 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({ const { serviceName, errorId } = params.path; const { environment, kuery, start, end } = params.query; - return getErrorSampleDetails({ + const { transaction, error } = await getErrorSampleDetails({ environment, errorId, kuery, @@ -214,6 +215,12 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({ start, end, }); + + if (!error) { + throw notFound(); + } + + return { error, transaction }; }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts index b0d3eabe0bab2..c606a6b045a93 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts @@ -8,6 +8,8 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../../common/utils/as_mutable_array'; import { ERROR_CULPRIT, ERROR_TYPE, @@ -19,6 +21,7 @@ import { SERVICE_NAME, TRANSACTION_NAME, TRANSACTION_TYPE, + AT_TIMESTAMP, } from '../../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../../common/utils/environment_query'; import { getErrorName } from '../../../../lib/helpers/get_error_name'; @@ -68,6 +71,16 @@ export async function getMobileCrashGroupMainStatistics({ ? { [maxTimestampAggKey]: sortDirection } : { _count: sortDirection }; + const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_crash_group_main_statistics', { apm: { events: [ProcessorEvent.error], @@ -99,22 +112,15 @@ export async function getMobileCrashGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { - '@timestamp': 'desc', + [AT_TIMESTAMP]: 'desc', }, }, }, ...(sortByLatestOccurrence - ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } } + ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } } : {}), }, }, @@ -123,14 +129,34 @@ export async function getMobileCrashGroupMainStatistics({ }); return ( - response.aggregations?.crash_groups.buckets.map((bucket) => ({ - groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(), - occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]?._source.error.culprit, - handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type, - })) ?? [] + response.aggregations?.crash_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + }, + }; + + return { + groupId: event.error?.grouping_key, + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), + occurrences: bucket.doc_count, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, + }; + }) ?? [] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts index f259e17d6154c..1181aa5b02870 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts @@ -8,7 +8,10 @@ import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types'; import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, ERROR_CULPRIT, ERROR_EXC_HANDLED, ERROR_EXC_MESSAGE, @@ -67,6 +70,16 @@ export async function getMobileErrorGroupMainStatistics({ ? { [maxTimestampAggKey]: sortDirection } : { _count: sortDirection }; + const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const); + + const optionalFields = asMutableArray([ + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const response = await apmEventClient.search('get_error_group_main_statistics', { apm: { events: [ProcessorEvent.error], @@ -100,22 +113,15 @@ export async function getMobileErrorGroupMainStatistics({ sample: { top_hits: { size: 1, - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], sort: { - '@timestamp': 'desc', + [AT_TIMESTAMP]: 'desc', }, }, }, ...(sortByLatestOccurrence - ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } } + ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } } : {}), }, }, @@ -124,14 +130,34 @@ export async function getMobileErrorGroupMainStatistics({ }); return ( - response.aggregations?.error_groups.buckets.map((bucket) => ({ - groupId: bucket.key as string, - name: getErrorName(bucket.sample.hits.hits[0]._source), - lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(), - occurrences: bucket.doc_count, - culprit: bucket.sample.hits.hits[0]?._source.error.culprit, - handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled, - type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type, - })) ?? [] + response.aggregations?.error_groups.buckets.map((bucket) => { + const errorSource = + 'error' in bucket.sample.hits.hits[0]._source + ? bucket.sample.hits.hits[0]._source + : undefined; + + const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields); + + const mergedEvent = { + ...event, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + }, + }; + + return { + groupId: event.error?.grouping_key, + name: getErrorName(mergedEvent), + lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(), + occurrences: bucket.doc_count, + culprit: mergedEvent.error.culprit, + handled: mergedEvent.error.exception?.[0].handled, + type: mergedEvent.error.exception?.[0].type, + }; + }) ?? [] ); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts index e766d56c44ae4..c683e308b73b8 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts @@ -7,9 +7,12 @@ import type { ESFilter } from '@kbn/es-types'; import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { isFiniteNumber } from '../../../../common/utils/is_finite_number'; import { Annotation, AnnotationType } from '../../../../common/annotations'; -import { SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm'; +import { AT_TIMESTAMP, SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { getBackwardCompatibleDocumentTypeFilter, @@ -66,6 +69,8 @@ export async function getDerivedServiceAnnotations({ if (versions.length <= 1) { return []; } + + const requiredFields = asMutableArray([AT_TIMESTAMP] as const); const annotations = await Promise.all( versions.map(async (version) => { const response = await apmEventClient.search('get_first_seen_of_version', { @@ -83,11 +88,21 @@ export async function getDerivedServiceAnnotations({ sort: { '@timestamp': 'asc', }, + fields: requiredFields, }, }); - const firstSeen = new Date(response.hits.hits[0]._source['@timestamp']).getTime(); + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields, + requiredFields + ); + + const timestamp = event?.[AT_TIMESTAMP]; + if (!timestamp) { + throw new Error('First seen for version was unexpectedly undefined or null.'); + } + const firstSeen = new Date(timestamp).getTime(); if (!isFiniteNumber(firstSeen)) { throw new Error('First seen for version was unexpectedly undefined or null.'); } @@ -99,7 +114,7 @@ export async function getDerivedServiceAnnotations({ return { type: AnnotationType.VERSION, id: version, - '@timestamp': firstSeen, + [AT_TIMESTAMP]: firstSeen, text: version, }; }) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts index 94c5bcbac4e66..dd272eadf57d6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts @@ -7,6 +7,8 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, SERVICE_NAME, @@ -16,23 +18,7 @@ import { } from '../../../common/es_fields/apm'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { getServerlessTypeFromCloudData, ServerlessType } from '../../../common/serverless'; - -interface ServiceAgent { - agent?: { - name: string; - }; - service?: { - runtime?: { - name?: string; - }; - }; - cloud?: { - provider?: string; - service?: { - name?: string; - }; - }; -} +import { maybe } from '../../../common/utils/maybe'; export interface ServiceAgentResponse { agentName?: string; @@ -51,6 +37,13 @@ export async function getServiceAgent({ start: number; end: number; }): Promise { + const fields = asMutableArray([ + AGENT_NAME, + SERVICE_RUNTIME_NAME, + CLOUD_PROVIDER, + CLOUD_SERVICE_NAME, + ] as const); + const params = { terminate_after: 1, apm: { @@ -90,6 +83,7 @@ export async function getServiceAgent({ ], }, }, + fields, sort: { _score: { order: 'desc' as const }, }, @@ -97,11 +91,14 @@ export async function getServiceAgent({ }; const response = await apmEventClient.search('get_service_agent_name', params); - if (response.hits.total.value === 0) { + const hit = maybe(response.hits.hits[0]); + if (!hit) { return {}; } - const { agent, service, cloud } = response.hits.hits[0]._source as ServiceAgent; + const event = unflattenKnownApmEventFields(hit.fields); + + const { agent, service, cloud } = event; const serverlessType = getServerlessTypeFromCloudData(cloud?.provider, cloud?.service?.name); return { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts index 400429617d803..d16910f5984fc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts @@ -6,19 +6,20 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { CONTAINER_ID, CONTAINER_IMAGE, KUBERNETES, KUBERNETES_POD_NAME, KUBERNETES_POD_UID, -} from '../../../common/es_fields/apm'; -import { KUBERNETES_CONTAINER_NAME, - KUBERNETES_NAMESPACE, KUBERNETES_REPLICASET_NAME, KUBERNETES_DEPLOYMENT_NAME, -} from '../../../common/es_fields/infra_metrics'; + KUBERNETES_CONTAINER_ID, + KUBERNETES_NAMESPACE, +} from '../../../common/es_fields/apm'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { maybe } from '../../../common/utils/maybe'; import { InfraMetricsClient } from '../../lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client'; @@ -51,9 +52,21 @@ export const getServiceInstanceContainerMetadata = async ({ { exists: { field: KUBERNETES_DEPLOYMENT_NAME } }, ]; + const fields = asMutableArray([ + KUBERNETES_POD_NAME, + KUBERNETES_POD_UID, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + ] as const); + const response = await infraMetricsClient.search({ size: 1, track_total_hits: false, + fields, query: { bool: { filter: [ @@ -69,7 +82,7 @@ export const getServiceInstanceContainerMetadata = async ({ }, }); - const sample = maybe(response.hits.hits[0])?._source as ServiceInstanceContainerMetadataDetails; + const sample = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields); return { kubernetes: { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts index daa49d2ed59c8..3c139f2aee0de 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts @@ -7,7 +7,16 @@ import { merge } from 'lodash'; import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; +import { + AGENT_NAME, + AT_TIMESTAMP, + METRICSET_NAME, + SERVICE_ENVIRONMENT, + SERVICE_NAME, + SERVICE_NODE_NAME, +} from '../../../common/es_fields/apm'; import { maybe } from '../../../common/utils/maybe'; import { getBackwardCompatibleDocumentTypeFilter, @@ -20,6 +29,13 @@ import { Container } from '../../../typings/es_schemas/raw/fields/container'; import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes'; import { Host } from '../../../typings/es_schemas/raw/fields/host'; import { Cloud } from '../../../typings/es_schemas/raw/fields/cloud'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; +import { + SERVICE_METADATA_CLOUD_KEYS, + SERVICE_METADATA_CONTAINER_KEYS, + SERVICE_METADATA_INFRA_METRICS_KEYS, + SERVICE_METADATA_SERVICE_KEYS, +} from '../../../common/service_metadata'; export interface ServiceInstanceMetadataDetailsResponse { '@timestamp': string; @@ -50,6 +66,18 @@ export async function getServiceInstanceMetadataDetails({ ...rangeQuery(start, end), ]; + const requiredKeys = asMutableArray([AT_TIMESTAMP, SERVICE_NAME, AGENT_NAME] as const); + + const optionalKeys = asMutableArray([ + SERVICE_ENVIRONMENT, + ...SERVICE_METADATA_SERVICE_KEYS, + ...SERVICE_METADATA_CLOUD_KEYS, + ...SERVICE_METADATA_CONTAINER_KEYS, + ...SERVICE_METADATA_INFRA_METRICS_KEYS, + ] as const); + + const fields = [...requiredKeys, ...optionalKeys]; + async function getApplicationMetricSample() { const response = await apmEventClient.search( 'get_service_instance_metadata_details_application_metric', @@ -66,11 +94,12 @@ export async function getServiceInstanceMetadataDetails({ filter: filter.concat({ term: { [METRICSET_NAME]: 'app' } }), }, }, + fields, }, } ); - return maybe(response.hits.hits[0]?._source); + return unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredKeys); } async function getTransactionEventSample() { @@ -85,11 +114,14 @@ export async function getServiceInstanceMetadataDetails({ terminate_after: 1, size: 1, query: { bool: { filter } }, + fields, }, } ); - return maybe(response.hits.hits[0]?._source); + return unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); } async function getTransactionMetricSample() { @@ -108,10 +140,14 @@ export async function getServiceInstanceMetadataDetails({ filter: filter.concat(getBackwardCompatibleDocumentTypeFilter(true)), }, }, + fields, }, } ); - return maybe(response.hits.hits[0]?._source); + + return unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); } // we can expect the most detail of application metrics, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index fb44638d8a6b0..0319ae66039e5 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -7,37 +7,26 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; import { environmentQuery } from '../../../common/utils/environment_query'; import { - AGENT, - CONTAINER, - CLOUD, CLOUD_AVAILABILITY_ZONE, CLOUD_REGION, CLOUD_MACHINE_TYPE, CLOUD_SERVICE_NAME, CONTAINER_ID, - HOST, - KUBERNETES, - SERVICE, SERVICE_NAME, SERVICE_NODE_NAME, SERVICE_VERSION, FAAS_ID, FAAS_TRIGGER_TYPE, - LABEL_TELEMETRY_AUTO_VERSION, } from '../../../common/es_fields/apm'; - import { ContainerType } from '../../../common/service_metadata'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { should } from './get_service_metadata_icons'; import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name'; - -type ServiceMetadataDetailsRaw = Pick< - TransactionRaw, - 'service' | 'agent' | 'host' | 'container' | 'kubernetes' | 'cloud' | 'labels' ->; +import { maybe } from '../../../common/utils/maybe'; export interface ServiceMetadataDetails { service?: { @@ -112,7 +101,6 @@ export async function getServiceMetadataDetails({ body: { track_total_hits: 1, size: 1, - _source: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION], query: { bool: { filter, should } }, aggs: { serviceVersions: { @@ -166,13 +154,17 @@ export async function getServiceMetadataDetails({ }, totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, + fields: ['*'], }, }; const response = await apmEventClient.search('get_service_metadata_details', params); - const hit = response.hits.hits[0]?._source as ServiceMetadataDetailsRaw | undefined; - if (!hit) { + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); + + if (!event) { return { service: undefined, container: undefined, @@ -180,7 +172,7 @@ export async function getServiceMetadataDetails({ }; } - const { service, agent, host, kubernetes, container, cloud, labels } = hit; + const { service, agent, host, kubernetes, container, cloud, labels } = event; const serviceMetadataDetails = { versions: response.aggregations?.serviceVersions.buckets.map((bucket) => bucket.key as string), diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 30580ddbf0ac8..ee0a857c9b719 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -7,12 +7,15 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; +import { maybe } from '../../../common/utils/maybe'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AGENT_NAME, CLOUD_PROVIDER, CLOUD_SERVICE_NAME, CONTAINER_ID, - KUBERNETES, SERVICE_NAME, KUBERNETES_POD_NAME, HOST_OS_PLATFORM, @@ -20,14 +23,11 @@ import { AGENT_VERSION, SERVICE_FRAMEWORK_NAME, } from '../../../common/es_fields/apm'; -import { ContainerType } from '../../../common/service_metadata'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; +import { ContainerType, SERVICE_METADATA_KUBERNETES_KEYS } from '../../../common/service_metadata'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless'; -type ServiceMetadataIconsRaw = Pick; - export interface ServiceMetadataIcons { agentName?: string; containerType?: ContainerType; @@ -61,6 +61,14 @@ export async function getServiceMetadataIcons({ }): Promise { const filter = [{ term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end)]; + const fields = asMutableArray([ + CLOUD_PROVIDER, + CONTAINER_ID, + AGENT_NAME, + CLOUD_SERVICE_NAME, + ...SERVICE_METADATA_KUBERNETES_KEYS, + ] as const); + const params = { apm: { events: [ @@ -72,8 +80,8 @@ export async function getServiceMetadataIcons({ body: { track_total_hits: 1, size: 1, - _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME], query: { bool: { filter, should } }, + fields, }, }; @@ -88,9 +96,11 @@ export async function getServiceMetadataIcons({ }; } - const { kubernetes, cloud, container, agent } = response.hits.hits[0] - ._source as ServiceMetadataIconsRaw; + const event = unflattenKnownApmEventFields( + maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent + ); + const { kubernetes, cloud, container, agent } = event ?? {}; let containerType: ContainerType; if (!!kubernetes) { containerType = 'Kubernetes'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts index 3f9cf1275cace..2ff34698c20bc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts @@ -7,6 +7,8 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { isEmpty } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { PROCESSOR_EVENT, SPAN_ID, @@ -16,8 +18,6 @@ import { TRACE_ID, TRANSACTION_ID, } from '../../../common/es_fields/apm'; -import type { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; -import type { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getBufferedTimerange } from './utils'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -39,12 +39,16 @@ async function fetchLinkedChildrenOfSpan({ end, }); + const requiredFields = asMutableArray([TRACE_ID, PROCESSOR_EVENT] as const); + const optionalFields = asMutableArray([SPAN_ID, TRANSACTION_ID] as const); + const response = await apmEventClient.search('fetch_linked_children_of_span', { apm: { events: [ProcessorEvent.span, ProcessorEvent.transaction], }, - _source: [SPAN_LINKS, TRACE_ID, SPAN_ID, PROCESSOR_EVENT, TRANSACTION_ID], + _source: [SPAN_LINKS], body: { + fields: [...requiredFields, ...optionalFields], track_total_hits: false, size: 1000, query: { @@ -58,19 +62,32 @@ async function fetchLinkedChildrenOfSpan({ }, }, }); + + const linkedChildren = response.hits.hits.map((hit) => { + const source = 'span' in hit._source ? hit._source : undefined; + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + return { + ...event, + span: { + ...event.span, + links: source?.span?.links ?? [], + }, + }; + }); // Filter out documents that don't have any span.links that match the combination of traceId and spanId - return response.hits.hits.filter(({ _source: source }) => { - const spanLinks = source.span?.links?.filter((spanLink) => { + return linkedChildren.filter((linkedChild) => { + const spanLinks = linkedChild?.span?.links?.filter((spanLink) => { return spanLink.trace.id === traceId && (spanId ? spanLink.span.id === spanId : true); }); return !isEmpty(spanLinks); }); } -function getSpanId(source: TransactionRaw | SpanRaw) { - return source.processor.event === ProcessorEvent.span - ? (source as SpanRaw).span.id - : (source as TransactionRaw).transaction?.id; +function getSpanId( + linkedChild: Awaited>[number] +): string { + return (linkedChild.span.id ?? linkedChild.transaction?.id) as string; } export async function getSpanLinksCountById({ @@ -90,8 +107,9 @@ export async function getSpanLinksCountById({ start, end, }); - return linkedChildren.reduce>((acc, { _source: source }) => { - source.span?.links?.forEach((link) => { + + return linkedChildren.reduce>((acc, item) => { + item.span?.links?.forEach((link) => { // Ignores span links that don't belong to this trace if (link.trace.id === traceId) { acc[link.span.id] = (acc[link.span.id] || 0) + 1; @@ -122,10 +140,10 @@ export async function getLinkedChildrenOfSpan({ end, }); - return linkedChildren.map(({ _source: source }) => { + return linkedChildren.map((item) => { return { - trace: { id: source.trace.id }, - span: { id: getSpanId(source) }, + trace: { id: item.trace.id }, + span: { id: getSpanId(item) }, }; }); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts index 2010cd5e86f2f..59e91e0b17e6b 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts @@ -56,7 +56,7 @@ export async function getLinkedParentsOfSpan({ }, }); - const source = response.hits.hits?.[0]?._source as TransactionRaw | SpanRaw; + const source = response.hits.hits?.[0]?._source as Pick; return source?.span?.links || []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts index 13f47764af375..669adb1008080 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts @@ -7,6 +7,8 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { chunk, compact, isEmpty, keyBy } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { SERVICE_NAME, SPAN_ID, @@ -25,8 +27,6 @@ import { import { Environment } from '../../../common/environment_rt'; import { SpanLinkDetails } from '../../../common/span_links'; import { SpanLink } from '../../../typings/es_schemas/raw/fields/span_links'; -import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw'; -import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getBufferedTimerange } from './utils'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; @@ -48,26 +48,35 @@ async function fetchSpanLinksDetails({ end, }); + const requiredFields = asMutableArray([ + TRACE_ID, + SERVICE_NAME, + AGENT_NAME, + PROCESSOR_EVENT, + ] as const); + + const requiredTxFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_DURATION, + ] as const); + + const requiredSpanFields = asMutableArray([ + SPAN_ID, + SPAN_NAME, + SPAN_DURATION, + SPAN_SUBTYPE, + SPAN_TYPE, + ] as const); + + const optionalFields = asMutableArray([SERVICE_ENVIRONMENT] as const); + const response = await apmEventClient.search('get_span_links_details', { apm: { events: [ProcessorEvent.span, ProcessorEvent.transaction], }, - _source: [ - TRACE_ID, - SPAN_ID, - TRANSACTION_ID, - SERVICE_NAME, - SPAN_NAME, - TRANSACTION_NAME, - TRANSACTION_DURATION, - SPAN_DURATION, - PROCESSOR_EVENT, - SPAN_SUBTYPE, - SPAN_TYPE, - AGENT_NAME, - SERVICE_ENVIRONMENT, - ], body: { + fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields], track_total_hits: false, size: 1000, query: { @@ -106,16 +115,67 @@ async function fetchSpanLinksDetails({ const spanIdsMap = keyBy(spanLinks, 'span.id'); - return response.hits.hits.filter(({ _source: source }) => { - // The above query might return other spans from the same transaction because siblings spans share the same transaction.id - // so, if it is a span we need to guarantee that the span.id is the same as the span links ids - if (source.processor.event === ProcessorEvent.span) { - const span = source as SpanRaw; - const hasSpanId = spanIdsMap[span.span.id] || false; - return hasSpanId; - } - return true; - }); + return response.hits.hits + .filter((hit) => { + // The above query might return other spans from the same transaction because siblings spans share the same transaction.id + // so, if it is a span we need to guarantee that the span.id is the same as the span links ids + if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) { + const spanLink = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + const hasSpanId = Boolean(spanIdsMap[spanLink.span.id] || false); + return hasSpanId; + } + return true; + }) + .map((hit) => { + const commonEvent = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const commonDetails = { + serviceName: commonEvent.service.name, + agentName: commonEvent.agent.name, + environment: commonEvent.service.environment as Environment, + transactionId: commonEvent.transaction?.id, + }; + + if (commonEvent.processor.event === ProcessorEvent.transaction) { + const event = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredTxFields, + ]); + return { + traceId: event.trace.id, + spanId: event.transaction.id, + processorEvent: commonEvent.processor.event, + transactionId: event.transaction.id, + details: { + ...commonDetails, + spanName: event.transaction.name, + duration: event.transaction.duration.us, + }, + }; + } else { + const event = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + return { + traceId: event.trace.id, + spanId: event.span.id, + processorEvent: commonEvent.processor.event, + details: { + ...commonDetails, + spanName: event.span.name, + duration: event.span.duration.us, + spanSubtype: event.span.subtype, + spanType: event.span.type, + }, + }; + } + }); } export async function getSpanLinksDetails({ @@ -153,39 +213,20 @@ export async function getSpanLinksDetails({ // Creates a map for all span links details found const spanLinksDetailsMap = linkedSpans.reduce>( - (acc, { _source: source }) => { - const commonDetails = { - serviceName: source.service.name, - agentName: source.agent.name, - environment: source.service.environment as Environment, - transactionId: source.transaction?.id, - }; - - if (source.processor.event === ProcessorEvent.transaction) { - const transaction = source as TransactionRaw; - const key = `${transaction.trace.id}:${transaction.transaction.id}`; + (acc, spanLink) => { + if (spanLink.processorEvent === ProcessorEvent.transaction) { + const key = `${spanLink.traceId}:${spanLink.transactionId}`; acc[key] = { - traceId: source.trace.id, - spanId: transaction.transaction.id, - details: { - ...commonDetails, - spanName: transaction.transaction.name, - duration: transaction.transaction.duration.us, - }, + traceId: spanLink.traceId, + spanId: spanLink.transactionId, + details: spanLink.details, }; } else { - const span = source as SpanRaw; - const key = `${span.trace.id}:${span.span.id}`; + const key = `${spanLink.traceId}:${spanLink.spanId}`; acc[key] = { - traceId: source.trace.id, - spanId: span.span.id, - details: { - ...commonDetails, - spanName: span.span.name, - duration: span.span.duration.us, - spanSubtype: span.span.subtype, - spanType: span.span.type, - }, + traceId: spanLink.traceId, + spanId: spanLink.spanId, + details: spanLink.details, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap index d64c33a421e19..58ec97112e8f6 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap @@ -12,15 +12,26 @@ Object { }, "body": Object { "_source": Array [ + "error.log.message", + "error.exception.message", + "error.exception.handled", + "error.exception.type", + ], + "fields": Array [ "timestamp.us", "trace.id", - "transaction.id", - "parent.id", "service.name", "error.id", - "error.log.message", - "error.exception", "error.grouping_key", + "processor.event", + "parent.id", + "transaction.id", + "span.id", + "error.culprit", + "error.log.message", + "error.exception.message", + "error.exception.handled", + "error.exception.type", ], "query": Object { "bool": Object { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts index d38a49745653a..55fb0aab47f38 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts @@ -10,12 +10,17 @@ import { SortResults } from '@elastic/elasticsearch/lib/api/types'; import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { last } from 'lodash'; +import { last, omit } from 'lodash'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { APMConfig } from '../..'; import { AGENT_NAME, CHILD_ID, - ERROR_EXCEPTION, + ERROR_CULPRIT, + ERROR_EXC_HANDLED, + ERROR_EXC_MESSAGE, + ERROR_EXC_TYPE, ERROR_GROUP_ID, ERROR_ID, ERROR_LOG_LEVEL, @@ -37,7 +42,7 @@ import { SPAN_SUBTYPE, SPAN_SYNC, SPAN_TYPE, - TIMESTAMP, + TIMESTAMP_US, TRACE_ID, TRANSACTION_DURATION, TRANSACTION_ID, @@ -84,6 +89,26 @@ export async function getTraceItems({ const maxTraceItems = maxTraceItemsFromUrlParam ?? config.ui.maxTraceItems; const excludedLogLevels = ['debug', 'info', 'warning']; + const requiredFields = asMutableArray([ + TIMESTAMP_US, + TRACE_ID, + SERVICE_NAME, + ERROR_ID, + ERROR_GROUP_ID, + PROCESSOR_EVENT, + ] as const); + + const optionalFields = asMutableArray([ + PARENT_ID, + TRANSACTION_ID, + SPAN_ID, + ERROR_CULPRIT, + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ] as const); + const errorResponsePromise = apmEventClient.search('get_errors_docs', { apm: { sources: [ @@ -96,23 +121,14 @@ export async function getTraceItems({ body: { track_total_hits: false, size: 1000, - _source: [ - TIMESTAMP, - TRACE_ID, - TRANSACTION_ID, - PARENT_ID, - SERVICE_NAME, - ERROR_ID, - ERROR_LOG_MESSAGE, - ERROR_EXCEPTION, - ERROR_GROUP_ID, - ], query: { bool: { filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)], must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, }, }, + fields: [...requiredFields, ...optionalFields], + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE], }, }); @@ -133,8 +149,32 @@ export async function getTraceItems({ const traceDocsTotal = traceResponse.total; const exceedsMax = traceDocsTotal > maxTraceItems; - const traceDocs = traceResponse.hits.map((hit) => hit._source); - const errorDocs = errorResponse.hits.hits.map((hit) => hit._source); + + const traceDocs = traceResponse.hits.map(({ hit }) => hit); + + const errorDocs = errorResponse.hits.hits.map((hit) => { + const errorSource = 'error' in hit._source ? hit._source : undefined; + + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const waterfallErrorEvent: WaterfallError = { + ...event, + parent: { + ...event?.parent, + id: event?.parent?.id ?? event?.span?.id, + }, + error: { + ...(event.error ?? {}), + exception: + (errorSource?.error.exception?.length ?? 0) > 1 + ? errorSource?.error.exception + : event?.error.exception && [event.error.exception], + log: errorSource?.error.log, + }, + }; + + return waterfallErrorEvent; + }); return { exceedsMax, @@ -220,41 +260,54 @@ async function getTraceDocsPerPage({ start: number; end: number; searchAfter?: SortResults; -}) { +}): Promise<{ + hits: Array<{ hit: WaterfallTransaction | WaterfallSpan; sort: SortResults | undefined }>; + total: number; +}> { const size = Math.min(maxTraceItems, MAX_ITEMS_PER_PAGE); + const requiredFields = asMutableArray([ + AGENT_NAME, + TIMESTAMP_US, + TRACE_ID, + SERVICE_NAME, + PROCESSOR_EVENT, + ] as const); + + const requiredTxFields = asMutableArray([ + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_TYPE, + ] as const); + + const requiredSpanFields = asMutableArray([ + SPAN_ID, + SPAN_TYPE, + SPAN_NAME, + SPAN_DURATION, + ] as const); + + const optionalFields = asMutableArray([ + PARENT_ID, + SERVICE_ENVIRONMENT, + EVENT_OUTCOME, + TRANSACTION_RESULT, + FAAS_COLDSTART, + SPAN_SUBTYPE, + SPAN_ACTION, + SPAN_COMPOSITE_COUNT, + SPAN_COMPOSITE_COMPRESSION_STRATEGY, + SPAN_COMPOSITE_SUM, + SPAN_SYNC, + CHILD_ID, + ] as const); + const body = { track_total_hits: true, size, search_after: searchAfter, - _source: [ - TIMESTAMP, - TRACE_ID, - PARENT_ID, - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - EVENT_OUTCOME, - PROCESSOR_EVENT, - TRANSACTION_DURATION, - TRANSACTION_ID, - TRANSACTION_NAME, - TRANSACTION_TYPE, - TRANSACTION_RESULT, - FAAS_COLDSTART, - SPAN_ID, - SPAN_TYPE, - SPAN_SUBTYPE, - SPAN_ACTION, - SPAN_NAME, - SPAN_DURATION, - SPAN_LINKS, - SPAN_COMPOSITE_COUNT, - SPAN_COMPOSITE_COMPRESSION_STRATEGY, - SPAN_COMPOSITE_SUM, - SPAN_SYNC, - CHILD_ID, - ], + _source: [SPAN_LINKS], query: { bool: { filter: [ @@ -266,6 +319,7 @@ async function getTraceDocsPerPage({ }, }, }, + fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields], sort: [ { _score: 'asc' }, { @@ -291,7 +345,51 @@ async function getTraceDocsPerPage({ }); return { - hits: res.hits.hits, + hits: res.hits.hits.map((hit) => { + const sort = hit.sort; + const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined; + + if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) { + const spanEvent = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredSpanFields, + ]); + + const spanWaterfallEvent: WaterfallSpan = { + ...omit(spanEvent, 'child'), + processor: { + event: 'span', + }, + span: { + ...spanEvent.span, + composite: spanEvent.span.composite + ? (spanEvent.span.composite as Required['composite']) + : undefined, + links: spanLinksSource, + }, + ...(spanEvent.child ? { child: spanEvent.child as WaterfallSpan['child'] } : {}), + }; + + return { sort, hit: spanWaterfallEvent }; + } + + const txEvent = unflattenKnownApmEventFields(hit.fields, [ + ...requiredFields, + ...requiredTxFields, + ]); + const txWaterfallEvent: WaterfallTransaction = { + ...txEvent, + processor: { + event: 'transaction', + }, + span: { + ...txEvent.span, + links: spanLinksSource, + }, + }; + + return { hit: txWaterfallEvent, sort }; + }), total: res.hits.total.value, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts index 033b666d0371c..dd1330dea4e48 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts @@ -89,10 +89,10 @@ export async function getTraceSamplesByQuery({ }, event_category_field: PROCESSOR_EVENT, query, - filter_path: 'hits.sequences.events._source.trace.id', + fields: [TRACE_ID], }) ).hits?.sequences?.flatMap((sequence) => - sequence.events.map((event) => (event._source as { trace: { id: string } }).trace.id) + sequence.events.map((event) => (event.fields as { [TRACE_ID]: [string] })[TRACE_ID][0]) ) ?? []; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts index 328201ec9d143..0814bcdc5738f 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts @@ -12,7 +12,10 @@ import { getSearchTransactionsEvents } from '../../lib/helpers/transactions'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, probabilityRt, rangeRt } from '../default_api_types'; import { getTransaction } from '../transactions/get_transaction'; -import { getRootTransactionByTraceId } from '../transactions/get_transaction_by_trace'; +import { + type TransactionDetailRedirectInfo, + getRootTransactionByTraceId, +} from '../transactions/get_transaction_by_trace'; import { getTopTracesPrimaryStats, TopTracesPrimaryStatsResponse, @@ -128,7 +131,7 @@ const rootTransactionByTraceIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction?: TransactionDetailRedirectInfo; }> => { const { params: { @@ -155,7 +158,7 @@ const transactionByIdRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction?: Transaction; }> => { const { params: { @@ -191,7 +194,7 @@ const transactionByNameRoute = createApmServerRoute({ handler: async ( resources ): Promise<{ - transaction: Transaction; + transaction?: TransactionDetailRedirectInfo; }> => { const { params: { @@ -295,7 +298,7 @@ const transactionFromTraceByIdRoute = createApmServerRoute({ query: rangeRt, }), options: { tags: ['access:apm'] }, - handler: async (resources): Promise => { + handler: async (resources): Promise => { const { params } = resources; const { path: { transactionId, traceId }, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap index deb1dec096f08..c7f832fe5ca65 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap @@ -11,6 +11,25 @@ Object { ], }, "body": Object { + "_source": Array [ + "span.links", + "transaction.agent.marks", + ], + "fields": Array [ + "trace.id", + "agent.name", + "processor.event", + "@timestamp", + "timestamp.us", + "service.name", + "transaction.id", + "transaction.duration.us", + "transaction.name", + "transaction.sampled", + "transaction.type", + "processor.name", + "service.language.name", + ], "query": Object { "bool": Object { "filter": Array [ @@ -311,6 +330,11 @@ Object { ], }, "body": Object { + "fields": Array [ + "transaction.id", + "trace.id", + "@timestamp", + ], "query": Object { "bool": Object { "filter": Array [ diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts index 4e1b7020a98a6..dc8ab0a6aab19 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts @@ -7,7 +7,11 @@ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { SPAN_ID, TRACE_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields'; +import { merge, omit } from 'lodash'; +import { maybe } from '../../../../common/utils/maybe'; +import { SPAN_ID, SPAN_STACKTRACE, TRACE_ID } from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { getTransaction } from '../get_transaction'; @@ -38,6 +42,8 @@ export async function getSpan({ track_total_hits: false, size: 1, terminate_after: 1, + fields: ['*'], + _source: [SPAN_STACKTRACE], query: { bool: { filter: asMutableArray([ @@ -60,5 +66,17 @@ export async function getSpan({ : undefined, ]); - return { span: spanResp.hits.hits[0]?._source, parentTransaction }; + const hit = maybe(spanResp.hits.hits[0]); + const spanFromSource = hit && 'span' in hit._source ? hit._source : undefined; + + const event = unflattenKnownApmEventFields(hit?.fields as undefined | FlattenedApmEvent); + + return { + span: event + ? merge({}, omit(event, 'span.links'), spanFromSource, { + processor: { event: 'span' as const, name: 'transaction' as const }, + }) + : undefined, + parentTransaction, + }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index 8854f3075e59b..8fc9d93ceff87 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -6,7 +6,26 @@ */ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; -import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import type { Transaction } from '@kbn/apm-types'; +import { maybe } from '../../../../common/utils/maybe'; +import { + AGENT_NAME, + PROCESSOR_EVENT, + SERVICE_NAME, + TIMESTAMP_US, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + AT_TIMESTAMP, + PROCESSOR_NAME, + SPAN_LINKS, + TRANSACTION_AGENT_MARKS, + SERVICE_LANGUAGE_NAME, +} from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../../common/document_type'; @@ -24,7 +43,23 @@ export async function getTransaction({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise { + const requiredFields = asMutableArray([ + TRACE_ID, + AGENT_NAME, + PROCESSOR_EVENT, + AT_TIMESTAMP, + TIMESTAMP_US, + SERVICE_NAME, + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_NAME, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + ] as const); + + const optionalFields = asMutableArray([PROCESSOR_NAME, SERVICE_LANGUAGE_NAME] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -47,8 +82,37 @@ export async function getTransaction({ ]), }, }, + fields: [...requiredFields, ...optionalFields], + _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS], }, }); - return resp.hits.hits[0]?._source; + const hit = maybe(resp.hits.hits[0]); + + if (!hit) { + return undefined; + } + + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + + const source = + 'span' in hit._source && 'transaction' in hit._source + ? (hit._source as { + transaction: Pick['transaction'], 'marks'>; + span?: Pick['span'], 'links'>; + }) + : undefined; + + return { + ...event, + transaction: { + ...event.transaction, + marks: source?.transaction.marks, + }, + processor: { + name: 'transaction', + event: 'transaction', + }, + span: source?.span, + }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts index 75b4655d70117..160e3f736580a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts @@ -6,11 +6,22 @@ */ import { rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; import { ApmDocumentType } from '../../../../common/document_type'; -import { SERVICE_NAME, TRANSACTION_NAME } from '../../../../common/es_fields/apm'; +import { + AT_TIMESTAMP, + SERVICE_NAME, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, +} from '../../../../common/es_fields/apm'; import { RollupInterval } from '../../../../common/rollup'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +import { TransactionDetailRedirectInfo } from '../get_transaction_by_trace'; export async function getTransactionByName({ transactionName, @@ -24,7 +35,17 @@ export async function getTransactionByName({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise { + const requiredFields = asMutableArray([ + AT_TIMESTAMP, + TRACE_ID, + TRANSACTION_ID, + TRANSACTION_TYPE, + TRANSACTION_NAME, + TRANSACTION_DURATION, + SERVICE_NAME, + ] as const); + const resp = await apmEventClient.search('get_transaction', { apm: { sources: [ @@ -47,8 +68,9 @@ export async function getTransactionByName({ ]), }, }, + fields: requiredFields, }, }); - return resp.hits.hits[0]?._source; + return unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields); } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts index d27be0489f8da..803ae19a2228e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts @@ -7,9 +7,40 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { rangeQuery } from '@kbn/observability-plugin/server'; -import { TRACE_ID, PARENT_ID } from '../../../../common/es_fields/apm'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { maybe } from '../../../../common/utils/maybe'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; +import { + TRACE_ID, + PARENT_ID, + AT_TIMESTAMP, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_TYPE, + SERVICE_NAME, +} from '../../../../common/es_fields/apm'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; +export interface TransactionDetailRedirectInfo { + [AT_TIMESTAMP]: string; + trace: { + id: string; + }; + transaction: { + id: string; + type: string; + name: string; + + duration: { + us: number; + }; + }; + service: { + name: string; + }; +} + export async function getRootTransactionByTraceId({ traceId, apmEventClient, @@ -20,7 +51,19 @@ export async function getRootTransactionByTraceId({ apmEventClient: APMEventClient; start: number; end: number; -}) { +}): Promise<{ + transaction: TransactionDetailRedirectInfo | undefined; +}> { + const requiredFields = asMutableArray([ + TRACE_ID, + TRANSACTION_ID, + TRANSACTION_NAME, + AT_TIMESTAMP, + TRANSACTION_TYPE, + TRANSACTION_DURATION, + SERVICE_NAME, + ] as const); + const params = { apm: { events: [ProcessorEvent.transaction as const], @@ -45,11 +88,15 @@ export async function getRootTransactionByTraceId({ filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)], }, }, + fields: requiredFields, }, }; const resp = await apmEventClient.search('get_root_transaction_by_trace_id', params); + + const event = unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields); + return { - transaction: resp.hits.hits[0]?._source, + transaction: event, }; } diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts index 191250d3781ee..18dad19635333 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts @@ -7,7 +7,10 @@ import { Sort, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { + AT_TIMESTAMP, SERVICE_NAME, TRACE_ID, TRANSACTION_ID, @@ -77,6 +80,8 @@ export async function getTraceSamples({ }); } + const requiredFields = asMutableArray([TRANSACTION_ID, TRACE_ID, AT_TIMESTAMP] as const); + const response = await apmEventClient.search('get_trace_samples_hits', { apm: { events: [ProcessorEvent.transaction], @@ -94,6 +99,7 @@ export async function getTraceSamples({ }, }, size: TRACE_SAMPLES_SIZE, + fields: requiredFields, sort: [ { _score: { @@ -101,7 +107,7 @@ export async function getTraceSamples({ }, }, { - '@timestamp': { + [AT_TIMESTAMP]: { order: 'desc', }, }, @@ -109,12 +115,15 @@ export async function getTraceSamples({ }, }); - const traceSamples = response.hits.hits.map((hit) => ({ - score: hit._score, - timestamp: hit._source['@timestamp'], - transactionId: hit._source.transaction.id, - traceId: hit._source.trace.id, - })); + const traceSamples = response.hits.hits.map((hit) => { + const event = unflattenKnownApmEventFields(hit.fields, requiredFields); + return { + score: hit._score, + timestamp: event[AT_TIMESTAMP], + transactionId: event.transaction.id, + traceId: event.trace.id, + }; + }); return { traceSamples }; }); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts index 2fac072a8cdb5..71ceac4002919 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils.ts @@ -15,3 +15,4 @@ export { } from './lib/helpers'; export { withApmSpan } from './utils/with_apm_span'; +export { unflattenKnownApmEventFields } from './utils/unflatten_known_fields'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts new file mode 100644 index 0000000000000..9dc861a5292df --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { unflattenKnownApmEventFields } from './unflatten_known_fields'; + +describe('unflattenKnownApmEventFields', () => { + it('should return an empty object when input is empty', () => { + const input = {}; + const expectedOutput = {}; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten a simple flat input', () => { + const input = { + '@timestamp': '2024-10-10T10:10:10.000Z', + }; + const expectedOutput = { + '@timestamp': '2024-10-10T10:10:10.000Z', + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should override unknown fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.name.text': 'node-svc', + }; + const expectedOutput = { + service: { + name: 'node-svc', + }, + }; + + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten multiple nested fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.version': '1.0.0', + 'service.environment': 'production', + 'agent.name': 'nodejs', + }; + const expectedOutput = { + service: { + name: 'node-svc', + version: '1.0.0', + environment: 'production', + }, + agent: { + name: 'nodejs', + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should handle multiple values for multi-valued fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.tags': ['foo', 'bar'], + }; + const expectedOutput = { + service: { + name: 'node-svc', + tags: ['foo', 'bar'], + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten with empty multi-valued fields', () => { + const input = { + 'service.name': 'node-svc', + 'service.tags': [], + }; + const expectedOutput = { + service: { + name: 'node-svc', + tags: [], + }, + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should retain unknown fields in the output', () => { + const input = { + 'service.name': 'node-svc', + 'unknown.texts': ['foo', 'bar'], + 'unknown.field': 'foo', + unknonwField: 'bar', + }; + const expectedOutput = { + service: { + name: 'node-svc', + }, + unknown: { + field: 'foo', + texts: ['foo', 'bar'], + }, + unknonwField: 'bar', + }; + expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput); + }); + + it('should correctly unflatten nested fields with mandatory field', () => { + const input = { + 'service.name': 'node-svc', + 'service.environment': undefined, + }; + + const requiredFields: ['service.name'] = ['service.name']; + + const expectedOutput = { + service: { + name: 'node-svc', + }, + }; + expect(unflattenKnownApmEventFields(input, requiredFields)).toEqual(expectedOutput); + }); + + it('should throw an exception when mandatory field is not in the input', () => { + const input = { + 'service.environment': 'PROD', + }; + + const requiredFields: ['service.name'] = ['service.name']; + + // @ts-expect-error + expect(() => unflattenKnownApmEventFields(input, requiredFields)).toThrowError( + 'Missing required fields service.name in event' + ); + }); +}); diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts new file mode 100644 index 0000000000000..b9a4322269828 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DedotObject } from '@kbn/utility-types'; +import * as APM_EVENT_FIELDS_MAP from '@kbn/apm-types/es_fields'; +import type { ValuesType } from 'utility-types'; +import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object'; +import { mergePlainObjects } from '@kbn/observability-utils/object/merge_plain_objects'; +import { castArray, isArray } from 'lodash'; +import { AgentName } from '@kbn/elastic-agent-utils'; +import { EventOutcome } from '@kbn/apm-types/src/es_schemas/raw/fields'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; + +const { + CLOUD, + AGENT, + SERVICE, + ERROR_EXCEPTION, + SPAN_LINKS, + HOST, + KUBERNETES, + CONTAINER, + TIER, + INDEX, + DATA_STEAM_TYPE, + VALUE_OTEL_JVM_PROCESS_MEMORY_HEAP, + VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP, + SPAN_LINKS_SPAN_ID, + SPAN_LINKS_TRACE_ID, + SPAN_STACKTRACE, + ...CONCRETE_FIELDS +} = APM_EVENT_FIELDS_MAP; + +const ALL_FIELDS = Object.values(CONCRETE_FIELDS); + +const KNOWN_MULTI_VALUED_FIELDS = [ + APM_EVENT_FIELDS_MAP.CHILD_ID, + APM_EVENT_FIELDS_MAP.PROCESS_ARGS, +] as const; + +type KnownField = ValuesType; + +type KnownSingleValuedField = Exclude; +type KnownMultiValuedField = ValuesType; + +const KNOWN_SINGLE_VALUED_FIELDS = ALL_FIELDS.filter( + (field): field is KnownSingleValuedField => !KNOWN_MULTI_VALUED_FIELDS.includes(field as any) +); + +interface TypeOverrideMap { + [APM_EVENT_FIELDS_MAP.SPAN_DURATION]: number; + [APM_EVENT_FIELDS_MAP.AGENT_NAME]: AgentName; + [APM_EVENT_FIELDS_MAP.EVENT_OUTCOME]: EventOutcome; + [APM_EVENT_FIELDS_MAP.FAAS_COLDSTART]: true; + [APM_EVENT_FIELDS_MAP.TRANSACTION_DURATION]: number; + [APM_EVENT_FIELDS_MAP.TIMESTAMP_US]: number; + [APM_EVENT_FIELDS_MAP.PROCESSOR_EVENT]: ProcessorEvent; + [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_COUNT]: number; + [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_SUM]: number; + [APM_EVENT_FIELDS_MAP.SPAN_SYNC]: boolean; + [APM_EVENT_FIELDS_MAP.TRANSACTION_SAMPLED]: boolean; + [APM_EVENT_FIELDS_MAP.PROCESSOR_NAME]: 'transaction' | 'metric' | 'error'; + [APM_EVENT_FIELDS_MAP.HTTP_RESPONSE_STATUS_CODE]: number; + [APM_EVENT_FIELDS_MAP.PROCESS_PID]: number; + [APM_EVENT_FIELDS_MAP.OBSERVER_VERSION_MAJOR]: number; + [APM_EVENT_FIELDS_MAP.ERROR_EXC_HANDLED]: boolean; +} + +type MaybeMultiValue = T extends KnownMultiValuedField ? U[] : U; + +type TypeOfKnownField = MaybeMultiValue< + T, + T extends keyof TypeOverrideMap ? TypeOverrideMap[T] : string +>; + +type MapToSingleOrMultiValue> = { + [TKey in keyof T]: TKey extends KnownField + ? T[TKey] extends undefined + ? TypeOfKnownField | undefined + : TypeOfKnownField + : unknown; +}; + +type UnflattenedKnownFields> = DedotObject< + MapToSingleOrMultiValue +>; + +export type FlattenedApmEvent = Record; + +export type UnflattenedApmEvent = UnflattenedKnownFields; + +export function unflattenKnownApmEventFields | undefined = undefined>( + fields: T +): T extends Record ? UnflattenedKnownFields : undefined; + +export function unflattenKnownApmEventFields< + T extends Record | undefined, + U extends Array> +>( + fields: T, + required: U +): T extends Record + ? UnflattenedKnownFields & + (U extends any[] + ? UnflattenedKnownFields<{ + [TKey in ValuesType]: keyof T extends TKey ? T[TKey] : unknown[]; + }> + : {}) + : undefined; + +export function unflattenKnownApmEventFields( + hitFields?: Record, + requiredFields?: string[] +) { + if (!hitFields) { + return undefined; + } + const missingRequiredFields = + requiredFields?.filter((key) => { + const value = hitFields?.[key]; + return value === null || value === undefined || (isArray(value) && value.length === 0); + }) ?? []; + + if (missingRequiredFields.length > 0) { + throw new Error(`Missing required fields ${missingRequiredFields.join(', ')} in event`); + } + + const copy: Record = mapToSingleOrMultiValue({ + ...hitFields, + }); + + const [knownFields, unknownFields] = Object.entries(copy).reduce( + (prev, [key, value]) => { + if (ALL_FIELDS.includes(key as KnownField)) { + prev[0][key as KnownField] = value; + } else { + prev[1][key] = value; + } + return prev; + }, + [{} as Record, {} as Record] + ); + + const unflattened = mergePlainObjects( + {}, + unflattenObject(unknownFields), + unflattenObject(knownFields) + ); + + return unflattened; +} + +export function mapToSingleOrMultiValue>( + fields: T +): MapToSingleOrMultiValue { + KNOWN_SINGLE_VALUED_FIELDS.forEach((field) => { + const value = fields[field]; + if (value !== null && value !== undefined) { + fields[field as keyof T] = castArray(value)[0]; + } + }); + + return fields; +} diff --git a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json index ea3ebf77b25be..aeeb73bee2857 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json @@ -20,6 +20,8 @@ "@kbn/apm-utils", "@kbn/core-http-server", "@kbn/security-plugin-types-server", - "@kbn/observability-utils" + "@kbn/observability-utils", + "@kbn/utility-types", + "@kbn/elastic-agent-utils" ] }