[APM][Otel] Use fields instead of _source on APM queries (#195242)

closes #192606 ## Summary v2 based on the work done in this PR #192608 and the suggestion from Dario #194424 This PR replaces the _source usage in APM queries with fields to support Otel data. The idea is to get rid of existing UI errors we have and make sure that otel data is shown correctly in the UI. One way to check it is using the [e2e PoC](https://github.com/elastic/otel-apm-e2e-poc/blob/main/README.md). --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: Jenny <[email protected]>
elastic · Oct 15, 2024 · 7235ed0 · 7235ed0
1 parent 90b4ba5
commit 7235ed0
Show file tree

Hide file tree

Showing 67 changed files with 1,612 additions and 385 deletions.
diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts
@@ -7,7 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export const TIMESTAMP = 'timestamp.us';
+export const TIMESTAMP_US = 'timestamp.us';
+export const AT_TIMESTAMP = '@timestamp';
 export const AGENT = 'agent';
 export const AGENT_NAME = 'agent.name';
 export const AGENT_VERSION = 'agent.version';
@@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider';
 export const CLOUD_REGION = 'cloud.region';
 export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
 export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
+export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
 export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
 export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
 export const CLOUD_SERVICE_NAME = 'cloud.service.name';
+export const CLOUD_PROJECT_NAME = 'cloud.project.name';
 
 export const EVENT_SUCCESS_COUNT = 'event.success_count';
 
@@ -48,10 +51,14 @@ export const USER_ID = 'user.id';
 export const USER_AGENT_ORIGINAL = 'user_agent.original';
 export const USER_AGENT_NAME = 'user_agent.name';
 
+export const OBSERVER_VERSION = 'observer.version';
+export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const OBSERVER_HOSTNAME = 'observer.hostname';
 export const OBSERVER_LISTENING = 'observer.listening';
 export const PROCESSOR_EVENT = 'processor.event';
+export const PROCESSOR_NAME = 'processor.name';
 
+export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks';
 export const TRANSACTION_DURATION = 'transaction.duration.us';
 export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram';
 export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary';
@@ -95,6 +102,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us';
 export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy';
 
 export const SPAN_SYNC = 'span.sync';
+export const SPAN_STACKTRACE = 'span.stacktrace';
 
 // Parent ID for a transaction or span
 export const PARENT_ID = 'parent.id';
@@ -110,6 +118,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i
 export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array
 export const ERROR_EXC_TYPE = 'error.exception.type';
 export const ERROR_PAGE_URL = 'error.page.url';
+export const ERROR_STACK_TRACE = 'error.stack_trace';
 export const ERROR_TYPE = 'error.type';
 
 // METRICS
@@ -153,6 +162,12 @@ export const CONTAINER_IMAGE = 'container.image.name';
 export const KUBERNETES = 'kubernetes';
 export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';
 export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
+export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
+export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
+export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
+export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
+export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
 
 export const FAAS_ID = 'faas.id';
 export const FAAS_NAME = 'faas.name';
@@ -198,3 +213,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name';
 export const CHILD_ID = 'child.id';
 
 export const LOG_LEVEL = 'log.level';
+
+// Process
+export const PROCESS_ARGS = 'process.args';
+export const PROCESS_PID = 'process.pid';
diff --git a/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts b/packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts
@@ -14,10 +14,10 @@ export interface APMBaseDoc {
   '@timestamp': string;
   agent: {
     name: string;
-    version: string;
+    version?: string;
   };
-  parent?: { id: string }; // parent ID is not available on root transactions
-  trace?: { id: string };
+  parent?: { id?: string }; // parent ID is not available on root transactions
+  trace?: { id?: string };
   labels?: {
     [key: string]: string | number | boolean;
   };

diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts
@@ -10,26 +10,26 @@
 export interface Cloud {
   availability_zone?: string;
   instance?: {
-    name: string;
-    id: string;
+    name?: string;
+    id?: string;
   };
   machine?: {
-    type: string;
+    type?: string;
   };
   project?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   provider?: string;
   region?: string;
   account?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   image?: {
-    id: string;
+    id?: string;
   };
   service?: {
-    name: string;
+    name?: string;
   };
 }
diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts
@@ -9,5 +9,7 @@
 
 export interface Container {
   id?: string | null;
-  image?: string | null;
+  image?: {
+    name?: string;
+  };
 }
diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts
@@ -8,7 +8,7 @@
  */
 
 export interface Http {
-  request?: { method: string; [key: string]: unknown };
-  response?: { status_code: number; [key: string]: unknown };
+  request?: { method?: string };
+  response?: { status_code?: number };
   version?: string;
 }
diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts
@@ -8,7 +8,7 @@
  */
 
 export interface Kubernetes {
-  pod?: { uid?: string | null; [key: string]: unknown };
+  pod?: { uid?: string | null; name?: string };
   namespace?: string;
   replicaset?: {
     name?: string;

diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts
@@ -13,6 +13,6 @@ export interface Observer {
   id?: string;
   name?: string;
   type?: string;
-  version: string;
-  version_major: number;
+  version?: string;
+  version_major?: number;
 }
diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts
@@ -9,5 +9,5 @@
 
 // only for RUM agent: shared by error and transaction
 export interface Page {
-  url: string;
+  url?: string;
 }
diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts
@@ -11,18 +11,18 @@ export interface Service {
   name: string;
   environment?: string;
   framework?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   node?: {
     name?: string;
   };
   runtime?: {
-    name: string;
-    version: string;
+    name?: string;
+    version?: string;
   };
   language?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   version?: string;

diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts
@@ -9,6 +9,6 @@
 
 export interface Url {
   domain?: string;
-  full: string;
+  full?: string;
   original?: string;
 }
diff --git a/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts b/packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts
@@ -8,5 +8,5 @@
  */
 
 export interface User {
-  id: string;
+  id?: string;
 }
diff --git a/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts b/packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts
@@ -14,5 +14,5 @@ export type { ElasticAgentName, OpenTelemetryAgentName, AgentName } from '@kbn/e
 export interface Agent {
   ephemeral_id?: string;
   name: AgentName;
-  version: string;
+  version?: string;
 }
diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts
@@ -21,6 +21,18 @@ describe('flattenObject', () => {
     });
   });
 
+  it('flattens arrays', () => {
+    expect(
+      flattenObject({
+        child: {
+          id: [1, 2],
+        },
+      })
+    ).toEqual({
+      'child.id': [1, 2],
+    });
+  });
+
   it('does not flatten arrays', () => {
     expect(
       flattenObject({

diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { unflattenObject } from './unflatten_object';
+
+describe('unflattenObject', () => {
+  it('unflattens deeply nested objects', () => {
+    expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({
+      first: {
+        second: {
+          third: 'third',
+        },
+      },
+    });
+  });
+
+  it('does not unflatten arrays', () => {
+    expect(
+      unflattenObject({
+        simpleArray: ['0', '1', '2'],
+        complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+        'nested.array': [0, 1, 2],
+        'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }],
+      })
+    ).toEqual({
+      simpleArray: ['0', '1', '2'],
+      complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+      nested: {
+        array: [0, 1, 2],
+      },
+      complex: {
+        nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }],
+      },
+    });
+  });
+});
diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/object/unflatten_object.ts
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { set } from '@kbn/safer-lodash-set';
+
+export function unflattenObject(source: Record<string, any>, target: Record<string, any> = {}) {
+  // eslint-disable-next-line guard-for-in
+  for (const key in source) {
+    const val = source[key as keyof typeof source];
+
+    if (Array.isArray(val)) {
+      const unflattenedArray = val.map((item) => {
+        if (item && typeof item === 'object' && !Array.isArray(item)) {
+          return unflattenObject(item);
+        }
+        return item;
+      });
+      set(target, key, unflattenedArray);
+    } else {
+      set(target, key, val);
+    }
+  }
+  return target;
+}
diff --git a/x-pack/packages/observability/observability_utils/tsconfig.json b/x-pack/packages/observability/observability_utils/tsconfig.json
@@ -21,5 +21,6 @@
     "@kbn/es-types",
     "@kbn/apm-utils",
     "@kbn/es-query",
+    "@kbn/safer-lodash-set",
   ]
 }