[8.x] [APM][Otel] Use `fields` instead of `_source` on APM queries (#195242)

packages/kbn-apm-types/src/es_fields/apm.ts

@@ -7,7 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export const TIMESTAMP = 'timestamp.us';
+export const TIMESTAMP_US = 'timestamp.us';
+export const AT_TIMESTAMP = '@timestamp';
 export const AGENT = 'agent';
 export const AGENT_NAME = 'agent.name';
 export const AGENT_VERSION = 'agent.version';
@@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider';
 export const CLOUD_REGION = 'cloud.region';
 export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
 export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
+export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
 export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
 export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
 export const CLOUD_SERVICE_NAME = 'cloud.service.name';
+export const CLOUD_PROJECT_NAME = 'cloud.project.name';
 
 export const EVENT_SUCCESS_COUNT = 'event.success_count';
 
@@ -48,10 +51,14 @@ export const USER_ID = 'user.id';
 export const USER_AGENT_ORIGINAL = 'user_agent.original';
 export const USER_AGENT_NAME = 'user_agent.name';
 
+export const OBSERVER_VERSION = 'observer.version';
+export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const OBSERVER_HOSTNAME = 'observer.hostname';
 export const OBSERVER_LISTENING = 'observer.listening';
 export const PROCESSOR_EVENT = 'processor.event';
+export const PROCESSOR_NAME = 'processor.name';
 
+export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks';
 export const TRANSACTION_DURATION = 'transaction.duration.us';
 export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram';
 export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary';
@@ -95,6 +102,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us';
 export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy';
 
 export const SPAN_SYNC = 'span.sync';
+export const SPAN_STACKTRACE = 'span.stacktrace';
 
 // Parent ID for a transaction or span
 export const PARENT_ID = 'parent.id';
@@ -110,6 +118,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i
 export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array
 export const ERROR_EXC_TYPE = 'error.exception.type';
 export const ERROR_PAGE_URL = 'error.page.url';
+export const ERROR_STACK_TRACE = 'error.stack_trace';
 export const ERROR_TYPE = 'error.type';
 
 // METRICS
@@ -153,6 +162,12 @@ export const CONTAINER_IMAGE = 'container.image.name';
 export const KUBERNETES = 'kubernetes';
 export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';
 export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
+export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
+export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
+export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
+export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
+export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
 
 export const FAAS_ID = 'faas.id';
 export const FAAS_NAME = 'faas.name';
@@ -198,3 +213,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name';
 export const CHILD_ID = 'child.id';
 
 export const LOG_LEVEL = 'log.level';
+
+// Process
+export const PROCESS_ARGS = 'process.args';
+export const PROCESS_PID = 'process.pid';

packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts

@@ -14,10 +14,10 @@ export interface APMBaseDoc {
   '@timestamp': string;
   agent: {
     name: string;
-    version: string;
+    version?: string;
   };
-  parent?: { id: string }; // parent ID is not available on root transactions
-  trace?: { id: string };
+  parent?: { id?: string }; // parent ID is not available on root transactions
+  trace?: { id?: string };
   labels?: {
     [key: string]: string | number | boolean;
   };

packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts

@@ -10,26 +10,26 @@
 export interface Cloud {
   availability_zone?: string;
   instance?: {
-    name: string;
-    id: string;
+    name?: string;
+    id?: string;
   };
   machine?: {
-    type: string;
+    type?: string;
   };
   project?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   provider?: string;
   region?: string;
   account?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   image?: {
-    id: string;
+    id?: string;
   };
   service?: {
-    name: string;
+    name?: string;
   };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts

@@ -9,5 +9,7 @@
 
 export interface Container {
   id?: string | null;
-  image?: string | null;
+  image?: {
+    name?: string;
+  };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts

@@ -8,7 +8,7 @@
  */
 
 export interface Http {
-  request?: { method: string; [key: string]: unknown };
-  response?: { status_code: number; [key: string]: unknown };
+  request?: { method?: string };
+  response?: { status_code?: number };
   version?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts

@@ -8,7 +8,7 @@
  */
 
 export interface Kubernetes {
-  pod?: { uid?: string | null; [key: string]: unknown };
+  pod?: { uid?: string | null; name?: string };
   namespace?: string;
   replicaset?: {
     name?: string;

packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts

@@ -13,6 +13,6 @@ export interface Observer {
   id?: string;
   name?: string;
   type?: string;
-  version: string;
-  version_major: number;
+  version?: string;
+  version_major?: number;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts

@@ -9,5 +9,5 @@
 
 // only for RUM agent: shared by error and transaction
 export interface Page {
-  url: string;
+  url?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts

@@ -11,18 +11,18 @@ export interface Service {
   name: string;
   environment?: string;
   framework?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   node?: {
     name?: string;
   };
   runtime?: {
-    name: string;
-    version: string;
+    name?: string;
+    version?: string;
   };
   language?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   version?: string;

packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts

@@ -9,6 +9,6 @@
 
 export interface Url {
   domain?: string;
-  full: string;
+  full?: string;
   original?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts

@@ -8,5 +8,5 @@
  */
 
 export interface User {
-  id: string;
+  id?: string;
 }

packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts

@@ -14,5 +14,5 @@ export type { ElasticAgentName, OpenTelemetryAgentName, AgentName } from '@kbn/e
 export interface Agent {
   ephemeral_id?: string;
   name: AgentName;
-  version: string;
+  version?: string;
 }

x-pack/packages/observability/observability_utils/object/flatten_object.test.ts

@@ -21,6 +21,18 @@ describe('flattenObject', () => {
     });
   });
 
+  it('flattens arrays', () => {
+    expect(
+      flattenObject({
+        child: {
+          id: [1, 2],
+        },
+      })
+    ).toEqual({
+      'child.id': [1, 2],
+    });
+  });
+
   it('does not flatten arrays', () => {
     expect(
       flattenObject({

x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { unflattenObject } from './unflatten_object';
+
+describe('unflattenObject', () => {
+  it('unflattens deeply nested objects', () => {
+    expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({
+      first: {
+        second: {
+          third: 'third',
+        },
+      },
+    });
+  });
+
+  it('does not unflatten arrays', () => {
+    expect(
+      unflattenObject({
+        simpleArray: ['0', '1', '2'],
+        complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+        'nested.array': [0, 1, 2],
+        'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }],
+      })
+    ).toEqual({
+      simpleArray: ['0', '1', '2'],
+      complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+      nested: {
+        array: [0, 1, 2],
+      },
+      complex: {
+        nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }],
+      },
+    });
+  });
+});

x-pack/packages/observability/observability_utils/object/unflatten_object.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { set } from '@kbn/safer-lodash-set';
+
+export function unflattenObject(source: Record<string, any>, target: Record<string, any> = {}) {
+  // eslint-disable-next-line guard-for-in
+  for (const key in source) {
+    const val = source[key as keyof typeof source];
+
+    if (Array.isArray(val)) {
+      const unflattenedArray = val.map((item) => {
+        if (item && typeof item === 'object' && !Array.isArray(item)) {
+          return unflattenObject(item);
+        }
+        return item;
+      });
+      set(target, key, unflattenedArray);
+    } else {
+      set(target, key, val);
+    }
+  }
+  return target;
+}

x-pack/packages/observability/observability_utils/tsconfig.json

@@ -21,5 +21,6 @@
     "@kbn/es-types",
     "@kbn/apm-utils",
     "@kbn/es-query",
+    "@kbn/safer-lodash-set",
   ]
 }