[APM][Otel] Use fields instead of _source on APM queries

packages/kbn-apm-types/src/es_fields/apm.ts

@@ -7,7 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export const TIMESTAMP = 'timestamp.us';
+export const TIMESTAMP_US = 'timestamp.us';
+export const AT_TIMESTAMP = '@timestamp';
 export const AGENT = 'agent';
 export const AGENT_NAME = 'agent.name';
 export const AGENT_VERSION = 'agent.version';
@@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider';
 export const CLOUD_REGION = 'cloud.region';
 export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
 export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
+export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
 export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
 export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
 export const CLOUD_SERVICE_NAME = 'cloud.service.name';
+export const CLOUD_PROJECT_NAME = 'cloud.project.name';
 
 export const EVENT_SUCCESS_COUNT = 'event.success_count';
 
@@ -48,10 +51,14 @@ export const USER_ID = 'user.id';
 export const USER_AGENT_ORIGINAL = 'user_agent.original';
 export const USER_AGENT_NAME = 'user_agent.name';
 
+export const OBSERVER_VERSION = 'observer.version';
+export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const OBSERVER_HOSTNAME = 'observer.hostname';
 export const OBSERVER_LISTENING = 'observer.listening';
 export const PROCESSOR_EVENT = 'processor.event';
+export const PROCESSOR_NAME = 'processor.name';
 
+export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks';
 export const TRANSACTION_DURATION = 'transaction.duration.us';
 export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram';
 export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary';
@@ -110,6 +117,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i
 export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array
 export const ERROR_EXC_TYPE = 'error.exception.type';
 export const ERROR_PAGE_URL = 'error.page.url';
+export const ERROR_STACK_TRACE = 'error.stack_trace';
 export const ERROR_TYPE = 'error.type';
 
 // METRICS
@@ -153,6 +161,12 @@ export const CONTAINER_IMAGE = 'container.image.name';
 export const KUBERNETES = 'kubernetes';
 export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';
 export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
+export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
+export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
+export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
+export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
+export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
 
 export const FAAS_ID = 'faas.id';
 export const FAAS_NAME = 'faas.name';
@@ -198,3 +212,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name';
 export const CHILD_ID = 'child.id';
 
 export const LOG_LEVEL = 'log.level';
+
+// Process
+export const PROCESS_ARGS = 'process.args';
+export const PROCESS_PID = 'process.pid';

packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts

@@ -10,26 +10,26 @@
 export interface Cloud {
   availability_zone?: string;
   instance?: {
-    name: string;
-    id: string;
+    name?: string;
+    id?: string;
   };
   machine?: {
-    type: string;
+    type?: string;
   };
   project?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   provider?: string;
   region?: string;
   account?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   image?: {
-    id: string;
+    id?: string;
   };
   service?: {
-    name: string;
+    name?: string;
   };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts

@@ -9,5 +9,7 @@
 
 export interface Container {
   id?: string | null;
-  image?: string | null;
+  image?: {
+    name?: string;
+  };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts

@@ -8,7 +8,7 @@
  */
 
 export interface Http {
-  request?: { method: string; [key: string]: unknown };
-  response?: { status_code: number; [key: string]: unknown };
+  request?: { method?: string };
+  response?: { status_code?: number };
   version?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts

@@ -8,8 +8,10 @@
  */
 
 export interface Kubernetes {
-  pod?: { uid?: string | null; [key: string]: unknown };
-  namespace?: string;
+  pod?: { uid?: string | null; name?: string };
+  namespace?: {
+    name?: string;
+  };
   replicaset?: {
     name?: string;
   };

packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts

@@ -13,6 +13,6 @@ export interface Observer {
   id?: string;
   name?: string;
   type?: string;
-  version: string;
-  version_major: number;
+  version?: string;
+  version_major?: number;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts

@@ -9,5 +9,5 @@
 
 // only for RUM agent: shared by error and transaction
 export interface Page {
-  url: string;
+  url?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts

@@ -11,18 +11,18 @@ export interface Service {
   name: string;
   environment?: string;
   framework?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   node?: {
     name?: string;
   };
   runtime?: {
-    name: string;
-    version: string;
+    name?: string;
+    version?: string;
   };
   language?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   version?: string;

packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts

@@ -9,6 +9,6 @@
 
 export interface Url {
   domain?: string;
-  full: string;
+  full?: string;
   original?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts

@@ -8,5 +8,5 @@
  */
 
 export interface User {
-  id: string;
+  id?: string;
 }

packages/kbn-elastic-agent-utils/src/agent_names.ts

@@ -36,7 +36,7 @@ export const ELASTIC_AGENT_NAMES: ElasticAgentName[] = [
   'rum-js',
   'android/java',
 ];
-
+export type OpenTelemetryGenericAgentName = `opentelemetry/${string}` | `otlp/${string}`;
 export type OpenTelemetryAgentName =
   | 'otlp'
   | 'opentelemetry'

x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { unflattenObject } from './unflatten_object';
+
+describe('unflattenObject', () => {
+  it('unflattens deeply nested objects', () => {
+    expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({
+      first: {
+        second: {
+          third: 'third',
+        },
+      },
+    });
+  });
+
+  it('does not unflatten arrays', () => {
+    expect(
+      unflattenObject({
+        simpleArray: ['0', '1', '2'],
+        complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+        'nested.array': [0, 1, 2],
+      })
+    ).toEqual({
+      simpleArray: ['0', '1', '2'],
+      complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+      nested: {
+        array: [0, 1, 2],
+      },
+    });
+  });
+});

x-pack/packages/observability/observability_utils/object/unflatten_object.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { set } from '@kbn/safer-lodash-set';
+
+export function unflattenObject(source: Record<string, any>, target: Record<string, any> = {}) {
+  // eslint-disable-next-line guard-for-in
+  for (const key in source) {
+    const val = source[key as keyof typeof source];
+    set(target, key, val);
+  }
+  return target;
+}

x-pack/packages/observability/observability_utils/tsconfig.json

@@ -21,5 +21,6 @@
     "@kbn/es-types",
     "@kbn/apm-utils",
     "@kbn/es-query",
+    "@kbn/safer-lodash-set",
   ]
 }

x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts

@@ -64,16 +64,17 @@ export interface WaterfallSpan {
     links?: SpanLink[];
   };
   transaction?: {
-    id: string;
+    id?: string;
   };
   child?: { id: string[] };
 }
 
 export interface WaterfallError {
   timestamp: TimestampUs;
-  trace?: { id: string };
-  transaction?: { id: string };
-  parent?: { id: string };
+  trace?: { id?: string };
+  transaction?: { id?: string };
+  parent?: { id?: string };
+  span?: { id?: string };
   error: {
     id: string;
     log?: {

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx

@@ -8,18 +8,36 @@ import { EuiFlexItem, EuiSpacer } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import type { Message } from '@kbn/observability-ai-assistant-plugin/public';
 import React, { useMemo, useState } from 'react';
+import { AT_TIMESTAMP } from '@kbn/apm-types';
 import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
 import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
-import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
 import { ErrorSampleDetailTabContent } from './error_sample_detail';
 import { exceptionStacktraceTab, logStacktraceTab } from './error_tabs';
 
 export function ErrorSampleContextualInsight({
   error,
   transaction,
 }: {
-  error: APMError;
-  transaction?: Transaction;
+  error: {
+    [AT_TIMESTAMP]: string;
+    error: Pick<APMError['error'], 'log' | 'exception' | 'id'>;
+    service: {
+      name: string;
+      environment?: string;
+      language?: {
+        name?: string;
+      };
+      runtime?: {
+        name?: string;
+        version?: string;
+      };
+    };
+  };
+  transaction?: {
+    transaction: {
+      name: string;
+    };
+  };
 }) {
   const { observabilityAIAssistant } = useApmPluginContext();