[APM][Otel] Use fields instead of _source on APM queries

packages/kbn-apm-types/src/es_fields/apm.ts

@@ -8,6 +8,7 @@
  */
 
 export const TIMESTAMP = 'timestamp.us';
+export const AT_TIMESTAMP = '@timestamp';
 export const AGENT = 'agent';
 export const AGENT_NAME = 'agent.name';
 export const AGENT_VERSION = 'agent.version';
@@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider';
 export const CLOUD_REGION = 'cloud.region';
 export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
 export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
+export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
 export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
 export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
 export const CLOUD_SERVICE_NAME = 'cloud.service.name';
+export const CLOUD_PROJECT_NAME = 'cloud.project.name';
 
 export const EVENT_SUCCESS_COUNT = 'event.success_count';
 
@@ -48,10 +51,14 @@ export const USER_ID = 'user.id';
 export const USER_AGENT_ORIGINAL = 'user_agent.original';
 export const USER_AGENT_NAME = 'user_agent.name';
 
+export const OBSERVER_VERSION = 'observer.version';
+export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const OBSERVER_HOSTNAME = 'observer.hostname';
 export const OBSERVER_LISTENING = 'observer.listening';
 export const PROCESSOR_EVENT = 'processor.event';
+export const PROCESSOR_NAME = 'processor.name';
 
+export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks';
 export const TRANSACTION_DURATION = 'transaction.duration.us';
 export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram';
 export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary';
@@ -110,6 +117,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i
 export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array
 export const ERROR_EXC_TYPE = 'error.exception.type';
 export const ERROR_PAGE_URL = 'error.page.url';
+export const ERROR_STACK_TRACE = 'error.stack_trace';
 export const ERROR_TYPE = 'error.type';
 
 // METRICS
@@ -153,6 +161,15 @@ export const CONTAINER_IMAGE = 'container.image.name';
 export const KUBERNETES = 'kubernetes';
 export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';
 export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
+export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
+export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
+export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
+export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
+export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment';
+export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
+export const KUBERNETES_REPLICASET = 'kubernetes.replicaset';
+export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
 
 export const FAAS_ID = 'faas.id';
 export const FAAS_NAME = 'faas.name';
@@ -198,3 +215,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name';
 export const CHILD_ID = 'child.id';
 
 export const LOG_LEVEL = 'log.level';
+
+// Process
+export const PROCESS_ARGS = 'process.args';
+export const PROCESS_PID = 'process.pid';

packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts

@@ -10,26 +10,26 @@
 export interface Cloud {
   availability_zone?: string;
   instance?: {
-    name: string;
-    id: string;
+    name?: string;
+    id?: string;
   };
   machine?: {
-    type: string;
+    type?: string;
   };
   project?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   provider?: string;
   region?: string;
   account?: {
-    id: string;
-    name: string;
+    id?: string;
+    name?: string;
   };
   image?: {
-    id: string;
+    id?: string;
   };
   service?: {
-    name: string;
+    name?: string;
   };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts

@@ -9,5 +9,7 @@
 
 export interface Container {
   id?: string | null;
-  image?: string | null;
+  image?: {
+    name?: string;
+  };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts

@@ -8,7 +8,7 @@
  */
 
 export interface Http {
-  request?: { method: string; [key: string]: unknown };
-  response?: { status_code: number; [key: string]: unknown };
+  request?: { method?: string };
+  response?: { status_code?: number };
   version?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts

@@ -8,7 +8,7 @@
  */
 
 export interface Kubernetes {
-  pod?: { uid?: string | null; [key: string]: unknown };
+  pod?: { uid?: string | null; name?: string };
   namespace?: string;
   replicaset?: {
     name?: string;

packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts

@@ -13,6 +13,6 @@ export interface Observer {
   id?: string;
   name?: string;
   type?: string;
-  version: string;
-  version_major: number;
+  version?: string;
+  version_major?: number;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts

@@ -9,5 +9,5 @@
 
 // only for RUM agent: shared by error and transaction
 export interface Page {
-  url: string;
+  url?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts

@@ -11,18 +11,18 @@ export interface Service {
   name: string;
   environment?: string;
   framework?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   node?: {
     name?: string;
   };
   runtime?: {
-    name: string;
-    version: string;
+    name?: string;
+    version?: string;
   };
   language?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   version?: string;

packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts

@@ -9,6 +9,6 @@
 
 export interface Url {
   domain?: string;
-  full: string;
+  full?: string;
   original?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts

@@ -8,5 +8,5 @@
  */
 
 export interface User {
-  id: string;
+  id?: string;
 }

x-pack/packages/observability/observability_utils/object/unflatten_object.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { set } from '@kbn/safer-lodash-set';
+
+export function unflattenObject(source: Record<string, any>, target: Record<string, any> = {}) {
+  // eslint-disable-next-line guard-for-in
+  for (const key in source) {
+    const val = source[key as keyof typeof source];
+    set(target, key, val);
+  }
+  return target;
+}

x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts

@@ -64,16 +64,17 @@ export interface WaterfallSpan {
     links?: SpanLink[];
   };
   transaction?: {
-    id: string;
+    id?: string;
   };
   child?: { id: string[] };
 }
 
 export interface WaterfallError {
   timestamp: TimestampUs;
-  trace?: { id: string };
-  transaction?: { id: string };
-  parent?: { id: string };
+  trace?: { id?: string };
+  transaction?: { id?: string };
+  parent?: { id?: string };
+  span?: { id?: string };
   error: {
     id: string;
     log?: {

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx

@@ -8,18 +8,36 @@ import { EuiFlexItem, EuiSpacer } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import type { Message } from '@kbn/observability-ai-assistant-plugin/public';
 import React, { useMemo, useState } from 'react';
+import { AT_TIMESTAMP } from '@kbn/apm-types';
 import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
 import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
-import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
 import { ErrorSampleDetailTabContent } from './error_sample_detail';
 import { exceptionStacktraceTab, logStacktraceTab } from './error_tabs';
 
 export function ErrorSampleContextualInsight({
   error,
   transaction,
 }: {
-  error: APMError;
-  transaction?: Transaction;
+  error: {
+    [AT_TIMESTAMP]: string;
+    error: Pick<APMError['error'], 'log' | 'exception' | 'id'>;
+    service: {
+      name: string;
+      environment?: string;
+      language?: {
+        name?: string;
+      };
+      runtime?: {
+        name?: string;
+        version?: string;
+      };
+    };
+  };
+  transaction?: {
+    transaction: {
+      name: string;
+    };
+  };
 }) {
   const { observabilityAIAssistant } = useApmPluginContext();
 

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx

@@ -29,7 +29,7 @@ import { first } from 'lodash';
 import React, { useEffect, useState } from 'react';
 import { useHistory } from 'react-router-dom';
 import useAsync from 'react-use/lib/useAsync';
-import { ERROR_GROUP_ID } from '../../../../../common/es_fields/apm';
+import { AT_TIMESTAMP, ERROR_GROUP_ID } from '../../../../../common/es_fields/apm';
 import { TraceSearchType } from '../../../../../common/trace_explorer';
 import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
 import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
@@ -348,14 +348,22 @@ export function ErrorSampleDetailTabContent({
   error,
   currentTab,
 }: {
-  error: APMError;
+  error: {
+    service: {
+      language?: {
+        name?: string;
+      };
+    };
+    [AT_TIMESTAMP]: string;
+    error: Pick<APMError['error'], 'id' | 'log' | 'stack_trace' | 'exception'>;
+  };
   currentTab: ErrorTab;
 }) {
   const codeLanguage = error?.service.language?.name;
   const exceptions = error?.error.exception || [];
   const logStackframes = error?.error.log?.stacktrace;
   const isPlaintextException =
-    !!error?.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace;
+    !!error.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace;
   switch (currentTab.key) {
     case ErrorTabKey.LogStackTrace:
       return <Stacktrace stackframes={logStackframes} codeLanguage={codeLanguage} />;

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx

@@ -41,7 +41,7 @@ export const metadataTab: ErrorTab = {
   }),
 };
 
-export function getTabs(error: APMError) {
+export function getTabs(error: { error: { log?: APMError['error']['log'] } }) {
   const hasLogStacktrace = !isEmpty(error?.error.log?.stacktrace);
   return [...(hasLogStacktrace ? [logStacktraceTab] : []), exceptionStacktraceTab, metadataTab];
 }

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx

@@ -18,7 +18,9 @@ const Label = euiStyled.div`
 `;
 
 interface Props {
-  error: APMError;
+  error: {
+    error: Pick<APMError['error'], 'log' | 'exception' | 'culprit'>;
+  };
 }
 export function SampleSummary({ error }: Props) {
   const logMessage = error.error.log?.message;

x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts

@@ -6,15 +6,15 @@
  */
 
 import { format } from 'url';
-import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
+import type { TransactionDetailRedirectInfo } from '../../../../server/routes/transactions/get_transaction_by_trace';
 
 export const getRedirectToTransactionDetailPageUrl = ({
   transaction,
   rangeFrom,
   rangeTo,
   waterfallItemId,
 }: {
-  transaction: Transaction;
+  transaction: TransactionDetailRedirectInfo;
   rangeFrom?: string;
   rangeTo?: string;
   waterfallItemId?: string;