[APM][Otel] Add Otel client based on PoC data (elastic#192293)

Closes [elastic#192115](elastic#192115) Closes [elastic#192465](elastic#192465) ## Summary This PR adds synthrace client for Otel native data and a simple scenario. This is the first step of adding it and in the future it will include more metrics and use cases. >[!NOTE] > To run ES the command needs "xpack.otel_data.registry.enabled=true" flag > `yarn es snapshot --license trial --E "xpack.otel_data.registry.enabled=true"` ## Next steps - We currently have only `service_destination` in the metrics indices we can include the other types in the future - After we have all the UI changes we can add more scenarios (also using the opentelemetry demo data and not only the e2e PoC example) ## Testing - Run ES: ```bash yarn es snapshot --license trial --E "xpack.otel_data.registry.enabled=true" ``` - Run Kibana: ```bash yarn start ``` >[!WARNING] If the e2e PoC is used the first 2 steps should be skipped - Run syntrace: ```bash node scripts/synthtrace otel_simple_trace.ts --clean ``` - Check indices in DevTools for the generated data: ```bash GET *metrics-generic.otel*/_search GET *traces-generic.otel*/_search GET *logs-generic.otel*/_search ``` - Check in the APM UI (all the tabs) >[!WARNING] Currently the UI changes done in APM are not merged so some errors are expected) https://github.com/user-attachments/assets/92f63610-82da-40f3-89bb-00be83c55377 --------- Co-authored-by: miriam.aparicio <[email protected]>
leemthompo · Oct 14, 2024 · 5067f15 · 5067f15
1 parent f787b85
commit 5067f15
Show file tree

Hide file tree

Showing 13 changed files with 546 additions and 4 deletions.
diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts
@@ -37,3 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator }
 export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs';
 export { type AssetDocument } from './src/lib/assets';
 export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics';
+export { otel, type OtelDocument } from './src/lib/otel';
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { OtelDocument } from '../../..';
+import { Serializable } from '../serializable';
+
+export interface OtelErrorDocument extends OtelDocument {
+  'event.name'?: string;
+  attributes?: {
+    'exception.message'?: string;
+    'error.stack_trace'?: string;
+    'exception.handled'?: boolean;
+    'exception.type'?: string;
+    'processor.event'?: string;
+    'timestamp.us'?: number;
+    'event.name'?: string;
+    'error.id'?: string;
+  };
+}
+
+export class OtelError extends Serializable<OtelErrorDocument> {
+  constructor(fields: OtelErrorDocument) {
+    super({
+      ...fields,
+    });
+  }
+}
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts
@@ -0,0 +1,213 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Fields } from '../entity';
+import { Serializable } from '../serializable';
+import { OtelError } from './error';
+import { OtelMetric } from './metric';
+import { OtelTransaction } from './transaction';
+
+interface OtelSharedResourceAttributes {
+  'service.name'?: string;
+  'agent.name'?: string;
+  'agent.version'?: string;
+  'metricset.interval'?: string;
+  'service.instance.id'?: string;
+  'telemetry.sdk.language'?: string;
+  'telemetry.sdk.name'?: string;
+  'telemetry.sdk.version'?: string;
+  'some.resource.attribute'?: string;
+}
+
+export interface OtelDocument extends Fields {
+  data_stream?: {
+    dataset: string;
+    namespace: string;
+    type: string;
+  };
+  attributes?: {
+    'timestamp.us'?: number;
+    'metricset.name'?: string;
+    [key: string]: any;
+  };
+  resource?: {
+    attributes?: OtelSharedResourceAttributes;
+    dropped_attributes_count?: number;
+    schema_url?: string;
+  };
+  scope?: {
+    attributes?: {
+      'service.framework.name'?: string;
+      'service.framework.version'?: string;
+    };
+    dropped_attributes_count?: number;
+    name?: string;
+  };
+  name?: string;
+  trace_id?: string;
+  trace?: { id: string };
+  span_id?: string;
+  span?: { id: string };
+  dropped_attributes_count?: number;
+  dropped_events_count?: number;
+  dropped_links_count?: number;
+  timestamp_us?: number;
+}
+
+class Otel extends Serializable<OtelDocument> {
+  constructor(fields: OtelDocument) {
+    super({
+      ...fields,
+    });
+  }
+
+  error(spanId: string) {
+    return new OtelError({
+      ...this.fields,
+      attributes: {
+        'exception.message': 'boom',
+        'exception.handled': false,
+        'exception.type': '*errors.errorString',
+        'error.stack_trace': 'Error: INTERNAL: Boom',
+        'processor.event': 'error',
+        'timestamp.us': 1726580752010657,
+        'event.name': 'exception',
+        'error.id': `error-${spanId}`,
+      },
+      data_stream: {
+        dataset: 'generic.otel',
+        namespace: 'default',
+        type: 'logs',
+      },
+      'event.name': 'exception',
+      dropped_attributes_count: 0,
+      resource: {
+        attributes: {
+          'agent.name': 'opentelemetry/go',
+          'agent.version': '1.28.0',
+          'service.name': 'sendotlp-synth',
+          'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a',
+        },
+        dropped_attributes_count: 0,
+        schema_url: 'https://opentelemetry.io/schemas/1.26.0',
+      },
+      scope: {
+        attributes: {
+          'service.framework.name': 'sendotlp-synth',
+          'service.framework.version': '',
+        },
+        dropped_attributes_count: 0,
+        name: 'sendotlp-synth',
+      },
+      span_id: spanId,
+    });
+  }
+
+  metric() {
+    return new OtelMetric({
+      ...this.fields,
+      attributes: {
+        'metricset.name': 'service_destination',
+        'processor.event': 'metric',
+        'event.outcome': 'success',
+        'service.target.name': 'foo_service',
+        'service.target.type': 'http',
+        'span.name': 'child1',
+        'span.destination.service.resource': 'foo_service:8080',
+      },
+      data_stream: {
+        dataset: 'service_destination.10m.otel',
+        namespace: 'default',
+        type: 'metrics',
+      },
+      metrics: {
+        service_summary: 2,
+      },
+      resource: {
+        attributes: {
+          'agent.name': 'otlp',
+          'agent.version': '1.28.0',
+          'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a',
+          'service.name': 'sendotlp-synth',
+          'metricset.interval': '10m',
+        },
+        dropped_attributes_count: 0,
+      },
+      scope: {
+        dropped_attributes_count: 0,
+        name: 'github.com/elastic/opentelemetry-collector-components/connector/spanmetricsconnectorv2',
+      },
+    });
+  }
+
+  // In Otel we have only spans (https://opentelemetry.io/docs/concepts/signals/traces/#spans)
+  // we call the root span a transaction to match our data model
+  transaction(id: string) {
+    return new OtelTransaction({
+      ...this.fields,
+      attributes: {
+        'event.outcome': 'success',
+        'event.success_count': 1,
+        'processor.event': 'transaction',
+        'timestamp.us': 1726580752010657,
+        'transaction.duration.us': 15202,
+        'transaction.id': id,
+        'transaction.name': 'parent-synth',
+        'transaction.representative_count': 1,
+        'transaction.result': 'HTTP 2xx',
+        'transaction.root': true,
+        'transaction.sampled': true,
+        'transaction.type': 'unknown',
+      },
+      data_stream: {
+        dataset: 'generic.otel',
+        namespace: 'default',
+        type: 'traces',
+      },
+      duration: 11742370,
+      kind: 'Internal',
+      name: 'parent-synth',
+      resource: {
+        attributes: {
+          'agent.name': 'otlp',
+          'agent.version': '1.28.0',
+          'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a',
+          'service.name': 'sendotlp-synth',
+        },
+        dropped_attributes_count: 0,
+        schema_url: 'https://opentelemetry.io/schemas/1.26.0',
+      },
+      scope: {
+        attributes: {
+          'service.framework.name': 'sendotlp-synth',
+          'service.framework.version': '',
+        },
+        dropped_attributes_count: 0,
+        name: 'sendotlp-synth',
+      },
+      span_id: id,
+      status: {
+        code: 'Unset',
+      },
+    });
+  }
+}
+
+export function create(id: string): Otel {
+  return new Otel({
+    trace_id: id,
+    dropped_attributes_count: 0,
+    dropped_events_count: 0,
+    dropped_links_count: 0,
+  });
+}
+
+export const otel = {
+  create,
+};
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { OtelDocument } from '.';
+import { Serializable } from '../serializable';
+
+export interface OtelMetricDocument extends OtelDocument {
+  attributes?: {
+    'metricset.name'?: string;
+    'processor.event'?: string;
+    'event.outcome'?: string;
+    'service.target.name'?: string;
+    'service.target.type'?: string;
+    'span.name'?: string;
+    'span.destination.service.resource'?: string;
+  };
+  metrics?: {
+    service_summary?: number;
+  };
+}
+export class OtelMetric extends Serializable<OtelMetricDocument> {
+  constructor(fields: OtelMetricDocument) {
+    super({
+      ...fields,
+    });
+  }
+}
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { OtelDocument } from '.';
+import { Serializable } from '../serializable';
+
+export interface OtelTransactionDocument extends OtelDocument {
+  attributes?: {
+    'event.outcome'?: string;
+    'event.success_count'?: number;
+    'processor.event'?: string;
+    'timestamp.us'?: number;
+    'transaction.duration.us'?: number;
+    'transaction.id'?: string;
+    'transaction.name'?: string;
+    'transaction.representative_count'?: number;
+    'transaction.result'?: string;
+    'transaction.root'?: boolean;
+    'transaction.sampled'?: boolean;
+    'transaction.type'?: string;
+  };
+  status?: {
+    code?: string;
+  };
+  dropped_events_count?: number;
+  dropped_links_count?: number;
+  duration?: number;
+  kind?: string;
+  name?: string;
+}
+
+export class OtelTransaction extends Serializable<OtelTransactionDocument> {
+  constructor(fields: OtelTransactionDocument) {
+    super({
+      ...fields,
+    });
+  }
+}
diff --git a/packages/kbn-apm-synthtrace/index.ts b/packages/kbn-apm-synthtrace/index.ts
@@ -17,6 +17,7 @@ export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_sy
 export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
 export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client';
 export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
+export { OtelSynthtraceEsClient } from './src/lib/otel/otel_synthtrace_es_client';
 export {
   addObserverVersionTransform,
   deleteSummaryFieldTransform,

diff --git a/packages/kbn-apm-synthtrace/src/cli/scenario.ts b/packages/kbn-apm-synthtrace/src/cli/scenario.ts
@@ -13,6 +13,7 @@ import {
   InfraSynthtraceEsClient,
   LogsSynthtraceEsClient,
   SyntheticsSynthtraceEsClient,
+  OtelSynthtraceEsClient,
 } from '../..';
 import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_client';
 import { Logger } from '../lib/utils/create_logger';
@@ -25,6 +26,7 @@ interface EsClients {
   infraEsClient: InfraSynthtraceEsClient;
   assetsEsClient: AssetsSynthtraceEsClient;
   syntheticsEsClient: SyntheticsSynthtraceEsClient;
+  otelEsClient: OtelSynthtraceEsClient;
 }
 
 type Generate<TFields> = (options: {

diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts
@@ -16,6 +16,7 @@ import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
 import { getAssetsEsClient } from './get_assets_es_client';
 import { getSyntheticsEsClient } from './get_synthetics_es_client';
+import { getOtelSynthtraceEsClient } from './get_otel_es_client';
 
 export async function bootstrap(runOptions: RunOptions) {
   const logger = createLogger(runOptions.logLevel);
@@ -68,13 +69,19 @@ export async function bootstrap(runOptions: RunOptions) {
     logger,
     concurrency: runOptions.concurrency,
   });
+  const otelEsClient = getOtelSynthtraceEsClient({
+    target: esUrl,
+    logger,
+    concurrency: runOptions.concurrency,
+  });
 
   if (runOptions.clean) {
     await apmEsClient.clean();
     await logsEsClient.clean();
     await infraEsClient.clean();
     await assetsEsClient.clean();
     await syntheticsEsClient.clean();
+    await otelEsClient.clean();
   }
 
   return {
@@ -84,6 +91,7 @@ export async function bootstrap(runOptions: RunOptions) {
     infraEsClient,
     assetsEsClient,
     syntheticsEsClient,
+    otelEsClient,
     version,
     kibanaUrl,
     esUrl,