From 268638f6965785c358160990257a7f67ad3d3527 Mon Sep 17 00:00:00 2001 From: Jenny Date: Fri, 6 Sep 2024 19:23:43 +0200 Subject: [PATCH 01/16] [WIP] Add Otel client based on PoC data --- packages/kbn-apm-synthtrace-client/index.ts | 1 + .../src/lib/otel/error.ts | 28 +++ .../src/lib/otel/index.ts | 238 ++++++++++++++++++ .../src/lib/otel/metric.ts | 28 +++ .../src/lib/otel/transaction.ts | 28 +++ packages/kbn-apm-synthtrace/index.ts | 1 + .../kbn-apm-synthtrace/src/cli/scenario.ts | 2 + .../src/cli/utils/bootstrap.ts | 7 + .../src/cli/utils/get_otel_es_client.ts | 33 +++ .../src/cli/utils/synthtrace_worker.ts | 17 +- .../src/lib/otel/otel_synthtrace_es_client.ts | 112 +++++++++ .../src/scenarios/otel_simple_trace.ts | 45 ++++ 12 files changed, 539 insertions(+), 1 deletion(-) create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts create mode 100644 packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts create mode 100644 packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts create mode 100644 packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts index 2553d3b08bfab..111eb6a3b3e40 100644 --- a/packages/kbn-apm-synthtrace-client/index.ts +++ b/packages/kbn-apm-synthtrace-client/index.ts @@ -36,3 +36,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs'; export { type AssetDocument } from './src/lib/assets'; export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics'; +export { otel, type OtelDocument, type OtelTrace } from './src/lib/otel'; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts new file mode 100644 index 0000000000000..190ff07a70fd8 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { Serializable } from '../serializable'; +import { generateShortId } from '../utils/generate_id'; + +export class OtelError extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + // 'processor.event': 'error', + // 'processor.name': 'error', + trace_id: generateShortId(), + }); + } + + timestamp(value: number) { + const ret = super.timestamp(value); + this.fields['timestamp.us'] = value * 1000; + return ret; + } +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts new file mode 100644 index 0000000000000..52c75af998a96 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Serializable } from '../serializable'; +import { OtelError } from './error'; +import { OtelMetric } from './metric'; +import { OtelTransaction } from './transaction'; + +export interface OtelTrace { + attributes?: { + 'event.outcome'?: string; + 'event.success_count'?: number; + 'processor.event'?: string; + 'timestamp.us'?: number; + 'transaction.duration.us'?: number; + 'transaction.id'?: string; + 'transaction.name'?: string; + 'transaction.representative_count'?: number; + 'transaction.result'?: string; + 'transaction.root'?: boolean; + 'transaction.sampled'?: boolean; + 'transaction.type'?: string; + // error + 'exception.message'?: string; + 'exception.type'?: string; + // metrics + 'metricset.interval'?: string; + 'metricset.name'?: string; + }; + data_stream?: { + dataset: string; + namespace: string; + type: string; + }; + dropped_attributes_count?: number; + dropped_events_count?: number; + dropped_links_count?: number; + duration?: number; + kind?: string; + name?: string; + resource?: { + attributes?: { + 'agent.name'?: string; + 'agent.version'?: string; + 'service.name'?: string; + 'some.resource.attribute'?: string; + 'telemetry.sdk.language'?: string; + 'telemetry.sdk.name'?: string; + 'telemetry.sdk.version'?: string; + // metrics + 'metricset.interval'?: string; + }; + dropped_attributes_count?: number; + schema_url?: string; + }; + scope?: { + attributes?: { + 'service.framework.name': string; + 'service.framework.version': string; + }; + dropped_attributes_count?: number; + name?: string; + }; + span_id?: string; + status?: { + code: string; + }; + trace_id: string; + '@timestamp'?: number | undefined; + 'timestamp.us'?: number | undefined; + // metrics + metrics?: { + service_summary?: number; + }; +} + +export type OtelDocument = OtelTrace; + +class Otel extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + }); + } + + timestamp(time: number) { + super.timestamp(time); + return this; + } + + error(id: string) { + return new OtelError({ + ...this.fields, + attributes: { + 'exception.message': 'boom', + 'exception.type': '*errors.errorString', + 'processor.event': 'error', + 'timestamp.us': 1725633628036123, + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'logs', + }, + dropped_attributes_count: 0, + name: 'exception', + resource: { + attributes: { + 'agent.name': 'opentelemetry/go', + 'agent.version': '1.28.0', + 'service.name': 'sendotlp-synth', + 'some.resource.attribute': 'resource.attr', + 'telemetry.sdk.language': 'go', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.28.0', + }, + dropped_attributes_count: 0, + schema_url: 'https://opentelemetry.io/schemas/1.26.0', + }, + scope: { + attributes: { + 'service.framework.name': 'sendotlp-synth', + 'service.framework.version': '', + }, + dropped_attributes_count: 0, + name: 'sendotlp-synth', + }, + span_id: '96d91c11a4e393fb', + trace_id: id, + }); + } + + metric(id: string) { + return new OtelMetric({ + ...this.fields, + attributes: { + 'metricset.interval': '1m', + 'metricset.name': 'service_summary', + 'processor.event': 'metric', + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'metrics', + }, + metrics: { + service_summary: 2, + }, + resource: { + attributes: { + 'agent.name': 'opentelemetry/go', + 'agent.version': '1.28.0', + 'metricset.interval': '1m', + 'service.name': 'sendotlp-synth', + 'some.resource.attribute': 'resource.attr', + 'telemetry.sdk.language': 'go', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.28.0', + }, + dropped_attributes_count: 0, + }, + scope: { + dropped_attributes_count: 0, + name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', + }, + trace_id: id, + }); + } + transaction(id: string) { + return new OtelTransaction({ + ...this.fields, + attributes: { + 'event.outcome': 'success', + 'event.success_count': 1, + 'processor.event': 'transaction', + 'timestamp.us': 1725464233071114, + 'transaction.duration.us': 15202, + 'transaction.id': 'da907d6b5267c8f6', + 'transaction.name': 'parent-synth', + 'transaction.representative_count': 1, + 'transaction.result': 'Success', + 'transaction.root': true, + 'transaction.sampled': true, + 'transaction.type': 'unknown', + }, + dropped_attributes_count: 0, + dropped_events_count: 0, + dropped_links_count: 0, + duration: 15202584, + kind: 'Internal', + name: 'parent-synth', + resource: { + attributes: { + 'agent.name': 'opentelemetry/go', + 'agent.version': '1.28.0', + 'service.name': 'sendotlp-synth', + 'some.resource.attribute': 'resource.attr', + 'telemetry.sdk.language': 'go', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.28.0', + }, + dropped_attributes_count: 0, + schema_url: 'https://opentelemetry.io/schemas/1.26.0', + }, + scope: { + attributes: { + 'service.framework.name': 'sendotlp-synth', + 'service.framework.version': '', + }, + dropped_attributes_count: 0, + name: 'sendotlp-synth', + }, + span_id: `${id}-da907d6b5267c8fa`, + status: { + code: 'Unset', + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'traces', + }, + trace_id: id, + }); + } +} + +export function create(): Otel { + return new Otel(); +} + +export const otel = { + create, +}; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts new file mode 100644 index 0000000000000..8e2c1a76b180a --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { Serializable } from '../serializable'; +// import { generateShortId } from '../utils/generate_id'; + +export class OtelMetric extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + // 'processor.event': 'error', + // 'processor.name': 'error', + // trace_id: generateShortId(), + }); + } + + timestamp(value: number) { + const ret = super.timestamp(value); + this.fields['timestamp.us'] = value * 1000; + return ret; + } +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts new file mode 100644 index 0000000000000..33f019f4b2fb8 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { Serializable } from '../serializable'; +// import { generateShortId } from '../utils/generate_id'; + +export class OtelTransaction extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + // 'processor.event': 'error', + // 'processor.name': 'error', + // trace_id: generateShortId(), + }); + } + + timestamp(value: number) { + const ret = super.timestamp(value); + this.fields['timestamp.us'] = value * 1000; + return ret; + } +} diff --git a/packages/kbn-apm-synthtrace/index.ts b/packages/kbn-apm-synthtrace/index.ts index 22b947cd1bbf3..f8ce0f578dff6 100644 --- a/packages/kbn-apm-synthtrace/index.ts +++ b/packages/kbn-apm-synthtrace/index.ts @@ -16,6 +16,7 @@ export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_sy export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client'; export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client'; export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client'; +export { OtelSynthtraceEsClient } from './src/lib/otel/otel_synthtrace_es_client'; export { addObserverVersionTransform, deleteSummaryFieldTransform, diff --git a/packages/kbn-apm-synthtrace/src/cli/scenario.ts b/packages/kbn-apm-synthtrace/src/cli/scenario.ts index ba5e17deb1c18..ed33eb7344fbc 100644 --- a/packages/kbn-apm-synthtrace/src/cli/scenario.ts +++ b/packages/kbn-apm-synthtrace/src/cli/scenario.ts @@ -12,6 +12,7 @@ import { InfraSynthtraceEsClient, LogsSynthtraceEsClient, SyntheticsSynthtraceEsClient, + OtelSynthtraceEsClient, } from '../..'; import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_client'; import { Logger } from '../lib/utils/create_logger'; @@ -24,6 +25,7 @@ interface EsClients { infraEsClient: InfraSynthtraceEsClient; assetsEsClient: AssetsSynthtraceEsClient; syntheticsEsClient: SyntheticsSynthtraceEsClient; + otelSynthtraceEsClient: OtelSynthtraceEsClient; } type Generate = (options: { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts index 5119625b7f261..35920e4b5e1e5 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts @@ -15,6 +15,7 @@ import { getServiceUrls } from './get_service_urls'; import { RunOptions } from './parse_run_cli_flags'; import { getAssetsEsClient } from './get_assets_es_client'; import { getSyntheticsEsClient } from './get_synthetics_es_client'; +import { getOtelSynthtraceEsClient } from './get_otel_es_client'; export async function bootstrap(runOptions: RunOptions) { const logger = createLogger(runOptions.logLevel); @@ -67,6 +68,11 @@ export async function bootstrap(runOptions: RunOptions) { logger, concurrency: runOptions.concurrency, }); + const otelEsClient = getOtelSynthtraceEsClient({ + target: esUrl, + logger, + concurrency: runOptions.concurrency, + }); if (runOptions.clean) { await apmEsClient.clean(); @@ -74,6 +80,7 @@ export async function bootstrap(runOptions: RunOptions) { await infraEsClient.clean(); await assetsEsClient.clean(); await syntheticsEsClient.clean(); + await otelEsClient.clean(); } return { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts new file mode 100644 index 0000000000000..3cdeaf9e8a134 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Client } from '@elastic/elasticsearch'; +import { Logger } from '../../lib/utils/create_logger'; +import { RunOptions } from './parse_run_cli_flags'; +import { getEsClientTlsSettings } from './ssl'; +import { OtelSynthtraceEsClient } from '../../lib/otel/otel_synthtrace_es_client'; + +export function getOtelSynthtraceEsClient({ + target, + logger, + concurrency, +}: Pick & { + target: string; + logger: Logger; +}) { + const client = new Client({ + node: target, + tls: getEsClientTlsSettings(target), + }); + + return new OtelSynthtraceEsClient({ + client, + logger, + concurrency, + }); +} diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts index 784931b5f6139..93d95918fddbc 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts @@ -18,6 +18,7 @@ import { getLogsEsClient } from './get_logs_es_client'; import { getInfraEsClient } from './get_infra_es_client'; import { getAssetsEsClient } from './get_assets_es_client'; import { getSyntheticsEsClient } from './get_synthetics_es_client'; +import { getOtelSynthtraceEsClient } from './get_otel_es_client'; export interface WorkerData { bucketFrom: Date; @@ -63,6 +64,12 @@ async function start() { logger, }); + const otelSynthtraceEsClient = getOtelSynthtraceEsClient({ + concurrency: runOptions.concurrency, + target: esUrl, + logger, + }); + const file = runOptions.file; const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger })); @@ -78,6 +85,7 @@ async function start() { infraEsClient, assetsEsClient, syntheticsEsClient, + otelSynthtraceEsClient, }); } @@ -86,7 +94,14 @@ async function start() { const generatorsAndClients = logger.perf('generate_scenario', () => generate({ range: timerange(bucketFrom, bucketTo), - clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient, syntheticsEsClient }, + clients: { + logsEsClient, + apmEsClient, + infraEsClient, + assetsEsClient, + syntheticsEsClient, + otelSynthtraceEsClient, + }, }) ); diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts new file mode 100644 index 0000000000000..ad5d9b7a3fb14 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Client } from '@elastic/elasticsearch'; +import { ESDocumentWithOperation } from '@kbn/apm-synthtrace-client'; +import { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { pipeline, Readable, Transform } from 'stream'; +import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client'; +import { getDedotTransform } from '../shared/get_dedot_transform'; +import { getSerializeTransform } from '../shared/get_serialize_transform'; +import { Logger } from '../utils/create_logger'; + +export type OtelSynthtraceEsClientOptions = Omit; + +export class OtelSynthtraceEsClient extends SynthtraceEsClient { + constructor(options: { client: Client; logger: Logger } & OtelSynthtraceEsClientOptions) { + super({ + ...options, + pipeline: otelPipeline(), + }); + this.dataStreams = [ + '.otel-*', + 'metrics-*', + 'logs-*.', + 'traces-*.', + 'generic.otel-*', + 'traces-otel-default', + 'metric-otel-*', + 'logs-otel.error-*', + '*-otel-*', + ]; + } +} + +function otelPipeline() { + return (base: Readable) => { + return pipeline( + base, + getSerializeTransform(), + getRoutingTransform(), + getDedotTransform(), + (err: unknown) => { + if (err) { + throw err; + } + } + ); + }; +} + +export function getRoutingTransform() { + return new Transform({ + objectMode: true, + transform(document: ESDocumentWithOperation, encoding, callback) { + const namespace = 'default'; + let index: string | undefined = 'traces-otel-default'; + console.log(document); + console.log('doc', document.attributes[`processor.event`]); + + switch (document.attributes['processor.event']) { + case 'transaction': + case 'span': + index = `traces-otel-${namespace}`; + document._index = `traces-otel-${namespace}`; + break; + + case 'error': + index = `logs-otel.error-${namespace}`; + document._index = `logs-otel.error-${namespace}`; + break; + + case 'metric': + const metricsetName = document.attributes['metricset.name']; + document._index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + + if (metricsetName === 'app') { + index = `metrics-otel.app.${document.attributes['service.name']}-${namespace}`; + } else if ( + metricsetName === 'transaction' || + metricsetName === 'service_transaction' || + metricsetName === 'service_destination' || + metricsetName === 'service_summary' + ) { + index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + } else { + index = `metrics-otel.internal-${namespace}`; + } + break; + default: + if (document['event.action'] != null) { + index = `logs-otel.app-${namespace}`; + } + break; + } + + console.log('index', index); + if (!index) { + const error = new Error('Cannot determine index for event'); + Object.assign(error, { document }); + } + + // document._index = index; + + callback(null, document); + }, + }); +} diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts new file mode 100644 index 0000000000000..4fbc68bd37592 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { ApmFields, otel, generateShortId } from '@kbn/apm-synthtrace-client'; +import { times } from 'lodash'; +import { Scenario } from '../cli/scenario'; +import { withClient } from '../lib/utils/with_client'; + +const scenario: Scenario = async (runOptions) => { + return { + generate: ({ range, clients: { otelSynthtraceEsClient } }) => { + const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; + const { logger } = runOptions; + + const otelDocs = times(numOtelTraces / 2).map((index) => otel.create()); + + const otelWithMetricsAndErrors = range + .interval('30s') + .rate(1) + .generator((timestamp) => + otelDocs.flatMap((oteld) => { + const id = generateShortId(); + return [ + oteld.metric(id).timestamp(timestamp), + oteld.transaction(id).timestamp(timestamp), + oteld.error(id).timestamp(timestamp), + ]; + }) + ); + + return [ + withClient( + otelSynthtraceEsClient, + logger.perf('generating_otel_otelTrace', () => otelWithMetricsAndErrors) + ), + ]; + }, + }; +}; + +export default scenario; From fdf43bd8f6f6bf8a3635d1cfe6d98a3967c94bd1 Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 9 Sep 2024 21:52:18 +0200 Subject: [PATCH 02/16] Add fields and changed indecies --- .../src/lib/otel/index.ts | 161 ++++++++++++++++++ .../src/lib/otel/otel_synthtrace_es_client.ts | 50 +++--- .../src/scenarios/otel_simple_trace.ts | 2 +- 3 files changed, 183 insertions(+), 30 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 52c75af998a96..442e942da691a 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -77,6 +77,7 @@ export interface OtelTrace { metrics?: { service_summary?: number; }; + 'service.name': string[]; } export type OtelDocument = OtelTrace; @@ -132,6 +133,51 @@ class Otel extends Serializable { }, span_id: '96d91c11a4e393fb', trace_id: id, + + fields: { + 'attributes.exception.message': ['boom'], + 'scope.name': ['sendotlp-synth'], + 'resource.attributes.agent.version': ['1.28.0'], + 'service.framework.version': [''], + 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], + // 'resource.attributes.agent.name.text': ['opentelemetry/go'], + 'resource.attributes.agent.name': ['opentelemetry/go'], + 'resource.attributes.telemetry.sdk.language': ['go'], + 'scope.dropped_attributes_count': [0], + 'service.language.name': ['go'], + 'telemetry.sdk.name': ['opentelemetry'], + 'attributes.timestamp.us': [1725906566779927], + 'telemetry.sdk.language': ['go'], + 'trace.id': ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + 'exception.message': ['boom'], + 'processor.event': ['error'], + 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], + 'agent.name': ['opentelemetry/go'], + 'telemetry.sdk.version': ['1.28.0'], + 'scope.attributes.service.framework.name': ['sendotlp-synth'], + trace_id: ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + 'service.name': ['sendotlp-synth'], + 'service.framework.name': ['sendotlp-synth'], + span_id: ['42fb9d5c7aff7d08'], + 'data_stream.namespace': ['default'], + 'exception.type': ['*errors.errorString'], + 'resource.attributes.some.resource.attribute': ['resource.attr'], + 'span.id': ['42fb9d5c7aff7d08'], + 'some.resource.attribute': ['resource.attr'], + 'data_stream.type': ['logs'], + 'attributes.processor.event': ['error'], + 'timestamp.us': [1725906566779927], + '@timestamp': ['2024-09-09T18:29:26.779Z'], + dropped_attributes_count: [0], + 'resource.attributes.telemetry.sdk.version': ['1.28.0'], + 'data_stream.dataset': ['generic.otel'], + name: ['exception'], + 'agent.version': ['1.28.0'], + 'attributes.exception.type': ['*errors.errorString'], + 'resource.dropped_attributes_count': [0], + 'scope.attributes.service.framework.version': [''], + 'resource.attributes.service.name': ['sendotlp-synth'], + }, }); } @@ -169,6 +215,53 @@ class Otel extends Serializable { name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', }, trace_id: id, + fields: { + 'resource.attributes.agent.version': ['1.28.0'], + 'scope.name': ['otelcol/spanmetricsconnectorv2'], + 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], + 'resource.attributes.agent.name': ['opentelemetry/go'], + 'resource.attributes.telemetry.sdk.language': ['go'], + 'scope.dropped_attributes_count': [0], + 'service.language.name': ['go'], + 'telemetry.sdk.name': ['opentelemetry'], + 'telemetry.sdk.language': ['go'], + 'transaction.root': [true], + 'processor.event': ['metric'], + 'resource.attributes.metricset.interval': ['1m'], + 'agent.name': ['opentelemetry/go'], + 'telemetry.sdk.version': ['1.28.0'], + 'attributes.metricset.name': ['service_transaction'], + 'metrics.transaction.duration.histogram': [ + { + values: [12500], + counts: [1], + }, + ], + 'attributes.transaction.root': [true], + 'service.name': ['sendotlp-synth'], + 'data_stream.namespace': ['default'], + 'resource.attributes.some.resource.attribute': ['resource.attr'], + 'some.resource.attribute': ['resource.attr'], + 'metricset.interval': ['1m'], + 'data_stream.type': ['metrics'], + 'transaction.duration.histogram': [ + { + values: [12500], + counts: [1], + }, + ], + 'transaction.type': ['unknown'], + 'metricset.name': ['service_transaction'], + 'attributes.processor.event': ['metric'], + '@timestamp': ['2024-09-09T15:49:44.920Z'], + 'resource.attributes.telemetry.sdk.version': ['1.28.0'], + 'data_stream.dataset': ['generic.otel'], + 'attributes.transaction.type': ['unknown'], + 'agent.version': ['1.28.0'], + 'attributes.metricset.interval': ['1m'], + 'resource.dropped_attributes_count': [0], + 'resource.attributes.service.name': ['sendotlp-synth'], + }, }); } transaction(id: string) { @@ -225,6 +318,74 @@ class Otel extends Serializable { type: 'traces', }, trace_id: id, + fields: { + // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], + 'scope.name': ['sendotlp-synth'], + 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], + 'resource.attributes.agent.name': ['opentelemetry/go'], + 'transaction.representative_count': [1], + 'scope.dropped_attributes_count': [0], + 'service.language.name': ['go'], + 'transaction.result': ['Success'], + 'transaction.id': ['de8004c8152af744'], + 'attributes.event.success_count': [1], + 'attributes.timestamp.us': [1725896984908682], + 'telemetry.sdk.language': ['go'], + 'processor.event': ['transaction'], + 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], + 'agent.name': ['opentelemetry/go'], + 'telemetry.sdk.version': ['1.28.0'], + 'event.outcome': ['success'], + 'scope.attributes.service.framework.name': ['sendotlp-synth'], + 'attributes.transaction.sampled': [true], + span_id: ['de8004c8152af744'], + kind: ['Internal'], + 'attributes.transaction.duration.us': [12026], + 'attributes.transaction.name': ['parent-synth'], + 'transaction.duration.us': [12026], + 'span.id': ['de8004c8152af744'], + 'data_stream.type': ['traces'], + 'timestamp.us': [1725896984908682], + 'resource.attributes.telemetry.sdk.version': ['1.28.0'], + name: ['parent-synth'], + 'agent.version': ['1.28.0'], + 'transaction.name': ['parent-synth'], + 'resource.dropped_attributes_count': [0], + 'scope.attributes.service.framework.version': [''], + 'resource.attributes.service.name': ['sendotlp-synth'], + 'attributes.transaction.representative_count': [1], + 'span.name': ['parent-synth'], + 'resource.attributes.agent.version': ['1.28.0'], + 'attributes.transaction.id': ['de8004c8152af744'], + 'service.framework.version': [''], + 'resource.attributes.telemetry.sdk.language': ['go'], + 'status.code': ['Unset'], + 'transaction.sampled': [true], + 'telemetry.sdk.name': ['opentelemetry'], + duration: [12026084], + 'attributes.transaction.result': ['Success'], + 'trace.id': ['b00875cbeb31ec7adcedc6b4213dd1ad'], + // 'attributes.transaction.name.text': ['parent-synth'], + 'event.success_count': [1], + 'transaction.root': [true], + // 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], + 'attributes.transaction.root': [true], + dropped_events_count: [0], + trace_id: ['b00875cbeb31ec7adcedc6b4213dd1ad'], + 'service.name': ['sendotlp-synth'], + 'service.framework.name': ['sendotlp-synth'], + 'data_stream.namespace': ['default'], + 'resource.attributes.some.resource.attribute': ['resource.attr'], + 'some.resource.attribute': ['resource.attr'], + dropped_links_count: [0], + 'transaction.type': ['unknown'], + 'attributes.processor.event': ['transaction'], + '@timestamp': ['2024-09-09T15:49:44.908Z'], + dropped_attributes_count: [0], + 'data_stream.dataset': ['generic.otel'], + 'attributes.event.outcome': ['success'], + 'attributes.transaction.type': ['unknown'], + }, }); } } diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index ad5d9b7a3fb14..fbb78677fa02e 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -23,17 +23,7 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient { ...options, pipeline: otelPipeline(), }); - this.dataStreams = [ - '.otel-*', - 'metrics-*', - 'logs-*.', - 'traces-*.', - 'generic.otel-*', - 'traces-otel-default', - 'metric-otel-*', - 'logs-otel.error-*', - '*-otel-*', - ]; + this.dataStreams = ['.otel-*-synth', '*-synth-*']; } } @@ -58,53 +48,55 @@ export function getRoutingTransform() { objectMode: true, transform(document: ESDocumentWithOperation, encoding, callback) { const namespace = 'default'; - let index: string | undefined = 'traces-otel-default'; - console.log(document); - console.log('doc', document.attributes[`processor.event`]); + let index: string | undefined; - switch (document.attributes['processor.event']) { + switch (document?.attributes?.['processor.event']) { case 'transaction': case 'span': - index = `traces-otel-${namespace}`; - document._index = `traces-otel-${namespace}`; + index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; + // document._index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; break; case 'error': - index = `logs-otel.error-${namespace}`; - document._index = `logs-otel.error-${namespace}`; + index = `.logs-otel.error-${namespace}-synth-2024.09.09-000001`; + // document._index = `logs-otel.error-${namespace}-synth-2024.09.09-000001`; break; case 'metric': const metricsetName = document.attributes['metricset.name']; - document._index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + document._index = `.ds-metrics-otel.service_summary.${document.attributes[ + 'metricset.interval' + ]!}-${namespace}-synth-2024.09.09-000001`; - if (metricsetName === 'app') { - index = `metrics-otel.app.${document.attributes['service.name']}-${namespace}`; - } else if ( + // if (metricsetName === 'app') { + // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}-2024.09.09-000001`; + // } else + if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' || metricsetName === 'service_destination' || metricsetName === 'service_summary' ) { - index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + index = `.ds-metrics-otel.${metricsetName}.${document.attributes[ + 'metricset.interval' + ]!}-${namespace}-2024.09.09-000001`; } else { index = `metrics-otel.internal-${namespace}`; } break; default: - if (document['event.action'] != null) { - index = `logs-otel.app-${namespace}`; - } + // if (document['event.action'] != null) { + // index = `logs-otel.app-${namespace}`; + // } break; } - console.log('index', index); if (!index) { const error = new Error('Cannot determine index for event'); Object.assign(error, { document }); } - // document._index = index; + document._index = index; callback(null, document); }, diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index 4fbc68bd37592..4d7be0af191ce 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -27,7 +27,7 @@ const scenario: Scenario = async (runOptions) => { return [ oteld.metric(id).timestamp(timestamp), oteld.transaction(id).timestamp(timestamp), - oteld.error(id).timestamp(timestamp), + // oteld.error(id).timestamp(timestamp), ]; }) ); From 1c471e41ec09c28d3c33cebd20dc7f1bf8646fb5 Mon Sep 17 00:00:00 2001 From: Jenny Date: Tue, 10 Sep 2024 17:24:42 +0200 Subject: [PATCH 03/16] Add service name and fields --- .../src/lib/otel/index.ts | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 442e942da691a..2540cbc23d57d 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -133,7 +133,7 @@ class Otel extends Serializable { }, span_id: '96d91c11a4e393fb', trace_id: id, - + 'service.name': ['sendotlp-synth'], fields: { 'attributes.exception.message': ['boom'], 'scope.name': ['sendotlp-synth'], @@ -148,21 +148,21 @@ class Otel extends Serializable { 'telemetry.sdk.name': ['opentelemetry'], 'attributes.timestamp.us': [1725906566779927], 'telemetry.sdk.language': ['go'], - 'trace.id': ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + 'trace.id': [id], 'exception.message': ['boom'], 'processor.event': ['error'], 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], 'agent.name': ['opentelemetry/go'], 'telemetry.sdk.version': ['1.28.0'], 'scope.attributes.service.framework.name': ['sendotlp-synth'], - trace_id: ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + trace_id: [id], 'service.name': ['sendotlp-synth'], 'service.framework.name': ['sendotlp-synth'], - span_id: ['42fb9d5c7aff7d08'], + span_id: [`${id}da907a`], 'data_stream.namespace': ['default'], 'exception.type': ['*errors.errorString'], 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'span.id': ['42fb9d5c7aff7d08'], + 'span.id': [`${id}da907a`], 'some.resource.attribute': ['resource.attr'], 'data_stream.type': ['logs'], 'attributes.processor.event': ['error'], @@ -177,6 +177,7 @@ class Otel extends Serializable { 'resource.dropped_attributes_count': [0], 'scope.attributes.service.framework.version': [''], 'resource.attributes.service.name': ['sendotlp-synth'], + 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], }, }); } @@ -215,6 +216,9 @@ class Otel extends Serializable { name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', }, trace_id: id, + service: { + name: ['sendotlp-synth'], + }, fields: { 'resource.attributes.agent.version': ['1.28.0'], 'scope.name': ['otelcol/spanmetricsconnectorv2'], @@ -308,7 +312,7 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, - span_id: `${id}-da907d6b5267c8fa`, + span_id: `${id}da907a`, status: { code: 'Unset', }, @@ -319,15 +323,17 @@ class Otel extends Serializable { }, trace_id: id, fields: { + // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], 'scope.name': ['sendotlp-synth'], 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], 'resource.attributes.agent.name': ['opentelemetry/go'], + // 'resource.attributes.agent.name.text': ['opentelemetry/go'], 'transaction.representative_count': [1], 'scope.dropped_attributes_count': [0], 'service.language.name': ['go'], 'transaction.result': ['Success'], - 'transaction.id': ['de8004c8152af744'], + 'transaction.id': [id], 'attributes.event.success_count': [1], 'attributes.timestamp.us': [1725896984908682], 'telemetry.sdk.language': ['go'], @@ -338,12 +344,12 @@ class Otel extends Serializable { 'event.outcome': ['success'], 'scope.attributes.service.framework.name': ['sendotlp-synth'], 'attributes.transaction.sampled': [true], - span_id: ['de8004c8152af744'], + span_id: [`${id}da907a`], kind: ['Internal'], 'attributes.transaction.duration.us': [12026], 'attributes.transaction.name': ['parent-synth'], 'transaction.duration.us': [12026], - 'span.id': ['de8004c8152af744'], + 'span.id': [`${id}da907a`], 'data_stream.type': ['traces'], 'timestamp.us': [1725896984908682], 'resource.attributes.telemetry.sdk.version': ['1.28.0'], @@ -352,11 +358,10 @@ class Otel extends Serializable { 'transaction.name': ['parent-synth'], 'resource.dropped_attributes_count': [0], 'scope.attributes.service.framework.version': [''], - 'resource.attributes.service.name': ['sendotlp-synth'], 'attributes.transaction.representative_count': [1], 'span.name': ['parent-synth'], 'resource.attributes.agent.version': ['1.28.0'], - 'attributes.transaction.id': ['de8004c8152af744'], + 'attributes.transaction.id': [id], 'service.framework.version': [''], 'resource.attributes.telemetry.sdk.language': ['go'], 'status.code': ['Unset'], @@ -364,14 +369,14 @@ class Otel extends Serializable { 'telemetry.sdk.name': ['opentelemetry'], duration: [12026084], 'attributes.transaction.result': ['Success'], - 'trace.id': ['b00875cbeb31ec7adcedc6b4213dd1ad'], + 'trace.id': [id], // 'attributes.transaction.name.text': ['parent-synth'], 'event.success_count': [1], 'transaction.root': [true], // 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], 'attributes.transaction.root': [true], dropped_events_count: [0], - trace_id: ['b00875cbeb31ec7adcedc6b4213dd1ad'], + trace_id: [id], 'service.name': ['sendotlp-synth'], 'service.framework.name': ['sendotlp-synth'], 'data_stream.namespace': ['default'], @@ -385,6 +390,8 @@ class Otel extends Serializable { 'data_stream.dataset': ['generic.otel'], 'attributes.event.outcome': ['success'], 'attributes.transaction.type': ['unknown'], + 'resource.attributes.service.name': ['sendotlp-synth'], + // 'resource.attributes.service.name.text': ['sendotlp-synth'], }, }); } From c023d457d6ba538776166d8ba616e23916459a84 Mon Sep 17 00:00:00 2001 From: Jenny Date: Fri, 6 Sep 2024 19:23:43 +0200 Subject: [PATCH 04/16] [WIP] Add Otel client based on PoC data --- packages/kbn-apm-synthtrace-client/index.ts | 1 + .../src/lib/otel/error.ts | 28 +++ .../src/lib/otel/index.ts | 238 ++++++++++++++++++ .../src/lib/otel/metric.ts | 28 +++ .../src/lib/otel/transaction.ts | 28 +++ packages/kbn-apm-synthtrace/index.ts | 1 + .../kbn-apm-synthtrace/src/cli/scenario.ts | 2 + .../src/cli/utils/bootstrap.ts | 7 + .../src/cli/utils/get_otel_es_client.ts | 33 +++ .../src/cli/utils/synthtrace_worker.ts | 17 +- .../src/lib/otel/otel_synthtrace_es_client.ts | 112 +++++++++ .../src/scenarios/otel_simple_trace.ts | 45 ++++ 12 files changed, 539 insertions(+), 1 deletion(-) create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts create mode 100644 packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts create mode 100644 packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts create mode 100644 packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts index 6ac3b6525ec00..a7eed2ebccda5 100644 --- a/packages/kbn-apm-synthtrace-client/index.ts +++ b/packages/kbn-apm-synthtrace-client/index.ts @@ -37,3 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs'; export { type AssetDocument } from './src/lib/assets'; export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics'; +export { otel, type OtelDocument, type OtelTrace } from './src/lib/otel'; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts new file mode 100644 index 0000000000000..190ff07a70fd8 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { Serializable } from '../serializable'; +import { generateShortId } from '../utils/generate_id'; + +export class OtelError extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + // 'processor.event': 'error', + // 'processor.name': 'error', + trace_id: generateShortId(), + }); + } + + timestamp(value: number) { + const ret = super.timestamp(value); + this.fields['timestamp.us'] = value * 1000; + return ret; + } +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts new file mode 100644 index 0000000000000..52c75af998a96 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Serializable } from '../serializable'; +import { OtelError } from './error'; +import { OtelMetric } from './metric'; +import { OtelTransaction } from './transaction'; + +export interface OtelTrace { + attributes?: { + 'event.outcome'?: string; + 'event.success_count'?: number; + 'processor.event'?: string; + 'timestamp.us'?: number; + 'transaction.duration.us'?: number; + 'transaction.id'?: string; + 'transaction.name'?: string; + 'transaction.representative_count'?: number; + 'transaction.result'?: string; + 'transaction.root'?: boolean; + 'transaction.sampled'?: boolean; + 'transaction.type'?: string; + // error + 'exception.message'?: string; + 'exception.type'?: string; + // metrics + 'metricset.interval'?: string; + 'metricset.name'?: string; + }; + data_stream?: { + dataset: string; + namespace: string; + type: string; + }; + dropped_attributes_count?: number; + dropped_events_count?: number; + dropped_links_count?: number; + duration?: number; + kind?: string; + name?: string; + resource?: { + attributes?: { + 'agent.name'?: string; + 'agent.version'?: string; + 'service.name'?: string; + 'some.resource.attribute'?: string; + 'telemetry.sdk.language'?: string; + 'telemetry.sdk.name'?: string; + 'telemetry.sdk.version'?: string; + // metrics + 'metricset.interval'?: string; + }; + dropped_attributes_count?: number; + schema_url?: string; + }; + scope?: { + attributes?: { + 'service.framework.name': string; + 'service.framework.version': string; + }; + dropped_attributes_count?: number; + name?: string; + }; + span_id?: string; + status?: { + code: string; + }; + trace_id: string; + '@timestamp'?: number | undefined; + 'timestamp.us'?: number | undefined; + // metrics + metrics?: { + service_summary?: number; + }; +} + +export type OtelDocument = OtelTrace; + +class Otel extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + }); + } + + timestamp(time: number) { + super.timestamp(time); + return this; + } + + error(id: string) { + return new OtelError({ + ...this.fields, + attributes: { + 'exception.message': 'boom', + 'exception.type': '*errors.errorString', + 'processor.event': 'error', + 'timestamp.us': 1725633628036123, + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'logs', + }, + dropped_attributes_count: 0, + name: 'exception', + resource: { + attributes: { + 'agent.name': 'opentelemetry/go', + 'agent.version': '1.28.0', + 'service.name': 'sendotlp-synth', + 'some.resource.attribute': 'resource.attr', + 'telemetry.sdk.language': 'go', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.28.0', + }, + dropped_attributes_count: 0, + schema_url: 'https://opentelemetry.io/schemas/1.26.0', + }, + scope: { + attributes: { + 'service.framework.name': 'sendotlp-synth', + 'service.framework.version': '', + }, + dropped_attributes_count: 0, + name: 'sendotlp-synth', + }, + span_id: '96d91c11a4e393fb', + trace_id: id, + }); + } + + metric(id: string) { + return new OtelMetric({ + ...this.fields, + attributes: { + 'metricset.interval': '1m', + 'metricset.name': 'service_summary', + 'processor.event': 'metric', + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'metrics', + }, + metrics: { + service_summary: 2, + }, + resource: { + attributes: { + 'agent.name': 'opentelemetry/go', + 'agent.version': '1.28.0', + 'metricset.interval': '1m', + 'service.name': 'sendotlp-synth', + 'some.resource.attribute': 'resource.attr', + 'telemetry.sdk.language': 'go', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.28.0', + }, + dropped_attributes_count: 0, + }, + scope: { + dropped_attributes_count: 0, + name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', + }, + trace_id: id, + }); + } + transaction(id: string) { + return new OtelTransaction({ + ...this.fields, + attributes: { + 'event.outcome': 'success', + 'event.success_count': 1, + 'processor.event': 'transaction', + 'timestamp.us': 1725464233071114, + 'transaction.duration.us': 15202, + 'transaction.id': 'da907d6b5267c8f6', + 'transaction.name': 'parent-synth', + 'transaction.representative_count': 1, + 'transaction.result': 'Success', + 'transaction.root': true, + 'transaction.sampled': true, + 'transaction.type': 'unknown', + }, + dropped_attributes_count: 0, + dropped_events_count: 0, + dropped_links_count: 0, + duration: 15202584, + kind: 'Internal', + name: 'parent-synth', + resource: { + attributes: { + 'agent.name': 'opentelemetry/go', + 'agent.version': '1.28.0', + 'service.name': 'sendotlp-synth', + 'some.resource.attribute': 'resource.attr', + 'telemetry.sdk.language': 'go', + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.version': '1.28.0', + }, + dropped_attributes_count: 0, + schema_url: 'https://opentelemetry.io/schemas/1.26.0', + }, + scope: { + attributes: { + 'service.framework.name': 'sendotlp-synth', + 'service.framework.version': '', + }, + dropped_attributes_count: 0, + name: 'sendotlp-synth', + }, + span_id: `${id}-da907d6b5267c8fa`, + status: { + code: 'Unset', + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'traces', + }, + trace_id: id, + }); + } +} + +export function create(): Otel { + return new Otel(); +} + +export const otel = { + create, +}; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts new file mode 100644 index 0000000000000..8e2c1a76b180a --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { Serializable } from '../serializable'; +// import { generateShortId } from '../utils/generate_id'; + +export class OtelMetric extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + // 'processor.event': 'error', + // 'processor.name': 'error', + // trace_id: generateShortId(), + }); + } + + timestamp(value: number) { + const ret = super.timestamp(value); + this.fields['timestamp.us'] = value * 1000; + return ret; + } +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts new file mode 100644 index 0000000000000..33f019f4b2fb8 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { Serializable } from '../serializable'; +// import { generateShortId } from '../utils/generate_id'; + +export class OtelTransaction extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + // 'processor.event': 'error', + // 'processor.name': 'error', + // trace_id: generateShortId(), + }); + } + + timestamp(value: number) { + const ret = super.timestamp(value); + this.fields['timestamp.us'] = value * 1000; + return ret; + } +} diff --git a/packages/kbn-apm-synthtrace/index.ts b/packages/kbn-apm-synthtrace/index.ts index b717b9a45af99..ebd35da3aa19e 100644 --- a/packages/kbn-apm-synthtrace/index.ts +++ b/packages/kbn-apm-synthtrace/index.ts @@ -17,6 +17,7 @@ export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_sy export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client'; export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client'; export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client'; +export { OtelSynthtraceEsClient } from './src/lib/otel/otel_synthtrace_es_client'; export { addObserverVersionTransform, deleteSummaryFieldTransform, diff --git a/packages/kbn-apm-synthtrace/src/cli/scenario.ts b/packages/kbn-apm-synthtrace/src/cli/scenario.ts index 85b0ee56fc9e8..4c7678507a338 100644 --- a/packages/kbn-apm-synthtrace/src/cli/scenario.ts +++ b/packages/kbn-apm-synthtrace/src/cli/scenario.ts @@ -13,6 +13,7 @@ import { InfraSynthtraceEsClient, LogsSynthtraceEsClient, SyntheticsSynthtraceEsClient, + OtelSynthtraceEsClient, } from '../..'; import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_client'; import { Logger } from '../lib/utils/create_logger'; @@ -25,6 +26,7 @@ interface EsClients { infraEsClient: InfraSynthtraceEsClient; assetsEsClient: AssetsSynthtraceEsClient; syntheticsEsClient: SyntheticsSynthtraceEsClient; + otelSynthtraceEsClient: OtelSynthtraceEsClient; } type Generate = (options: { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts index 6c6b065dabfc7..ebdfaf7c8d62a 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts @@ -16,6 +16,7 @@ import { getServiceUrls } from './get_service_urls'; import { RunOptions } from './parse_run_cli_flags'; import { getAssetsEsClient } from './get_assets_es_client'; import { getSyntheticsEsClient } from './get_synthetics_es_client'; +import { getOtelSynthtraceEsClient } from './get_otel_es_client'; export async function bootstrap(runOptions: RunOptions) { const logger = createLogger(runOptions.logLevel); @@ -68,6 +69,11 @@ export async function bootstrap(runOptions: RunOptions) { logger, concurrency: runOptions.concurrency, }); + const otelEsClient = getOtelSynthtraceEsClient({ + target: esUrl, + logger, + concurrency: runOptions.concurrency, + }); if (runOptions.clean) { await apmEsClient.clean(); @@ -75,6 +81,7 @@ export async function bootstrap(runOptions: RunOptions) { await infraEsClient.clean(); await assetsEsClient.clean(); await syntheticsEsClient.clean(); + await otelEsClient.clean(); } return { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts new file mode 100644 index 0000000000000..3cdeaf9e8a134 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Client } from '@elastic/elasticsearch'; +import { Logger } from '../../lib/utils/create_logger'; +import { RunOptions } from './parse_run_cli_flags'; +import { getEsClientTlsSettings } from './ssl'; +import { OtelSynthtraceEsClient } from '../../lib/otel/otel_synthtrace_es_client'; + +export function getOtelSynthtraceEsClient({ + target, + logger, + concurrency, +}: Pick & { + target: string; + logger: Logger; +}) { + const client = new Client({ + node: target, + tls: getEsClientTlsSettings(target), + }); + + return new OtelSynthtraceEsClient({ + client, + logger, + concurrency, + }); +} diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts index a5defe4b6e1b4..5e9301027a27e 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts @@ -20,6 +20,7 @@ import { getLogsEsClient } from './get_logs_es_client'; import { getInfraEsClient } from './get_infra_es_client'; import { getAssetsEsClient } from './get_assets_es_client'; import { getSyntheticsEsClient } from './get_synthetics_es_client'; +import { getOtelSynthtraceEsClient } from './get_otel_es_client'; export interface WorkerData { bucketFrom: Date; @@ -65,6 +66,12 @@ async function start() { logger, }); + const otelSynthtraceEsClient = getOtelSynthtraceEsClient({ + concurrency: runOptions.concurrency, + target: esUrl, + logger, + }); + const file = runOptions.file; const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger })); @@ -80,6 +87,7 @@ async function start() { infraEsClient, assetsEsClient, syntheticsEsClient, + otelSynthtraceEsClient, }); } @@ -88,7 +96,14 @@ async function start() { const generatorsAndClients = logger.perf('generate_scenario', () => generate({ range: timerange(bucketFrom, bucketTo), - clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient, syntheticsEsClient }, + clients: { + logsEsClient, + apmEsClient, + infraEsClient, + assetsEsClient, + syntheticsEsClient, + otelSynthtraceEsClient, + }, }) ); diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts new file mode 100644 index 0000000000000..ad5d9b7a3fb14 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Client } from '@elastic/elasticsearch'; +import { ESDocumentWithOperation } from '@kbn/apm-synthtrace-client'; +import { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { pipeline, Readable, Transform } from 'stream'; +import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client'; +import { getDedotTransform } from '../shared/get_dedot_transform'; +import { getSerializeTransform } from '../shared/get_serialize_transform'; +import { Logger } from '../utils/create_logger'; + +export type OtelSynthtraceEsClientOptions = Omit; + +export class OtelSynthtraceEsClient extends SynthtraceEsClient { + constructor(options: { client: Client; logger: Logger } & OtelSynthtraceEsClientOptions) { + super({ + ...options, + pipeline: otelPipeline(), + }); + this.dataStreams = [ + '.otel-*', + 'metrics-*', + 'logs-*.', + 'traces-*.', + 'generic.otel-*', + 'traces-otel-default', + 'metric-otel-*', + 'logs-otel.error-*', + '*-otel-*', + ]; + } +} + +function otelPipeline() { + return (base: Readable) => { + return pipeline( + base, + getSerializeTransform(), + getRoutingTransform(), + getDedotTransform(), + (err: unknown) => { + if (err) { + throw err; + } + } + ); + }; +} + +export function getRoutingTransform() { + return new Transform({ + objectMode: true, + transform(document: ESDocumentWithOperation, encoding, callback) { + const namespace = 'default'; + let index: string | undefined = 'traces-otel-default'; + console.log(document); + console.log('doc', document.attributes[`processor.event`]); + + switch (document.attributes['processor.event']) { + case 'transaction': + case 'span': + index = `traces-otel-${namespace}`; + document._index = `traces-otel-${namespace}`; + break; + + case 'error': + index = `logs-otel.error-${namespace}`; + document._index = `logs-otel.error-${namespace}`; + break; + + case 'metric': + const metricsetName = document.attributes['metricset.name']; + document._index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + + if (metricsetName === 'app') { + index = `metrics-otel.app.${document.attributes['service.name']}-${namespace}`; + } else if ( + metricsetName === 'transaction' || + metricsetName === 'service_transaction' || + metricsetName === 'service_destination' || + metricsetName === 'service_summary' + ) { + index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + } else { + index = `metrics-otel.internal-${namespace}`; + } + break; + default: + if (document['event.action'] != null) { + index = `logs-otel.app-${namespace}`; + } + break; + } + + console.log('index', index); + if (!index) { + const error = new Error('Cannot determine index for event'); + Object.assign(error, { document }); + } + + // document._index = index; + + callback(null, document); + }, + }); +} diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts new file mode 100644 index 0000000000000..4fbc68bd37592 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { ApmFields, otel, generateShortId } from '@kbn/apm-synthtrace-client'; +import { times } from 'lodash'; +import { Scenario } from '../cli/scenario'; +import { withClient } from '../lib/utils/with_client'; + +const scenario: Scenario = async (runOptions) => { + return { + generate: ({ range, clients: { otelSynthtraceEsClient } }) => { + const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; + const { logger } = runOptions; + + const otelDocs = times(numOtelTraces / 2).map((index) => otel.create()); + + const otelWithMetricsAndErrors = range + .interval('30s') + .rate(1) + .generator((timestamp) => + otelDocs.flatMap((oteld) => { + const id = generateShortId(); + return [ + oteld.metric(id).timestamp(timestamp), + oteld.transaction(id).timestamp(timestamp), + oteld.error(id).timestamp(timestamp), + ]; + }) + ); + + return [ + withClient( + otelSynthtraceEsClient, + logger.perf('generating_otel_otelTrace', () => otelWithMetricsAndErrors) + ), + ]; + }, + }; +}; + +export default scenario; From 309487f38f3cc11497676fcee036286ccfcff659 Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 9 Sep 2024 21:52:18 +0200 Subject: [PATCH 05/16] Add fields and changed indecies --- .../src/lib/otel/index.ts | 161 ++++++++++++++++++ .../src/lib/otel/otel_synthtrace_es_client.ts | 50 +++--- .../src/scenarios/otel_simple_trace.ts | 2 +- 3 files changed, 183 insertions(+), 30 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 52c75af998a96..442e942da691a 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -77,6 +77,7 @@ export interface OtelTrace { metrics?: { service_summary?: number; }; + 'service.name': string[]; } export type OtelDocument = OtelTrace; @@ -132,6 +133,51 @@ class Otel extends Serializable { }, span_id: '96d91c11a4e393fb', trace_id: id, + + fields: { + 'attributes.exception.message': ['boom'], + 'scope.name': ['sendotlp-synth'], + 'resource.attributes.agent.version': ['1.28.0'], + 'service.framework.version': [''], + 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], + // 'resource.attributes.agent.name.text': ['opentelemetry/go'], + 'resource.attributes.agent.name': ['opentelemetry/go'], + 'resource.attributes.telemetry.sdk.language': ['go'], + 'scope.dropped_attributes_count': [0], + 'service.language.name': ['go'], + 'telemetry.sdk.name': ['opentelemetry'], + 'attributes.timestamp.us': [1725906566779927], + 'telemetry.sdk.language': ['go'], + 'trace.id': ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + 'exception.message': ['boom'], + 'processor.event': ['error'], + 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], + 'agent.name': ['opentelemetry/go'], + 'telemetry.sdk.version': ['1.28.0'], + 'scope.attributes.service.framework.name': ['sendotlp-synth'], + trace_id: ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + 'service.name': ['sendotlp-synth'], + 'service.framework.name': ['sendotlp-synth'], + span_id: ['42fb9d5c7aff7d08'], + 'data_stream.namespace': ['default'], + 'exception.type': ['*errors.errorString'], + 'resource.attributes.some.resource.attribute': ['resource.attr'], + 'span.id': ['42fb9d5c7aff7d08'], + 'some.resource.attribute': ['resource.attr'], + 'data_stream.type': ['logs'], + 'attributes.processor.event': ['error'], + 'timestamp.us': [1725906566779927], + '@timestamp': ['2024-09-09T18:29:26.779Z'], + dropped_attributes_count: [0], + 'resource.attributes.telemetry.sdk.version': ['1.28.0'], + 'data_stream.dataset': ['generic.otel'], + name: ['exception'], + 'agent.version': ['1.28.0'], + 'attributes.exception.type': ['*errors.errorString'], + 'resource.dropped_attributes_count': [0], + 'scope.attributes.service.framework.version': [''], + 'resource.attributes.service.name': ['sendotlp-synth'], + }, }); } @@ -169,6 +215,53 @@ class Otel extends Serializable { name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', }, trace_id: id, + fields: { + 'resource.attributes.agent.version': ['1.28.0'], + 'scope.name': ['otelcol/spanmetricsconnectorv2'], + 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], + 'resource.attributes.agent.name': ['opentelemetry/go'], + 'resource.attributes.telemetry.sdk.language': ['go'], + 'scope.dropped_attributes_count': [0], + 'service.language.name': ['go'], + 'telemetry.sdk.name': ['opentelemetry'], + 'telemetry.sdk.language': ['go'], + 'transaction.root': [true], + 'processor.event': ['metric'], + 'resource.attributes.metricset.interval': ['1m'], + 'agent.name': ['opentelemetry/go'], + 'telemetry.sdk.version': ['1.28.0'], + 'attributes.metricset.name': ['service_transaction'], + 'metrics.transaction.duration.histogram': [ + { + values: [12500], + counts: [1], + }, + ], + 'attributes.transaction.root': [true], + 'service.name': ['sendotlp-synth'], + 'data_stream.namespace': ['default'], + 'resource.attributes.some.resource.attribute': ['resource.attr'], + 'some.resource.attribute': ['resource.attr'], + 'metricset.interval': ['1m'], + 'data_stream.type': ['metrics'], + 'transaction.duration.histogram': [ + { + values: [12500], + counts: [1], + }, + ], + 'transaction.type': ['unknown'], + 'metricset.name': ['service_transaction'], + 'attributes.processor.event': ['metric'], + '@timestamp': ['2024-09-09T15:49:44.920Z'], + 'resource.attributes.telemetry.sdk.version': ['1.28.0'], + 'data_stream.dataset': ['generic.otel'], + 'attributes.transaction.type': ['unknown'], + 'agent.version': ['1.28.0'], + 'attributes.metricset.interval': ['1m'], + 'resource.dropped_attributes_count': [0], + 'resource.attributes.service.name': ['sendotlp-synth'], + }, }); } transaction(id: string) { @@ -225,6 +318,74 @@ class Otel extends Serializable { type: 'traces', }, trace_id: id, + fields: { + // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], + 'scope.name': ['sendotlp-synth'], + 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], + 'resource.attributes.agent.name': ['opentelemetry/go'], + 'transaction.representative_count': [1], + 'scope.dropped_attributes_count': [0], + 'service.language.name': ['go'], + 'transaction.result': ['Success'], + 'transaction.id': ['de8004c8152af744'], + 'attributes.event.success_count': [1], + 'attributes.timestamp.us': [1725896984908682], + 'telemetry.sdk.language': ['go'], + 'processor.event': ['transaction'], + 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], + 'agent.name': ['opentelemetry/go'], + 'telemetry.sdk.version': ['1.28.0'], + 'event.outcome': ['success'], + 'scope.attributes.service.framework.name': ['sendotlp-synth'], + 'attributes.transaction.sampled': [true], + span_id: ['de8004c8152af744'], + kind: ['Internal'], + 'attributes.transaction.duration.us': [12026], + 'attributes.transaction.name': ['parent-synth'], + 'transaction.duration.us': [12026], + 'span.id': ['de8004c8152af744'], + 'data_stream.type': ['traces'], + 'timestamp.us': [1725896984908682], + 'resource.attributes.telemetry.sdk.version': ['1.28.0'], + name: ['parent-synth'], + 'agent.version': ['1.28.0'], + 'transaction.name': ['parent-synth'], + 'resource.dropped_attributes_count': [0], + 'scope.attributes.service.framework.version': [''], + 'resource.attributes.service.name': ['sendotlp-synth'], + 'attributes.transaction.representative_count': [1], + 'span.name': ['parent-synth'], + 'resource.attributes.agent.version': ['1.28.0'], + 'attributes.transaction.id': ['de8004c8152af744'], + 'service.framework.version': [''], + 'resource.attributes.telemetry.sdk.language': ['go'], + 'status.code': ['Unset'], + 'transaction.sampled': [true], + 'telemetry.sdk.name': ['opentelemetry'], + duration: [12026084], + 'attributes.transaction.result': ['Success'], + 'trace.id': ['b00875cbeb31ec7adcedc6b4213dd1ad'], + // 'attributes.transaction.name.text': ['parent-synth'], + 'event.success_count': [1], + 'transaction.root': [true], + // 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], + 'attributes.transaction.root': [true], + dropped_events_count: [0], + trace_id: ['b00875cbeb31ec7adcedc6b4213dd1ad'], + 'service.name': ['sendotlp-synth'], + 'service.framework.name': ['sendotlp-synth'], + 'data_stream.namespace': ['default'], + 'resource.attributes.some.resource.attribute': ['resource.attr'], + 'some.resource.attribute': ['resource.attr'], + dropped_links_count: [0], + 'transaction.type': ['unknown'], + 'attributes.processor.event': ['transaction'], + '@timestamp': ['2024-09-09T15:49:44.908Z'], + dropped_attributes_count: [0], + 'data_stream.dataset': ['generic.otel'], + 'attributes.event.outcome': ['success'], + 'attributes.transaction.type': ['unknown'], + }, }); } } diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index ad5d9b7a3fb14..fbb78677fa02e 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -23,17 +23,7 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient { ...options, pipeline: otelPipeline(), }); - this.dataStreams = [ - '.otel-*', - 'metrics-*', - 'logs-*.', - 'traces-*.', - 'generic.otel-*', - 'traces-otel-default', - 'metric-otel-*', - 'logs-otel.error-*', - '*-otel-*', - ]; + this.dataStreams = ['.otel-*-synth', '*-synth-*']; } } @@ -58,53 +48,55 @@ export function getRoutingTransform() { objectMode: true, transform(document: ESDocumentWithOperation, encoding, callback) { const namespace = 'default'; - let index: string | undefined = 'traces-otel-default'; - console.log(document); - console.log('doc', document.attributes[`processor.event`]); + let index: string | undefined; - switch (document.attributes['processor.event']) { + switch (document?.attributes?.['processor.event']) { case 'transaction': case 'span': - index = `traces-otel-${namespace}`; - document._index = `traces-otel-${namespace}`; + index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; + // document._index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; break; case 'error': - index = `logs-otel.error-${namespace}`; - document._index = `logs-otel.error-${namespace}`; + index = `.logs-otel.error-${namespace}-synth-2024.09.09-000001`; + // document._index = `logs-otel.error-${namespace}-synth-2024.09.09-000001`; break; case 'metric': const metricsetName = document.attributes['metricset.name']; - document._index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + document._index = `.ds-metrics-otel.service_summary.${document.attributes[ + 'metricset.interval' + ]!}-${namespace}-synth-2024.09.09-000001`; - if (metricsetName === 'app') { - index = `metrics-otel.app.${document.attributes['service.name']}-${namespace}`; - } else if ( + // if (metricsetName === 'app') { + // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}-2024.09.09-000001`; + // } else + if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' || metricsetName === 'service_destination' || metricsetName === 'service_summary' ) { - index = `metrics-otel.${metricsetName}.${document.attributes['metricset.interval']!}-${namespace}`; + index = `.ds-metrics-otel.${metricsetName}.${document.attributes[ + 'metricset.interval' + ]!}-${namespace}-2024.09.09-000001`; } else { index = `metrics-otel.internal-${namespace}`; } break; default: - if (document['event.action'] != null) { - index = `logs-otel.app-${namespace}`; - } + // if (document['event.action'] != null) { + // index = `logs-otel.app-${namespace}`; + // } break; } - console.log('index', index); if (!index) { const error = new Error('Cannot determine index for event'); Object.assign(error, { document }); } - // document._index = index; + document._index = index; callback(null, document); }, diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index 4fbc68bd37592..4d7be0af191ce 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -27,7 +27,7 @@ const scenario: Scenario = async (runOptions) => { return [ oteld.metric(id).timestamp(timestamp), oteld.transaction(id).timestamp(timestamp), - oteld.error(id).timestamp(timestamp), + // oteld.error(id).timestamp(timestamp), ]; }) ); From e214bfcce8366a12d857ae7a102b2fcdc4997764 Mon Sep 17 00:00:00 2001 From: Jenny Date: Tue, 10 Sep 2024 17:24:42 +0200 Subject: [PATCH 06/16] Add service name and fields --- .../src/lib/otel/index.ts | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 442e942da691a..2540cbc23d57d 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -133,7 +133,7 @@ class Otel extends Serializable { }, span_id: '96d91c11a4e393fb', trace_id: id, - + 'service.name': ['sendotlp-synth'], fields: { 'attributes.exception.message': ['boom'], 'scope.name': ['sendotlp-synth'], @@ -148,21 +148,21 @@ class Otel extends Serializable { 'telemetry.sdk.name': ['opentelemetry'], 'attributes.timestamp.us': [1725906566779927], 'telemetry.sdk.language': ['go'], - 'trace.id': ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + 'trace.id': [id], 'exception.message': ['boom'], 'processor.event': ['error'], 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], 'agent.name': ['opentelemetry/go'], 'telemetry.sdk.version': ['1.28.0'], 'scope.attributes.service.framework.name': ['sendotlp-synth'], - trace_id: ['b60ad822c9498bb5fc0db4a8ba05f9ce'], + trace_id: [id], 'service.name': ['sendotlp-synth'], 'service.framework.name': ['sendotlp-synth'], - span_id: ['42fb9d5c7aff7d08'], + span_id: [`${id}da907a`], 'data_stream.namespace': ['default'], 'exception.type': ['*errors.errorString'], 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'span.id': ['42fb9d5c7aff7d08'], + 'span.id': [`${id}da907a`], 'some.resource.attribute': ['resource.attr'], 'data_stream.type': ['logs'], 'attributes.processor.event': ['error'], @@ -177,6 +177,7 @@ class Otel extends Serializable { 'resource.dropped_attributes_count': [0], 'scope.attributes.service.framework.version': [''], 'resource.attributes.service.name': ['sendotlp-synth'], + 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], }, }); } @@ -215,6 +216,9 @@ class Otel extends Serializable { name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', }, trace_id: id, + service: { + name: ['sendotlp-synth'], + }, fields: { 'resource.attributes.agent.version': ['1.28.0'], 'scope.name': ['otelcol/spanmetricsconnectorv2'], @@ -308,7 +312,7 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, - span_id: `${id}-da907d6b5267c8fa`, + span_id: `${id}da907a`, status: { code: 'Unset', }, @@ -319,15 +323,17 @@ class Otel extends Serializable { }, trace_id: id, fields: { + // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], 'scope.name': ['sendotlp-synth'], 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], 'resource.attributes.agent.name': ['opentelemetry/go'], + // 'resource.attributes.agent.name.text': ['opentelemetry/go'], 'transaction.representative_count': [1], 'scope.dropped_attributes_count': [0], 'service.language.name': ['go'], 'transaction.result': ['Success'], - 'transaction.id': ['de8004c8152af744'], + 'transaction.id': [id], 'attributes.event.success_count': [1], 'attributes.timestamp.us': [1725896984908682], 'telemetry.sdk.language': ['go'], @@ -338,12 +344,12 @@ class Otel extends Serializable { 'event.outcome': ['success'], 'scope.attributes.service.framework.name': ['sendotlp-synth'], 'attributes.transaction.sampled': [true], - span_id: ['de8004c8152af744'], + span_id: [`${id}da907a`], kind: ['Internal'], 'attributes.transaction.duration.us': [12026], 'attributes.transaction.name': ['parent-synth'], 'transaction.duration.us': [12026], - 'span.id': ['de8004c8152af744'], + 'span.id': [`${id}da907a`], 'data_stream.type': ['traces'], 'timestamp.us': [1725896984908682], 'resource.attributes.telemetry.sdk.version': ['1.28.0'], @@ -352,11 +358,10 @@ class Otel extends Serializable { 'transaction.name': ['parent-synth'], 'resource.dropped_attributes_count': [0], 'scope.attributes.service.framework.version': [''], - 'resource.attributes.service.name': ['sendotlp-synth'], 'attributes.transaction.representative_count': [1], 'span.name': ['parent-synth'], 'resource.attributes.agent.version': ['1.28.0'], - 'attributes.transaction.id': ['de8004c8152af744'], + 'attributes.transaction.id': [id], 'service.framework.version': [''], 'resource.attributes.telemetry.sdk.language': ['go'], 'status.code': ['Unset'], @@ -364,14 +369,14 @@ class Otel extends Serializable { 'telemetry.sdk.name': ['opentelemetry'], duration: [12026084], 'attributes.transaction.result': ['Success'], - 'trace.id': ['b00875cbeb31ec7adcedc6b4213dd1ad'], + 'trace.id': [id], // 'attributes.transaction.name.text': ['parent-synth'], 'event.success_count': [1], 'transaction.root': [true], // 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], 'attributes.transaction.root': [true], dropped_events_count: [0], - trace_id: ['b00875cbeb31ec7adcedc6b4213dd1ad'], + trace_id: [id], 'service.name': ['sendotlp-synth'], 'service.framework.name': ['sendotlp-synth'], 'data_stream.namespace': ['default'], @@ -385,6 +390,8 @@ class Otel extends Serializable { 'data_stream.dataset': ['generic.otel'], 'attributes.event.outcome': ['success'], 'attributes.transaction.type': ['unknown'], + 'resource.attributes.service.name': ['sendotlp-synth'], + // 'resource.attributes.service.name.text': ['sendotlp-synth'], }, }); } From 5c95114f9702ed65e1f3180cc6b5e441f34378d5 Mon Sep 17 00:00:00 2001 From: "miriam.aparicio" Date: Tue, 24 Sep 2024 10:34:31 +0100 Subject: [PATCH 07/16] rename indices and refactor code --- packages/kbn-apm-synthtrace-client/index.ts | 2 +- .../src/lib/otel/error.ts | 26 +- .../src/lib/otel/index.ts | 301 +++--------------- .../src/lib/otel/metric.ts | 26 +- .../src/lib/otel/transaction.ts | 41 ++- .../src/lib/otel/otel_synthtrace_es_client.ts | 22 +- .../src/scenarios/otel_simple_trace.ts | 12 +- 7 files changed, 121 insertions(+), 309 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts index a7eed2ebccda5..d3d24a8940a3b 100644 --- a/packages/kbn-apm-synthtrace-client/index.ts +++ b/packages/kbn-apm-synthtrace-client/index.ts @@ -37,4 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs'; export { type AssetDocument } from './src/lib/assets'; export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics'; -export { otel, type OtelDocument, type OtelTrace } from './src/lib/otel'; +export { otel, type OtelDocument } from './src/lib/otel'; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts index 190ff07a70fd8..efc25735792b7 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -6,23 +6,23 @@ * Side Public License, v 1. */ -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import type { OtelDocument } from '../../..'; import { Serializable } from '../serializable'; -import { generateShortId } from '../utils/generate_id'; -export class OtelError extends Serializable { - constructor(fields: OtelDocument) { +export interface OtelErrorDocument extends OtelDocument { + name?: string; + attributes?: { + 'exception.message'?: string; + 'exception.type'?: string; + 'processor.event'?: string; + 'timestamp.us'?: number; + }; +} + +export class OtelError extends Serializable { + constructor(fields: OtelErrorDocument) { super({ ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - trace_id: generateShortId(), }); } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 2540cbc23d57d..c5aa3ead42c8a 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -6,82 +6,55 @@ * Side Public License, v 1. */ +import { Fields } from '../entity'; import { Serializable } from '../serializable'; import { OtelError } from './error'; import { OtelMetric } from './metric'; import { OtelTransaction } from './transaction'; -export interface OtelTrace { - attributes?: { - 'event.outcome'?: string; - 'event.success_count'?: number; - 'processor.event'?: string; - 'timestamp.us'?: number; - 'transaction.duration.us'?: number; - 'transaction.id'?: string; - 'transaction.name'?: string; - 'transaction.representative_count'?: number; - 'transaction.result'?: string; - 'transaction.root'?: boolean; - 'transaction.sampled'?: boolean; - 'transaction.type'?: string; - // error - 'exception.message'?: string; - 'exception.type'?: string; - // metrics - 'metricset.interval'?: string; - 'metricset.name'?: string; - }; +export interface SharedAttributes { + 'service.name'?: string; + 'agent.name'?: string; + 'agent.version'?: string; + 'metricset.interval'?: string; + 'service.instance.id'?: string; + 'telemetry.sdk.language'?: string; + 'telemetry.sdk.name'?: string; + 'telemetry.sdk.version'?: string; + 'some.resource.attribute'?: string; + 'timestamp.us'?: number; +} + +export interface OtelDocument extends Fields { data_stream?: { dataset: string; namespace: string; type: string; }; - dropped_attributes_count?: number; - dropped_events_count?: number; - dropped_links_count?: number; - duration?: number; - kind?: string; - name?: string; + attributes?: { + [key: string]: any; + }; resource?: { - attributes?: { - 'agent.name'?: string; - 'agent.version'?: string; - 'service.name'?: string; - 'some.resource.attribute'?: string; - 'telemetry.sdk.language'?: string; - 'telemetry.sdk.name'?: string; - 'telemetry.sdk.version'?: string; - // metrics - 'metricset.interval'?: string; - }; + attributes?: SharedAttributes; dropped_attributes_count?: number; schema_url?: string; }; scope?: { attributes?: { - 'service.framework.name': string; - 'service.framework.version': string; + 'service.framework.name'?: string; + 'service.framework.version'?: string; }; dropped_attributes_count?: number; name?: string; }; + name?: string; + trace_id?: string; span_id?: string; - status?: { - code: string; - }; - trace_id: string; - '@timestamp'?: number | undefined; - 'timestamp.us'?: number | undefined; - // metrics - metrics?: { - service_summary?: number; - }; - 'service.name': string[]; + dropped_attributes_count?: number; + dropped_events_count?: number; + dropped_links_count?: number; } -export type OtelDocument = OtelTrace; - class Otel extends Serializable { constructor(fields: OtelDocument) { super({ @@ -89,11 +62,6 @@ class Otel extends Serializable { }); } - timestamp(time: number) { - super.timestamp(time); - return this; - } - error(id: string) { return new OtelError({ ...this.fields, @@ -132,57 +100,10 @@ class Otel extends Serializable { name: 'sendotlp-synth', }, span_id: '96d91c11a4e393fb', - trace_id: id, - 'service.name': ['sendotlp-synth'], - fields: { - 'attributes.exception.message': ['boom'], - 'scope.name': ['sendotlp-synth'], - 'resource.attributes.agent.version': ['1.28.0'], - 'service.framework.version': [''], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - // 'resource.attributes.agent.name.text': ['opentelemetry/go'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'telemetry.sdk.name': ['opentelemetry'], - 'attributes.timestamp.us': [1725906566779927], - 'telemetry.sdk.language': ['go'], - 'trace.id': [id], - 'exception.message': ['boom'], - 'processor.event': ['error'], - 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'scope.attributes.service.framework.name': ['sendotlp-synth'], - trace_id: [id], - 'service.name': ['sendotlp-synth'], - 'service.framework.name': ['sendotlp-synth'], - span_id: [`${id}da907a`], - 'data_stream.namespace': ['default'], - 'exception.type': ['*errors.errorString'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'span.id': [`${id}da907a`], - 'some.resource.attribute': ['resource.attr'], - 'data_stream.type': ['logs'], - 'attributes.processor.event': ['error'], - 'timestamp.us': [1725906566779927], - '@timestamp': ['2024-09-09T18:29:26.779Z'], - dropped_attributes_count: [0], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - 'data_stream.dataset': ['generic.otel'], - name: ['exception'], - 'agent.version': ['1.28.0'], - 'attributes.exception.type': ['*errors.errorString'], - 'resource.dropped_attributes_count': [0], - 'scope.attributes.service.framework.version': [''], - 'resource.attributes.service.name': ['sendotlp-synth'], - 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], - }, }); } - metric(id: string) { + metric() { return new OtelMetric({ ...this.fields, attributes: { @@ -200,14 +121,12 @@ class Otel extends Serializable { }, resource: { attributes: { - 'agent.name': 'opentelemetry/go', - 'agent.version': '1.28.0', + 'agent.name': 'otlp', + 'agent.version': 'unknown', 'metricset.interval': '1m', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', }, dropped_attributes_count: 0, }, @@ -215,59 +134,9 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', }, - trace_id: id, - service: { - name: ['sendotlp-synth'], - }, - fields: { - 'resource.attributes.agent.version': ['1.28.0'], - 'scope.name': ['otelcol/spanmetricsconnectorv2'], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'telemetry.sdk.name': ['opentelemetry'], - 'telemetry.sdk.language': ['go'], - 'transaction.root': [true], - 'processor.event': ['metric'], - 'resource.attributes.metricset.interval': ['1m'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'attributes.metricset.name': ['service_transaction'], - 'metrics.transaction.duration.histogram': [ - { - values: [12500], - counts: [1], - }, - ], - 'attributes.transaction.root': [true], - 'service.name': ['sendotlp-synth'], - 'data_stream.namespace': ['default'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'some.resource.attribute': ['resource.attr'], - 'metricset.interval': ['1m'], - 'data_stream.type': ['metrics'], - 'transaction.duration.histogram': [ - { - values: [12500], - counts: [1], - }, - ], - 'transaction.type': ['unknown'], - 'metricset.name': ['service_transaction'], - 'attributes.processor.event': ['metric'], - '@timestamp': ['2024-09-09T15:49:44.920Z'], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - 'data_stream.dataset': ['generic.otel'], - 'attributes.transaction.type': ['unknown'], - 'agent.version': ['1.28.0'], - 'attributes.metricset.interval': ['1m'], - 'resource.dropped_attributes_count': [0], - 'resource.attributes.service.name': ['sendotlp-synth'], - }, }); } + transaction(id: string) { return new OtelTransaction({ ...this.fields, @@ -275,9 +144,9 @@ class Otel extends Serializable { 'event.outcome': 'success', 'event.success_count': 1, 'processor.event': 'transaction', - 'timestamp.us': 1725464233071114, + 'timestamp.us': 1726580752010657, 'transaction.duration.us': 15202, - 'transaction.id': 'da907d6b5267c8f6', + 'transaction.id': id, 'transaction.name': 'parent-synth', 'transaction.representative_count': 1, 'transaction.result': 'Success', @@ -285,21 +154,21 @@ class Otel extends Serializable { 'transaction.sampled': true, 'transaction.type': 'unknown', }, - dropped_attributes_count: 0, - dropped_events_count: 0, - dropped_links_count: 0, - duration: 15202584, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'traces', + }, + duration: 11742370, kind: 'Internal', name: 'parent-synth', resource: { attributes: { - 'agent.name': 'opentelemetry/go', + 'agent.name': 'otlp', 'agent.version': '1.28.0', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', }, dropped_attributes_count: 0, schema_url: 'https://opentelemetry.io/schemas/1.26.0', @@ -312,93 +181,21 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, - span_id: `${id}da907a`, + span_id: id, status: { code: 'Unset', }, - data_stream: { - dataset: 'generic.otel', - namespace: 'default', - type: 'traces', - }, - trace_id: id, - fields: { - // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], - // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], - 'scope.name': ['sendotlp-synth'], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - // 'resource.attributes.agent.name.text': ['opentelemetry/go'], - 'transaction.representative_count': [1], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'transaction.result': ['Success'], - 'transaction.id': [id], - 'attributes.event.success_count': [1], - 'attributes.timestamp.us': [1725896984908682], - 'telemetry.sdk.language': ['go'], - 'processor.event': ['transaction'], - 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'event.outcome': ['success'], - 'scope.attributes.service.framework.name': ['sendotlp-synth'], - 'attributes.transaction.sampled': [true], - span_id: [`${id}da907a`], - kind: ['Internal'], - 'attributes.transaction.duration.us': [12026], - 'attributes.transaction.name': ['parent-synth'], - 'transaction.duration.us': [12026], - 'span.id': [`${id}da907a`], - 'data_stream.type': ['traces'], - 'timestamp.us': [1725896984908682], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - name: ['parent-synth'], - 'agent.version': ['1.28.0'], - 'transaction.name': ['parent-synth'], - 'resource.dropped_attributes_count': [0], - 'scope.attributes.service.framework.version': [''], - 'attributes.transaction.representative_count': [1], - 'span.name': ['parent-synth'], - 'resource.attributes.agent.version': ['1.28.0'], - 'attributes.transaction.id': [id], - 'service.framework.version': [''], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'status.code': ['Unset'], - 'transaction.sampled': [true], - 'telemetry.sdk.name': ['opentelemetry'], - duration: [12026084], - 'attributes.transaction.result': ['Success'], - 'trace.id': [id], - // 'attributes.transaction.name.text': ['parent-synth'], - 'event.success_count': [1], - 'transaction.root': [true], - // 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], - 'attributes.transaction.root': [true], - dropped_events_count: [0], - trace_id: [id], - 'service.name': ['sendotlp-synth'], - 'service.framework.name': ['sendotlp-synth'], - 'data_stream.namespace': ['default'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'some.resource.attribute': ['resource.attr'], - dropped_links_count: [0], - 'transaction.type': ['unknown'], - 'attributes.processor.event': ['transaction'], - '@timestamp': ['2024-09-09T15:49:44.908Z'], - dropped_attributes_count: [0], - 'data_stream.dataset': ['generic.otel'], - 'attributes.event.outcome': ['success'], - 'attributes.transaction.type': ['unknown'], - 'resource.attributes.service.name': ['sendotlp-synth'], - // 'resource.attributes.service.name.text': ['sendotlp-synth'], - }, }); } } -export function create(): Otel { - return new Otel(); +export function create(id: string): Otel { + return new Otel({ + trace_id: id, + dropped_attributes_count: 0, + dropped_events_count: 0, + dropped_links_count: 0, + }); } export const otel = { diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts index 8e2c1a76b180a..8f69cc2c6a480 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -6,23 +6,23 @@ * Side Public License, v 1. */ -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { OtelDocument } from '.'; import { Serializable } from '../serializable'; -// import { generateShortId } from '../utils/generate_id'; -export class OtelMetric extends Serializable { - constructor(fields: OtelDocument) { +export interface OtelMetricDocument extends OtelDocument { + attributes?: { + 'metricset.interval'?: string; + 'metricset.name'?: string; + 'processor.event'?: string; + }; + metrics?: { + service_summary?: number; + }; +} +export class OtelMetric extends Serializable { + constructor(fields: OtelMetricDocument) { super({ ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - // trace_id: generateShortId(), }); } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts index 33f019f4b2fb8..b917ce172c23f 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -6,23 +6,38 @@ * Side Public License, v 1. */ -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { OtelDocument } from '.'; import { Serializable } from '../serializable'; -// import { generateShortId } from '../utils/generate_id'; -export class OtelTransaction extends Serializable { - constructor(fields: OtelDocument) { +export interface OtelTransactionDocument extends OtelDocument { + attributes?: { + 'event.outcome'?: string; + 'event.success_count'?: number; + 'processor.event'?: string; + 'timestamp.us'?: number; + 'transaction.duration.us'?: number; + 'transaction.id'?: string; + 'transaction.name'?: string; + 'transaction.representative_count'?: number; + 'transaction.result'?: string; + 'transaction.root'?: boolean; + 'transaction.sampled'?: boolean; + 'transaction.type'?: string; + }; + status?: { + code?: string; + }; + dropped_events_count?: number; + dropped_links_count?: number; + duration?: number; + kind?: string; + name?: string; +} + +export class OtelTransaction extends Serializable { + constructor(fields: OtelTransactionDocument) { super({ ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - // trace_id: generateShortId(), }); } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } } diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index fbb78677fa02e..3693e4c711ac8 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -23,7 +23,7 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient { ...options, pipeline: otelPipeline(), }); - this.dataStreams = ['.otel-*-synth', '*-synth-*']; + this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*']; } } @@ -53,33 +53,33 @@ export function getRoutingTransform() { switch (document?.attributes?.['processor.event']) { case 'transaction': case 'span': - index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; - // document._index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; + index = `traces-generic.otel-${namespace}-synth`; + // document._index = `.ds-traces-generic.otel-${namespace}-synth`; break; case 'error': - index = `.logs-otel.error-${namespace}-synth-2024.09.09-000001`; - // document._index = `logs-otel.error-${namespace}-synth-2024.09.09-000001`; + index = `.logs-otel.error-${namespace}-synth`; + // document._index = `logs-otel.error-${namespace}-synth`; break; case 'metric': const metricsetName = document.attributes['metricset.name']; - document._index = `.ds-metrics-otel.service_summary.${document.attributes[ + document._index = `metrics-otel.service_summary.${document.attributes[ 'metricset.interval' - ]!}-${namespace}-synth-2024.09.09-000001`; + ]!}-${namespace}-synth`; // if (metricsetName === 'app') { - // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}-2024.09.09-000001`; - // } else + // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}`; + // } else if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' || metricsetName === 'service_destination' || metricsetName === 'service_summary' ) { - index = `.ds-metrics-otel.${metricsetName}.${document.attributes[ + index = `metrics-otel.${metricsetName}.${document.attributes[ 'metricset.interval' - ]!}-${namespace}-2024.09.09-000001`; + ]!}-${namespace}-synth`; } else { index = `metrics-otel.internal-${namespace}`; } diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index 4d7be0af191ce..b527ab7df8afb 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -5,28 +5,28 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { ApmFields, otel, generateShortId } from '@kbn/apm-synthtrace-client'; +import { otel, generateShortId, OtelDocument } from '@kbn/apm-synthtrace-client'; import { times } from 'lodash'; import { Scenario } from '../cli/scenario'; import { withClient } from '../lib/utils/with_client'; -const scenario: Scenario = async (runOptions) => { +const scenario: Scenario = async (runOptions) => { return { generate: ({ range, clients: { otelSynthtraceEsClient } }) => { const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; const { logger } = runOptions; + const traceId = generateShortId(); - const otelDocs = times(numOtelTraces / 2).map((index) => otel.create()); + const otelDocs = times(numOtelTraces / 2).map((index) => otel.create(traceId)); const otelWithMetricsAndErrors = range .interval('30s') .rate(1) .generator((timestamp) => otelDocs.flatMap((oteld) => { - const id = generateShortId(); return [ - oteld.metric(id).timestamp(timestamp), - oteld.transaction(id).timestamp(timestamp), + oteld.metric().timestamp(timestamp), + oteld.transaction(traceId).timestamp(timestamp), // oteld.error(id).timestamp(timestamp), ]; }) From 2ab2b7850e48aecd2e12b6dfcac1d0ad4b53add5 Mon Sep 17 00:00:00 2001 From: "miriam.aparicio" Date: Thu, 26 Sep 2024 12:04:09 +0100 Subject: [PATCH 08/16] review fields for errors --- .../src/lib/otel/error.ts | 12 +++++---- .../src/lib/otel/index.ts | 27 ++++++++++++------- .../src/lib/otel/metric.ts | 9 ++++--- .../src/lib/otel/transaction.ts | 9 ++++--- .../src/cli/utils/get_otel_es_client.ts | 9 ++++--- .../src/lib/otel/otel_synthtrace_es_client.ts | 20 +++++--------- .../src/scenarios/otel_simple_trace.ts | 13 +++++---- 7 files changed, 53 insertions(+), 46 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts index efc25735792b7..76ebe9d95a44f 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -1,21 +1,23 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import type { OtelDocument } from '../../..'; import { Serializable } from '../serializable'; export interface OtelErrorDocument extends OtelDocument { - name?: string; + 'event.name'?: string; attributes?: { 'exception.message'?: string; 'exception.type'?: string; 'processor.event'?: string; 'timestamp.us'?: number; + 'event.name'?: string; }; } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index c5aa3ead42c8a..36a26fca5baf2 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import { Fields } from '../entity'; @@ -49,10 +50,13 @@ export interface OtelDocument extends Fields { }; name?: string; trace_id?: string; + trace?: { id: string }; // / I saw in fields this two representacion of trace id span_id?: string; + span?: { id: string }; // / I saw in fields this two representacion of span id dropped_attributes_count?: number; dropped_events_count?: number; dropped_links_count?: number; + timestamp_us?: number; } class Otel extends Serializable { @@ -62,7 +66,7 @@ class Otel extends Serializable { }); } - error(id: string) { + error({ spanId, traceId }: { spanId: string; traceId: string }) { return new OtelError({ ...this.fields, attributes: { @@ -70,23 +74,22 @@ class Otel extends Serializable { 'exception.type': '*errors.errorString', 'processor.event': 'error', 'timestamp.us': 1725633628036123, + 'event.name': 'exception', }, data_stream: { dataset: 'generic.otel', namespace: 'default', type: 'logs', }, + 'event.name': 'exception', dropped_attributes_count: 0, - name: 'exception', resource: { attributes: { 'agent.name': 'opentelemetry/go', 'agent.version': '1.28.0', 'service.name': 'sendotlp-synth', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', }, dropped_attributes_count: 0, schema_url: 'https://opentelemetry.io/schemas/1.26.0', @@ -99,7 +102,11 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, - span_id: '96d91c11a4e393fb', + trace_id: traceId, + trace: { id: traceId }, + span_id: spanId, + span: { id: spanId }, + timestamp_us: 1725633628036123, }); } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts index 8f69cc2c6a480..dfa5da49d4bbf 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import { OtelDocument } from '.'; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts index b917ce172c23f..d6f7c7111b187 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import { OtelDocument } from '.'; diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts index 3cdeaf9e8a134..0671ea66c472f 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import { Client } from '@elastic/elasticsearch'; diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index 3693e4c711ac8..c778afac26ef5 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ import { Client } from '@elastic/elasticsearch'; @@ -54,23 +55,14 @@ export function getRoutingTransform() { case 'transaction': case 'span': index = `traces-generic.otel-${namespace}-synth`; - // document._index = `.ds-traces-generic.otel-${namespace}-synth`; break; case 'error': - index = `.logs-otel.error-${namespace}-synth`; - // document._index = `logs-otel.error-${namespace}-synth`; + index = `.logs-generic-otel-${namespace}-synth`; break; case 'metric': const metricsetName = document.attributes['metricset.name']; - document._index = `metrics-otel.service_summary.${document.attributes[ - 'metricset.interval' - ]!}-${namespace}-synth`; - - // if (metricsetName === 'app') { - // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}`; - // } else if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' || diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index b527ab7df8afb..fb041c6945080 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -1,10 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ + import { otel, generateShortId, OtelDocument } from '@kbn/apm-synthtrace-client'; import { times } from 'lodash'; import { Scenario } from '../cli/scenario'; @@ -16,6 +18,7 @@ const scenario: Scenario = async (runOptions) => { const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; const { logger } = runOptions; const traceId = generateShortId(); + const spanId = generateShortId(); const otelDocs = times(numOtelTraces / 2).map((index) => otel.create(traceId)); @@ -27,7 +30,7 @@ const scenario: Scenario = async (runOptions) => { return [ oteld.metric().timestamp(timestamp), oteld.transaction(traceId).timestamp(timestamp), - // oteld.error(id).timestamp(timestamp), + oteld.error({ spanId, traceId }).timestamp(timestamp), ]; }) ); From bff034b9d600009394d0a5397f518d09c111d6f4 Mon Sep 17 00:00:00 2001 From: "miriam.aparicio" Date: Thu, 26 Sep 2024 13:03:17 +0100 Subject: [PATCH 09/16] remove redundant trace.is --- packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts | 5 ++--- .../kbn-apm-synthtrace-client/src/lib/otel/transaction.ts | 1 + .../kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 36a26fca5baf2..141d174df1b86 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -66,7 +66,7 @@ class Otel extends Serializable { }); } - error({ spanId, traceId }: { spanId: string; traceId: string }) { + error(spanId: string) { return new OtelError({ ...this.fields, attributes: { @@ -102,8 +102,6 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, - trace_id: traceId, - trace: { id: traceId }, span_id: spanId, span: { id: spanId }, timestamp_us: 1725633628036123, @@ -188,6 +186,7 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, + transaction: { id }, span_id: id, status: { code: 'Unset', diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts index d6f7c7111b187..f5e4b49992293 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -33,6 +33,7 @@ export interface OtelTransactionDocument extends OtelDocument { duration?: number; kind?: string; name?: string; + transaction: { id: string }; } export class OtelTransaction extends Serializable { diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index fb041c6945080..ab1755d2e7b78 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -29,8 +29,8 @@ const scenario: Scenario = async (runOptions) => { otelDocs.flatMap((oteld) => { return [ oteld.metric().timestamp(timestamp), - oteld.transaction(traceId).timestamp(timestamp), - oteld.error({ spanId, traceId }).timestamp(timestamp), + oteld.transaction(spanId).timestamp(timestamp), + oteld.error(spanId).timestamp(timestamp), ]; }) ); From 71a607d31cedb5b182ef061006df6fe108fd7785 Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 7 Oct 2024 14:38:53 +0200 Subject: [PATCH 10/16] Resolve conflicts --- packages/kbn-apm-synthtrace-client/index.ts | 4 - .../src/lib/otel/error.ts | 28 -- .../src/lib/otel/index.ts | 336 ------------------ .../src/lib/otel/metric.ts | 28 -- .../src/lib/otel/transaction.ts | 28 -- .../src/cli/utils/get_otel_es_client.ts | 7 - .../src/lib/otel/otel_synthtrace_es_client.ts | 37 -- .../src/scenarios/otel_simple_trace.ts | 26 -- 8 files changed, 494 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts index 4c48629d9bdd2..d3d24a8940a3b 100644 --- a/packages/kbn-apm-synthtrace-client/index.ts +++ b/packages/kbn-apm-synthtrace-client/index.ts @@ -37,8 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs'; export { type AssetDocument } from './src/lib/assets'; export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics'; -<<<<<<< HEAD -export { otel, type OtelDocument, type OtelTrace } from './src/lib/otel'; -======= export { otel, type OtelDocument } from './src/lib/otel'; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts index 2a97aeb1fd7c0..76ebe9d95a44f 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -1,32 +1,5 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -<<<<<<< HEAD - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; -import { Serializable } from '../serializable'; -import { generateShortId } from '../utils/generate_id'; - -export class OtelError extends Serializable { - constructor(fields: OtelDocument) { - super({ - ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - trace_id: generateShortId(), - }); - } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } -======= * or more contributor license agreements. Licensed under the "Elastic License * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side * Public License v 1"; you may not use this file except in compliance with, at @@ -54,5 +27,4 @@ export class OtelError extends Serializable { ...fields, }); } ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 8a20feef877fa..141d174df1b86 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -1,13 +1,5 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -<<<<<<< HEAD - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -======= * or more contributor license agreements. Licensed under the "Elastic License * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side * Public License v 1"; you may not use this file except in compliance with, at @@ -16,35 +8,11 @@ */ import { Fields } from '../entity'; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 import { Serializable } from '../serializable'; import { OtelError } from './error'; import { OtelMetric } from './metric'; import { OtelTransaction } from './transaction'; -<<<<<<< HEAD -export interface OtelTrace { - attributes?: { - 'event.outcome'?: string; - 'event.success_count'?: number; - 'processor.event'?: string; - 'timestamp.us'?: number; - 'transaction.duration.us'?: number; - 'transaction.id'?: string; - 'transaction.name'?: string; - 'transaction.representative_count'?: number; - 'transaction.result'?: string; - 'transaction.root'?: boolean; - 'transaction.sampled'?: boolean; - 'transaction.type'?: string; - // error - 'exception.message'?: string; - 'exception.type'?: string; - // metrics - 'metricset.interval'?: string; - 'metricset.name'?: string; - }; -======= export interface SharedAttributes { 'service.name'?: string; 'agent.name'?: string; @@ -59,72 +27,27 @@ export interface SharedAttributes { } export interface OtelDocument extends Fields { ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 data_stream?: { dataset: string; namespace: string; type: string; }; -<<<<<<< HEAD - dropped_attributes_count?: number; - dropped_events_count?: number; - dropped_links_count?: number; - duration?: number; - kind?: string; - name?: string; - resource?: { - attributes?: { - 'agent.name'?: string; - 'agent.version'?: string; - 'service.name'?: string; - 'some.resource.attribute'?: string; - 'telemetry.sdk.language'?: string; - 'telemetry.sdk.name'?: string; - 'telemetry.sdk.version'?: string; - // metrics - 'metricset.interval'?: string; - }; -======= attributes?: { [key: string]: any; }; resource?: { attributes?: SharedAttributes; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 dropped_attributes_count?: number; schema_url?: string; }; scope?: { attributes?: { -<<<<<<< HEAD - 'service.framework.name': string; - 'service.framework.version': string; -======= 'service.framework.name'?: string; 'service.framework.version'?: string; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 }; dropped_attributes_count?: number; name?: string; }; -<<<<<<< HEAD - span_id?: string; - status?: { - code: string; - }; - trace_id: string; - '@timestamp'?: number | undefined; - 'timestamp.us'?: number | undefined; - // metrics - metrics?: { - service_summary?: number; - }; - 'service.name': string[]; -} - -export type OtelDocument = OtelTrace; - -======= name?: string; trace_id?: string; trace?: { id: string }; // / I saw in fields this two representacion of trace id @@ -136,7 +59,6 @@ export type OtelDocument = OtelTrace; timestamp_us?: number; } ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 class Otel extends Serializable { constructor(fields: OtelDocument) { super({ @@ -144,16 +66,7 @@ class Otel extends Serializable { }); } -<<<<<<< HEAD - timestamp(time: number) { - super.timestamp(time); - return this; - } - - error(id: string) { -======= error(spanId: string) { ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 return new OtelError({ ...this.fields, attributes: { @@ -161,37 +74,22 @@ class Otel extends Serializable { 'exception.type': '*errors.errorString', 'processor.event': 'error', 'timestamp.us': 1725633628036123, -<<<<<<< HEAD -======= 'event.name': 'exception', ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 }, data_stream: { dataset: 'generic.otel', namespace: 'default', type: 'logs', }, -<<<<<<< HEAD - dropped_attributes_count: 0, - name: 'exception', -======= 'event.name': 'exception', dropped_attributes_count: 0, ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 resource: { attributes: { 'agent.name': 'opentelemetry/go', 'agent.version': '1.28.0', 'service.name': 'sendotlp-synth', -<<<<<<< HEAD - 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', -======= 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'some.resource.attribute': 'resource.attr', ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 }, dropped_attributes_count: 0, schema_url: 'https://opentelemetry.io/schemas/1.26.0', @@ -204,60 +102,6 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, -<<<<<<< HEAD - span_id: '96d91c11a4e393fb', - trace_id: id, - 'service.name': ['sendotlp-synth'], - fields: { - 'attributes.exception.message': ['boom'], - 'scope.name': ['sendotlp-synth'], - 'resource.attributes.agent.version': ['1.28.0'], - 'service.framework.version': [''], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - // 'resource.attributes.agent.name.text': ['opentelemetry/go'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'telemetry.sdk.name': ['opentelemetry'], - 'attributes.timestamp.us': [1725906566779927], - 'telemetry.sdk.language': ['go'], - 'trace.id': [id], - 'exception.message': ['boom'], - 'processor.event': ['error'], - 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'scope.attributes.service.framework.name': ['sendotlp-synth'], - trace_id: [id], - 'service.name': ['sendotlp-synth'], - 'service.framework.name': ['sendotlp-synth'], - span_id: [`${id}da907a`], - 'data_stream.namespace': ['default'], - 'exception.type': ['*errors.errorString'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'span.id': [`${id}da907a`], - 'some.resource.attribute': ['resource.attr'], - 'data_stream.type': ['logs'], - 'attributes.processor.event': ['error'], - 'timestamp.us': [1725906566779927], - '@timestamp': ['2024-09-09T18:29:26.779Z'], - dropped_attributes_count: [0], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - 'data_stream.dataset': ['generic.otel'], - name: ['exception'], - 'agent.version': ['1.28.0'], - 'attributes.exception.type': ['*errors.errorString'], - 'resource.dropped_attributes_count': [0], - 'scope.attributes.service.framework.version': [''], - 'resource.attributes.service.name': ['sendotlp-synth'], - 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], - }, - }); - } - - metric(id: string) { -======= span_id: spanId, span: { id: spanId }, timestamp_us: 1725633628036123, @@ -265,7 +109,6 @@ class Otel extends Serializable { } metric() { ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 return new OtelMetric({ ...this.fields, attributes: { @@ -283,23 +126,12 @@ class Otel extends Serializable { }, resource: { attributes: { -<<<<<<< HEAD - 'agent.name': 'opentelemetry/go', - 'agent.version': '1.28.0', - 'metricset.interval': '1m', - 'service.name': 'sendotlp-synth', - 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', -======= 'agent.name': 'otlp', 'agent.version': 'unknown', 'metricset.interval': '1m', 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', 'some.resource.attribute': 'resource.attr', ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 }, dropped_attributes_count: 0, }, @@ -307,65 +139,9 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', }, -<<<<<<< HEAD - trace_id: id, - service: { - name: ['sendotlp-synth'], - }, - fields: { - 'resource.attributes.agent.version': ['1.28.0'], - 'scope.name': ['otelcol/spanmetricsconnectorv2'], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'telemetry.sdk.name': ['opentelemetry'], - 'telemetry.sdk.language': ['go'], - 'transaction.root': [true], - 'processor.event': ['metric'], - 'resource.attributes.metricset.interval': ['1m'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'attributes.metricset.name': ['service_transaction'], - 'metrics.transaction.duration.histogram': [ - { - values: [12500], - counts: [1], - }, - ], - 'attributes.transaction.root': [true], - 'service.name': ['sendotlp-synth'], - 'data_stream.namespace': ['default'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'some.resource.attribute': ['resource.attr'], - 'metricset.interval': ['1m'], - 'data_stream.type': ['metrics'], - 'transaction.duration.histogram': [ - { - values: [12500], - counts: [1], - }, - ], - 'transaction.type': ['unknown'], - 'metricset.name': ['service_transaction'], - 'attributes.processor.event': ['metric'], - '@timestamp': ['2024-09-09T15:49:44.920Z'], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - 'data_stream.dataset': ['generic.otel'], - 'attributes.transaction.type': ['unknown'], - 'agent.version': ['1.28.0'], - 'attributes.metricset.interval': ['1m'], - 'resource.dropped_attributes_count': [0], - 'resource.attributes.service.name': ['sendotlp-synth'], - }, - }); - } -======= }); } ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 transaction(id: string) { return new OtelTransaction({ ...this.fields, @@ -373,15 +149,9 @@ class Otel extends Serializable { 'event.outcome': 'success', 'event.success_count': 1, 'processor.event': 'transaction', -<<<<<<< HEAD - 'timestamp.us': 1725464233071114, - 'transaction.duration.us': 15202, - 'transaction.id': 'da907d6b5267c8f6', -======= 'timestamp.us': 1726580752010657, 'transaction.duration.us': 15202, 'transaction.id': id, ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 'transaction.name': 'parent-synth', 'transaction.representative_count': 1, 'transaction.result': 'Success', @@ -389,38 +159,21 @@ class Otel extends Serializable { 'transaction.sampled': true, 'transaction.type': 'unknown', }, -<<<<<<< HEAD - dropped_attributes_count: 0, - dropped_events_count: 0, - dropped_links_count: 0, - duration: 15202584, -======= data_stream: { dataset: 'generic.otel', namespace: 'default', type: 'traces', }, duration: 11742370, ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 kind: 'Internal', name: 'parent-synth', resource: { attributes: { -<<<<<<< HEAD - 'agent.name': 'opentelemetry/go', - 'agent.version': '1.28.0', - 'service.name': 'sendotlp-synth', - 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', -======= 'agent.name': 'otlp', 'agent.version': '1.28.0', 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', 'some.resource.attribute': 'resource.attr', ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 }, dropped_attributes_count: 0, schema_url: 'https://opentelemetry.io/schemas/1.26.0', @@ -433,103 +186,15 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, -<<<<<<< HEAD - span_id: `${id}da907a`, - status: { - code: 'Unset', - }, - data_stream: { - dataset: 'generic.otel', - namespace: 'default', - type: 'traces', - }, - trace_id: id, - fields: { - // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], - // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], - 'scope.name': ['sendotlp-synth'], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - // 'resource.attributes.agent.name.text': ['opentelemetry/go'], - 'transaction.representative_count': [1], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'transaction.result': ['Success'], - 'transaction.id': [id], - 'attributes.event.success_count': [1], - 'attributes.timestamp.us': [1725896984908682], - 'telemetry.sdk.language': ['go'], - 'processor.event': ['transaction'], - 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'event.outcome': ['success'], - 'scope.attributes.service.framework.name': ['sendotlp-synth'], - 'attributes.transaction.sampled': [true], - span_id: [`${id}da907a`], - kind: ['Internal'], - 'attributes.transaction.duration.us': [12026], - 'attributes.transaction.name': ['parent-synth'], - 'transaction.duration.us': [12026], - 'span.id': [`${id}da907a`], - 'data_stream.type': ['traces'], - 'timestamp.us': [1725896984908682], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - name: ['parent-synth'], - 'agent.version': ['1.28.0'], - 'transaction.name': ['parent-synth'], - 'resource.dropped_attributes_count': [0], - 'scope.attributes.service.framework.version': [''], - 'attributes.transaction.representative_count': [1], - 'span.name': ['parent-synth'], - 'resource.attributes.agent.version': ['1.28.0'], - 'attributes.transaction.id': [id], - 'service.framework.version': [''], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'status.code': ['Unset'], - 'transaction.sampled': [true], - 'telemetry.sdk.name': ['opentelemetry'], - duration: [12026084], - 'attributes.transaction.result': ['Success'], - 'trace.id': [id], - // 'attributes.transaction.name.text': ['parent-synth'], - 'event.success_count': [1], - 'transaction.root': [true], - // 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], - 'attributes.transaction.root': [true], - dropped_events_count: [0], - trace_id: [id], - 'service.name': ['sendotlp-synth'], - 'service.framework.name': ['sendotlp-synth'], - 'data_stream.namespace': ['default'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'some.resource.attribute': ['resource.attr'], - dropped_links_count: [0], - 'transaction.type': ['unknown'], - 'attributes.processor.event': ['transaction'], - '@timestamp': ['2024-09-09T15:49:44.908Z'], - dropped_attributes_count: [0], - 'data_stream.dataset': ['generic.otel'], - 'attributes.event.outcome': ['success'], - 'attributes.transaction.type': ['unknown'], - 'resource.attributes.service.name': ['sendotlp-synth'], - // 'resource.attributes.service.name.text': ['sendotlp-synth'], - }, -======= transaction: { id }, span_id: id, status: { code: 'Unset', }, ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 }); } } -<<<<<<< HEAD -export function create(): Otel { - return new Otel(); -======= export function create(id: string): Otel { return new Otel({ trace_id: id, @@ -537,7 +202,6 @@ export function create(id: string): Otel { dropped_events_count: 0, dropped_links_count: 0, }); ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 } export const otel = { diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts index ac59ea52a8768..dfa5da49d4bbf 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -1,32 +1,5 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -<<<<<<< HEAD - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; -import { Serializable } from '../serializable'; -// import { generateShortId } from '../utils/generate_id'; - -export class OtelMetric extends Serializable { - constructor(fields: OtelDocument) { - super({ - ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - // trace_id: generateShortId(), - }); - } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } -======= * or more contributor license agreements. Licensed under the "Elastic License * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side * Public License v 1"; you may not use this file except in compliance with, at @@ -53,5 +26,4 @@ export class OtelMetric extends Serializable { ...fields, }); } ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts index 18b657bb266bd..f5e4b49992293 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -1,32 +1,5 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -<<<<<<< HEAD - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; -import { Serializable } from '../serializable'; -// import { generateShortId } from '../utils/generate_id'; - -export class OtelTransaction extends Serializable { - constructor(fields: OtelDocument) { - super({ - ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - // trace_id: generateShortId(), - }); - } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } -======= * or more contributor license agreements. Licensed under the "Elastic License * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side * Public License v 1"; you may not use this file except in compliance with, at @@ -69,5 +42,4 @@ export class OtelTransaction extends Serializable { ...fields, }); } ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 } diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts index b60caebf47312..0671ea66c472f 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts @@ -1,17 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -<<<<<<< HEAD - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. -======= * or more contributor license agreements. Licensed under the "Elastic License * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side * Public License v 1"; you may not use this file except in compliance with, at * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 */ import { Client } from '@elastic/elasticsearch'; diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index ebdca19e2cd12..c778afac26ef5 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -1,17 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -<<<<<<< HEAD - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. -======= * or more contributor license agreements. Licensed under the "Elastic License * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side * Public License v 1"; you may not use this file except in compliance with, at * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 */ import { Client } from '@elastic/elasticsearch'; @@ -31,11 +24,7 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient { ...options, pipeline: otelPipeline(), }); -<<<<<<< HEAD - this.dataStreams = ['.otel-*-synth', '*-synth-*']; -======= this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*']; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 } } @@ -65,50 +54,24 @@ export function getRoutingTransform() { switch (document?.attributes?.['processor.event']) { case 'transaction': case 'span': -<<<<<<< HEAD - index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; - // document._index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; - break; - - case 'error': - index = `.logs-otel.error-${namespace}-synth-2024.09.09-000001`; - // document._index = `logs-otel.error-${namespace}-synth-2024.09.09-000001`; -======= index = `traces-generic.otel-${namespace}-synth`; break; case 'error': index = `.logs-generic-otel-${namespace}-synth`; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 break; case 'metric': const metricsetName = document.attributes['metricset.name']; -<<<<<<< HEAD - document._index = `.ds-metrics-otel.service_summary.${document.attributes[ - 'metricset.interval' - ]!}-${namespace}-synth-2024.09.09-000001`; - - // if (metricsetName === 'app') { - // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}-2024.09.09-000001`; - // } else -======= ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' || metricsetName === 'service_destination' || metricsetName === 'service_summary' ) { -<<<<<<< HEAD - index = `.ds-metrics-otel.${metricsetName}.${document.attributes[ - 'metricset.interval' - ]!}-${namespace}-2024.09.09-000001`; -======= index = `metrics-otel.${metricsetName}.${document.attributes[ 'metricset.interval' ]!}-${namespace}-synth`; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 } else { index = `metrics-otel.internal-${namespace}`; } diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index 217a5bda5efb6..ab1755d2e7b78 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -1,13 +1,5 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -<<<<<<< HEAD - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import { ApmFields, otel, generateShortId } from '@kbn/apm-synthtrace-client'; -======= * or more contributor license agreements. Licensed under the "Elastic License * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side * Public License v 1"; you may not use this file except in compliance with, at @@ -16,47 +8,29 @@ import { ApmFields, otel, generateShortId } from '@kbn/apm-synthtrace-client'; */ import { otel, generateShortId, OtelDocument } from '@kbn/apm-synthtrace-client'; ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 import { times } from 'lodash'; import { Scenario } from '../cli/scenario'; import { withClient } from '../lib/utils/with_client'; -<<<<<<< HEAD -const scenario: Scenario = async (runOptions) => { -======= const scenario: Scenario = async (runOptions) => { ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 return { generate: ({ range, clients: { otelSynthtraceEsClient } }) => { const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; const { logger } = runOptions; -<<<<<<< HEAD - - const otelDocs = times(numOtelTraces / 2).map((index) => otel.create()); -======= const traceId = generateShortId(); const spanId = generateShortId(); const otelDocs = times(numOtelTraces / 2).map((index) => otel.create(traceId)); ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 const otelWithMetricsAndErrors = range .interval('30s') .rate(1) .generator((timestamp) => otelDocs.flatMap((oteld) => { -<<<<<<< HEAD - const id = generateShortId(); - return [ - oteld.metric(id).timestamp(timestamp), - oteld.transaction(id).timestamp(timestamp), - // oteld.error(id).timestamp(timestamp), -======= return [ oteld.metric().timestamp(timestamp), oteld.transaction(spanId).timestamp(timestamp), oteld.error(spanId).timestamp(timestamp), ->>>>>>> 46b62aa3664b26f4a589e4d58e39fc542d15b9a5 ]; }) ); From 0d084204fa2a60b5cff9e5630c576cd54bc2265a Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 7 Oct 2024 20:35:02 +0200 Subject: [PATCH 11/16] Fix error docs and update metrics and traces --- .../src/lib/otel/error.ts | 1 + .../src/lib/otel/index.ts | 27 ++++++++++--------- .../src/lib/otel/metric.ts | 6 ++++- .../src/lib/otel/otel_synthtrace_es_client.ts | 16 +++++------ 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts index 76ebe9d95a44f..e013e9948cd8f 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -14,6 +14,7 @@ export interface OtelErrorDocument extends OtelDocument { 'event.name'?: string; attributes?: { 'exception.message'?: string; + 'error.stack_trace'?: string; 'exception.type'?: string; 'processor.event'?: string; 'timestamp.us'?: number; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 141d174df1b86..ba722c72c3ae0 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -18,6 +18,7 @@ export interface SharedAttributes { 'agent.name'?: string; 'agent.version'?: string; 'metricset.interval'?: string; + 'metricset.name'?: string; 'service.instance.id'?: string; 'telemetry.sdk.language'?: string; 'telemetry.sdk.name'?: string; @@ -50,9 +51,9 @@ export interface OtelDocument extends Fields { }; name?: string; trace_id?: string; - trace?: { id: string }; // / I saw in fields this two representacion of trace id + trace?: { id: string }; span_id?: string; - span?: { id: string }; // / I saw in fields this two representacion of span id + span?: { id: string }; dropped_attributes_count?: number; dropped_events_count?: number; dropped_links_count?: number; @@ -72,8 +73,9 @@ class Otel extends Serializable { attributes: { 'exception.message': 'boom', 'exception.type': '*errors.errorString', + 'error.stack_trace': 'Error: INTERNAL: Boom', 'processor.event': 'error', - 'timestamp.us': 1725633628036123, + 'timestamp.us': 1726580752010657, 'event.name': 'exception', }, data_stream: { @@ -103,8 +105,6 @@ class Otel extends Serializable { name: 'sendotlp-synth', }, span_id: spanId, - span: { id: spanId }, - timestamp_us: 1725633628036123, }); } @@ -112,12 +112,15 @@ class Otel extends Serializable { return new OtelMetric({ ...this.fields, attributes: { - 'metricset.interval': '1m', - 'metricset.name': 'service_summary', + 'metricset.name': 'service_destination', 'processor.event': 'metric', + 'event.outcome': 'success', + 'service.target.name': 'foo_service', + 'service.target.type': 'http', + 'span.name': 'child1', }, data_stream: { - dataset: 'generic.otel', + dataset: 'service_destination.10m.otel', namespace: 'default', type: 'metrics', }, @@ -127,17 +130,16 @@ class Otel extends Serializable { resource: { attributes: { 'agent.name': 'otlp', - 'agent.version': 'unknown', - 'metricset.interval': '1m', + 'agent.version': '1.28.0', 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', - 'some.resource.attribute': 'resource.attr', + 'metricset.interval': '10m', }, dropped_attributes_count: 0, }, scope: { dropped_attributes_count: 0, - name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', + name: 'github.com/elastic/opentelemetry-collector-components/connector/spanmetricsconnectorv2', }, }); } @@ -173,7 +175,6 @@ class Otel extends Serializable { 'agent.version': '1.28.0', 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', - 'some.resource.attribute': 'resource.attr', }, dropped_attributes_count: 0, schema_url: 'https://opentelemetry.io/schemas/1.26.0', diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts index dfa5da49d4bbf..518ed19324f5b 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -12,9 +12,13 @@ import { Serializable } from '../serializable'; export interface OtelMetricDocument extends OtelDocument { attributes?: { - 'metricset.interval'?: string; 'metricset.name'?: string; 'processor.event'?: string; + // service_destination + 'event.outcome'?: string; + 'service.target.name'?: string; + 'service.target.type'?: string; + 'span.name'?: string; }; metrics?: { service_summary?: number; diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index c778afac26ef5..c2c29690bc5e3 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -24,7 +24,7 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient { ...options, pipeline: otelPipeline(), }); - this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*']; + this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*', 'logs-generic.otel*']; } } @@ -58,28 +58,28 @@ export function getRoutingTransform() { break; case 'error': - index = `.logs-generic-otel-${namespace}-synth`; + index = `logs-generic.otel-${namespace}-synth`; break; case 'metric': - const metricsetName = document.attributes['metricset.name']; + const metricsetName = document?.resource?.attributes?.['metricset.name']; if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' || metricsetName === 'service_destination' || metricsetName === 'service_summary' ) { - index = `metrics-otel.${metricsetName}.${document.attributes[ + index = `metrics-generic.otel.${metricsetName}.${document.attributes[ 'metricset.interval' ]!}-${namespace}-synth`; } else { - index = `metrics-otel.internal-${namespace}`; + index = `metrics-generic.otel.internal-${namespace}-synth`; } break; default: - // if (document['event.action'] != null) { - // index = `logs-otel.app-${namespace}`; - // } + if (document?.attributes?.['event.action'] != null) { + index = `logs-generic.otel-${namespace}-synth`; + } break; } From 832abf11b62e2bf6c3e3f8b22a67ebacd4761142 Mon Sep 17 00:00:00 2001 From: Jenny Date: Tue, 8 Oct 2024 10:18:28 +0200 Subject: [PATCH 12/16] Fix types --- .../kbn-apm-synthtrace/src/cli/scenario.ts | 2 +- .../src/cli/utils/bootstrap.ts | 1 + .../src/cli/utils/start_live_data_upload.ts | 20 ++++++++++++++++--- .../src/cli/utils/synthtrace_worker.ts | 6 +++--- .../src/scenarios/otel_simple_trace.ts | 4 ++-- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/kbn-apm-synthtrace/src/cli/scenario.ts b/packages/kbn-apm-synthtrace/src/cli/scenario.ts index 4c7678507a338..4f1550b8bdbc8 100644 --- a/packages/kbn-apm-synthtrace/src/cli/scenario.ts +++ b/packages/kbn-apm-synthtrace/src/cli/scenario.ts @@ -26,7 +26,7 @@ interface EsClients { infraEsClient: InfraSynthtraceEsClient; assetsEsClient: AssetsSynthtraceEsClient; syntheticsEsClient: SyntheticsSynthtraceEsClient; - otelSynthtraceEsClient: OtelSynthtraceEsClient; + otelEsClient: OtelSynthtraceEsClient; } type Generate = (options: { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts index ebdfaf7c8d62a..22d07f73c56cb 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts @@ -91,6 +91,7 @@ export async function bootstrap(runOptions: RunOptions) { infraEsClient, assetsEsClient, syntheticsEsClient, + otelEsClient, version, kibanaUrl, esUrl, diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts b/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts index 90fa0189469ad..79c9907dc13d1 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts @@ -26,8 +26,15 @@ export async function startLiveDataUpload({ }) { const file = runOptions.file; - const { logger, apmEsClient, logsEsClient, infraEsClient, assetsEsClient, syntheticsEsClient } = - await bootstrap(runOptions); + const { + logger, + apmEsClient, + logsEsClient, + infraEsClient, + assetsEsClient, + syntheticsEsClient, + otelEsClient, + } = await bootstrap(runOptions); const scenario = await getScenario({ file, logger }); const { generate } = await scenario({ ...runOptions, logger }); @@ -65,7 +72,14 @@ export async function startLiveDataUpload({ const generatorsAndClients = generate({ range: timerange(bucketFrom.getTime(), bucketTo.getTime()), - clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient, syntheticsEsClient }, + clients: { + logsEsClient, + apmEsClient, + infraEsClient, + assetsEsClient, + syntheticsEsClient, + otelEsClient, + }, }); const generatorsAndClientsArray = castArray(generatorsAndClients); diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts index 5e9301027a27e..78c89d110c892 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts @@ -66,7 +66,7 @@ async function start() { logger, }); - const otelSynthtraceEsClient = getOtelSynthtraceEsClient({ + const otelEsClient = getOtelSynthtraceEsClient({ concurrency: runOptions.concurrency, target: esUrl, logger, @@ -87,7 +87,7 @@ async function start() { infraEsClient, assetsEsClient, syntheticsEsClient, - otelSynthtraceEsClient, + otelEsClient, }); } @@ -102,7 +102,7 @@ async function start() { infraEsClient, assetsEsClient, syntheticsEsClient, - otelSynthtraceEsClient, + otelEsClient, }, }) ); diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index ab1755d2e7b78..51fe8a5a72384 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -14,7 +14,7 @@ import { withClient } from '../lib/utils/with_client'; const scenario: Scenario = async (runOptions) => { return { - generate: ({ range, clients: { otelSynthtraceEsClient } }) => { + generate: ({ range, clients: { otelEsClient } }) => { const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; const { logger } = runOptions; const traceId = generateShortId(); @@ -37,7 +37,7 @@ const scenario: Scenario = async (runOptions) => { return [ withClient( - otelSynthtraceEsClient, + otelEsClient, logger.perf('generating_otel_otelTrace', () => otelWithMetricsAndErrors) ), ]; From 6a59171545943f745334a85a116f095de1579318 Mon Sep 17 00:00:00 2001 From: Jenny Date: Tue, 8 Oct 2024 17:01:28 +0200 Subject: [PATCH 13/16] Rename to OtelSharedResourceAttributes --- packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index ba722c72c3ae0..7347cd011cb3a 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -13,7 +13,7 @@ import { OtelError } from './error'; import { OtelMetric } from './metric'; import { OtelTransaction } from './transaction'; -export interface SharedAttributes { +interface OtelSharedResourceAttributes { 'service.name'?: string; 'agent.name'?: string; 'agent.version'?: string; @@ -37,7 +37,7 @@ export interface OtelDocument extends Fields { [key: string]: any; }; resource?: { - attributes?: SharedAttributes; + attributes?: OtelSharedResourceAttributes; dropped_attributes_count?: number; schema_url?: string; }; From 3a66c4477ab6da2b8ba03e7ba1bcd5ade79a3b77 Mon Sep 17 00:00:00 2001 From: Jenny Date: Fri, 11 Oct 2024 19:28:50 +0200 Subject: [PATCH 14/16] CR changes --- .../kbn-apm-synthtrace-client/src/lib/otel/error.ts | 1 + .../kbn-apm-synthtrace-client/src/lib/otel/index.ts | 11 ++++++----- .../kbn-apm-synthtrace-client/src/lib/otel/metric.ts | 2 +- .../src/lib/otel/transaction.ts | 1 - .../src/scenarios/otel_simple_trace.ts | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts index e013e9948cd8f..070c2804e4a78 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -19,6 +19,7 @@ export interface OtelErrorDocument extends OtelDocument { 'processor.event'?: string; 'timestamp.us'?: number; 'event.name'?: string; + 'error.id'?: string; }; } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 7347cd011cb3a..b61e0f0666bb8 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -18,13 +18,11 @@ interface OtelSharedResourceAttributes { 'agent.name'?: string; 'agent.version'?: string; 'metricset.interval'?: string; - 'metricset.name'?: string; 'service.instance.id'?: string; 'telemetry.sdk.language'?: string; 'telemetry.sdk.name'?: string; 'telemetry.sdk.version'?: string; 'some.resource.attribute'?: string; - 'timestamp.us'?: number; } export interface OtelDocument extends Fields { @@ -34,6 +32,7 @@ export interface OtelDocument extends Fields { type: string; }; attributes?: { + 'timestamp.us'?: number; [key: string]: any; }; resource?: { @@ -77,6 +76,7 @@ class Otel extends Serializable { 'processor.event': 'error', 'timestamp.us': 1726580752010657, 'event.name': 'exception', + 'error.id': `error-${spanId}`, }, data_stream: { dataset: 'generic.otel', @@ -91,7 +91,6 @@ class Otel extends Serializable { 'agent.version': '1.28.0', 'service.name': 'sendotlp-synth', 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', - 'some.resource.attribute': 'resource.attr', }, dropped_attributes_count: 0, schema_url: 'https://opentelemetry.io/schemas/1.26.0', @@ -118,6 +117,7 @@ class Otel extends Serializable { 'service.target.name': 'foo_service', 'service.target.type': 'http', 'span.name': 'child1', + 'span.destination.service.resource': 'foo_service:8080', }, data_stream: { dataset: 'service_destination.10m.otel', @@ -144,6 +144,8 @@ class Otel extends Serializable { }); } + // In Otel we have only spans (https://opentelemetry.io/docs/concepts/signals/traces/#spans) + // we call the root span a transaction to match our data model transaction(id: string) { return new OtelTransaction({ ...this.fields, @@ -156,7 +158,7 @@ class Otel extends Serializable { 'transaction.id': id, 'transaction.name': 'parent-synth', 'transaction.representative_count': 1, - 'transaction.result': 'Success', + 'transaction.result': 'HTTP 2xx', 'transaction.root': true, 'transaction.sampled': true, 'transaction.type': 'unknown', @@ -187,7 +189,6 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, - transaction: { id }, span_id: id, status: { code: 'Unset', diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts index 518ed19324f5b..2f238b36c5aca 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -14,11 +14,11 @@ export interface OtelMetricDocument extends OtelDocument { attributes?: { 'metricset.name'?: string; 'processor.event'?: string; - // service_destination 'event.outcome'?: string; 'service.target.name'?: string; 'service.target.type'?: string; 'span.name'?: string; + 'span.destination.service.resource'?: string; }; metrics?: { service_summary?: number; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts index f5e4b49992293..d6f7c7111b187 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -33,7 +33,6 @@ export interface OtelTransactionDocument extends OtelDocument { duration?: number; kind?: string; name?: string; - transaction: { id: string }; } export class OtelTransaction extends Serializable { diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index 51fe8a5a72384..7721a8651905f 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -38,7 +38,7 @@ const scenario: Scenario = async (runOptions) => { return [ withClient( otelEsClient, - logger.perf('generating_otel_otelTrace', () => otelWithMetricsAndErrors) + logger.perf('generating_otel_trace', () => otelWithMetricsAndErrors) ), ]; }, From 61f38b792eab9f08189e37264c33dbaec20ce591 Mon Sep 17 00:00:00 2001 From: Jenny Date: Fri, 11 Oct 2024 19:42:00 +0200 Subject: [PATCH 15/16] Add exception.handled --- packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts | 1 + packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts index 070c2804e4a78..63265d45fe886 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -15,6 +15,7 @@ export interface OtelErrorDocument extends OtelDocument { attributes?: { 'exception.message'?: string; 'error.stack_trace'?: string; + 'exception.handled'?: boolean; 'exception.type'?: string; 'processor.event'?: string; 'timestamp.us'?: number; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index b61e0f0666bb8..f3e416d7041bd 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -71,6 +71,7 @@ class Otel extends Serializable { ...this.fields, attributes: { 'exception.message': 'boom', + 'exception.handled': false, 'exception.type': '*errors.errorString', 'error.stack_trace': 'Error: INTERNAL: Boom', 'processor.event': 'error', From 7ce3a70e9aa50bb04f74c99acbb2949237045e93 Mon Sep 17 00:00:00 2001 From: Jenny Date: Mon, 14 Oct 2024 10:55:50 +0200 Subject: [PATCH 16/16] Move metricset.name to attributes type --- packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts | 1 + .../src/lib/otel/otel_synthtrace_es_client.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index f3e416d7041bd..86bb74dd94ff4 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -33,6 +33,7 @@ export interface OtelDocument extends Fields { }; attributes?: { 'timestamp.us'?: number; + 'metricset.name'?: string; [key: string]: any; }; resource?: { diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index c2c29690bc5e3..e2162925e3c72 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -62,7 +62,7 @@ export function getRoutingTransform() { break; case 'metric': - const metricsetName = document?.resource?.attributes?.['metricset.name']; + const metricsetName = document?.attributes?.['metricset.name']; if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' ||