From 5c95114f9702ed65e1f3180cc6b5e441f34378d5 Mon Sep 17 00:00:00 2001 From: "miriam.aparicio" Date: Tue, 24 Sep 2024 10:34:31 +0100 Subject: [PATCH] rename indices and refactor code --- packages/kbn-apm-synthtrace-client/index.ts | 2 +- .../src/lib/otel/error.ts | 26 +- .../src/lib/otel/index.ts | 301 +++--------------- .../src/lib/otel/metric.ts | 26 +- .../src/lib/otel/transaction.ts | 41 ++- .../src/lib/otel/otel_synthtrace_es_client.ts | 22 +- .../src/scenarios/otel_simple_trace.ts | 12 +- 7 files changed, 121 insertions(+), 309 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts index a7eed2ebccda5..d3d24a8940a3b 100644 --- a/packages/kbn-apm-synthtrace-client/index.ts +++ b/packages/kbn-apm-synthtrace-client/index.ts @@ -37,4 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs'; export { type AssetDocument } from './src/lib/assets'; export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics'; -export { otel, type OtelDocument, type OtelTrace } from './src/lib/otel'; +export { otel, type OtelDocument } from './src/lib/otel'; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts index 190ff07a70fd8..efc25735792b7 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -6,23 +6,23 @@ * Side Public License, v 1. */ -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import type { OtelDocument } from '../../..'; import { Serializable } from '../serializable'; -import { generateShortId } from '../utils/generate_id'; -export class OtelError extends Serializable { - constructor(fields: OtelDocument) { +export interface OtelErrorDocument extends OtelDocument { + name?: string; + attributes?: { + 'exception.message'?: string; + 'exception.type'?: string; + 'processor.event'?: string; + 'timestamp.us'?: number; + }; +} + +export class OtelError extends Serializable { + constructor(fields: OtelErrorDocument) { super({ ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - trace_id: generateShortId(), }); } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts index 2540cbc23d57d..c5aa3ead42c8a 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -6,82 +6,55 @@ * Side Public License, v 1. */ +import { Fields } from '../entity'; import { Serializable } from '../serializable'; import { OtelError } from './error'; import { OtelMetric } from './metric'; import { OtelTransaction } from './transaction'; -export interface OtelTrace { - attributes?: { - 'event.outcome'?: string; - 'event.success_count'?: number; - 'processor.event'?: string; - 'timestamp.us'?: number; - 'transaction.duration.us'?: number; - 'transaction.id'?: string; - 'transaction.name'?: string; - 'transaction.representative_count'?: number; - 'transaction.result'?: string; - 'transaction.root'?: boolean; - 'transaction.sampled'?: boolean; - 'transaction.type'?: string; - // error - 'exception.message'?: string; - 'exception.type'?: string; - // metrics - 'metricset.interval'?: string; - 'metricset.name'?: string; - }; +export interface SharedAttributes { + 'service.name'?: string; + 'agent.name'?: string; + 'agent.version'?: string; + 'metricset.interval'?: string; + 'service.instance.id'?: string; + 'telemetry.sdk.language'?: string; + 'telemetry.sdk.name'?: string; + 'telemetry.sdk.version'?: string; + 'some.resource.attribute'?: string; + 'timestamp.us'?: number; +} + +export interface OtelDocument extends Fields { data_stream?: { dataset: string; namespace: string; type: string; }; - dropped_attributes_count?: number; - dropped_events_count?: number; - dropped_links_count?: number; - duration?: number; - kind?: string; - name?: string; + attributes?: { + [key: string]: any; + }; resource?: { - attributes?: { - 'agent.name'?: string; - 'agent.version'?: string; - 'service.name'?: string; - 'some.resource.attribute'?: string; - 'telemetry.sdk.language'?: string; - 'telemetry.sdk.name'?: string; - 'telemetry.sdk.version'?: string; - // metrics - 'metricset.interval'?: string; - }; + attributes?: SharedAttributes; dropped_attributes_count?: number; schema_url?: string; }; scope?: { attributes?: { - 'service.framework.name': string; - 'service.framework.version': string; + 'service.framework.name'?: string; + 'service.framework.version'?: string; }; dropped_attributes_count?: number; name?: string; }; + name?: string; + trace_id?: string; span_id?: string; - status?: { - code: string; - }; - trace_id: string; - '@timestamp'?: number | undefined; - 'timestamp.us'?: number | undefined; - // metrics - metrics?: { - service_summary?: number; - }; - 'service.name': string[]; + dropped_attributes_count?: number; + dropped_events_count?: number; + dropped_links_count?: number; } -export type OtelDocument = OtelTrace; - class Otel extends Serializable { constructor(fields: OtelDocument) { super({ @@ -89,11 +62,6 @@ class Otel extends Serializable { }); } - timestamp(time: number) { - super.timestamp(time); - return this; - } - error(id: string) { return new OtelError({ ...this.fields, @@ -132,57 +100,10 @@ class Otel extends Serializable { name: 'sendotlp-synth', }, span_id: '96d91c11a4e393fb', - trace_id: id, - 'service.name': ['sendotlp-synth'], - fields: { - 'attributes.exception.message': ['boom'], - 'scope.name': ['sendotlp-synth'], - 'resource.attributes.agent.version': ['1.28.0'], - 'service.framework.version': [''], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - // 'resource.attributes.agent.name.text': ['opentelemetry/go'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'telemetry.sdk.name': ['opentelemetry'], - 'attributes.timestamp.us': [1725906566779927], - 'telemetry.sdk.language': ['go'], - 'trace.id': [id], - 'exception.message': ['boom'], - 'processor.event': ['error'], - 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'scope.attributes.service.framework.name': ['sendotlp-synth'], - trace_id: [id], - 'service.name': ['sendotlp-synth'], - 'service.framework.name': ['sendotlp-synth'], - span_id: [`${id}da907a`], - 'data_stream.namespace': ['default'], - 'exception.type': ['*errors.errorString'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'span.id': [`${id}da907a`], - 'some.resource.attribute': ['resource.attr'], - 'data_stream.type': ['logs'], - 'attributes.processor.event': ['error'], - 'timestamp.us': [1725906566779927], - '@timestamp': ['2024-09-09T18:29:26.779Z'], - dropped_attributes_count: [0], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - 'data_stream.dataset': ['generic.otel'], - name: ['exception'], - 'agent.version': ['1.28.0'], - 'attributes.exception.type': ['*errors.errorString'], - 'resource.dropped_attributes_count': [0], - 'scope.attributes.service.framework.version': [''], - 'resource.attributes.service.name': ['sendotlp-synth'], - 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], - }, }); } - metric(id: string) { + metric() { return new OtelMetric({ ...this.fields, attributes: { @@ -200,14 +121,12 @@ class Otel extends Serializable { }, resource: { attributes: { - 'agent.name': 'opentelemetry/go', - 'agent.version': '1.28.0', + 'agent.name': 'otlp', + 'agent.version': 'unknown', 'metricset.interval': '1m', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', }, dropped_attributes_count: 0, }, @@ -215,59 +134,9 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector', }, - trace_id: id, - service: { - name: ['sendotlp-synth'], - }, - fields: { - 'resource.attributes.agent.version': ['1.28.0'], - 'scope.name': ['otelcol/spanmetricsconnectorv2'], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'telemetry.sdk.name': ['opentelemetry'], - 'telemetry.sdk.language': ['go'], - 'transaction.root': [true], - 'processor.event': ['metric'], - 'resource.attributes.metricset.interval': ['1m'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'attributes.metricset.name': ['service_transaction'], - 'metrics.transaction.duration.histogram': [ - { - values: [12500], - counts: [1], - }, - ], - 'attributes.transaction.root': [true], - 'service.name': ['sendotlp-synth'], - 'data_stream.namespace': ['default'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'some.resource.attribute': ['resource.attr'], - 'metricset.interval': ['1m'], - 'data_stream.type': ['metrics'], - 'transaction.duration.histogram': [ - { - values: [12500], - counts: [1], - }, - ], - 'transaction.type': ['unknown'], - 'metricset.name': ['service_transaction'], - 'attributes.processor.event': ['metric'], - '@timestamp': ['2024-09-09T15:49:44.920Z'], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - 'data_stream.dataset': ['generic.otel'], - 'attributes.transaction.type': ['unknown'], - 'agent.version': ['1.28.0'], - 'attributes.metricset.interval': ['1m'], - 'resource.dropped_attributes_count': [0], - 'resource.attributes.service.name': ['sendotlp-synth'], - }, }); } + transaction(id: string) { return new OtelTransaction({ ...this.fields, @@ -275,9 +144,9 @@ class Otel extends Serializable { 'event.outcome': 'success', 'event.success_count': 1, 'processor.event': 'transaction', - 'timestamp.us': 1725464233071114, + 'timestamp.us': 1726580752010657, 'transaction.duration.us': 15202, - 'transaction.id': 'da907d6b5267c8f6', + 'transaction.id': id, 'transaction.name': 'parent-synth', 'transaction.representative_count': 1, 'transaction.result': 'Success', @@ -285,21 +154,21 @@ class Otel extends Serializable { 'transaction.sampled': true, 'transaction.type': 'unknown', }, - dropped_attributes_count: 0, - dropped_events_count: 0, - dropped_links_count: 0, - duration: 15202584, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'traces', + }, + duration: 11742370, kind: 'Internal', name: 'parent-synth', resource: { attributes: { - 'agent.name': 'opentelemetry/go', + 'agent.name': 'otlp', 'agent.version': '1.28.0', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', 'service.name': 'sendotlp-synth', 'some.resource.attribute': 'resource.attr', - 'telemetry.sdk.language': 'go', - 'telemetry.sdk.name': 'opentelemetry', - 'telemetry.sdk.version': '1.28.0', }, dropped_attributes_count: 0, schema_url: 'https://opentelemetry.io/schemas/1.26.0', @@ -312,93 +181,21 @@ class Otel extends Serializable { dropped_attributes_count: 0, name: 'sendotlp-synth', }, - span_id: `${id}da907a`, + span_id: id, status: { code: 'Unset', }, - data_stream: { - dataset: 'generic.otel', - namespace: 'default', - type: 'traces', - }, - trace_id: id, - fields: { - // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], - // 'resource.attributes.telemetry.sdk.name.text': ['opentelemetry'], - 'scope.name': ['sendotlp-synth'], - 'resource.attributes.telemetry.sdk.name': ['opentelemetry'], - 'resource.attributes.agent.name': ['opentelemetry/go'], - // 'resource.attributes.agent.name.text': ['opentelemetry/go'], - 'transaction.representative_count': [1], - 'scope.dropped_attributes_count': [0], - 'service.language.name': ['go'], - 'transaction.result': ['Success'], - 'transaction.id': [id], - 'attributes.event.success_count': [1], - 'attributes.timestamp.us': [1725896984908682], - 'telemetry.sdk.language': ['go'], - 'processor.event': ['transaction'], - 'resource.schema_url': ['https://opentelemetry.io/schemas/1.26.0'], - 'agent.name': ['opentelemetry/go'], - 'telemetry.sdk.version': ['1.28.0'], - 'event.outcome': ['success'], - 'scope.attributes.service.framework.name': ['sendotlp-synth'], - 'attributes.transaction.sampled': [true], - span_id: [`${id}da907a`], - kind: ['Internal'], - 'attributes.transaction.duration.us': [12026], - 'attributes.transaction.name': ['parent-synth'], - 'transaction.duration.us': [12026], - 'span.id': [`${id}da907a`], - 'data_stream.type': ['traces'], - 'timestamp.us': [1725896984908682], - 'resource.attributes.telemetry.sdk.version': ['1.28.0'], - name: ['parent-synth'], - 'agent.version': ['1.28.0'], - 'transaction.name': ['parent-synth'], - 'resource.dropped_attributes_count': [0], - 'scope.attributes.service.framework.version': [''], - 'attributes.transaction.representative_count': [1], - 'span.name': ['parent-synth'], - 'resource.attributes.agent.version': ['1.28.0'], - 'attributes.transaction.id': [id], - 'service.framework.version': [''], - 'resource.attributes.telemetry.sdk.language': ['go'], - 'status.code': ['Unset'], - 'transaction.sampled': [true], - 'telemetry.sdk.name': ['opentelemetry'], - duration: [12026084], - 'attributes.transaction.result': ['Success'], - 'trace.id': [id], - // 'attributes.transaction.name.text': ['parent-synth'], - 'event.success_count': [1], - 'transaction.root': [true], - // 'scope.attributes.service.framework.name.text': ['sendotlp-synth'], - 'attributes.transaction.root': [true], - dropped_events_count: [0], - trace_id: [id], - 'service.name': ['sendotlp-synth'], - 'service.framework.name': ['sendotlp-synth'], - 'data_stream.namespace': ['default'], - 'resource.attributes.some.resource.attribute': ['resource.attr'], - 'some.resource.attribute': ['resource.attr'], - dropped_links_count: [0], - 'transaction.type': ['unknown'], - 'attributes.processor.event': ['transaction'], - '@timestamp': ['2024-09-09T15:49:44.908Z'], - dropped_attributes_count: [0], - 'data_stream.dataset': ['generic.otel'], - 'attributes.event.outcome': ['success'], - 'attributes.transaction.type': ['unknown'], - 'resource.attributes.service.name': ['sendotlp-synth'], - // 'resource.attributes.service.name.text': ['sendotlp-synth'], - }, }); } } -export function create(): Otel { - return new Otel(); +export function create(id: string): Otel { + return new Otel({ + trace_id: id, + dropped_attributes_count: 0, + dropped_events_count: 0, + dropped_links_count: 0, + }); } export const otel = { diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts index 8e2c1a76b180a..8f69cc2c6a480 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -6,23 +6,23 @@ * Side Public License, v 1. */ -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { OtelDocument } from '.'; import { Serializable } from '../serializable'; -// import { generateShortId } from '../utils/generate_id'; -export class OtelMetric extends Serializable { - constructor(fields: OtelDocument) { +export interface OtelMetricDocument extends OtelDocument { + attributes?: { + 'metricset.interval'?: string; + 'metricset.name'?: string; + 'processor.event'?: string; + }; + metrics?: { + service_summary?: number; + }; +} +export class OtelMetric extends Serializable { + constructor(fields: OtelMetricDocument) { super({ ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - // trace_id: generateShortId(), }); } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } } diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts index 33f019f4b2fb8..b917ce172c23f 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -6,23 +6,38 @@ * Side Public License, v 1. */ -import type { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { OtelDocument } from '.'; import { Serializable } from '../serializable'; -// import { generateShortId } from '../utils/generate_id'; -export class OtelTransaction extends Serializable { - constructor(fields: OtelDocument) { +export interface OtelTransactionDocument extends OtelDocument { + attributes?: { + 'event.outcome'?: string; + 'event.success_count'?: number; + 'processor.event'?: string; + 'timestamp.us'?: number; + 'transaction.duration.us'?: number; + 'transaction.id'?: string; + 'transaction.name'?: string; + 'transaction.representative_count'?: number; + 'transaction.result'?: string; + 'transaction.root'?: boolean; + 'transaction.sampled'?: boolean; + 'transaction.type'?: string; + }; + status?: { + code?: string; + }; + dropped_events_count?: number; + dropped_links_count?: number; + duration?: number; + kind?: string; + name?: string; +} + +export class OtelTransaction extends Serializable { + constructor(fields: OtelTransactionDocument) { super({ ...fields, - // 'processor.event': 'error', - // 'processor.name': 'error', - // trace_id: generateShortId(), }); } - - timestamp(value: number) { - const ret = super.timestamp(value); - this.fields['timestamp.us'] = value * 1000; - return ret; - } } diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts index fbb78677fa02e..3693e4c711ac8 100644 --- a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -23,7 +23,7 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient { ...options, pipeline: otelPipeline(), }); - this.dataStreams = ['.otel-*-synth', '*-synth-*']; + this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*']; } } @@ -53,33 +53,33 @@ export function getRoutingTransform() { switch (document?.attributes?.['processor.event']) { case 'transaction': case 'span': - index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; - // document._index = `.ds-traces-generic.otel-${namespace}-synth-2024.09.09-000001`; + index = `traces-generic.otel-${namespace}-synth`; + // document._index = `.ds-traces-generic.otel-${namespace}-synth`; break; case 'error': - index = `.logs-otel.error-${namespace}-synth-2024.09.09-000001`; - // document._index = `logs-otel.error-${namespace}-synth-2024.09.09-000001`; + index = `.logs-otel.error-${namespace}-synth`; + // document._index = `logs-otel.error-${namespace}-synth`; break; case 'metric': const metricsetName = document.attributes['metricset.name']; - document._index = `.ds-metrics-otel.service_summary.${document.attributes[ + document._index = `metrics-otel.service_summary.${document.attributes[ 'metricset.interval' - ]!}-${namespace}-synth-2024.09.09-000001`; + ]!}-${namespace}-synth`; // if (metricsetName === 'app') { - // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}-2024.09.09-000001`; - // } else + // index = `metrics-otel.app.${document?.attributes?.['service.name']}-${namespace}`; + // } else if ( metricsetName === 'transaction' || metricsetName === 'service_transaction' || metricsetName === 'service_destination' || metricsetName === 'service_summary' ) { - index = `.ds-metrics-otel.${metricsetName}.${document.attributes[ + index = `metrics-otel.${metricsetName}.${document.attributes[ 'metricset.interval' - ]!}-${namespace}-2024.09.09-000001`; + ]!}-${namespace}-synth`; } else { index = `metrics-otel.internal-${namespace}`; } diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts index 4d7be0af191ce..b527ab7df8afb 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -5,28 +5,28 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { ApmFields, otel, generateShortId } from '@kbn/apm-synthtrace-client'; +import { otel, generateShortId, OtelDocument } from '@kbn/apm-synthtrace-client'; import { times } from 'lodash'; import { Scenario } from '../cli/scenario'; import { withClient } from '../lib/utils/with_client'; -const scenario: Scenario = async (runOptions) => { +const scenario: Scenario = async (runOptions) => { return { generate: ({ range, clients: { otelSynthtraceEsClient } }) => { const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; const { logger } = runOptions; + const traceId = generateShortId(); - const otelDocs = times(numOtelTraces / 2).map((index) => otel.create()); + const otelDocs = times(numOtelTraces / 2).map((index) => otel.create(traceId)); const otelWithMetricsAndErrors = range .interval('30s') .rate(1) .generator((timestamp) => otelDocs.flatMap((oteld) => { - const id = generateShortId(); return [ - oteld.metric(id).timestamp(timestamp), - oteld.transaction(id).timestamp(timestamp), + oteld.metric().timestamp(timestamp), + oteld.transaction(traceId).timestamp(timestamp), // oteld.error(id).timestamp(timestamp), ]; })