From b82203a57cc6898d9571a7c9a30285e6f69be541 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 16 Jan 2024 09:34:14 +0100 Subject: [PATCH] [APM] Improve logic to determine when to use the transaction.duration.summary field (#171315) fixes https://github.com/elastic/kibana/issues/167578 ## Summary This pull request addresses an issue related to the rendering of latency charts in the UI when using APM Servers with versions both pre-8.7 and 8.7 or higher. it changes the `GET /internal/apm/time_range_metadata` to will only return `hasDurationSummary` `true` when **all** documents within a give time range are produced with `transaction.duration.summary` field. ### Services Inventory | before | after | |--------|--------| | image | image | ### Service Overview image ```json { "isUsingServiceDestinationMetrics": false, "sources": [ { "documentType": "serviceTransactionMetric", "rollupInterval": "1m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "serviceTransactionMetric", "rollupInterval": "10m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "serviceTransactionMetric", "rollupInterval": "60m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionMetric", "rollupInterval": "1m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionMetric", "rollupInterval": "10m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionMetric", "rollupInterval": "60m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionEvent", "rollupInterval": "none", "hasDocs": true, "hasDurationSummaryField": false } ] } ``` **Time range with only APM Server >= 8.7 produced documents** image ```json { "isUsingServiceDestinationMetrics": false, "sources": [ { "documentType": "serviceTransactionMetric", "rollupInterval": "1m", "hasDocs": true, "hasDurationSummaryField": true }, { "documentType": "serviceTransactionMetric", "rollupInterval": "10m", "hasDocs": true, "hasDurationSummaryField": true }, { "documentType": "serviceTransactionMetric", "rollupInterval": "60m", "hasDocs": true, "hasDurationSummaryField": true }, { "documentType": "transactionMetric", "rollupInterval": "1m", "hasDocs": true, "hasDurationSummaryField": true }, { "documentType": "transactionMetric", "rollupInterval": "10m", "hasDocs": true, "hasDurationSummaryField": true }, { "documentType": "transactionMetric", "rollupInterval": "60m", "hasDocs": true, "hasDurationSummaryField": true }, { "documentType": "transactionEvent", "rollupInterval": "none", "hasDocs": true, "hasDurationSummaryField": false } ] } ``` **Time range with only APM Server < 8.7 produced documents** image ```json { "isUsingServiceDestinationMetrics": false, "sources": [ { "documentType": "serviceTransactionMetric", "rollupInterval": "1m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "serviceTransactionMetric", "rollupInterval": "10m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "serviceTransactionMetric", "rollupInterval": "60m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionMetric", "rollupInterval": "1m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionMetric", "rollupInterval": "10m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionMetric", "rollupInterval": "60m", "hasDocs": true, "hasDurationSummaryField": false }, { "documentType": "transactionEvent", "rollupInterval": "none", "hasDocs": true, "hasDurationSummaryField": false } ] } ``` ### How to test - Setup local Kibana and ES instance - run Synthtrace to produce documents simulating APM Server < 8.7 ``` node scripts/synthtrace service_summary_field_version_dependent.ts --versionOverride=8.6.2 --from=now-15m --to=now ``` - run Synthtrace to produce documents simulating APM Server >= 8.7 ``` node scripts/synthtrace service_summary_field_version_dependent.ts --versionOverride=8.9.2 --from=now-15m --to=now ``` - Navigate to APM > Service and Trace and click through the services --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...service_summary_field_version_dependent.ts | 71 +++++ .../lib/helpers/get_document_sources.ts | 263 +++++++++--------- .../server/lib/helpers/transactions/index.ts | 35 ++- .../get_service_transaction_stats.ts | 2 +- .../many_apm_server_versions.spec.ts | 47 +++- .../time_range_metadata.spec.ts | 84 ++++-- 6 files changed, 335 insertions(+), 167 deletions(-) create mode 100644 packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts diff --git a/packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts b/packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts new file mode 100644 index 0000000000000..37999bff114c2 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ApmFields, apm } from '@kbn/apm-synthtrace-client'; +import { random } from 'lodash'; +import { pipeline, Readable } from 'stream'; +import semver from 'semver'; +import { Scenario } from '../cli/scenario'; +import { + addObserverVersionTransform, + deleteSummaryFieldTransform, +} from '../lib/utils/transform_helpers'; +import { withClient } from '../lib/utils/with_client'; + +const scenario: Scenario = async ({ logger, versionOverride }) => { + const version = versionOverride as string; + const isLegacy = versionOverride && semver.lt(version, '8.7.0'); + return { + bootstrap: async ({ apmEsClient }) => { + if (isLegacy) { + apmEsClient.pipeline((base: Readable) => { + const defaultPipeline = apmEsClient.getDefaultPipeline()( + base + ) as unknown as NodeJS.ReadableStream; + + return pipeline( + defaultPipeline, + addObserverVersionTransform(version), + deleteSummaryFieldTransform(), + (err) => { + if (err) { + logger.error(err); + } + } + ); + }); + } + }, + generate: ({ range, clients: { apmEsClient } }) => { + const successfulTimestamps = range.ratePerMinute(6); + const instance = apm + .service({ + name: `java${isLegacy ? '-legacy' : ''}`, + environment: 'production', + agentName: 'java', + }) + .instance(`instance`); + + return withClient( + apmEsClient, + successfulTimestamps.generator((timestamp) => { + const randomHigh = random(1000, 4000); + const randomLow = random(100, randomHigh / 5); + const duration = random(randomLow, randomHigh); + return instance + .transaction({ transactionName: 'GET /order/{id}' }) + .timestamp(timestamp) + .duration(duration) + .success(); + }) + ); + }, + }; +}; + +export default scenario; diff --git a/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts b/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts index 02258283ac554..97b8f17b3712b 100644 --- a/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts +++ b/x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts @@ -10,8 +10,22 @@ import { ApmDocumentType } from '../../../common/document_type'; import { RollupInterval } from '../../../common/rollup'; import { APMEventClient } from './create_es_client/create_apm_event_client'; import { getConfigForDocumentType } from './create_es_client/document_type'; -import { TRANSACTION_DURATION_SUMMARY } from '../../../common/es_fields/apm'; import { TimeRangeMetadata } from '../../../common/time_range_metadata'; +import { getDurationLegacyFilter } from './transactions'; + +const QUERY_INDEX = { + BEFORE: 0, + CURRENT: 1, + DURATION_SUMMARY: 2, +} as const; + +interface DocumentTypeData { + documentType: ApmDocumentType; + rollupInterval: RollupInterval; + hasDocBefore: boolean; + hasDocAfter: boolean; + allHaveDurationSummary: boolean; +} const getRequest = ({ documentType, @@ -64,129 +78,156 @@ export async function getDocumentSources({ kuery: string; enableServiceTransactionMetrics: boolean; enableContinuousRollups: boolean; -}) { - const currentRange = rangeQuery(start, end); - const diff = end - start; - const kql = kqlQuery(kuery); - const beforeRange = rangeQuery(start - diff, end - diff); - - const sourcesToCheck = [ +}): Promise { + const documentTypesToCheck = [ ...(enableServiceTransactionMetrics ? [ApmDocumentType.ServiceTransactionMetric as const] : []), ApmDocumentType.TransactionMetric as const, - ].flatMap((documentType) => { - const docTypeConfig = getConfigForDocumentType(documentType); - - return ( - enableContinuousRollups - ? docTypeConfig.rollupIntervals - : [RollupInterval.OneMinute] - ).flatMap((rollupInterval) => { - return { - documentType, - rollupInterval, - meta: { - checkSummaryFieldExists: false, - }, - before: getRequest({ - documentType, - rollupInterval, - filters: [...kql, ...beforeRange], - }), - current: getRequest({ - documentType, - rollupInterval, - filters: [...kql, ...currentRange], - }), - }; - }); + ]; + + const documentTypesInfo = await getDocumentTypesInfo({ + apmEventClient, + start, + end, + kuery, + enableContinuousRollups, + documentTypesToCheck, }); - const sourcesToCheckWithSummary = [ - ApmDocumentType.TransactionMetric as const, - ].flatMap((documentType) => { - const docTypeConfig = getConfigForDocumentType(documentType); - - return ( - enableContinuousRollups - ? docTypeConfig.rollupIntervals - : [RollupInterval.OneMinute] - ).flatMap((rollupInterval) => { - const summaryExistsFilter = { - bool: { - filter: [ - { - exists: { - field: TRANSACTION_DURATION_SUMMARY, - }, - }, - ], - }, - }; + const hasAnySourceDocBefore = documentTypesInfo.some( + (source) => source.hasDocBefore + ); - return { - documentType, - rollupInterval, - meta: { - checkSummaryFieldExists: true, - }, - before: getRequest({ - documentType, - rollupInterval, - filters: [...kql, ...beforeRange, summaryExistsFilter], - }), - current: getRequest({ - documentType, - rollupInterval, - filters: [...kql, ...currentRange, summaryExistsFilter], - }), - }; - }); + return [ + ...mapToSources(documentTypesInfo, hasAnySourceDocBefore), + { + documentType: ApmDocumentType.TransactionEvent, + rollupInterval: RollupInterval.None, + hasDocs: true, + hasDurationSummaryField: false, + }, + ]; +} + +const getDocumentTypesInfo = async ({ + apmEventClient, + start, + end, + kuery, + enableContinuousRollups, + documentTypesToCheck, +}: { + apmEventClient: APMEventClient; + start: number; + end: number; + kuery: string; + enableContinuousRollups: boolean; + documentTypesToCheck: ApmDocumentType[]; +}) => { + const getRequests = getDocumentTypeRequestsFn({ + enableContinuousRollups, + start, + end, + kuery, }); - const allSourcesToCheck = [...sourcesToCheck, ...sourcesToCheckWithSummary]; + const sourceRequests = documentTypesToCheck.flatMap(getRequests); - const allSearches = allSourcesToCheck.flatMap(({ before, current }) => [ - before, - current, - ]); + const allSearches = sourceRequests + .flatMap(({ before, current, durationSummaryCheck }) => [ + before, + current, + durationSummaryCheck, + ]) + .filter( + (request): request is ReturnType => + request !== undefined + ); const allResponses = ( await apmEventClient.msearch('get_document_availability', ...allSearches) ).responses; - const checkedSources = allSourcesToCheck.map((source, index) => { - const { documentType, rollupInterval } = source; - const responseBefore = allResponses[index * 2]; - const responseAfter = allResponses[index * 2 + 1]; - - const hasDocBefore = responseBefore.hits.total.value > 0; - const hasDocAfter = responseAfter.hits.total.value > 0; + return sourceRequests.map(({ documentType, rollupInterval, ...queries }) => { + const numberOfQueries = Object.values(queries).filter(Boolean).length; + // allResponses is sorted by the order of the requests in sourceRequests + const docTypeResponses = allResponses.splice(0, numberOfQueries); return { documentType, rollupInterval, - hasDocBefore, - hasDocAfter, - checkSummaryFieldExists: source.meta.checkSummaryFieldExists, + hasDocBefore: docTypeResponses[QUERY_INDEX.BEFORE].hits.total.value > 0, + hasDocAfter: docTypeResponses[QUERY_INDEX.CURRENT].hits.total.value > 0, + allHaveDurationSummary: docTypeResponses[QUERY_INDEX.DURATION_SUMMARY] + ? docTypeResponses[QUERY_INDEX.DURATION_SUMMARY].hits.total.value === 0 + : true, }; }); +}; - const hasAnySourceDocBefore = checkedSources.some( - (source) => source.hasDocBefore - ); +const getDocumentTypeRequestsFn = + ({ + enableContinuousRollups, + start, + end, + kuery, + }: { + enableContinuousRollups: boolean; + start: number; + end: number; + kuery: string; + }) => + (documentType: ApmDocumentType) => { + const currentRange = rangeQuery(start, end); + const diff = end - start; + const kql = kqlQuery(kuery); + const beforeRange = rangeQuery(start - diff, end - diff); + + const rollupIntervals = enableContinuousRollups + ? getConfigForDocumentType(documentType).rollupIntervals + : [RollupInterval.OneMinute]; + + return rollupIntervals.map((rollupInterval) => ({ + documentType, + rollupInterval, + before: getRequest({ + documentType, + rollupInterval, + filters: [...kql, ...beforeRange], + }), + current: getRequest({ + documentType, + rollupInterval, + filters: [...kql, ...currentRange], + }), + ...(documentType !== ApmDocumentType.ServiceTransactionMetric + ? { + durationSummaryCheck: getRequest({ + documentType, + rollupInterval, + filters: [...kql, ...currentRange, getDurationLegacyFilter()], + }), + } + : {}), + })); + }; - const sourcesWithHasDocs = checkedSources.map((checkedSource) => { +const mapToSources = ( + sources: DocumentTypeData[], + hasAnySourceDocBefore: boolean +) => { + return sources.map((source) => { const { documentType, hasDocAfter, hasDocBefore, rollupInterval, - checkSummaryFieldExists, - } = checkedSource; + allHaveDurationSummary, + } = source; const hasDocBeforeOrAfter = hasDocBefore || hasDocAfter; + // If there is any data before, we require that data is available before // this time range to mark this source as available. If we don't do that, // users that upgrade to a version that starts generating service tx metrics @@ -194,40 +235,12 @@ export async function getDocumentSources({ // If we only check before, users with a new deployment will use raw transaction // events. const hasDocs = hasAnySourceDocBefore ? hasDocBefore : hasDocBeforeOrAfter; + return { documentType, rollupInterval, - checkSummaryFieldExists, hasDocs, + hasDurationSummaryField: allHaveDurationSummary, }; }); - - const sources: TimeRangeMetadata['sources'] = sourcesWithHasDocs - .filter((source) => !source.checkSummaryFieldExists) - .map((checkedSource) => { - const { documentType, hasDocs, rollupInterval } = checkedSource; - return { - documentType, - rollupInterval, - hasDocs, - hasDurationSummaryField: - documentType === ApmDocumentType.ServiceTransactionMetric || - Boolean( - sourcesWithHasDocs.find((eSource) => { - return ( - eSource.documentType === documentType && - eSource.rollupInterval === rollupInterval && - eSource.checkSummaryFieldExists - ); - })?.hasDocs - ), - }; - }); - - return sources.concat({ - documentType: ApmDocumentType.TransactionEvent, - rollupInterval: RollupInterval.None, - hasDocs: true, - hasDurationSummaryField: false, - }); -} +}; diff --git a/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts b/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts index 182d2419c2543..dfe449e5fa635 100644 --- a/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/transactions/index.ts @@ -7,6 +7,7 @@ import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { SearchAggregatedTransactionSetting } from '../../../../common/aggregated_transactions'; import { TRANSACTION_DURATION, @@ -90,13 +91,12 @@ export async function getSearchTransactionsEvents({ } } -export function getDurationFieldForTransactions( +export function isSummaryFieldSupportedByDocType( typeOrSearchAgggregatedTransactions: | ApmDocumentType.ServiceTransactionMetric | ApmDocumentType.TransactionMetric | ApmDocumentType.TransactionEvent - | boolean, - useDurationSummaryField?: boolean + | boolean ) { let type: ApmDocumentType; @@ -108,10 +108,20 @@ export function getDurationFieldForTransactions( type = typeOrSearchAgggregatedTransactions; } - if ( + return ( type === ApmDocumentType.ServiceTransactionMetric || type === ApmDocumentType.TransactionMetric - ) { + ); +} +export function getDurationFieldForTransactions( + typeOrSearchAgggregatedTransactions: + | ApmDocumentType.ServiceTransactionMetric + | ApmDocumentType.TransactionMetric + | ApmDocumentType.TransactionEvent + | boolean, + useDurationSummaryField?: boolean +) { + if (isSummaryFieldSupportedByDocType(typeOrSearchAgggregatedTransactions)) { if (useDurationSummaryField) { return TRANSACTION_DURATION_SUMMARY; } @@ -163,3 +173,18 @@ export function isRootTransaction(searchAggregatedTransactions: boolean) { }, }; } + +export function getDurationLegacyFilter(): QueryDslQueryContainer { + return { + bool: { + must: [ + { + bool: { + filter: [{ exists: { field: TRANSACTION_DURATION_HISTOGRAM } }], + must_not: [{ exists: { field: TRANSACTION_DURATION_SUMMARY } }], + }, + }, + ], + }, + }; +} diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts index 6362b4a3c5e5a..7fbeb1cc9d24c 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts @@ -189,6 +189,6 @@ export async function getServiceTransactionStats({ }; }) ?? [], serviceOverflowCount: - response.aggregations?.sample?.overflowCount.value || 0, + response.aggregations?.sample?.overflowCount?.value || 0, }; } diff --git a/x-pack/test/apm_api_integration/tests/time_range_metadata/many_apm_server_versions.spec.ts b/x-pack/test/apm_api_integration/tests/time_range_metadata/many_apm_server_versions.spec.ts index 7f91aaf93d804..012fa07ca6f6c 100644 --- a/x-pack/test/apm_api_integration/tests/time_range_metadata/many_apm_server_versions.spec.ts +++ b/x-pack/test/apm_api_integration/tests/time_range_metadata/many_apm_server_versions.spec.ts @@ -112,23 +112,48 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, }); - const allHasSummaryField = response.body.sources.every((source) => { - if (source.documentType === 'transactionEvent') { - return true; - } + const allHasSummaryField = response.body.sources + .filter( + (source) => + source.documentType === ApmDocumentType.TransactionMetric && + source.rollupInterval !== RollupInterval.OneMinute + ) + .every((source) => { + return source.hasDurationSummaryField; + }); + expect(allHasSummaryField).to.eql(true); + }); + + it('does not support transaction.duration.summary when the field is not supported by all APM server versions', async () => { + const response = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/time_range_metadata', + params: { + query: { + start: startLegacy.toISOString(), + end: end.toISOString(), + enableContinuousRollups: true, + enableServiceTransactionMetrics: true, + useSpanName: false, + kuery: '', + }, + }, + }); + + const allHasSummaryField = response.body.sources.every((source) => { return source.hasDurationSummaryField; }); - expect(allHasSummaryField).to.eql(true); + expect(allHasSummaryField).to.eql(false); }); - it('does not have latency data for synth-java-legacy service', async () => { + it('does not have latency data for synth-java-legacy', async () => { const res = await getLatencyChartForService({ serviceName: 'synth-java-legacy', start, end, apmApiClient, + useDurationSummary: true, }); expect(res.body.currentPeriod.latencyTimeseries.map(({ y }) => y)).to.eql([ @@ -147,6 +172,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { start, end, apmApiClient, + useDurationSummary: true, }); expect(res.body.currentPeriod.latencyTimeseries.map(({ y }) => y)).to.eql([ @@ -169,11 +195,13 @@ function getLatencyChartForService({ start, end, apmApiClient, + useDurationSummary, }: { serviceName: string; start: moment.Moment; end: moment.Moment; apmApiClient: ApmApiClient; + useDurationSummary: boolean; }) { return apmApiClient.readUser({ endpoint: `GET /internal/apm/services/{serviceName}/transactions/charts/latency`, @@ -189,7 +217,7 @@ function getLatencyChartForService({ documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, bucketSizeInSeconds: 60, - useDurationSummary: true, + useDurationSummary, }, }, }); @@ -227,8 +255,9 @@ function generateTraceDataForService({ ); const apmPipeline = (base: Readable) => { - // @ts-expect-error - const defaultPipeline: NodeJS.ReadableStream = synthtrace.getDefaultPipeline()(base); + const defaultPipeline = synthtrace.getDefaultPipeline()( + base + ) as unknown as NodeJS.ReadableStream; return pipeline( defaultPipeline, diff --git a/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts b/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts index 32303b7b0bb4c..6de17c22c2309 100644 --- a/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts +++ b/x-pack/test/apm_api_integration/tests/time_range_metadata/time_range_metadata.spec.ts @@ -11,8 +11,13 @@ import { omit, sortBy } from 'lodash'; import moment, { Moment } from 'moment'; import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; -import { deleteSummaryFieldTransform } from '@kbn/apm-synthtrace'; +import { + addObserverVersionTransform, + ApmSynthtraceEsClient, + deleteSummaryFieldTransform, +} from '@kbn/apm-synthtrace'; import { Readable, pipeline } from 'stream'; +import { ToolingLog } from '@kbn/tooling-log'; import { FtrProviderContext } from '../../common/ftr_provider_context'; export default function ApiTest({ getService }: FtrProviderContext) { @@ -85,27 +90,21 @@ export default function ApiTest({ getService }: FtrProviderContext) { const withSummaryFieldEnd = moment(withoutSummaryFieldEnd).add(2, 'hours'); before(async () => { - const previousTxEvents = getTransactionEvents( - withoutSummaryFieldStart, - withoutSummaryFieldEnd - ); - - const apmPipeline = (base: Readable) => { - // @ts-expect-error - const defaultPipeline: NodeJS.ReadableStream = - synthtraceEsClient.getDefaultPipeline()(base); - - return pipeline(defaultPipeline, deleteSummaryFieldTransform(), (err) => { - if (err) { - log.error(err); - } - }); - }; - - await synthtraceEsClient.index(previousTxEvents, apmPipeline); + await getTransactionEvents({ + start: withoutSummaryFieldStart, + end: withoutSummaryFieldEnd, + isLegacy: true, + synthtrace: synthtraceEsClient, + logger: log, + }); - const txEvents = getTransactionEvents(withSummaryFieldStart, withSummaryFieldEnd); - await synthtraceEsClient.index(txEvents); + await getTransactionEvents({ + start: withSummaryFieldStart, + end: withSummaryFieldEnd, + isLegacy: false, + synthtrace: synthtraceEsClient, + logger: log, + }); }); after(() => { @@ -123,7 +122,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { response.sources.filter( (source) => source.documentType === ApmDocumentType.TransactionMetric && - source.hasDurationSummaryField === true + source.hasDurationSummaryField ).length ).to.eql(3); }); @@ -138,9 +137,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { response.sources.filter( (source) => source.documentType === ApmDocumentType.TransactionMetric && - source.hasDurationSummaryField === false + !source.hasDurationSummaryField ).length - ).to.eql(3); + ).to.eql(2); }); }); }); @@ -494,7 +493,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { documentType: ApmDocumentType.TransactionMetric, rollupInterval: RollupInterval.OneMinute, hasDocs: false, - hasDurationSummaryField: false, + hasDurationSummaryField: true, }, { documentType: ApmDocumentType.TransactionMetric, @@ -513,7 +512,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); } -function getTransactionEvents(start: Moment, end: Moment) { +function getTransactionEvents({ + start, + end, + synthtrace, + logger, + isLegacy = false, +}: { + start: Moment; + end: Moment; + synthtrace: ApmSynthtraceEsClient; + logger: ToolingLog; + isLegacy?: boolean; +}) { const serviceName = 'synth-go'; const transactionName = 'GET /api/product/list'; const GO_PROD_RATE = 15; @@ -523,7 +534,7 @@ function getTransactionEvents(start: Moment, end: Moment) { .service({ name: serviceName, environment: 'production', agentName: 'go' }) .instance('instance-a'); - return [ + const events = [ timerange(start, end) .interval('1m') .rate(GO_PROD_RATE) @@ -546,4 +557,23 @@ function getTransactionEvents(start: Moment, end: Moment) { .failure() ), ]; + + const apmPipeline = (base: Readable) => { + const defaultPipeline = synthtrace.getDefaultPipeline()( + base + ) as unknown as NodeJS.ReadableStream; + + return pipeline( + defaultPipeline, + addObserverVersionTransform('8.5.0'), + deleteSummaryFieldTransform(), + (err) => { + if (err) { + logger.error(err); + } + } + ); + }; + + return synthtrace.index(events, isLegacy ? apmPipeline : undefined); }