[APM] Improve logic to determine when to use the transaction.duration.summary field

packages/kbn-apm-synthtrace/src/scenarios/service_summary_field_version_dependent.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ApmFields, apm } from '@kbn/apm-synthtrace-client';
+import { random } from 'lodash';
+import { pipeline, Readable } from 'stream';
+import semver from 'semver';
+import { Scenario } from '../cli/scenario';
+import { deleteSummaryFieldTransform } from '../lib/utils/transform_helpers';
+import { withClient } from '../lib/utils/with_client';
+
+const scenario: Scenario<ApmFields> = async ({ logger, versionOverride }) => {
+  const isLegacy = versionOverride && semver.lt(versionOverride, '8.7.0');
+  return {
+    bootstrap: async ({ apmEsClient }) => {
+      if (isLegacy) {
+        apmEsClient.pipeline((base: Readable) => {
+          const defaultPipeline = apmEsClient.getDefaultPipeline()(
+            base
+          ) as unknown as NodeJS.ReadableStream;
+
+          return pipeline(defaultPipeline, deleteSummaryFieldTransform(), (err) => {
+            if (err) {
+              logger.error(err);
+            }
+          });
+        });
+      }
+    },
+    generate: ({ range, clients: { apmEsClient } }) => {
+      const successfulTimestamps = range.ratePerMinute(6);
+      const instance = apm
+        .service({
+          name: `java${isLegacy ? '-legacy' : ''}`,
+          environment: 'production',
+          agentName: 'java',
+        })
+        .instance(`instance`);
+
+      return withClient(
+        apmEsClient,
+        successfulTimestamps.generator((timestamp) => {
+          const randomHigh = random(1000, 4000);
+          const randomLow = random(100, randomHigh / 5);
+          const duration = random(randomLow, randomHigh);
+          return instance
+            .transaction({ transactionName: 'GET /order/{id}' })
+            .timestamp(timestamp)
+            .duration(duration)
+            .success();
+        })
+      );
+    },
+  };
+};
+
+export default scenario;

x-pack/plugins/apm/public/context/time_range_metadata/time_range_metadata_context.tsx

@@ -16,6 +16,7 @@ import { useApmRoutePath } from '../../hooks/use_apm_route_path';
 import { FetcherResult, useFetcher } from '../../hooks/use_fetcher';
 import { useTimeRange } from '../../hooks/use_time_range';
 import { useApmPluginContext } from '../apm_plugin/use_apm_plugin_context';
+import { useEnvironmentsContext } from '../environments_context/use_environments_context';
 
 export const TimeRangeMetadataContext = createContext<
   FetcherResult<TimeRangeMetadata> | undefined
@@ -30,9 +31,11 @@ export function ApmTimeRangeMetadataContextProvider({
     core: { uiSettings },
   } = useApmPluginContext();
 
-  const { query } = useApmParams('/*');
+  const { query, path } = useApmParams('/*');
+  const { environment } = useEnvironmentsContext();
 
   const kuery = 'kuery' in query && query.kuery ? query.kuery : '';
+  const serviceName = 'serviceName' in path ? path.serviceName : undefined;
 
   const range =
     'rangeFrom' in query && 'rangeTo' in query
@@ -58,6 +61,8 @@ export function ApmTimeRangeMetadataContextProvider({
       start={start}
       end={end}
       kuery={kuery}
+      serviceName={serviceName}
+      environment={environment}
     >
       {children}
     </TimeRangeMetadataContextProvider>
@@ -71,13 +76,17 @@ export function TimeRangeMetadataContextProvider({
   start,
   end,
   kuery,
+  serviceName,
+  environment,
 }: {
   children?: React.ReactNode;
   uiSettings: IUiSettingsClient;
   useSpanName: boolean;
   start: string;
   end: string;
   kuery: string;
+  serviceName?: string;
+  environment?: string;
 }) {
   const enableServiceTransactionMetrics = uiSettings.get<boolean>(
     apmEnableServiceMetrics,
@@ -100,6 +109,8 @@ export function TimeRangeMetadataContextProvider({
             useSpanName,
             enableServiceTransactionMetrics,
             enableContinuousRollups,
+            serviceName,
+            environment,
           },
         },
       });
@@ -111,6 +122,8 @@ export function TimeRangeMetadataContextProvider({
       useSpanName,
       enableServiceTransactionMetrics,
       enableContinuousRollups,
+      serviceName,
+      environment,
     ]
   );
 

x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts

@@ -6,12 +6,21 @@
  */
 import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { Environment } from '../../../common/environment_rt';
 import { ApmDocumentType } from '../../../common/document_type';
 import { RollupInterval } from '../../../common/rollup';
 import { APMEventClient } from './create_es_client/create_apm_event_client';
 import { getConfigForDocumentType } from './create_es_client/document_type';
-import { TRANSACTION_DURATION_SUMMARY } from '../../../common/es_fields/apm';
 import { TimeRangeMetadata } from '../../../common/time_range_metadata';
+import {
+  getDurationSummaryFilter,
+  getDurationLegacyFilter,
+} from './transactions';
+import {
+  SERVICE_ENVIRONMENT,
+  SERVICE_NAME,
+} from '../../../common/es_fields/apm';
 
 const getRequest = ({
   documentType,
@@ -57,25 +66,50 @@ export async function getDocumentSources({
   kuery,
   enableServiceTransactionMetrics,
   enableContinuousRollups,
+  environment,
+  serviceName,
 }: {
   apmEventClient: APMEventClient;
   start: number;
   end: number;
   kuery: string;
   enableServiceTransactionMetrics: boolean;
   enableContinuousRollups: boolean;
+  environment?: Environment;
+  serviceName?: string;
 }) {
   const currentRange = rangeQuery(start, end);
   const diff = end - start;
   const kql = kqlQuery(kuery);
   const beforeRange = rangeQuery(start - diff, end - diff);
+  const serviceFilter: QueryDslQueryContainer[] | undefined = serviceName
+    ? [
+        {
+          term: {
+            [SERVICE_NAME]: serviceName,
+          },
+        },
+        ...(environment &&
+        environment !== 'ENVIRONMENT_ALL' &&
+        environment !== 'ENVIRONMENT_NOT_DEFINED'
+          ? [
+              {
+                term: {
+                  [SERVICE_ENVIRONMENT]: environment,
+                },
+              },
+            ]
+          : []),
+      ]
+    : undefined;
 
-  const sourcesToCheck = [
+  const documentTypesToCheck = [
     ...(enableServiceTransactionMetrics
       ? [ApmDocumentType.ServiceTransactionMetric as const]
       : []),
     ApmDocumentType.TransactionMetric as const,
-  ].flatMap((documentType) => {
+  ];
+  const sourcesToCheck = documentTypesToCheck.flatMap((documentType) => {
     const docTypeConfig = getConfigForDocumentType(documentType);
 
     return (
@@ -103,47 +137,43 @@ export async function getDocumentSources({
     });
   });
 
-  const sourcesToCheckWithSummary = [
-    ApmDocumentType.TransactionMetric as const,
-  ].flatMap((documentType) => {
-    const docTypeConfig = getConfigForDocumentType(documentType);
+  const sourcesToCheckWithSummary = documentTypesToCheck.flatMap(
+    (documentType) => {
+      const docTypeConfig = getConfigForDocumentType(documentType);
 
-    return (
-      enableContinuousRollups
-        ? docTypeConfig.rollupIntervals
-        : [RollupInterval.OneMinute]
-    ).flatMap((rollupInterval) => {
-      const summaryExistsFilter = {
-        bool: {
-          filter: [
-            {
-              exists: {
-                field: TRANSACTION_DURATION_SUMMARY,
-              },
-            },
-          ],
-        },
-      };
-
-      return {
-        documentType,
-        rollupInterval,
-        meta: {
-          checkSummaryFieldExists: true,
-        },
-        before: getRequest({
+      return (
+        enableContinuousRollups
+          ? docTypeConfig.rollupIntervals
+          : [RollupInterval.OneMinute]
+      ).flatMap((rollupInterval) => {
+        return {
           documentType,
           rollupInterval,
-          filters: [...kql, ...beforeRange, summaryExistsFilter],
-        }),
-        current: getRequest({
-          documentType,
-          rollupInterval,
-          filters: [...kql, ...currentRange, summaryExistsFilter],
-        }),
-      };
-    });
-  });
+          meta: {
+            checkSummaryFieldExists: true,
+          },
+          before: getRequest({
+            documentType,
+            rollupInterval,
+            filters: [
+              ...kql,
+              ...currentRange,
+              getDurationLegacyFilter(serviceFilter),
+            ],
+          }),
+          current: getRequest({
+            documentType,
+            rollupInterval,
+            filters: [
+              ...kql,
+              ...currentRange,
+              getDurationSummaryFilter(serviceFilter),
+            ],
+          }),
+        };
+      });
+    }
+  );
 
   const allSourcesToCheck = [...sourcesToCheck, ...sourcesToCheckWithSummary];
 
@@ -187,40 +217,43 @@ export async function getDocumentSources({
     } = checkedSource;
 
     const hasDocBeforeOrAfter = hasDocBefore || hasDocAfter;
+
     // If there is any data before, we require that data is available before
     // this time range to mark this source as available. If we don't do that,
     // users that upgrade to a version that starts generating service tx metrics
     // will see a mostly empty screen for a while after upgrading.
     // If we only check before, users with a new deployment will use raw transaction
     // events.
     const hasDocs = hasAnySourceDocBefore ? hasDocBefore : hasDocBeforeOrAfter;
+    const hasOnlyDocsWithSummary = hasDocBefore ? false : hasDocAfter;
+
     return {
       documentType,
       rollupInterval,
       checkSummaryFieldExists,
-      hasDocs,
+      hasDocs: checkSummaryFieldExists ? hasOnlyDocsWithSummary : hasDocs,
     };
   });
 
   const sources: TimeRangeMetadata['sources'] = sourcesWithHasDocs
     .filter((source) => !source.checkSummaryFieldExists)
     .map((checkedSource) => {
       const { documentType, hasDocs, rollupInterval } = checkedSource;
+
+      const hasDurationSummaryField = sourcesWithHasDocs.some((eSource) => {
+        return (
+          eSource.documentType === documentType &&
+          eSource.rollupInterval === rollupInterval &&
+          eSource.checkSummaryFieldExists &&
+          eSource.hasDocs
+        );
+      });
+
       return {
         documentType,
         rollupInterval,
         hasDocs,
-        hasDurationSummaryField:
-          documentType === ApmDocumentType.ServiceTransactionMetric ||
-          Boolean(
-            sourcesWithHasDocs.find((eSource) => {
-              return (
-                eSource.documentType === documentType &&
-                eSource.rollupInterval === rollupInterval &&
-                eSource.checkSummaryFieldExists
-              );
-            })?.hasDocs
-          ),
+        hasDurationSummaryField,
       };
     });