[APM] Modify has_any_service telemetry (#166925)

# Summary fixes elastic/observability-bi#65 ## Context Observability-bi uses the [signal](https://docs.elastic.dev/observability-bi/observability-marker#signals) `has_apm_services` based on the telemetry field `has_any_services` to determine whether a cluster has documents for at least one APM service for the given day. ## Problem Previously we determined if the cluster has any service by checking ONLY the officially supported agents. There might be cases where the cluster has agents that are not on the agent's list. Leading to the wrong result. Modified to check if there is any document with the field `service.name` and added `has_any_services_per_official_agent` to track the official agent's services ### Side effects Modifying `has_any_service` telemetry (most likely) will increase the metrics using this field. --------- Co-authored-by: Achyut Jhunjhunwala <[email protected]>
elastic · Oct 13, 2023 · c83f5db · c83f5db
1 parent bd3c83e
commit c83f5db
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 4 deletions.
diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap
diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts
@@ -678,8 +678,37 @@ export const tasks: TelemetryTask[] = [
         Promise.resolve({} as Record<AgentName, number>)
       );
 
+      const services = await telemetryClient.search({
+        index: [
+          indices.error,
+          indices.span,
+          indices.metric,
+          indices.transaction,
+        ],
+        body: {
+          size: 0,
+          track_total_hits: true,
+          terminate_after: 1,
+          query: {
+            bool: {
+              filter: [
+                {
+                  exists: {
+                    field: SERVICE_NAME,
+                  },
+                },
+                range1d,
+              ],
+            },
+          },
+          timeout,
+        },
+      });
+
       return {
-        has_any_services: sum(Object.values(servicesPerAgent)) > 0,
+        has_any_services_per_official_agent:
+          sum(Object.values(servicesPerAgent)) > 0,
+        has_any_services: services?.hits?.total?.value > 0,
         services_per_agent: servicesPerAgent,
       };
     },

diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts
@@ -567,11 +567,18 @@ export const apmPerServiceSchema: MakeSchemaFrom<APMPerService, true> = {
 
 export const apmSchema: MakeSchemaFrom<APMUsage, true> = {
   ...apmPerAgentSchema,
+  has_any_services_per_official_agent: {
+    type: 'boolean',
+    _meta: {
+      description:
+        'Indicates whether any service is being monitored. This is determined by checking all officially supported agents within the last day',
+    },
+  },
   has_any_services: {
     type: 'boolean',
     _meta: {
       description:
-        'Indicates whether any service is being monitored. This is determined by checking all agents within the last day',
+        'Indicates whether any service is being monitored within the last day.',
     },
   },
   version: {

diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts
@@ -70,6 +70,7 @@ export interface APMPerService {
 }
 
 export interface APMUsage {
+  has_any_services_per_official_agent: boolean;
   has_any_services: boolean;
   services_per_agent: Record<AgentName, number>;
   version: {

diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
@@ -4230,10 +4230,16 @@
             }
           }
         },
+        "has_any_services_per_official_agent": {
+          "type": "boolean",
+          "_meta": {
+            "description": "Indicates whether any service is being monitored. This is determined by checking all officially supported agents within the last day"
+          }
+        },
         "has_any_services": {
           "type": "boolean",
           "_meta": {
-            "description": "Indicates whether any service is being monitored. This is determined by checking all agents within the last day"
+            "description": "Indicates whether any service is being monitored within the last day."
           }
         },
         "version": {