[8.x] [Logs Data Telemetry] Create background job to collect and send…

… logs data telemetry (#189380) (#193877) # Backport This will backport the following commits from `main` to `8.x`: - [[Logs Data Telemetry] Create background job to collect and send logs data telemetry (#189380)](#189380)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Abdul Wahab Zahid <[email protected]>
elastic · Sep 24, 2024 · 8d66d6c · 8d66d6c
1 parent 5624f8f
commit 8d66d6c
Show file tree

Hide file tree

Showing 12 changed files with 1,848 additions and 6 deletions.
diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/constants.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/constants.ts
@@ -55,6 +55,7 @@ export const DATA_DATASETS_INDEX_PATTERNS = [
   { pattern: 'telegraf*', patternName: 'telegraf' },
   { pattern: 'prometheusbeat*', patternName: 'prometheusbeat' },
   { pattern: 'fluentbit*', patternName: 'fluentbit' },
+  { pattern: 'fluent-bit*', patternName: 'fluentbit' },
   { pattern: '*nginx*', patternName: 'nginx' },
   { pattern: '*apache*', patternName: 'apache' }, // Already in Security (keeping it in here for documentation)
   { pattern: '*logs*', patternName: 'generic-logs' },

diff --git a/x-pack/plugins/observability_solution/dataset_quality/kibana.jsonc b/x-pack/plugins/observability_solution/dataset_quality/kibana.jsonc
@@ -23,9 +23,10 @@
       "fieldFormats",
       "dataViews",
       "lens",
-      "fieldsMetadata"
+      "fieldsMetadata",
+      "taskManager"
     ],
-    "optionalPlugins": [],
+    "optionalPlugins": ["telemetry"],
     "requiredBundles": ["unifiedHistogram", "discover"],
     "extraPublicDirs": [
       "common"

diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/plugin.ts b/x-pack/plugins/observability_solution/dataset_quality/server/plugin.ts
@@ -5,8 +5,9 @@
  * 2.0.
  */
 
-import { CoreSetup, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server';
+import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server';
 import { mapValues } from 'lodash';
+import { DataTelemetryService } from './services';
 import { getDatasetQualityServerRouteRepository } from './routes';
 import { registerRoutes } from './routes/register_routes';
 import { DatasetQualityRouteHandlerResources } from './routes/types';
@@ -18,9 +19,11 @@ import {
 
 export class DatasetQualityServerPlugin implements Plugin {
   private readonly logger: Logger;
+  private readonly dataTelemetryService: DataTelemetryService;
 
   constructor(initializerContext: PluginInitializerContext) {
     this.logger = initializerContext.logger.get();
+    this.dataTelemetryService = new DataTelemetryService(this.logger);
   }
 
   setup(
@@ -53,10 +56,18 @@ export class DatasetQualityServerPlugin implements Plugin {
       getEsCapabilities,
     });
 
+    // Setup Data Telemetry Service
+    this.dataTelemetryService.setup(core.analytics, plugins.taskManager);
+
     return {};
   }
 
-  start() {
+  start(core: CoreStart, plugins: DatasetQualityPluginStartDependencies) {
+    // Start Data Telemetry Service
+    this.dataTelemetryService.start(plugins.telemetry, core, plugins.taskManager).catch((error) => {
+      this.logger.error(`[Data Telemetry Service]: ${error}`);
+    });
+
     return {};
   }
 }
diff --git a/...lugins/observability_solution/dataset_quality/server/services/data_telemetry/constants.ts b/...lugins/observability_solution/dataset_quality/server/services/data_telemetry/constants.ts
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from '@kbn/telemetry-plugin/server/telemetry_collection/get_data_telemetry/constants';
+
+import { DatasetIndexPattern } from './types';
+
+export const LOGS_DATA_TELEMETRY_TASK_TYPE = 'logs-data-telemetry';
+export const LOGS_DATA_TELEMETRY_TASK_ID = 'logs-data-telemetry:collect-and-report-task-2';
+
+export const TELEMETRY_TASK_INTERVAL = 24 * 60; // 24 hours (in minutes)
+export const TELEMETRY_TASK_TIMEOUT = 10; // 10 minutes
+
+export const BREATHE_DELAY_SHORT = 1000; // 1 seconds
+export const BREATHE_DELAY_MEDIUM = 5 * 1000; // 5 seconds
+
+export const MAX_STREAMS_TO_REPORT = 1000;
+
+export const NON_LOG_SIGNALS = ['metrics', 'traces', 'internal', 'synthetics'];
+export const EXCLUDE_ELASTIC_LOGS = ['logs-synth', 'logs-elastic', 'logs-endpoint'];
+
+export const TELEMETRY_CHANNEL = 'logs-data-telemetry';
+
+const LOGS_INDEX_PATTERN_NAMES = [
+  'filebeat',
+  'generic-filebeat',
+  'metricbeat',
+  'generic-metricbeat',
+  'apm',
+  'functionbeat',
+  'generic-functionbeat',
+  'heartbeat',
+  'generic-heartbeat',
+  'logstash',
+  'generic-logstash',
+  'fluentd',
+  'telegraf',
+  'prometheusbeat',
+  'fluentbit',
+  'nginx',
+  'apache',
+  'generic-logs',
+];
+
+const TELEMETRY_PATTERNS_BY_NAME = DATA_DATASETS_INDEX_PATTERNS_UNIQUE.reduce((acc, pattern) => {
+  acc[pattern.patternName] = [pattern, ...(acc[pattern.patternName] || [])];
+  return acc;
+}, {} as Record<string, DatasetIndexPattern[]>);
+
+export const LOGS_DATASET_INDEX_PATTERNS = LOGS_INDEX_PATTERN_NAMES.flatMap<DatasetIndexPattern>(
+  (patternName) => TELEMETRY_PATTERNS_BY_NAME[patternName] || []
+);
+
+export const LEVEL_2_RESOURCE_FIELDS = [
+  'host.name',
+  'service.name',
+  'host',
+  'hostname',
+  'host_name',
+];
+
+export const PROMINENT_LOG_ECS_FIELDS = [
+  'log.level',
+  'log.logger',
+  'log.origin.file.name',
+  'log.origin.function',
+  'log.origin.file.line',
+  'event.action',
+  'event.category',
+  'event.dataset',
+  'event.kind',
+  'log.file.path',
+];
+
+export const DATA_TELEMETRY_FIELDS = [
+  'container.id',
+  'log.level',
+  'container.name',
+  'host.name',
+  'host.hostname',
+  'kubernetes.pod.name',
+  'kubernetes.pod.uid',
+  'cloud.provider',
+  'agent.type',
+  'event.dataset',
+  'event.category',
+  'event.module',
+  'service.name',
+  'service.type',
+  'service.version',
+  'message',
+  'event.original',
+  'error.message',
+  '@timestamp',
+  'data_stream.dataset',
+  'data_stream.namespace',
+  'data_stream.type',
+];