[Telemetry][Security Solution] Index metadata collector (elastic#194004)

## Summary Implements a security_solution task scheduled to run once a day to collect the following information: 1. Datastreams stats 2. Indices stats 3. ILMs stats 4. ILM configs The task allows a runtime configuration to limit the number of indices and data streams to analyze or event to disable the feature entirely. Once the data is gathered, the task sends it as EBT events. --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Elastic Machine <[email protected]>
kdelemme · Dec 13, 2024 · 36b344a · 36b344a
1 parent 2fd8994
commit 36b344a
Show file tree

Hide file tree

Showing 43 changed files with 3,087 additions and 136 deletions.
diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml
@@ -89,6 +89,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts

diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml
@@ -70,6 +70,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts

diff --git a/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts b/x-pack/plugins/security_solution/server/integration_tests/configuration.test.ts
@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import Path from 'path';
+import axios from 'axios';
+
+import { cloneDeep } from 'lodash';
+
+import { telemetryConfiguration } from '../lib/telemetry/configuration';
+import {
+  TaskManagerPlugin,
+  type TaskManagerStartContract,
+} from '@kbn/task-manager-plugin/server/plugin';
+
+import {
+  setupTestServers,
+  removeFile,
+  mockAxiosPost,
+  DEFAULT_GET_ROUTES,
+  mockAxiosGet,
+  getRandomInt,
+} from './lib/helpers';
+
+import {
+  type TestElasticsearchUtils,
+  type TestKibanaUtils,
+} from '@kbn/core-test-helpers-kbn-server';
+import { Plugin as SecuritySolutionPlugin } from '../plugin';
+import { getTelemetryTasks, runSoonConfigTask } from './lib/telemetry_helpers';
+import type { SecurityTelemetryTask } from '../lib/telemetry/task';
+
+jest.mock('axios');
+
+const logFilePath = Path.join(__dirname, 'config.logs.log');
+const taskManagerStartSpy = jest.spyOn(TaskManagerPlugin.prototype, 'start');
+const securitySolutionStartSpy = jest.spyOn(SecuritySolutionPlugin.prototype, 'start');
+
+const mockedAxiosGet = jest.spyOn(axios, 'get');
+const mockedAxiosPost = jest.spyOn(axios, 'post');
+
+const securitySolutionPlugin = jest.spyOn(SecuritySolutionPlugin.prototype, 'start');
+
+describe('configuration', () => {
+  let esServer: TestElasticsearchUtils;
+  let kibanaServer: TestKibanaUtils;
+  let taskManagerPlugin: TaskManagerStartContract;
+  let tasks: SecurityTelemetryTask[];
+
+  beforeAll(async () => {
+    await removeFile(logFilePath);
+
+    const servers = await setupTestServers(logFilePath);
+
+    esServer = servers.esServer;
+    kibanaServer = servers.kibanaServer;
+
+    expect(taskManagerStartSpy).toHaveBeenCalledTimes(1);
+    taskManagerPlugin = taskManagerStartSpy.mock.results[0].value;
+
+    expect(securitySolutionStartSpy).toHaveBeenCalledTimes(1);
+
+    tasks = getTelemetryTasks(securitySolutionStartSpy);
+
+    expect(securitySolutionPlugin).toHaveBeenCalledTimes(1);
+  });
+
+  afterAll(async () => {
+    if (kibanaServer) {
+      await kibanaServer.stop();
+    }
+    if (esServer) {
+      await esServer.stop();
+    }
+  });
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+    mockAxiosPost(mockedAxiosPost);
+  });
+
+  afterEach(async () => {});
+
+  describe('configuration task', () => {
+    it('should keep default values when no new config was provided', async () => {
+      const before = cloneDeep(telemetryConfiguration);
+
+      await runSoonConfigTask(tasks, taskManagerPlugin);
+
+      expect(telemetryConfiguration).toEqual(before);
+    });
+
+    it('should update values with new manifest', async () => {
+      const expected = {
+        telemetry_max_buffer_size: getRandomInt(1, 100),
+        max_security_list_telemetry_batch: getRandomInt(1, 100),
+        max_endpoint_telemetry_batch: getRandomInt(1, 100),
+        max_detection_rule_telemetry_batch: getRandomInt(1, 100),
+        max_detection_alerts_batch: getRandomInt(1, 100),
+        use_async_sender: true,
+        pagination_config: {
+          max_page_size_bytes: getRandomInt(1, 100),
+          num_docs_to_sample: getRandomInt(1, 100),
+        },
+        sender_channels: {
+          default: {
+            buffer_time_span_millis: getRandomInt(1, 100),
+            inflight_events_threshold: getRandomInt(1, 100),
+            max_payload_size_bytes: getRandomInt(1, 100),
+          },
+        },
+        indices_metadata_config: {
+          indices_threshold: getRandomInt(1, 100),
+          datastreams_threshold: getRandomInt(1, 100),
+          max_prefixes: getRandomInt(1, 100),
+          max_group_size: getRandomInt(1, 100),
+        },
+      };
+
+      mockAxiosGet(mockedAxiosGet, [
+        ...DEFAULT_GET_ROUTES,
+        [/.*telemetry-buffer-and-batch-sizes-v1.*/, { status: 200, data: cloneDeep(expected) }],
+      ]);
+
+      await runSoonConfigTask(tasks, taskManagerPlugin);
+
+      expect(telemetryConfiguration.telemetry_max_buffer_size).toEqual(
+        expected.telemetry_max_buffer_size
+      );
+      expect(telemetryConfiguration.max_security_list_telemetry_batch).toEqual(
+        expected.max_security_list_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_endpoint_telemetry_batch).toEqual(
+        expected.max_endpoint_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_detection_rule_telemetry_batch).toEqual(
+        expected.max_detection_rule_telemetry_batch
+      );
+      expect(telemetryConfiguration.max_detection_alerts_batch).toEqual(
+        expected.max_detection_alerts_batch
+      );
+      expect(telemetryConfiguration.use_async_sender).toEqual(expected.use_async_sender);
+      expect(telemetryConfiguration.sender_channels).toEqual(expected.sender_channels);
+      expect(telemetryConfiguration.pagination_config).toEqual(expected.pagination_config);
+      expect(telemetryConfiguration.indices_metadata_config).toEqual(
+        expected.indices_metadata_config
+      );
+    });
+  });
+});
diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/helpers.ts
@@ -10,6 +10,20 @@ import Util from 'util';
 import type { ElasticsearchClient } from '@kbn/core/server';
 import deepmerge from 'deepmerge';
 import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
+
+export const DEFAULT_GET_ROUTES: Array<[RegExp, unknown]> = [
+  [new RegExp('.*/ping$'), { status: 200 }],
+  [
+    /.*kibana\/manifest\/artifacts.*/,
+    {
+      status: 200,
+      data: 'x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/kibana-artifacts.zip',
+    },
+  ],
+];
+
+export const DEFAULT_POST_ROUTES: Array<[RegExp, unknown]> = [[/.*/, { status: 200 }]];
+
 const asyncUnlink = Util.promisify(Fs.unlink);
 
 /**
@@ -127,3 +141,35 @@ export function updateTimestamps(data: object[]): object[] {
     return { ...d, '@timestamp': new Date(currentTimeMillis + (i + 1) * 100) };
   });
 }
+
+export function mockAxiosPost(
+  postSpy: jest.SpyInstance,
+  routes: Array<[RegExp, unknown]> = DEFAULT_POST_ROUTES
+) {
+  postSpy.mockImplementation(async (url: string) => {
+    for (const [route, returnValue] of routes) {
+      if (route.test(url)) {
+        return returnValue;
+      }
+    }
+    return { status: 404 };
+  });
+}
+
+export function mockAxiosGet(
+  getSpy: jest.SpyInstance,
+  routes: Array<[RegExp, unknown]> = DEFAULT_GET_ROUTES
+) {
+  getSpy.mockImplementation(async (url: string) => {
+    for (const [route, returnValue] of routes) {
+      if (route.test(url)) {
+        return returnValue;
+      }
+    }
+    return { status: 404 };
+  });
+}
+
+export function getRandomInt(min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min + 1)) + min;
+}
diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts
@@ -25,13 +25,13 @@ import {
   deleteExceptionListItem,
 } from '@kbn/lists-plugin/server/services/exception_lists';
 import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants';
+import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
 
 import { packagePolicyService } from '@kbn/fleet-plugin/server/services';
 
 import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
 import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock';
-import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants';
-import { bulkInsert, updateTimestamps } from './helpers';
+import { bulkInsert, eventually, updateTimestamps } from './helpers';
 import { TelemetryEventsSender } from '../../lib/telemetry/sender';
 import type {
   SecuritySolutionPluginStart,
@@ -41,6 +41,7 @@ import type { SecurityTelemetryTask } from '../../lib/telemetry/task';
 import { Plugin as SecuritySolutionPlugin } from '../../plugin';
 import { AsyncTelemetryEventsSender } from '../../lib/telemetry/async_sender';
 import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/receiver';
+import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants';
 import mockEndpointAlert from '../__mocks__/endpoint-alert.json';
 import mockedRule from '../__mocks__/rule.json';
 import fleetAgents from '../__mocks__/fleet-agents.json';
@@ -417,3 +418,24 @@ export function getTelemetryTaskType(task: SecurityTelemetryTask): string {
     return '';
   }
 }
+
+export async function runSoonConfigTask(
+  tasks: SecurityTelemetryTask[],
+  taskManagerPlugin: TaskManagerStartContract
+) {
+  const configTaskType = 'security:telemetry-configuration';
+  const configTask = getTelemetryTask(tasks, configTaskType);
+  const runAfter = new Date();
+  await eventually(async () => {
+    await taskManagerPlugin.runSoon(configTask.getTaskId());
+  });
+
+  // wait until the task finishes
+  await eventually(async () => {
+    const hasRun = await taskManagerPlugin
+      .get(configTask.getTaskId())
+      .then((t) => new Date(t.state.lastExecutionTimestamp) > runAfter)
+      .catch(() => false);
+    expect(hasRun).toBe(true);
+  });
+}