[Security Solution][Endpoint] Add FTR API tests that validates creati…

…on of DOT indices (elastic#197899)

## Summary

- Adds new FTR API test suite for validating that DOT indices are
created whenever a policy in fleet is created/updated
- Renamed and moved `DEFAULT_DIAGNOSTIC_INDEX` `const` to security
solution top-level `common` directory for better reuse
- Moved utility function that builds an index name with the `namespace`
included to top-level `common` directory for better reuse
- Created some additional scripting methods in the Fleet services module
for updating fleet policies

(cherry picked from commit a5d571a)

# Conflicts:
#	x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -93,6 +93,7 @@ disabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/serverless.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -83,6 +83,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/ess.config.ts

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -55,6 +55,9 @@ export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
 export const ENDPOINT_HEARTBEAT_INDEX = '.logs-endpoint.heartbeat-default';
 export const ENDPOINT_HEARTBEAT_INDEX_PATTERN = '.logs-endpoint.heartbeat-*';
 
+// Endpoint diagnostics index
+export const DEFAULT_DIAGNOSTIC_INDEX_PATTERN = '.logs-endpoint.diagnostic.collection-*' as const;
+
 // File storage indexes supporting endpoint Upload/download
 export const FILE_STORAGE_METADATA_INDEX = getFileMetadataIndexName('endpoint');
 export const FILE_STORAGE_DATA_INDEX = getFileDataIndexName('endpoint');

x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.test.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { buildIndexNameWithNamespace } from './index_name_utilities';
+
+describe('index name utilities', () => {
+  describe('buildIndexNameWithNamespace()', () => {
+    test.each(['logs-endpoint.foo', 'logs-endpoint.foo-', 'logs-endpoint.foo-*'])(
+      `should build correct index name for: %s`,
+      (prefix) => {
+        expect(buildIndexNameWithNamespace(prefix, 'bar')).toEqual('logs-endpoint.foo-bar');
+      }
+    );
+  });
+});

x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Builds an index name that includes the `namespace` using the provided index name prefix or pattern.
+ *
+ * @param indexNamePrefixOrPattern
+ * @param namespace
+ *
+ * @example
+ *
+ * buildIndexNameWithNamespace('logs-foo.bar-*', 'default'); // == 'logs-foo.bar-default'
+ * buildIndexNameWithNamespace('logs-foo.bar', 'default'); // == 'logs-foo.bar-default'
+ * buildIndexNameWithNamespace('logs-foo.bar-', 'default'); // == 'logs-foo.bar-default'
+ */
+export const buildIndexNameWithNamespace = (
+  indexNamePrefixOrPattern: string,
+  namespace: string
+): string => {
+  if (indexNamePrefixOrPattern.endsWith('*')) {
+    const hasDash = indexNamePrefixOrPattern.endsWith('-*');
+    return `${indexNamePrefixOrPattern.substring(0, indexNamePrefixOrPattern.length - 1)}${
+      hasDash ? '' : '-'
+    }${namespace}`;
+  }
+
+  return `${indexNamePrefixOrPattern}${
+    indexNamePrefixOrPattern.endsWith('-') ? '' : '-'
+  }${namespace}`;
+};

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts

@@ -21,10 +21,12 @@ import type {
   GetAgentsResponse,
   GetInfoResponse,
   GetOneAgentPolicyResponse,
+  GetOnePackagePolicyResponse,
   GetPackagePoliciesRequest,
   GetPackagePoliciesResponse,
   PackagePolicy,
   PostFleetSetupResponse,
+  UpdatePackagePolicyResponse,
 } from '@kbn/fleet-plugin/common';
 import {
   AGENT_API_ROUTES,
@@ -39,6 +41,7 @@ import {
   PACKAGE_POLICY_API_ROUTES,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   SETUP_API_ROUTE,
+  packagePolicyRouteService,
 } from '@kbn/fleet-plugin/common';
 import type { ToolingLog } from '@kbn/tooling-log';
 import type { KbnClient } from '@kbn/test';
@@ -57,11 +60,14 @@ import type {
   GetEnrollmentAPIKeysResponse,
   GetOutputsResponse,
   PostAgentUnenrollResponse,
+  UpdateAgentPolicyRequest,
+  UpdateAgentPolicyResponse,
 } from '@kbn/fleet-plugin/common/types';
 import semver from 'semver';
 import axios from 'axios';
 import { userInfo } from 'os';
 import pRetry from 'p-retry';
+import { getPolicyDataForUpdate } from '../../../common/endpoint/service/policy';
 import { fetchActiveSpace } from './spaces';
 import { fetchKibanaStatus } from '../../../common/endpoint/utils/kibana_status';
 import { isFleetServerRunning } from './fleet_server/fleet_server_services';
@@ -76,6 +82,7 @@ import {
 } from '../../../common/endpoint/data_loaders/utils';
 import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error';
 import { FleetAgentGenerator } from '../../../common/endpoint/data_generators/fleet_agent_generator';
+import type { PolicyData } from '../../../common/endpoint/types';
 
 const fleetGenerator = new FleetAgentGenerator();
 const CURRENT_USERNAME = userInfo().username.toLowerCase();
@@ -101,6 +108,39 @@ export const randomAgentPolicyName = (() => {
  */
 const isValidArtifactVersion = (version: string) => !!version.match(/^\d+\.\d+\.\d+(-SNAPSHOT)?$/);
 
+const getAgentPolicyDataForUpdate = (
+  agentPolicy: AgentPolicy
+): UpdateAgentPolicyRequest['body'] => {
+  return pick(agentPolicy, [
+    'advanced_settings',
+    'agent_features',
+    'data_output_id',
+    'description',
+    'download_source_id',
+    'fleet_server_host_id',
+    'global_data_tags',
+    'has_fleet_server',
+    'id',
+    'inactivity_timeout',
+    'is_default',
+    'is_default_fleet_server',
+    'is_managed',
+    'is_protected',
+    'keep_monitoring_alive',
+    'monitoring_diagnostics',
+    'monitoring_enabled',
+    'monitoring_http',
+    'monitoring_output_id',
+    'monitoring_pprof_enabled',
+    'name',
+    'namespace',
+    'overrides',
+    'space_ids',
+    'supports_agentless',
+    'unenroll_timeout',
+  ]) as UpdateAgentPolicyRequest['body'];
+};
+
 export const checkInFleetAgent = async (
   esClient: Client,
   agentId: string,
@@ -1369,3 +1409,93 @@ export const enableFleetSpaceAwareness = memoize(async (kbnClient: KbnClient): P
     })
     .catch(catchAxiosErrorFormatAndThrow);
 });
+
+/**
+ * Fetches a single integratino policy by id
+ * @param kbnClient
+ * @param policyId
+ */
+export const fetchIntegrationPolicy = async (
+  kbnClient: KbnClient,
+  policyId: string
+): Promise<GetOnePackagePolicyResponse['item']> => {
+  return kbnClient
+    .request<GetOnePackagePolicyResponse>({
+      path: packagePolicyRouteService.getInfoPath(policyId),
+      method: 'GET',
+      headers: { 'elastic-api-version': '2023-10-31' },
+    })
+    .catch(catchAxiosErrorFormatAndThrow)
+    .then((response) => response.data.item);
+};
+
+/**
+ * Update a fleet integration policy (aka: package policy)
+ * @param kbnClient
+ */
+export const updateIntegrationPolicy = async (
+  kbnClient: KbnClient,
+  /** The Integration policy id */
+  id: string,
+  policyData: Partial<CreatePackagePolicyRequest['body']>,
+  /** If set to `true`, then `policyData` can be a partial set of updates and not the full policy data */
+  patch: boolean = false
+): Promise<UpdatePackagePolicyResponse['item']> => {
+  let fullPolicyData = policyData;
+
+  if (patch) {
+    const currentSavedPolicy = await fetchIntegrationPolicy(kbnClient, id);
+    fullPolicyData = getPolicyDataForUpdate(currentSavedPolicy as PolicyData);
+    Object.assign(fullPolicyData, policyData);
+  }
+
+  return kbnClient
+    .request<UpdatePackagePolicyResponse>({
+      path: packagePolicyRouteService.getUpdatePath(id),
+      method: 'PUT',
+      body: fullPolicyData,
+      headers: { 'elastic-api-version': '2023-10-31' },
+    })
+    .catch(catchAxiosErrorFormatAndThrow)
+    .then((response) => response.data.item);
+};
+
+/**
+ * Updates a Fleet agent policy
+ * @param kbnClient
+ * @param id
+ * @param policyData
+ * @param patch
+ */
+export const updateAgentPolicy = async (
+  kbnClient: KbnClient,
+  /** Fleet Agent Policy ID */
+  id: string,
+  /** The updated agent policy data. Could be a `partial` update if `patch` arguments below is true */
+  policyData: Partial<UpdateAgentPolicyRequest['body']>,
+  /**
+   * If set to `true`, the `policyData` provided on input will first be merged with the latest version
+   * of the policy and then the updated applied
+   */
+  patch: boolean = false
+): Promise<UpdateAgentPolicyResponse['item']> => {
+  let fullPolicyData = policyData;
+
+  if (patch) {
+    const currentSavedPolicy = await fetchAgentPolicy(kbnClient, id);
+
+    fullPolicyData = getAgentPolicyDataForUpdate(currentSavedPolicy);
+    delete fullPolicyData.id;
+    Object.assign(fullPolicyData, policyData);
+  }
+
+  return kbnClient
+    .request<UpdateAgentPolicyResponse>({
+      path: agentPolicyRouteService.getUpdatePath(id),
+      method: 'PUT',
+      body: fullPolicyData,
+      headers: { 'elastic-api-version': '2023-10-31' },
+    })
+    .catch(catchAxiosErrorFormatAndThrow)
+    .then((response) => response.data.item);
+};

x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts

@@ -6,33 +6,18 @@
  */
 
 import pMap from 'p-map';
+import { buildIndexNameWithNamespace } from '../../../common/endpoint/utils/index_name_utilities';
 import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services';
 import { catchAndWrapError } from '../../endpoint/utils';
 import type { SimpleMemCacheInterface } from '../../endpoint/lib/simple_mem_cache';
 import { SimpleMemCache } from '../../endpoint/lib/simple_mem_cache';
 import {
+  DEFAULT_DIAGNOSTIC_INDEX_PATTERN,
   ENDPOINT_ACTION_RESPONSES_DS,
   ENDPOINT_HEARTBEAT_INDEX_PATTERN,
 } from '../../../common/endpoint/constants';
-import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants';
 import { stringify } from '../../endpoint/utils/stringify';
 
-const buildIndexNameWithNamespace = (
-  indexNamePrefixOrPattern: string,
-  namespace: string
-): string => {
-  if (indexNamePrefixOrPattern.endsWith('*')) {
-    const hasDash = indexNamePrefixOrPattern.endsWith('-*');
-    return `${indexNamePrefixOrPattern.substring(0, indexNamePrefixOrPattern.length - 1)}${
-      hasDash ? '' : '-'
-    }${namespace}`;
-  }
-
-  return `${indexNamePrefixOrPattern}${
-    indexNamePrefixOrPattern.endsWith('-') ? '' : '-'
-  }${namespace}`;
-};
-
 const cache = new SimpleMemCache({
   // Cache of created Datastreams last for 12h, at which point it is checked again.
   // This is just a safeguard case (for whatever reason) the index is deleted
@@ -82,10 +67,8 @@ export const createPolicyDataStreamsIfNeeded: PolicyDataStreamsCreator = async (
     string[]
   >((acc, namespaceList) => {
     for (const namespace of namespaceList) {
-      acc.push(
-        buildIndexNameWithNamespace(DEFAULT_DIAGNOSTIC_INDEX, namespace),
-        buildIndexNameWithNamespace(ENDPOINT_ACTION_RESPONSES_DS, namespace)
-      );
+        acc.add(buildIndexNameWithNamespace(DEFAULT_DIAGNOSTIC_INDEX_PATTERN, namespace));
+        acc.add(buildIndexNameWithNamespace(ENDPOINT_ACTION_RESPONSES_DS, namespace));
 
       if (endpointServices.isServerless()) {
         acc.push(buildIndexNameWithNamespace(ENDPOINT_HEARTBEAT_INDEX_PATTERN, namespace));

x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts

@@ -29,6 +29,7 @@ import { packagePolicyService } from '@kbn/fleet-plugin/server/services';
 
 import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
 import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock';
+import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants';
 import { bulkInsert, updateTimestamps } from './helpers';
 import { TelemetryEventsSender } from '../../lib/telemetry/sender';
 import type {
@@ -39,7 +40,6 @@ import type { SecurityTelemetryTask } from '../../lib/telemetry/task';
 import { Plugin as SecuritySolutionPlugin } from '../../plugin';
 import { AsyncTelemetryEventsSender } from '../../lib/telemetry/async_sender';
 import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/receiver';
-import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants';
 import mockEndpointAlert from '../__mocks__/endpoint-alert.json';
 import mockedRule from '../__mocks__/rule.json';
 import fleetAgents from '../__mocks__/fleet-agents.json';
@@ -141,7 +141,7 @@ export function getTelemetryTask(
 }
 
 export async function createMockedEndpointAlert(esClient: ElasticsearchClient) {
-  const index = `${DEFAULT_DIAGNOSTIC_INDEX.replace('-*', '')}-001`;
+  const index = `${DEFAULT_DIAGNOSTIC_INDEX_PATTERN.replace('-*', '')}-001`;
 
   await esClient.indices.create({ index, body: { settings: { hidden: true } } });
 
@@ -217,7 +217,7 @@ export async function dropEndpointIndices(esClient: ElasticsearchClient) {
 }
 
 export async function cleanupMockedEndpointAlerts(esClient: ElasticsearchClient) {
-  const index = `${DEFAULT_DIAGNOSTIC_INDEX.replace('-*', '')}-001`;
+  const index = `${DEFAULT_DIAGNOSTIC_INDEX_PATTERN.replace('-*', '')}-001`;
 
   await esClient.indices.delete({ index }).catch(() => {
     // ignore errors

x-pack/plugins/security_solution/server/lib/telemetry/constants.ts

@@ -27,8 +27,6 @@ export const INSIGHTS_CHANNEL = 'security-insights-v1';
 
 export const TASK_METRICS_CHANNEL = 'task-metrics';
 
-export const DEFAULT_DIAGNOSTIC_INDEX = '.logs-endpoint.diagnostic.collection-*' as const;
-
 export const DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS = {
   linux: {
     advanced: {

x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts

@@ -48,6 +48,7 @@ import type {
 } from '@kbn/fleet-plugin/server';
 import type { ExceptionListClient } from '@kbn/lists-plugin/server';
 import moment from 'moment';
+import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants';
 import type { ExperimentalFeatures } from '../../../common';
 import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services';
 import {
@@ -85,7 +86,6 @@ import type {
 import { telemetryConfiguration } from './configuration';
 import { ENDPOINT_METRICS_INDEX } from '../../../common/constants';
 import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants';
-import { DEFAULT_DIAGNOSTIC_INDEX } from './constants';
 import type { TelemetryLogger } from './telemetry_logger';
 
 export interface ITelemetryReceiver {
@@ -546,7 +546,7 @@ export class TelemetryReceiver implements ITelemetryReceiver {
       to: executeTo,
     } as LogMeta);
 
-    let pitId = await this.openPointInTime(DEFAULT_DIAGNOSTIC_INDEX);
+    let pitId = await this.openPointInTime(DEFAULT_DIAGNOSTIC_INDEX_PATTERN);
     let fetchMore = true;
     let searchAfter: SortResults | undefined;
 

x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts

@@ -6,11 +6,12 @@
  */
 
 import type { Logger } from '@kbn/core/server';
+import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../../common/endpoint/constants';
 import type { ITelemetryEventsSender } from '../sender';
 import type { ITelemetryReceiver } from '../receiver';
 import type { TaskExecutionPeriod } from '../task';
 import type { ITaskMetricsService } from '../task_metrics.types';
-import { DEFAULT_DIAGNOSTIC_INDEX, TELEMETRY_CHANNEL_TIMELINE } from '../constants';
+import { TELEMETRY_CHANNEL_TIMELINE } from '../constants';
 import { ranges, TelemetryTimelineFetcher, newTelemetryLogger } from '../helpers';
 
 export function createTelemetryDiagnosticTimelineTaskConfig() {
@@ -43,7 +44,7 @@ export function createTelemetryDiagnosticTimelineTaskConfig() {
         const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod);
 
         const alerts = await receiver.fetchTimelineAlerts(
-          DEFAULT_DIAGNOSTIC_INDEX,
+          DEFAULT_DIAGNOSTIC_INDEX_PATTERN,
           rangeFrom,
           rangeTo
         );