[Fleet] POC - Add support for OTEL policies and integrations

x-pack/plugins/fleet/common/constants/package_policy.ts

@@ -17,3 +17,6 @@ export const inputsFormat = {
 } as const;
 
 export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise';
+
+export const OTEL_POLICY_SAVED_OBJECT_TYPE = 'fleet-otel-policies';
+export const OTEL_INTEGRATION_SAVED_OBJECT_TYPE = 'fleet-otel-integrations';

x-pack/plugins/fleet/common/constants/routes.ts

@@ -16,6 +16,7 @@ export const PACKAGE_POLICY_API_ROOT = `${API_ROOT}/package_policies`;
 export const AGENT_POLICY_API_ROOT = `${API_ROOT}/agent_policies`;
 export const K8S_API_ROOT = `${API_ROOT}/kubernetes`;
 export const DOWNLOAD_SOURCE_API_ROOT = `${API_ROOT}/agent_download_sources`;
+export const OTEL_API_ROOT = `${API_ROOT}/otel`;
 
 export const LIMITED_CONCURRENCY_ROUTE_TAG = 'ingest:limited-concurrency';
 
@@ -215,6 +216,13 @@ export const DOWNLOAD_SOURCE_API_ROUTES = {
   DELETE_PATTERN: `${API_ROOT}/agent_download_sources/{sourceId}`,
 };
 
+// OTEL policies and templates routes
+export const OTEL_INTEGRATIONS_ROUTES = {
+  CREATE_PATTERN: `${OTEL_API_ROOT}/integrations/{name}`,
+};
+export const OTEL_POLICIES_ROUTES = {
+  CREATE_PATTERN: `${OTEL_API_ROOT}/policies`,
+};
 export const CREATE_STANDALONE_AGENT_API_KEY_ROUTE = `${INTERNAL_ROOT}/create_standalone_agent_api_key`;
 
 // Fleet debug routes

x-pack/plugins/fleet/common/types/models/agent_policy.ts

@@ -12,6 +12,7 @@ import type { MonitoringType, PolicySecretReference, ValueOf } from '..';
 
 import type { PackagePolicy, PackagePolicyPackage } from './package_policy';
 import type { Output } from './output';
+import type { OtelPolicy } from './otel';
 
 export type AgentPolicyStatus = typeof agentPolicyStatuses;
 
@@ -74,6 +75,7 @@ export interface AgentPolicy extends Omit<NewAgentPolicy, 'id'> {
   space_ids?: string[] | undefined;
   status: ValueOf<AgentPolicyStatus>;
   package_policies?: PackagePolicy[];
+  otel_policies?: OtelPolicy[];
   is_managed: boolean; // required for created policy
   updated_at: string;
   updated_by: string;

x-pack/plugins/fleet/common/types/models/otel.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface OtelPolicy {
+  id: string;
+  name: string;
+  description?: string;
+  namespace?: string;
+  policy_ids: string[];
+  output_id?: string | null;
+  integration?: {
+    name: string;
+    version?: string;
+  };
+  vars?: Record<string, string | string[]>;
+  pipelines?: string[];
+  revision: number;
+  created_at: string;
+  created_by: string;
+  updated_at: string;
+  updated_by: string;
+}
+
+export interface OtelInputReceivers {
+  receivers?: {
+    [extendedName: string]: {
+      name?: string;
+      pipelines?: string[];
+      parameters?: Record<string, string | string[]>;
+    };
+  };
+}
+
+export interface OtelInputExtensions {
+  extensions?: {
+    config_integrations?: { integrations?: string };
+    file_integrations?: { path?: string };
+  };
+}

x-pack/plugins/fleet/server/routes/index.ts

@@ -28,6 +28,7 @@ import { registerRoutes as registerMessageSigningServiceRoutes } from './message
 import { registerRoutes as registerUninstallTokenRoutes } from './uninstall_token';
 import { registerRoutes as registerStandaloneAgentApiKeyRoutes } from './standalone_agent_api_key';
 import { registerRoutes as registerDebugRoutes } from './debug';
+import { registerRoutes as registerOtelRoutes } from './otel';
 
 export function registerRoutes(fleetAuthzRouter: FleetAuthzRouter, config: FleetConfigType) {
   // Always register app routes for permissions checking
@@ -51,6 +52,7 @@ export function registerRoutes(fleetAuthzRouter: FleetAuthzRouter, config: Fleet
   registerUninstallTokenRoutes(fleetAuthzRouter, config);
   registerStandaloneAgentApiKeyRoutes(fleetAuthzRouter);
   registerDebugRoutes(fleetAuthzRouter);
+  registerOtelRoutes(fleetAuthzRouter);
 
   // Conditional config routes
   if (config.agents.enabled) {

x-pack/plugins/fleet/server/routes/otel/handlers.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { TypeOf } from '@kbn/config-schema';
+import { v4 as uuidv4 } from 'uuid';
+import { load } from 'js-yaml';
+
+import type {
+  FleetRequestHandler,
+  OtelPolicySOAttributes,
+  OtelIntegrationSOAttributes,
+} from '../../types';
+import type {
+  CreateOtelPolicyRequestSchema,
+  InstallOtelIntegrationRequestSchema,
+} from '../../types/rest_spec/otel';
+import { defaultFleetErrorHandler } from '../../errors';
+import {
+  OTEL_POLICY_SAVED_OBJECT_TYPE,
+  OTEL_INTEGRATION_SAVED_OBJECT_TYPE,
+} from '../../../common/constants';
+import { agentPolicyService, appContextService } from '../../services';
+
+export const createOtelPolicyHandler: FleetRequestHandler<
+  undefined,
+  undefined,
+  TypeOf<typeof CreateOtelPolicyRequestSchema.body>
+> = async (context, request, response) => {
+  const fleetContext = await context.fleet;
+  const coreContext = await context.core;
+  const logger = appContextService.getLogger();
+  const user = appContextService.getSecurityCore().authc.getCurrentUser(request) || undefined;
+  const soClient = fleetContext.internalSoClient;
+  const esClient = coreContext.elasticsearch.client.asInternalUser;
+
+  const { id, ...newPolicy } = request.body;
+  const policyId = id || uuidv4();
+
+  try {
+    const isoDate = new Date().toISOString();
+    const newSo = await soClient.create<OtelPolicySOAttributes>(
+      OTEL_POLICY_SAVED_OBJECT_TYPE,
+      {
+        ...newPolicy,
+        revision: 1,
+        created_at: isoDate,
+        created_by: user?.username ?? 'system',
+        updated_at: isoDate,
+        updated_by: user?.username ?? 'system',
+      },
+      { id: policyId }
+    );
+
+    // bump agent policy revision
+    for (const newPolicyId of newPolicy.policy_ids) {
+      await agentPolicyService.bumpRevision(soClient, esClient, newPolicyId, {
+        user,
+      });
+    }
+
+    const createdPolicy = { id: newSo.id, version: newSo.version, ...newSo.attributes };
+    logger.debug(`Created new otel policy with id ${newSo.id} and version ${newSo.version}`);
+
+    return response.ok({
+      body: {
+        item: createdPolicy,
+      },
+    });
+  } catch (error) {
+    return defaultFleetErrorHandler({ error, response });
+  }
+};
+
+export const createOtelIntegrationPolicyHandler: FleetRequestHandler<
+  TypeOf<typeof InstallOtelIntegrationRequestSchema.params>,
+  undefined,
+  TypeOf<typeof InstallOtelIntegrationRequestSchema.body>
+> = async (context, request, response) => {
+  const fleetContext = await context.fleet;
+  const soClient = fleetContext.internalSoClient;
+  const user = appContextService.getSecurityCore().authc.getCurrentUser(request) || undefined;
+  const logger = appContextService.getLogger();
+
+  const { name } = request.params;
+  const jsonConfig = request?.body ? load(request.body) : {};
+
+  try {
+    const isoDate = new Date().toISOString();
+    const template = {
+      name,
+      config: jsonConfig,
+    };
+    const newSo = await soClient.create<OtelIntegrationSOAttributes>(
+      OTEL_INTEGRATION_SAVED_OBJECT_TYPE,
+      {
+        ...template,
+        created_at: isoDate,
+        created_by: user?.username ?? 'system',
+        updated_at: isoDate,
+        updated_by: user?.username ?? 'system',
+      }
+    );
+    logger.debug(
+      `Created new otel ${name} integration with id ${newSo.id} and version ${newSo.version}`
+    );
+    return response.ok({ body: { item: template } });
+  } catch (error) {
+    return defaultFleetErrorHandler({ error, response });
+  }
+};

x-pack/plugins/fleet/server/routes/otel/index.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { FleetAuthzRouter } from '../../services/security';
+import {
+  API_VERSIONS,
+  OTEL_POLICIES_ROUTES,
+  OTEL_INTEGRATIONS_ROUTES,
+} from '../../../common/constants';
+import {
+  CreateOtelPolicyRequestSchema,
+  InstallOtelIntegrationRequestSchema,
+} from '../../types/rest_spec/otel';
+
+const MAX_FILE_SIZE_BYTES = 104857600; // 100MB
+
+import { createOtelPolicyHandler, createOtelIntegrationPolicyHandler } from './handlers';
+
+export const registerRoutes = (router: FleetAuthzRouter) => {
+  // Create policy
+  router.versioned
+    .post({
+      path: OTEL_POLICIES_ROUTES.CREATE_PATTERN,
+      fleetAuthz: {
+        integrations: { writeIntegrationPolicies: true },
+      },
+      description: 'Create new otel policy',
+      options: {
+        tags: ['oas-tag:Fleet Otel policies'],
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        validate: {
+          request: CreateOtelPolicyRequestSchema,
+        },
+      },
+      createOtelPolicyHandler
+    );
+
+  // create integration
+  router.versioned
+    .post({
+      path: OTEL_INTEGRATIONS_ROUTES.CREATE_PATTERN,
+      fleetAuthz: {
+        integrations: { writeIntegrationPolicies: true },
+      },
+      description: 'Create new otel integration',
+      options: {
+        tags: ['oas-tag:Fleet Otel integrations'],
+        body: {
+          accepts: ['application/x-yaml', 'text/x-yaml'],
+          parse: false,
+          maxBytes: MAX_FILE_SIZE_BYTES,
+        },
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        validate: {
+          request: InstallOtelIntegrationRequestSchema,
+        },
+      },
+      createOtelIntegrationPolicyHandler
+    );
+};

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -12,6 +12,8 @@ import type { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-obje
 import {
   LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+  OTEL_POLICY_SAVED_OBJECT_TYPE,
+  OTEL_INTEGRATION_SAVED_OBJECT_TYPE,
 } from '../../common/constants';
 
 import {
@@ -792,6 +794,57 @@ export const getSavedObjectTypes = (
         },
       },
     },
+    [OTEL_POLICY_SAVED_OBJECT_TYPE]: {
+      name: OTEL_POLICY_SAVED_OBJECT_TYPE,
+      indexPattern: INGEST_SAVED_OBJECT_INDEX,
+      hidden: false,
+      namespaceType: 'multiple',
+      management: {
+        importableAndExportable: false,
+      },
+      mappings: {
+        properties: {
+          name: { type: 'keyword' },
+          description: { type: 'text' },
+          namespace: { type: 'keyword' },
+          policy_ids: { type: 'keyword' },
+          output_id: { type: 'keyword' },
+          integration: {
+            properties: {
+              name: { type: 'keyword' },
+              version: { type: 'keyword' },
+            },
+          },
+          vars: { type: 'flattened' },
+          pipelines: { type: 'keyword' },
+          revision: { type: 'integer' },
+          updated_at: { type: 'date' },
+          updated_by: { type: 'keyword' },
+          created_at: { type: 'date' },
+          created_by: { type: 'keyword' },
+        },
+      },
+    },
+    [OTEL_INTEGRATION_SAVED_OBJECT_TYPE]: {
+      name: OTEL_INTEGRATION_SAVED_OBJECT_TYPE,
+      indexPattern: INGEST_SAVED_OBJECT_INDEX,
+      hidden: false,
+      namespaceType: 'multiple',
+      management: {
+        importableAndExportable: false,
+      },
+      mappings: {
+        properties: {
+          name: { type: 'keyword' },
+          version: { type: 'keyword' },
+          config: { type: 'flattened' },
+          updated_at: { type: 'date' },
+          updated_by: { type: 'keyword' },
+          created_at: { type: 'date' },
+          created_by: { type: 'keyword' },
+        },
+      },
+    },
     [PACKAGES_SAVED_OBJECT_TYPE]: {
       name: PACKAGES_SAVED_OBJECT_TYPE,
       indexPattern: INGEST_SAVED_OBJECT_INDEX,

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts

@@ -53,6 +53,7 @@ import {
   DEFAULT_CLUSTER_PERMISSIONS,
 } from './package_policies_to_agent_permissions';
 import { fetchRelatedSavedObjects } from './related_saved_objects';
+import { otelPoliciesToAgentInputs } from './otel_policies_to_agent_inputs';
 
 async function fetchAgentPolicy(soClient: SavedObjectsClientContract, id: string) {
   try {
@@ -120,13 +121,16 @@ export async function getFullAgentPolicy(
     })
   );
 
-  const inputs = await storedPackagePoliciesToAgentInputs(
+  const packagePolicyInputs = await storedPackagePoliciesToAgentInputs(
     agentPolicy.package_policies as PackagePolicy[],
     packageInfoCache,
     getOutputIdForAgentPolicy(dataOutput),
     agentPolicy.namespace,
     agentPolicy.global_data_tags
   );
+  const otelInputs = await otelPoliciesToAgentInputs(soClient, agentPolicy);
+  const inputs = [...packagePolicyInputs, ...(otelInputs ? otelInputs : [])];
+
   const features = (agentPolicy.agent_features || []).reduce((acc, { name, ...featureConfig }) => {
     acc[name] = featureConfig;
     return acc;