[Fleet] POC - Add support for OTEL policies and integrations

x-pack/plugins/fleet/common/constants/package_policy.ts

@@ -17,3 +17,5 @@ export const inputsFormat = {
 } as const;
 
 export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise';
+
+export const OTEL_POLICY_SAVED_OBJECT_TYPE = 'fleet-otel-policies';

x-pack/plugins/fleet/common/constants/routes.ts

@@ -16,6 +16,7 @@ export const PACKAGE_POLICY_API_ROOT = `${API_ROOT}/package_policies`;
 export const AGENT_POLICY_API_ROOT = `${API_ROOT}/agent_policies`;
 export const K8S_API_ROOT = `${API_ROOT}/kubernetes`;
 export const DOWNLOAD_SOURCE_API_ROOT = `${API_ROOT}/agent_download_sources`;
+export const OTEL_API_ROOT = `${API_ROOT}/otel`;
 
 export const LIMITED_CONCURRENCY_ROUTE_TAG = 'ingest:limited-concurrency';
 
@@ -215,6 +216,13 @@ export const DOWNLOAD_SOURCE_API_ROUTES = {
   DELETE_PATTERN: `${API_ROOT}/agent_download_sources/{sourceId}`,
 };
 
+// OTEL policies and templates routes
+export const OTEL_INTEGRATIONS_ROUTES = {
+  CREATE_PATTERN: `${OTEL_API_ROOT}/templates/{pkgName}`,
+};
+export const OTEL_POLICIES_ROUTES = {
+  CREATE_PATTERN: `${OTEL_API_ROOT}/policies`,
+};
 export const CREATE_STANDALONE_AGENT_API_KEY_ROUTE = `${INTERNAL_ROOT}/create_standalone_agent_api_key`;
 
 // Fleet debug routes

x-pack/plugins/fleet/server/routes/index.ts

@@ -28,6 +28,7 @@ import { registerRoutes as registerMessageSigningServiceRoutes } from './message
 import { registerRoutes as registerUninstallTokenRoutes } from './uninstall_token';
 import { registerRoutes as registerStandaloneAgentApiKeyRoutes } from './standalone_agent_api_key';
 import { registerRoutes as registerDebugRoutes } from './debug';
+import { registerRoutes as registerOtelRoutes } from './otel';
 
 export function registerRoutes(fleetAuthzRouter: FleetAuthzRouter, config: FleetConfigType) {
   // Always register app routes for permissions checking
@@ -51,6 +52,7 @@ export function registerRoutes(fleetAuthzRouter: FleetAuthzRouter, config: Fleet
   registerUninstallTokenRoutes(fleetAuthzRouter, config);
   registerStandaloneAgentApiKeyRoutes(fleetAuthzRouter);
   registerDebugRoutes(fleetAuthzRouter);
+  registerOtelRoutes(fleetAuthzRouter);
 
   // Conditional config routes
   if (config.agents.enabled) {

x-pack/plugins/fleet/server/routes/otel/handlers.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { TypeOf } from '@kbn/config-schema';
+import { v4 as uuidv4 } from 'uuid';
+
+import type { FleetRequestHandler, OtelPolicySOAttributes } from '../../types';
+import type { CreateOtelPolicyRequestSchema } from '../../types/rest_spec/otel';
+import { defaultFleetErrorHandler } from '../../errors';
+import { OTEL_POLICY_SAVED_OBJECT_TYPE } from '../../../common/constants';
+import { appContextService } from '../../services';
+
+export const createOtelPolicyHandler: FleetRequestHandler<
+  undefined,
+  undefined,
+  TypeOf<typeof CreateOtelPolicyRequestSchema.body>
+> = async (context, request, response) => {
+  const fleetContext = await context.fleet;
+
+  const logger = appContextService.getLogger();
+  const user = appContextService.getSecurityCore().authc.getCurrentUser(request) || undefined;
+  const soClient = fleetContext.internalSoClient;
+  const { id, ...newPolicy } = request.body;
+  const policyId = id || uuidv4();
+
+  try {
+    const isoDate = new Date().toISOString();
+    const newSo = await soClient.create<OtelPolicySOAttributes>(
+      OTEL_POLICY_SAVED_OBJECT_TYPE,
+      {
+        ...newPolicy,
+        revision: 1,
+        created_at: isoDate,
+        created_by: user?.username ?? 'system',
+        updated_at: isoDate,
+        updated_by: user?.username ?? 'system',
+      },
+      { id: policyId }
+    );
+
+    // if (options?.bumpRevision ?? true) {
+    //   for (const policyId of newPolicy.policy_ids) {
+    //     await agentPolicyService.bumpRevision(soClient, esClient, policyId, {
+    //       user: options?.user,
+    //     });
+    //   }
+    // }
+
+    const createdPolicy = { id: newSo.id, version: newSo.version, ...newSo.attributes };
+    logger.debug(`Created new otel policy with id ${newSo.id} and version ${newSo.version}`);
+
+    return response.ok({
+      body: {
+        item: createdPolicy,
+      },
+    });
+  } catch (error) {
+    return defaultFleetErrorHandler({ error, response });
+  }
+};

x-pack/plugins/fleet/server/routes/otel/index.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { FleetAuthzRouter } from '../../services/security';
+import { API_VERSIONS, OTEL_POLICIES_ROUTES } from '../../../common/constants';
+import { CreateOtelPolicyRequestSchema } from '../../types/rest_spec/otel';
+
+import { createOtelPolicyHandler } from './handlers';
+
+export const registerRoutes = (router: FleetAuthzRouter) => {
+  // Create
+  router.versioned
+    .post({
+      path: OTEL_POLICIES_ROUTES.CREATE_PATTERN,
+      fleetAuthz: {
+        integrations: { writeIntegrationPolicies: true },
+      },
+      description: 'Create new otel policy',
+      options: {
+        tags: ['oas-tag:Fleet Otel policies'],
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        validate: {
+          request: CreateOtelPolicyRequestSchema,
+        },
+      },
+      createOtelPolicyHandler
+    );
+};

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -12,6 +12,7 @@ import type { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-obje
 import {
   LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+  OTEL_POLICY_SAVED_OBJECT_TYPE,
 } from '../../common/constants';
 
 import {
@@ -792,6 +793,37 @@ export const getSavedObjectTypes = (
         },
       },
     },
+    [OTEL_POLICY_SAVED_OBJECT_TYPE]: {
+      name: OTEL_POLICY_SAVED_OBJECT_TYPE,
+      indexPattern: INGEST_SAVED_OBJECT_INDEX,
+      hidden: false,
+      namespaceType: 'multiple',
+      management: {
+        importableAndExportable: false,
+      },
+      mappings: {
+        properties: {
+          name: { type: 'keyword' },
+          description: { type: 'text' },
+          namespace: { type: 'keyword' },
+          policy_ids: { type: 'keyword' },
+          output_id: { type: 'keyword' },
+          integration: {
+            properties: {
+              name: { type: 'keyword' },
+              version: { type: 'keyword' },
+            },
+          },
+          vars: { type: 'flattened' },
+          pipelines: { type: 'keyword' },
+          revision: { type: 'integer' },
+          updated_at: { type: 'date' },
+          updated_by: { type: 'keyword' },
+          created_at: { type: 'date' },
+          created_by: { type: 'keyword' },
+        },
+      },
+    },
     [PACKAGES_SAVED_OBJECT_TYPE]: {
       name: PACKAGES_SAVED_OBJECT_TYPE,
       indexPattern: INGEST_SAVED_OBJECT_INDEX,

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts

@@ -136,7 +136,7 @@ export async function getFullAgentPolicy(
   const packagePolicySecretReferences = (agentPolicy?.package_policies || []).flatMap(
     (policy) => policy.secret_references || []
   );
-
+  // add data from otel policies
   const fullAgentPolicy: FullAgentPolicy = {
     id: agentPolicy.id,
     outputs: {

x-pack/plugins/fleet/server/types/models/index.ts

@@ -12,3 +12,4 @@ export * from './output';
 export * from './enrollment_api_key';
 export * from './preconfiguration';
 export * from './download_sources';
+export * from './otel';

x-pack/plugins/fleet/server/types/models/otel.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { schema } from '@kbn/config-schema';
+
+export const VarsRecordSchema = schema.recordOf(
+  schema.string(),
+  schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
+  {
+    meta: {
+      description: 'Otel policy variable',
+    },
+  }
+);

x-pack/plugins/fleet/server/types/rest_spec/otel.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+import { VarsRecordSchema } from '../models';
+
+const createOtelPolicyRequestBodySchema = schema.object({
+  id: schema.maybe(
+    schema.string({
+      meta: {
+        description: 'Package policy unique identifier',
+      },
+    })
+  ),
+  name: schema.string({
+    meta: {
+      description: 'Otel policy name (should be unique)',
+    },
+  }),
+  description: schema.maybe(
+    schema.string({
+      meta: {
+        description: 'Otel policy description',
+      },
+    })
+  ),
+  namespace: schema.maybe(
+    schema.string({
+      meta: {
+        description: 'Otel policy namespace',
+      },
+    })
+  ),
+  output_id: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])),
+  policy_ids: schema.arrayOf(
+    schema.string({
+      meta: {
+        description: 'Agent policy IDs where that Otel policy will be added',
+      },
+    })
+  ),
+  integration: schema.maybe(
+    schema.object({
+      name: schema.string({
+        meta: {
+          description: 'Integration name',
+        },
+      }),
+      version: schema.maybe(
+        schema.string({
+          meta: {
+            description: 'Integration version',
+          },
+        })
+      ),
+    })
+  ),
+  pipelines: schema.maybe(schema.arrayOf(schema.string())),
+  vars: schema.maybe(VarsRecordSchema),
+});
+
+export const CreateOtelPolicyRequestSchema = {
+  body: createOtelPolicyRequestBodySchema,
+};

x-pack/plugins/fleet/server/types/so_attributes.ts

@@ -139,6 +139,25 @@ export interface PackagePolicySOAttributes {
   overrides?: any | null;
 }
 
+export interface OtelPolicySOAttributes {
+  name: string;
+  description?: string;
+  namespace?: string;
+  policy_ids: string[];
+  output_id?: string | null;
+  integration?: {
+    name: string;
+    version?: string;
+  };
+  vars?: Record<string, string | string[]>;
+  pipelines?: string[];
+  revision: number;
+  created_at: string;
+  created_by: string;
+  updated_at: string;
+  updated_by: string;
+}
+
 interface OutputSoBaseAttributes {
   is_default: boolean;
   is_default_monitoring: boolean;