[Observability] Use Fleet's default data output when onboarding integ…

…rations using auto-detect flow (elastic#201158)

Resolves elastic#199751

## Summary

Use Fleet's default data output when onboarding integrations using
auto-detect flow.

## Screenshot

### Fleet output settings

<img width="1411" alt="Screenshot 2024-11-21 at 15 10 24"
src="https://github.com/user-attachments/assets/ac193552-7f18-4566-a84b-061df45c13f3">

### Generated Agent config

```
$ cat /Library/Elastic/Agent/elastic-agent.yml

outputs:
  default:
    type: elasticsearch
    hosts:
      - https://192.168.1.73:9200
    ssl.ca_trusted_fingerprint: c48c98cdf7f85d7cc29f834704011c1002b5251d594fc0bb08e6177544fe304a
    api_key: b1BkR1Q1TUIyOUpfMWhaS2NCUXA6SS1Jb3dncGVReVNpcEdzOGpSVmlzQQ==
    preset: balanced
```

## Testing

1. Go to Fleet > Settings > Outputs
2. Edit the default output and enter dummy data into the "Elasticsearch
CA trusted fingerprint" field
3. Go through the auto-detect onboarding flow
4. Inspect the `elastic-agent.yml` file written to disk. It should
contain the default output configured in Fleet including
`ca_trusted_fingerprint` setting

(cherry picked from commit 845aa8c)

# Conflicts:
#	x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts

x-pack/plugins/fleet/server/mocks/index.ts

@@ -29,6 +29,7 @@ import { createFleetActionsClientMock } from '../services/actions/mocks';
 import { createFleetFilesClientFactoryMock } from '../services/files/mocks';
 
 import { createArtifactsClientMock } from '../services/artifacts/mocks';
+import { createOutputClientMock } from '../services/output_client.mock';
 
 import type { PackagePolicyClient } from '../services/package_policy_service';
 import type { AgentPolicyServiceInterface } from '../services';
@@ -301,6 +302,7 @@ export const createFleetStartContractMock = (): DeeplyMockedKeys<FleetStartContr
     uninstallTokenService: createUninstallTokenServiceMock(),
     createFleetActionsClient: jest.fn((_) => fleetActionsClient),
     getPackageSpecTagId: jest.fn(getPackageSpecTagId),
+    createOutputClient: jest.fn(async (_) => createOutputClientMock()),
   };
 
   return startContract;

x-pack/plugins/fleet/server/plugin.ts

@@ -84,6 +84,8 @@ import {
   MessageSigningService,
 } from './services/security';
 
+import { OutputClient, type OutputClientInterface } from './services/output_client';
+
 import {
   ASSETS_SAVED_OBJECT_TYPE,
   DOWNLOAD_SOURCE_SAVED_OBJECT_TYPE,
@@ -262,6 +264,12 @@ export interface FleetStartContract {
   Function exported to allow creating unique ids for saved object tags
    */
   getPackageSpecTagId: (spaceId: string, pkgName: string, tagName: string) => string;
+
+  /**
+   * Create a Fleet Output Client instance
+   * @param packageName
+   */
+  createOutputClient: (request: KibanaRequest) => Promise<OutputClientInterface>;
 }
 
 export class FleetPlugin
@@ -837,6 +845,11 @@ export class FleetPlugin
         return new FleetActionsClient(core.elasticsearch.client.asInternalUser, packageName);
       },
       getPackageSpecTagId,
+      async createOutputClient(request: KibanaRequest) {
+        const soClient = appContextService.getSavedObjects().getScopedClient(request);
+        const authz = await getAuthzFromRequest(request);
+        return new OutputClient(soClient, authz);
+      },
     };
   }
 

x-pack/plugins/fleet/server/services/output_client.mock.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { OutputClientInterface } from './output_client';
+
+export const createOutputClientMock = (): jest.Mocked<OutputClientInterface> => {
+  return {
+    getDefaultDataOutputId: jest.fn(),
+    get: jest.fn(),
+  };
+};

x-pack/plugins/fleet/server/services/output_client.test.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { savedObjectsClientMock } from '@kbn/core/server/mocks';
+
+import { createFleetAuthzMock } from '../../common/mocks';
+
+import { OutputClient } from './output_client';
+import { outputService } from './output';
+
+jest.mock('./output');
+
+const mockedOutputService = outputService as jest.Mocked<typeof outputService>;
+
+describe('OutputClient', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('getDefaultDataOutputId()', () => {
+    it('should call output service `getDefaultDataOutputId()` method', async () => {
+      const soClient = savedObjectsClientMock.create();
+      const authz = createFleetAuthzMock();
+      const outputClient = new OutputClient(soClient, authz);
+      await outputClient.getDefaultDataOutputId();
+
+      expect(mockedOutputService.getDefaultDataOutputId).toHaveBeenCalledWith(soClient);
+    });
+
+    it('should throw error when no `fleet.readSettings` and no `fleet.readAgentPolicies` privileges', async () => {
+      const soClient = savedObjectsClientMock.create();
+      const authz = createFleetAuthzMock();
+      authz.fleet.readSettings = false;
+      authz.fleet.readAgentPolicies = false;
+      const outputClient = new OutputClient(soClient, authz);
+
+      await expect(outputClient.getDefaultDataOutputId()).rejects.toMatchInlineSnapshot(
+        `[OutputUnauthorizedError]`
+      );
+      expect(mockedOutputService.getDefaultDataOutputId).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('get()', () => {
+    it('should call output service `get()` method', async () => {
+      const soClient = savedObjectsClientMock.create();
+      const authz = createFleetAuthzMock();
+      const outputClient = new OutputClient(soClient, authz);
+      await outputClient.get('default');
+
+      expect(mockedOutputService.get).toHaveBeenCalledWith(soClient, 'default');
+    });
+
+    it('should throw error when no `fleet.readSettings` and no `fleet.readAgentPolicies` privileges', async () => {
+      const soClient = savedObjectsClientMock.create();
+      const authz = createFleetAuthzMock();
+      authz.fleet.readSettings = false;
+      authz.fleet.readAgentPolicies = false;
+      const outputClient = new OutputClient(soClient, authz);
+
+      await expect(outputClient.get('default')).rejects.toMatchInlineSnapshot(
+        `[OutputUnauthorizedError]`
+      );
+      expect(mockedOutputService.get).not.toHaveBeenCalled();
+    });
+  });
+});

x-pack/plugins/fleet/server/services/output_client.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectsClientContract } from '@kbn/core/server';
+
+import type { FleetAuthz } from '../../common';
+
+import { OutputUnauthorizedError } from '../errors';
+import type { Output } from '../types';
+
+import { outputService } from './output';
+
+export { transformOutputToFullPolicyOutput } from './agent_policies/full_agent_policy';
+
+export interface OutputClientInterface {
+  getDefaultDataOutputId(): Promise<string | null>;
+  get(outputId: string): Promise<Output>;
+}
+
+export class OutputClient implements OutputClientInterface {
+  constructor(private soClient: SavedObjectsClientContract, private authz: FleetAuthz) {}
+
+  async getDefaultDataOutputId() {
+    if (!this.authz.fleet.readSettings && !this.authz.fleet.readAgentPolicies) {
+      throw new OutputUnauthorizedError();
+    }
+    return outputService.getDefaultDataOutputId(this.soClient);
+  }
+
+  async get(outputId: string) {
+    if (!this.authz.fleet.readSettings && !this.authz.fleet.readAgentPolicies) {
+      throw new OutputUnauthorizedError();
+    }
+    return outputService.get(this.soClient, outputId);
+  }
+}

x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts

@@ -13,15 +13,15 @@ import {
   type PackageClient,
 } from '@kbn/fleet-plugin/server';
 import { safeDump } from 'js-yaml';
-import { PackageDataStreamTypes } from '@kbn/fleet-plugin/common/types';
+import { PackageDataStreamTypes, Output } from '@kbn/fleet-plugin/common/types';
+import { transformOutputToFullPolicyOutput } from '@kbn/fleet-plugin/server/services/output_client';
 import { getObservabilityOnboardingFlow, saveObservabilityOnboardingFlow } from '../../lib/state';
 import type { SavedObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status';
 import { ObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status';
 import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
 import { getHasLogs } from './get_has_logs';
 import { getKibanaUrl } from '../../lib/get_fallback_urls';
 import { getAgentVersionInfo } from '../../lib/get_agent_version';
-import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { ElasticAgentStepPayload, InstalledIntegration, StepProgressPayloadRT } from '../types';
 import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key';
 import { createInstallApiKey } from '../../lib/api_key/create_install_api_key';
@@ -329,6 +329,13 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
       throw Boom.notFound(`Onboarding session '${params.path.onboardingId}' not found.`);
     }
 
+    const outputClient = await fleetStart.createOutputClient(request);
+    const defaultOutputId = await outputClient.getDefaultDataOutputId();
+    if (!defaultOutputId) {
+      throw Boom.notFound('Default data output not found');
+    }
+    const output = await outputClient.get(defaultOutputId);
+
     const integrationsToInstall = parseIntegrationsTSV(params.body);
     if (!integrationsToInstall.length) {
       return response.badRequest({
@@ -381,15 +388,11 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
       },
     });
 
-    const elasticsearchUrl = plugins.cloud?.setup?.elasticsearchUrl
-      ? [plugins.cloud?.setup?.elasticsearchUrl]
-      : await getFallbackESUrl(services.esLegacyConfigService);
-
     return response.ok({
       headers: {
         'content-type': 'application/x-tar',
       },
-      body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
+      body: generateAgentConfigTar(output, installedIntegrations),
     });
   },
 });
@@ -565,14 +568,9 @@ function parseRegistryIntegrationMetadata(
   }
 }
 
-const generateAgentConfigTar = ({
-  elasticsearchUrl,
-  installedIntegrations,
-}: {
-  elasticsearchUrl: string[];
-  installedIntegrations: InstalledIntegration[];
-}) => {
+function generateAgentConfigTar(output: Output, installedIntegrations: InstalledIntegration[]) {
   const now = new Date();
+
   return makeTar([
     {
       type: 'File',
@@ -581,11 +579,7 @@ const generateAgentConfigTar = ({
       mtime: now,
       data: safeDump({
         outputs: {
-          default: {
-            type: 'elasticsearch',
-            hosts: elasticsearchUrl,
-            api_key: '${API_KEY}', // Placeholder to be replaced by bash script with the actual API key
-          },
+          default: transformOutputToFullPolicyOutput(output, undefined, true),
         },
       }),
     },
@@ -603,7 +597,7 @@ const generateAgentConfigTar = ({
       data: integration.config,
     })),
   ]);
-};
+}
 
 export const flowRouteRepository = {
   ...createFlowRoute,