From 3ad240427237598c52deb40b39161701084c6205 Mon Sep 17 00:00:00 2001 From: Thom Heymann <190132+thomheymann@users.noreply.github.com> Date: Mon, 25 Nov 2024 11:14:49 +0000 Subject: [PATCH] [Observability] Use Fleet's default data output when onboarding integrations using auto-detect flow (#201158) Resolves #199751 ## Summary Use Fleet's default data output when onboarding integrations using auto-detect flow. ## Screenshot ### Fleet output settings Screenshot 2024-11-21 at 15 10 24 ### Generated Agent config ``` $ cat /Library/Elastic/Agent/elastic-agent.yml outputs: default: type: elasticsearch hosts: - https://192.168.1.73:9200 ssl.ca_trusted_fingerprint: c48c98cdf7f85d7cc29f834704011c1002b5251d594fc0bb08e6177544fe304a api_key: b1BkR1Q1TUIyOUpfMWhaS2NCUXA6SS1Jb3dncGVReVNpcEdzOGpSVmlzQQ== preset: balanced ``` ## Testing 1. Go to Fleet > Settings > Outputs 2. Edit the default output and enter dummy data into the "Elasticsearch CA trusted fingerprint" field 3. Go through the auto-detect onboarding flow 4. Inspect the `elastic-agent.yml` file written to disk. It should contain the default output configured in Fleet including `ca_trusted_fingerprint` setting --- x-pack/plugins/fleet/server/mocks/index.ts | 2 + x-pack/plugins/fleet/server/plugin.ts | 13 ++++ .../server/services/output_client.mock.ts | 21 ++++++ .../server/services/output_client.test.ts | 71 +++++++++++++++++++ .../fleet/server/services/output_client.ts | 40 +++++++++++ .../server/routes/flow/route.ts | 34 ++++----- 6 files changed, 161 insertions(+), 20 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/output_client.mock.ts create mode 100644 x-pack/plugins/fleet/server/services/output_client.test.ts create mode 100644 x-pack/plugins/fleet/server/services/output_client.ts diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 8d452b394dd18..db100c5fcf7ec 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -29,6 +29,7 @@ import { createFleetActionsClientMock } from '../services/actions/mocks'; import { createFleetFilesClientFactoryMock } from '../services/files/mocks'; import { createArtifactsClientMock } from '../services/artifacts/mocks'; +import { createOutputClientMock } from '../services/output_client.mock'; import type { PackagePolicyClient } from '../services/package_policy_service'; import type { AgentPolicyServiceInterface } from '../services'; @@ -303,6 +304,7 @@ export const createFleetStartContractMock = (): DeeplyMockedKeys fleetActionsClient), getPackageSpecTagId: jest.fn(getPackageSpecTagId), + createOutputClient: jest.fn(async (_) => createOutputClientMock()), }; return startContract; diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 7716d6b21d9e3..1620df27b82c3 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -84,6 +84,8 @@ import { MessageSigningService, } from './services/security'; +import { OutputClient, type OutputClientInterface } from './services/output_client'; + import { ASSETS_SAVED_OBJECT_TYPE, DOWNLOAD_SOURCE_SAVED_OBJECT_TYPE, @@ -262,6 +264,12 @@ export interface FleetStartContract { Function exported to allow creating unique ids for saved object tags */ getPackageSpecTagId: (spaceId: string, pkgName: string, tagName: string) => string; + + /** + * Create a Fleet Output Client instance + * @param packageName + */ + createOutputClient: (request: KibanaRequest) => Promise; } export class FleetPlugin @@ -837,6 +845,11 @@ export class FleetPlugin return new FleetActionsClient(core.elasticsearch.client.asInternalUser, packageName); }, getPackageSpecTagId, + async createOutputClient(request: KibanaRequest) { + const soClient = appContextService.getSavedObjects().getScopedClient(request); + const authz = await getAuthzFromRequest(request); + return new OutputClient(soClient, authz); + }, }; } diff --git a/x-pack/plugins/fleet/server/services/output_client.mock.ts b/x-pack/plugins/fleet/server/services/output_client.mock.ts new file mode 100644 index 0000000000000..ba684a2aff615 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/output_client.mock.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { OutputClientInterface } from './output_client'; + +export const createOutputClientMock = (): jest.Mocked => { + return { + getDefaultDataOutputId: jest.fn(), + get: jest.fn(), + }; +}; diff --git a/x-pack/plugins/fleet/server/services/output_client.test.ts b/x-pack/plugins/fleet/server/services/output_client.test.ts new file mode 100644 index 0000000000000..f3d4b83bea4bc --- /dev/null +++ b/x-pack/plugins/fleet/server/services/output_client.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; + +import { createFleetAuthzMock } from '../../common/mocks'; + +import { OutputClient } from './output_client'; +import { outputService } from './output'; + +jest.mock('./output'); + +const mockedOutputService = outputService as jest.Mocked; + +describe('OutputClient', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('getDefaultDataOutputId()', () => { + it('should call output service `getDefaultDataOutputId()` method', async () => { + const soClient = savedObjectsClientMock.create(); + const authz = createFleetAuthzMock(); + const outputClient = new OutputClient(soClient, authz); + await outputClient.getDefaultDataOutputId(); + + expect(mockedOutputService.getDefaultDataOutputId).toHaveBeenCalledWith(soClient); + }); + + it('should throw error when no `fleet.readSettings` and no `fleet.readAgentPolicies` privileges', async () => { + const soClient = savedObjectsClientMock.create(); + const authz = createFleetAuthzMock(); + authz.fleet.readSettings = false; + authz.fleet.readAgentPolicies = false; + const outputClient = new OutputClient(soClient, authz); + + await expect(outputClient.getDefaultDataOutputId()).rejects.toMatchInlineSnapshot( + `[OutputUnauthorizedError]` + ); + expect(mockedOutputService.getDefaultDataOutputId).not.toHaveBeenCalled(); + }); + }); + + describe('get()', () => { + it('should call output service `get()` method', async () => { + const soClient = savedObjectsClientMock.create(); + const authz = createFleetAuthzMock(); + const outputClient = new OutputClient(soClient, authz); + await outputClient.get('default'); + + expect(mockedOutputService.get).toHaveBeenCalledWith(soClient, 'default'); + }); + + it('should throw error when no `fleet.readSettings` and no `fleet.readAgentPolicies` privileges', async () => { + const soClient = savedObjectsClientMock.create(); + const authz = createFleetAuthzMock(); + authz.fleet.readSettings = false; + authz.fleet.readAgentPolicies = false; + const outputClient = new OutputClient(soClient, authz); + + await expect(outputClient.get('default')).rejects.toMatchInlineSnapshot( + `[OutputUnauthorizedError]` + ); + expect(mockedOutputService.get).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/output_client.ts b/x-pack/plugins/fleet/server/services/output_client.ts new file mode 100644 index 0000000000000..574446e1f32a7 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/output_client.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; + +import type { FleetAuthz } from '../../common'; + +import { OutputUnauthorizedError } from '../errors'; +import type { Output } from '../types'; + +import { outputService } from './output'; + +export { transformOutputToFullPolicyOutput } from './agent_policies/full_agent_policy'; + +export interface OutputClientInterface { + getDefaultDataOutputId(): Promise; + get(outputId: string): Promise; +} + +export class OutputClient implements OutputClientInterface { + constructor(private soClient: SavedObjectsClientContract, private authz: FleetAuthz) {} + + async getDefaultDataOutputId() { + if (!this.authz.fleet.readSettings && !this.authz.fleet.readAgentPolicies) { + throw new OutputUnauthorizedError(); + } + return outputService.getDefaultDataOutputId(this.soClient); + } + + async get(outputId: string) { + if (!this.authz.fleet.readSettings && !this.authz.fleet.readAgentPolicies) { + throw new OutputUnauthorizedError(); + } + return outputService.get(this.soClient, outputId); + } +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts index 372eaf7972373..738c9f1fefd57 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts @@ -13,7 +13,8 @@ import { type PackageClient, } from '@kbn/fleet-plugin/server'; import { dump } from 'js-yaml'; -import { PackageDataStreamTypes } from '@kbn/fleet-plugin/common/types'; +import { PackageDataStreamTypes, Output } from '@kbn/fleet-plugin/common/types'; +import { transformOutputToFullPolicyOutput } from '@kbn/fleet-plugin/server/services/output_client'; import { getObservabilityOnboardingFlow, saveObservabilityOnboardingFlow } from '../../lib/state'; import type { SavedObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status'; import { ObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status'; @@ -21,7 +22,6 @@ import { createObservabilityOnboardingServerRoute } from '../create_observabilit import { getHasLogs } from './get_has_logs'; import { getKibanaUrl } from '../../lib/get_fallback_urls'; import { getAgentVersionInfo } from '../../lib/get_agent_version'; -import { getFallbackESUrl } from '../../lib/get_fallback_urls'; import { ElasticAgentStepPayload, InstalledIntegration, StepProgressPayloadRT } from '../types'; import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key'; import { createInstallApiKey } from '../../lib/api_key/create_install_api_key'; @@ -329,6 +329,13 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({ throw Boom.notFound(`Onboarding session '${params.path.onboardingId}' not found.`); } + const outputClient = await fleetStart.createOutputClient(request); + const defaultOutputId = await outputClient.getDefaultDataOutputId(); + if (!defaultOutputId) { + throw Boom.notFound('Default data output not found'); + } + const output = await outputClient.get(defaultOutputId); + const integrationsToInstall = parseIntegrationsTSV(params.body); if (!integrationsToInstall.length) { return response.badRequest({ @@ -381,15 +388,11 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({ }, }); - const elasticsearchUrl = plugins.cloud?.setup?.elasticsearchUrl - ? [plugins.cloud?.setup?.elasticsearchUrl] - : await getFallbackESUrl(services.esLegacyConfigService); - return response.ok({ headers: { 'content-type': 'application/x-tar', }, - body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }), + body: generateAgentConfigTar(output, installedIntegrations), }); }, }); @@ -565,14 +568,9 @@ function parseRegistryIntegrationMetadata( } } -const generateAgentConfigTar = ({ - elasticsearchUrl, - installedIntegrations, -}: { - elasticsearchUrl: string[]; - installedIntegrations: InstalledIntegration[]; -}) => { +function generateAgentConfigTar(output: Output, installedIntegrations: InstalledIntegration[]) { const now = new Date(); + return makeTar([ { type: 'File', @@ -581,11 +579,7 @@ const generateAgentConfigTar = ({ mtime: now, data: dump({ outputs: { - default: { - type: 'elasticsearch', - hosts: elasticsearchUrl, - api_key: '${API_KEY}', // Placeholder to be replaced by bash script with the actual API key - }, + default: transformOutputToFullPolicyOutput(output, undefined, true), }, }), }, @@ -603,7 +597,7 @@ const generateAgentConfigTar = ({ data: integration.config, })), ]); -}; +} export const flowRouteRepository = { ...createFlowRoute,