From 45242fe92a83a4a69a9a5cf491814f8384c4d327 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Tue, 12 Dec 2023 15:41:38 +0100 Subject: [PATCH 01/12] [Fleet] Expose install package (#173148) --- .../services/epm/package_service.mock.ts | 1 + .../server/services/epm/package_service.ts | 57 ++++++++++++++++--- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts index 4007ad7545ec..3eb689dfa1a2 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts @@ -10,6 +10,7 @@ import type { PackageClient, PackageService } from './package_service'; const createClientMock = (): jest.Mocked => ({ getInstallation: jest.fn(), ensureInstalledPackage: jest.fn(), + installPackage: jest.fn(), fetchFindLatestPackage: jest.fn(), readBundledPackage: jest.fn(), getPackage: jest.fn(), diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index eee1afb37dca..a535af9636d1 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -8,34 +8,40 @@ /* eslint-disable max-classes-per-file */ import type { - KibanaRequest, ElasticsearchClient, - SavedObjectsClientContract, + KibanaRequest, Logger, + SavedObjectsClientContract, } from '@kbn/core/server'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; + import { HTTPAuthorizationHeader } from '../../../common/http_authorization_header'; import type { PackageList } from '../../../common'; import type { + ArchivePackage, + BundledPackage, CategoryId, EsAssetReference, InstallablePackage, Installation, RegistryPackage, - ArchivePackage, - BundledPackage, } from '../../types'; import type { FleetAuthzRouteConfig } from '../security/types'; -import { checkSuperuser, getAuthzFromRequest, doesNotHaveRequiredFleetAuthz } from '../security'; -import { FleetUnauthorizedError, FleetError } from '../../errors'; +import { checkSuperuser, doesNotHaveRequiredFleetAuthz, getAuthzFromRequest } from '../security'; +import { FleetError, FleetUnauthorizedError } from '../../errors'; import { INSTALL_PACKAGES_AUTHZ, READ_PACKAGE_INFO_AUTHZ } from '../../routes/epm'; -import { installTransforms, isTransform } from './elasticsearch/transform/install'; +import type { InstallResult } from '../../../common'; + import type { FetchFindLatestPackageOptions } from './registry'; +import * as Registry from './registry'; import { fetchFindLatestPackageOrThrow, getPackage } from './registry'; -import { ensureInstalledPackage, getInstallation, getPackages } from './packages'; + +import { installTransforms, isTransform } from './elasticsearch/transform/install'; +import { ensureInstalledPackage, getInstallation, getPackages, installPackage } from './packages'; import { generatePackageInfoFromArchiveBuffer } from './archive'; export type InstalledAssetType = EsAssetReference; @@ -52,8 +58,16 @@ export interface PackageClient { pkgName: string; pkgVersion?: string; spaceId?: string; + force?: boolean; }): Promise; + installPackage(options: { + pkgName: string; + pkgVersion?: string; + spaceId?: string; + force?: boolean; + }): Promise; + fetchFindLatestPackage( packageName: string, options?: FetchFindLatestPackageOptions @@ -151,6 +165,7 @@ class PackageClientImpl implements PackageClient { pkgName: string; pkgVersion?: string; spaceId?: string; + force?: boolean; }): Promise { await this.#runPreflight(INSTALL_PACKAGES_AUTHZ); @@ -160,6 +175,32 @@ class PackageClientImpl implements PackageClient { savedObjectsClient: this.internalSoClient, }); } + public async installPackage(options: { + pkgName: string; + pkgVersion?: string; + spaceId?: string; + force?: boolean; + }): Promise { + await this.#runPreflight(INSTALL_PACKAGES_AUTHZ); + + const { pkgName, pkgVersion, spaceId = DEFAULT_SPACE_ID, force = false } = options; + + // If pkgVersion isn't specified, find the latest package version + const pkgKeyProps = pkgVersion + ? { name: pkgName, version: pkgVersion } + : await Registry.fetchFindLatestPackageOrThrow(pkgName, { prerelease: true }); + const pkgkey = Registry.pkgToPkgKey(pkgKeyProps); + + return await installPackage({ + force, + pkgkey, + spaceId, + installSource: 'registry', + esClient: this.internalEsClient, + savedObjectsClient: this.internalSoClient, + neverIgnoreVerificationError: !force, + }); + } public async fetchFindLatestPackage( packageName: string, From 05bfe53cb3a2fe33ecb9eec4a6fcb19a492aaadf Mon Sep 17 00:00:00 2001 From: Jordan <51442161+JordanSh@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:56:24 +0200 Subject: [PATCH 02/12] [Cloud Security] Azure org as default value (#173047) --- .../public/components/fleet_extensions/policy_template_form.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.tsx b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.tsx index 89ec1007f7d5..17474fd9cd23 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.tsx @@ -442,7 +442,7 @@ const AzureAccountTypeSelect = ({ updatePolicy( getPosturePolicy(newPolicy, input.type, { 'azure.account_type': { - value: AZURE_SINGLE_ACCOUNT, + value: isAzureOrganizationDisabled ? AZURE_SINGLE_ACCOUNT : AZURE_ORGANIZATION_ACCOUNT, type: 'text', }, 'azure.credentials.type': { From 6443b571642c453a9232a04297fddfb0a918c0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Tue, 12 Dec 2023 15:09:21 +0000 Subject: [PATCH 03/12] [ProfilingxAPM] Passing selected time range to links (#173160) I forgot to pass the selected time range to the Profiling links in the APM UI. Fixing it here. --- .../components/app/profiling_overview/index.tsx | 15 ++++++++++++++- .../profiling_overview/profiling_flamegraph.tsx | 6 ++++++ .../profiling_top_functions.tsx | 6 ++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/apm/public/components/app/profiling_overview/index.tsx b/x-pack/plugins/apm/public/components/app/profiling_overview/index.tsx index 3e460448ad42..9e8a0e2baf46 100644 --- a/x-pack/plugins/apm/public/components/app/profiling_overview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/profiling_overview/index.tsx @@ -78,6 +78,8 @@ export function ProfilingOverview() { environment={environment} dataSource={preferred?.source} kuery={kuery} + rangeFrom={rangeFrom} + rangeTo={rangeTo} /> ), @@ -99,12 +101,23 @@ export function ProfilingOverview() { endIndex={10} dataSource={preferred?.source} kuery={kuery} + rangeFrom={rangeFrom} + rangeTo={rangeTo} /> ), }, ]; - }, [end, environment, kuery, preferred?.source, serviceName, start]); + }, [ + end, + environment, + kuery, + preferred?.source, + rangeFrom, + rangeTo, + serviceName, + start, + ]); if (isLoading) { return ( diff --git a/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_flamegraph.tsx b/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_flamegraph.tsx index 8dcda14783a8..bbae54abdf70 100644 --- a/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_flamegraph.tsx +++ b/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_flamegraph.tsx @@ -40,6 +40,8 @@ interface Props { ApmDocumentType.TransactionMetric | ApmDocumentType.TransactionEvent >; kuery: string; + rangeFrom: string; + rangeTo: string; } export function ProfilingFlamegraph({ @@ -49,6 +51,8 @@ export function ProfilingFlamegraph({ environment, dataSource, kuery, + rangeFrom, + rangeTo, }: Props) { const { profilingLocators } = useProfilingPlugin(); @@ -93,6 +97,8 @@ export function ProfilingFlamegraph({ data-test-subj="apmProfilingFlamegraphGoToFlamegraphLink" href={profilingLocators?.flamegraphLocator.getRedirectUrl({ kuery: mergeKueries([`(${hostNamesKueryFormat})`, kuery]), + rangeFrom, + rangeTo, })} > {i18n.translate('xpack.apm.profiling.flamegraph.link', { diff --git a/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_top_functions.tsx b/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_top_functions.tsx index 7b428802fbfc..0462af188d3f 100644 --- a/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_top_functions.tsx +++ b/x-pack/plugins/apm/public/components/app/profiling_overview/profiling_top_functions.tsx @@ -31,6 +31,8 @@ interface Props { ApmDocumentType.TransactionMetric | ApmDocumentType.TransactionEvent >; kuery: string; + rangeFrom: string; + rangeTo: string; } export function ProfilingTopNFunctions({ @@ -42,6 +44,8 @@ export function ProfilingTopNFunctions({ endIndex, dataSource, kuery, + rangeFrom, + rangeTo, }: Props) { const { profilingLocators } = useProfilingPlugin(); @@ -97,6 +101,8 @@ export function ProfilingTopNFunctions({ data-test-subj="apmProfilingTopNFunctionsGoToUniversalProfilingFlamegraphLink" href={profilingLocators?.topNFunctionsLocator.getRedirectUrl({ kuery: mergeKueries([`(${hostNamesKueryFormat})`, kuery]), + rangeFrom, + rangeTo, })} > {i18n.translate('xpack.apm.profiling.topnFunctions.link', { From f8fb553d79a6469d365fa73d16f015f5ade5e2f3 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:46:53 +0100 Subject: [PATCH 04/12] [Fleet] fix text wrapping in output error callout + hide cancel change btn (#173149) ## Summary 1. Fixed text wrapping in output error callout when the hostname is very long. image 2. Hiding Cancel X change button when creating secret for the first time Closes https://github.com/elastic/kibana/issues/173034 image --- .../output_form_secret_form_row.test.tsx | 16 ++++++++++++++++ .../output_form_secret_form_row.tsx | 4 ++-- .../edit_output_flyout/output_health.test.tsx | 2 +- .../edit_output_flyout/output_health.tsx | 4 ++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.test.tsx index 94bb49d75128..53945f86e09e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.test.tsx @@ -75,4 +75,20 @@ describe('SecretFormRow', () => { expect(onUsePlainText).toHaveBeenCalled(); }); + + it('should not display the cancel change button when no initial value is provided', () => { + const { queryByTestId } = render( + + + + ); + + expect(queryByTestId('secretCancelChangeBtn')).not.toBeInTheDocument(); + }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.tsx index f483503af9e4..868c80b895fa 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_secret_form_row.tsx @@ -41,7 +41,7 @@ export const SecretFormRow: React.FC<{ onUsePlainText, cancelEdit, }) => { - const hasInitialValue = initialValue !== undefined; + const hasInitialValue = !!initialValue; const [editMode, setEditMode] = useState(!initialValue); const valueHiddenPanel = ( @@ -98,7 +98,7 @@ export const SecretFormRow: React.FC<{ <> {children} {hasInitialValue && ( - + {cancelButton} )} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.test.tsx index 7aa29322229d..a666dda3bac4 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.test.tsx @@ -72,7 +72,7 @@ describe('OutputHealth', () => { await waitFor(async () => { expect(utils.getByTestId('outputHealthDegradedCallout').textContent).toContain( - 'Unable to connect to "Remote ES" at https://remote-es:443. Please check the details are correct.' + 'Unable to connect to "Remote ES" at https://remote-es:443.Please check the details are correct.' ); }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.tsx index c26a122287d0..0d71bb075cf2 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.tsx @@ -51,7 +51,7 @@ export const OutputHealth: React.FunctionComponent = ({ output, showBadge iconType="error" data-test-subj="outputHealthDegradedCallout" > -

+

{i18n.translate('xpack.fleet.output.calloutText', { defaultMessage: 'Unable to connect to "{name}" at {host}.', values: { @@ -59,7 +59,7 @@ export const OutputHealth: React.FunctionComponent = ({ output, showBadge host: output.hosts?.join(',') ?? '', }, })} -

{' '} +

{i18n.translate('xpack.fleet.output.calloutPromptText', { defaultMessage: 'Please check the details are correct.', From fa3b6f4c9da88e6ddbd1fae7c502d00353c55134 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 12 Dec 2023 11:05:46 -0500 Subject: [PATCH 05/12] [Fleet] Fix get file handler for bundled package (#172182) (#173058) --- .../server/routes/epm/file_handler.test.ts | 245 ++++++++++++++++++ .../fleet/server/routes/epm/file_handler.ts | 141 ++++++++++ .../fleet/server/routes/epm/handlers.ts | 84 +----- .../plugins/fleet/server/routes/epm/index.ts | 2 +- x-pack/plugins/fleet/tsconfig.json | 1 + 5 files changed, 389 insertions(+), 84 deletions(-) create mode 100644 x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts create mode 100644 x-pack/plugins/fleet/server/routes/epm/file_handler.ts diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts new file mode 100644 index 000000000000..56d6d8c127bc --- /dev/null +++ b/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { httpServerMock } from '@kbn/core-http-server-mocks'; +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; +import { Headers } from 'node-fetch'; + +import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_packages'; +import { getFile, getInstallation } from '../../services/epm/packages/get'; +import type { FleetRequestHandlerContext } from '../..'; +import { appContextService } from '../../services'; +import { unpackBufferEntries, getArchiveEntry } from '../../services/epm/archive'; +import { getAsset } from '../../services/epm/archive/storage'; + +import { getFileHandler } from './file_handler'; + +jest.mock('../../services/app_context'); +jest.mock('../../services/epm/archive'); +jest.mock('../../services/epm/archive/storage'); +jest.mock('../../services/epm/packages/bundled_packages'); +jest.mock('../../services/epm/packages/get'); + +const mockedGetBundledPackageByPkgKey = jest.mocked(getBundledPackageByPkgKey); +const mockedGetInstallation = jest.mocked(getInstallation); +const mockedGetFile = jest.mocked(getFile); +const mockedGetArchiveEntry = jest.mocked(getArchiveEntry); +const mockedUnpackBufferEntries = jest.mocked(unpackBufferEntries); +const mockedGetAsset = jest.mocked(getAsset); + +function mockContext() { + const mockSavedObjectsClient = savedObjectsClientMock.create(); + const mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + return { + fleet: { + internalSOClient: async () => mockSavedObjectsClient, + }, + core: { + savedObjects: { + client: mockSavedObjectsClient, + }, + elasticsearch: { + client: { + asInternalUser: mockElasticsearchClient, + }, + }, + }, + } as unknown as FleetRequestHandlerContext; +} + +describe('getFileHandler', () => { + beforeEach(() => { + const logger = loggingSystemMock.createLogger(); + jest.mocked(appContextService).getLogger.mockReturnValue(logger); + mockedGetBundledPackageByPkgKey.mockReset(); + mockedUnpackBufferEntries.mockReset(); + mockedGetFile.mockReset(); + mockedGetInstallation.mockReset(); + mockedGetArchiveEntry.mockReset(); + mockedGetAsset.mockReset(); + }); + + it('should return the file for bundled package and an existing file', async () => { + mockedGetBundledPackageByPkgKey.mockResolvedValue({ + getBuffer: () => Promise.resolve(), + } as any); + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'README.md', + }, + }); + const buffer = Buffer.from(`TEST`); + mockedUnpackBufferEntries.mockResolvedValue([ + { + path: 'test-1.0.0/README.md', + buffer, + }, + ]); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 200, + body: buffer, + headers: expect.objectContaining({ + 'content-type': 'text/markdown; charset=utf-8', + }), + }) + ); + }); + + it('should a 404 for bundled package with a non existing file', async () => { + mockedGetBundledPackageByPkgKey.mockResolvedValue({ + getBuffer: () => Promise.resolve(), + } as any); + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'idonotexists.md', + }, + }); + mockedUnpackBufferEntries.mockResolvedValue([ + { + path: 'test-1.0.0/README.md', + buffer: Buffer.from(`TEST`), + }, + ]); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 404, + body: 'bundled package file not found: idonotexists.md', + }) + ); + }); + + it('should proxy registry 200 for non bundled and non installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'idonotexists.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetFile.mockResolvedValue({ + status: 200, + // @ts-expect-error + body: 'test', + headers: new Headers({ + raw: '', + 'content-type': 'text/markdown', + }), + }); + + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 200, + body: 'test', + headers: expect.objectContaining({ + 'content-type': 'text/markdown', + }), + }) + ); + }); + + it('should proxy registry 404 for non bundled and non installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'idonotexists.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetFile.mockResolvedValue({ + status: 404, + // @ts-expect-error + body: 'not found', + headers: new Headers({ + raw: '', + 'content-type': 'text', + }), + }); + + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 404, + body: 'not found', + headers: expect.objectContaining({ + 'content-type': 'text', + }), + }) + ); + }); + + it('should return the file from installation for installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'README.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetInstallation.mockResolvedValue({ version: '1.0.0' } as any); + mockedGetArchiveEntry.mockReturnValue(Buffer.from('test')); + + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 200, + headers: expect.objectContaining({ + 'content-type': 'text/markdown; charset=utf-8', + }), + }) + ); + }); + + it('should a 404 if the file from installation do not exists for installed package', async () => { + const request = httpServerMock.createKibanaRequest({ + params: { + pkgName: 'test', + pkgVersion: '1.0.0', + filePath: 'README.md', + }, + }); + const response = httpServerMock.createResponseFactory(); + const context = mockContext(); + + mockedGetInstallation.mockResolvedValue({ version: '1.0.0' } as any); + await getFileHandler(context, request, response); + + expect(response.custom).toBeCalledWith( + expect.objectContaining({ + statusCode: 404, + body: 'installed package file not found: README.md', + }) + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts new file mode 100644 index 000000000000..4b6b74628aa4 --- /dev/null +++ b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; + +import type { TypeOf } from '@kbn/config-schema'; +import mime from 'mime-types'; +import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/core/server'; + +import type { GetFileRequestSchema, FleetRequestHandler } from '../../types'; +import { getFile, getInstallation } from '../../services/epm/packages'; +import { defaultFleetErrorHandler } from '../../errors'; +import { getArchiveEntry } from '../../services/epm/archive'; +import { getAsset } from '../../services/epm/archive/storage'; +import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_packages'; +import { pkgToPkgKey } from '../../services/epm/registry'; +import { unpackBufferEntries } from '../../services/epm/archive'; + +const CACHE_CONTROL_10_MINUTES_HEADER: HttpResponseOptions['headers'] = { + 'cache-control': 'max-age=600', +}; +export const getFileHandler: FleetRequestHandler< + TypeOf +> = async (context, request, response) => { + try { + const { pkgName, pkgVersion, filePath } = request.params; + const savedObjectsClient = (await context.fleet).internalSoClient; + + const installation = await getInstallation({ savedObjectsClient, pkgName }); + const useLocalFile = pkgVersion === installation?.version; + const assetPath = `${pkgName}-${pkgVersion}/${filePath}`; + + if (useLocalFile) { + const fileBuffer = getArchiveEntry(assetPath); + // only pull local installation if we don't have it cached + const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath })); + + // error, if neither is available + if (!fileBuffer && !storedAsset) { + return response.custom({ + body: `installed package file not found: ${filePath}`, + statusCode: 404, + }); + } + + // if storedAsset is not available, fileBuffer *must* be + // b/c we error if we don't have at least one, and storedAsset is the least likely + const { buffer, contentType } = storedAsset + ? { + contentType: storedAsset.media_type, + buffer: storedAsset.data_utf8 + ? Buffer.from(storedAsset.data_utf8, 'utf8') + : Buffer.from(storedAsset.data_base64, 'base64'), + } + : { + contentType: mime.contentType(path.extname(assetPath)), + buffer: fileBuffer, + }; + + if (!contentType) { + return response.custom({ + body: `unknown content type for file: ${filePath}`, + statusCode: 400, + }); + } + + return response.custom({ + body: buffer, + statusCode: 200, + headers: { + ...CACHE_CONTROL_10_MINUTES_HEADER, + 'content-type': contentType, + }, + }); + } + + const bundledPackage = await getBundledPackageByPkgKey( + pkgToPkgKey({ name: pkgName, version: pkgVersion }) + ); + if (bundledPackage) { + const bufferEntries = await unpackBufferEntries( + await bundledPackage.getBuffer(), + 'application/zip' + ); + + const fileBuffer = bufferEntries.find((entry) => entry.path === assetPath)?.buffer; + + if (!fileBuffer) { + return response.custom({ + body: `bundled package file not found: ${filePath}`, + statusCode: 404, + }); + } + + // if storedAsset is not available, fileBuffer *must* be + // b/c we error if we don't have at least one, and storedAsset is the least likely + const { buffer, contentType } = { + contentType: mime.contentType(path.extname(assetPath)), + buffer: fileBuffer, + }; + + if (!contentType) { + return response.custom({ + body: `unknown content type for file: ${filePath}`, + statusCode: 400, + }); + } + + return response.custom({ + body: buffer, + statusCode: 200, + headers: { + ...CACHE_CONTROL_10_MINUTES_HEADER, + 'content-type': contentType, + }, + }); + } else { + const registryResponse = await getFile(pkgName, pkgVersion, filePath); + const headersToProxy: KnownHeaders[] = ['content-type']; + const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => { + const value = registryResponse.headers.get(knownHeader); + if (value !== null) { + headers[knownHeader] = value; + } + return headers; + }, {} as ResponseHeaders); + + return response.custom({ + body: registryResponse.body, + statusCode: registryResponse.status, + headers: { ...CACHE_CONTROL_10_MINUTES_HEADER, ...proxiedHeaders }, + }); + } + } catch (error) { + return defaultFleetErrorHandler({ error, response }); + } +}; diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index c03272119dd1..6fadeff5180c 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -5,12 +5,9 @@ * 2.0. */ -import path from 'path'; - import type { TypeOf } from '@kbn/config-schema'; -import mime from 'mime-types'; import semverValid from 'semver/functions/valid'; -import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/core/server'; +import type { HttpResponseOptions } from '@kbn/core/server'; import { pick } from 'lodash'; @@ -41,7 +38,6 @@ import type { GetPackagesRequestSchema, GetInstalledPackagesRequestSchema, GetDataStreamsRequestSchema, - GetFileRequestSchema, GetInfoRequestSchema, InstallPackageFromRegistryRequestSchema, InstallPackageByUploadRequestSchema, @@ -60,21 +56,17 @@ import { getCategories, getPackages, getInstalledPackages, - getFile, getPackageInfo, isBulkInstallError, installPackage, removeInstallation, getLimitedPackages, - getInstallation, getBulkAssets, getTemplateInputs, } from '../../services/epm/packages'; import type { BulkInstallResponse } from '../../services/epm/packages'; import { defaultFleetErrorHandler, fleetErrorToResponseOptions, FleetError } from '../../errors'; import { appContextService, checkAllowedPackages } from '../../services'; -import { getArchiveEntry } from '../../services/epm/archive/cache'; -import { getAsset } from '../../services/epm/archive/storage'; import { getPackageUsageStats } from '../../services/epm/packages/get'; import { updatePackage } from '../../services/epm/packages/update'; import { getGpgKeyIdOrUndefined } from '../../services/epm/packages/package_verification'; @@ -206,80 +198,6 @@ export const getLimitedListHandler: FleetRequestHandler< } }; -export const getFileHandler: FleetRequestHandler< - TypeOf -> = async (context, request, response) => { - try { - const { pkgName, pkgVersion, filePath } = request.params; - const savedObjectsClient = (await context.fleet).internalSoClient; - const installation = await getInstallation({ savedObjectsClient, pkgName }); - const useLocalFile = pkgVersion === installation?.version; - - if (useLocalFile) { - const assetPath = `${pkgName}-${pkgVersion}/${filePath}`; - const fileBuffer = getArchiveEntry(assetPath); - // only pull local installation if we don't have it cached - const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath })); - - // error, if neither is available - if (!fileBuffer && !storedAsset) { - return response.custom({ - body: `installed package file not found: ${filePath}`, - statusCode: 404, - }); - } - - // if storedAsset is not available, fileBuffer *must* be - // b/c we error if we don't have at least one, and storedAsset is the least likely - const { buffer, contentType } = storedAsset - ? { - contentType: storedAsset.media_type, - buffer: storedAsset.data_utf8 - ? Buffer.from(storedAsset.data_utf8, 'utf8') - : Buffer.from(storedAsset.data_base64, 'base64'), - } - : { - contentType: mime.contentType(path.extname(assetPath)), - buffer: fileBuffer, - }; - - if (!contentType) { - return response.custom({ - body: `unknown content type for file: ${filePath}`, - statusCode: 400, - }); - } - - return response.custom({ - body: buffer, - statusCode: 200, - headers: { - ...CACHE_CONTROL_10_MINUTES_HEADER, - 'content-type': contentType, - }, - }); - } else { - const registryResponse = await getFile(pkgName, pkgVersion, filePath); - const headersToProxy: KnownHeaders[] = ['content-type']; - const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => { - const value = registryResponse.headers.get(knownHeader); - if (value !== null) { - headers[knownHeader] = value; - } - return headers; - }, {} as ResponseHeaders); - - return response.custom({ - body: registryResponse.body, - statusCode: registryResponse.status, - headers: { ...CACHE_CONTROL_10_MINUTES_HEADER, ...proxiedHeaders }, - }); - } - } catch (error) { - return defaultFleetErrorHandler({ error, response }); - } -}; - export const getInfoHandler: FleetRequestHandler< TypeOf, TypeOf diff --git a/x-pack/plugins/fleet/server/routes/epm/index.ts b/x-pack/plugins/fleet/server/routes/epm/index.ts index 6e0000bf4ccb..5245381a409d 100644 --- a/x-pack/plugins/fleet/server/routes/epm/index.ts +++ b/x-pack/plugins/fleet/server/routes/epm/index.ts @@ -55,7 +55,6 @@ import { getListHandler, getInstalledListHandler, getLimitedListHandler, - getFileHandler, getInfoHandler, getBulkAssetsHandler, installPackageFromRegistryHandler, @@ -70,6 +69,7 @@ import { createCustomIntegrationHandler, getInputsHandler, } from './handlers'; +import { getFileHandler } from './file_handler'; const MAX_FILE_SIZE_BYTES = 104857600; // 100MB diff --git a/x-pack/plugins/fleet/tsconfig.json b/x-pack/plugins/fleet/tsconfig.json index 4d3c850b9e75..4503d328e150 100644 --- a/x-pack/plugins/fleet/tsconfig.json +++ b/x-pack/plugins/fleet/tsconfig.json @@ -102,5 +102,6 @@ "@kbn/dashboard-plugin", "@kbn/cloud", "@kbn/config", + "@kbn/core-http-server-mocks", ] } From f90a489cac01c9a91669576c30d8b533bb445e3e Mon Sep 17 00:00:00 2001 From: Cristina Amico Date: Tue, 12 Dec 2023 17:14:02 +0100 Subject: [PATCH 06/12] [Fleet][spacetime] Improve fleet logging and error handling (#172657) Solves https://github.com/elastic/kibana/issues/170953 ## Summary Improving logging in various parts of Fleet: - Package lifecycle - Package policy service - Agent policy service - Outputs - Fleet Server Hosts - Proxies Also improving errors handling by using more specific typed errors instead of generic error. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../verify_test_packages.test.ts | 21 ++++++++ .../plugins/fleet/server/errors/handlers.ts | 30 +++++++---- x-pack/plugins/fleet/server/errors/index.ts | 31 ++++++++--- .../server/integration_tests/helpers/index.ts | 4 +- .../server/routes/agent/source_uri_utils.ts | 5 +- .../routes/agent/upgrade_handler.test.ts | 2 +- .../server/routes/agent/upgrade_handler.ts | 23 ++++---- .../server/routes/package_policy/handlers.ts | 13 +++-- .../agent_policies/outputs_helpers.ts | 4 +- .../package_policies_to_agent_permissions.ts | 7 ++- .../agent_policies/related_saved_objects.ts | 7 +-- .../server/services/agent_policy.test.ts | 9 +++- .../fleet/server/services/agent_policy.ts | 49 ++++++++++------- .../fleet/server/services/agents/crud.ts | 12 +++-- .../fleet/server/services/agents/reassign.ts | 11 ++-- .../services/agents/update_agent_tags.test.ts | 6 +-- .../agents/update_agent_tags_action_runner.ts | 8 ++- .../api_keys/enrollment_api_key.test.ts | 14 +++++ .../services/api_keys/enrollment_api_key.ts | 26 ++++++--- .../server/services/download_source.test.ts | 9 +++- .../fleet/server/services/download_source.ts | 17 ++++-- .../server/services/epm/agent/agent.test.ts | 21 ++++++++ .../fleet/server/services/epm/agent/agent.ts | 9 +++- .../server/services/epm/archive/cache.ts | 4 +- .../server/services/epm/archive/index.ts | 8 ++- .../server/services/epm/archive/parse.test.ts | 20 +++++++ .../server/services/epm/archive/parse.ts | 15 +++++- .../services/epm/packages/_install_package.ts | 41 +++++++++++--- .../services/epm/packages/install.test.ts | 2 +- .../server/services/epm/packages/install.ts | 13 +++-- .../fleet/server/services/files/utils.ts | 5 +- .../server/services/fleet_proxies.test.ts | 17 ++++++ .../fleet/server/services/fleet_proxies.ts | 15 +++++- .../server/services/fleet_server_host.test.ts | 20 +++++++ .../server/services/fleet_server_host.ts | 19 +++++-- .../plugins/fleet/server/services/output.ts | 23 +++++--- .../fleet/server/services/package_policy.ts | 53 +++++++++++++------ .../fleet/server/services/preconfiguration.ts | 14 ++--- .../preconfiguration/fleet_server_host.ts | 4 +- .../preconfiguration/reset_agent_policies.ts | 3 +- .../security/message_signing_service.ts | 7 +-- x-pack/plugins/fleet/server/services/setup.ts | 3 +- .../setup/clean_old_fleet_indices.tsx | 1 + .../apis/enrollment_api_keys/crud.ts | 16 +++--- 44 files changed, 481 insertions(+), 160 deletions(-) diff --git a/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts b/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts index 219d101d6e1a..b8d1ba7e9bec 100644 --- a/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts +++ b/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts @@ -5,9 +5,30 @@ * 2.0. */ +import { securityMock } from '@kbn/security-plugin/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +import type { Logger } from '@kbn/core/server'; + +import { appContextService } from '../../server/services/app_context'; + import { verifyAllTestPackages } from './verify_test_packages'; +jest.mock('../../server/services/app_context'); + +const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + +let mockedLogger: jest.Mocked; + describe('Test packages', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + test('All test packages should be valid (node scripts/verify_test_packages) ', async () => { const { errors } = await verifyAllTestPackages(); expect(errors).toEqual([]); diff --git a/x-pack/plugins/fleet/server/errors/handlers.ts b/x-pack/plugins/fleet/server/errors/handlers.ts index 3bfe94537c58..c85bfeced9db 100644 --- a/x-pack/plugins/fleet/server/errors/handlers.ts +++ b/x-pack/plugins/fleet/server/errors/handlers.ts @@ -20,18 +20,14 @@ import { UninstallTokenError } from '../../common/errors'; import { appContextService } from '../services'; import { - AgentNotFoundError, - AgentActionNotFoundError, AgentPolicyNameExistsError, ConcurrentInstallOperationError, FleetError, - PackageNotFoundError, PackageUnsupportedMediaTypeError, RegistryConnectionError, RegistryError, RegistryResponseError, PackageFailedVerificationError, - PackagePolicyNotFoundError, FleetUnauthorizedError, PackagePolicyNameExistsError, PackageOutdatedError, @@ -41,6 +37,11 @@ import { PackageESError, KibanaSOReferenceError, PackageAlreadyInstalledError, + AgentPolicyInvalidError, + EnrollmentKeyNameExistsError, + AgentRequestInvalidError, + PackagePolicyRequestError, + FleetNotFoundError, } from '.'; type IngestErrorHandler = ( @@ -71,24 +72,31 @@ const getHTTPResponseCode = (error: FleetError): number => { if (error instanceof KibanaSOReferenceError) { return 400; } + if (error instanceof AgentPolicyInvalidError) { + return 400; + } + if (error instanceof AgentRequestInvalidError) { + return 400; + } + if (error instanceof PackagePolicyRequestError) { + return 400; + } // Unauthorized if (error instanceof FleetUnauthorizedError) { return 403; } // Not Found - if (error instanceof PackageNotFoundError || error instanceof PackagePolicyNotFoundError) { - return 404; - } - if (error instanceof AgentNotFoundError) { - return 404; - } - if (error instanceof AgentActionNotFoundError) { + if (error instanceof FleetNotFoundError) { return 404; } + // Conflict if (error instanceof AgentPolicyNameExistsError) { return 409; } + if (error instanceof EnrollmentKeyNameExistsError) { + return 409; + } if (error instanceof ConcurrentInstallOperationError) { return 409; } diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index 7f607f469277..ce7245672e62 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -28,7 +28,7 @@ export class RegistryResponseError extends RegistryError { } // Package errors -export class PackageNotFoundError extends FleetError {} + export class PackageOutdatedError extends FleetError {} export class PackageFailedVerificationError extends FleetError { constructor(pkgName: string, pkgVersion: string) { @@ -43,20 +43,24 @@ export class PackageInvalidArchiveError extends FleetError {} export class PackageRemovalError extends FleetError {} export class PackageESError extends FleetError {} export class ConcurrentInstallOperationError extends FleetError {} -export class BundledPackageLocationNotFoundError extends FleetError {} + export class KibanaSOReferenceError extends FleetError {} export class PackageAlreadyInstalledError extends FleetError {} export class AgentPolicyError extends FleetError {} -export class AgentPolicyNotFoundError extends FleetError {} -export class AgentNotFoundError extends FleetError {} -export class AgentActionNotFoundError extends FleetError {} +export class AgentRequestInvalidError extends FleetError {} +export class AgentPolicyInvalidError extends FleetError {} + export class AgentPolicyNameExistsError extends AgentPolicyError {} export class AgentReassignmentError extends FleetError {} export class PackagePolicyIneligibleForUpgradeError extends FleetError {} export class PackagePolicyValidationError extends FleetError {} export class PackagePolicyNameExistsError extends FleetError {} -export class PackagePolicyNotFoundError extends FleetError {} +export class BundledPackageLocationNotFoundError extends FleetError {} + +export class PackagePolicyRequestError extends FleetError {} + +export class EnrollmentKeyNameExistsError extends FleetError {} export class HostedAgentPolicyRestrictionRelatedError extends FleetError { constructor(message = 'Cannot perform that action') { super( @@ -75,12 +79,27 @@ export class FleetEncryptedSavedObjectEncryptionKeyRequired extends FleetError { export class FleetSetupError extends FleetError {} export class GenerateServiceTokenError extends FleetError {} export class FleetUnauthorizedError extends FleetError {} +export class FleetNotFoundError extends FleetError {} export class OutputUnauthorizedError extends FleetError {} export class OutputInvalidError extends FleetError {} export class OutputLicenceError extends FleetError {} export class DownloadSourceError extends FleetError {} +// Not found errors +export class AgentNotFoundError extends FleetNotFoundError {} +export class AgentPolicyNotFoundError extends FleetNotFoundError {} +export class AgentActionNotFoundError extends FleetNotFoundError {} +export class DownloadSourceNotFound extends FleetNotFoundError {} +export class EnrollmentKeyNotFoundError extends FleetNotFoundError {} +export class FleetServerHostNotFoundError extends FleetNotFoundError {} +export class SigningServiceNotFoundError extends FleetNotFoundError {} +export class InputNotFoundError extends FleetNotFoundError {} +export class OutputNotFoundError extends FleetNotFoundError {} +export class PackageNotFoundError extends FleetNotFoundError {} +export class PackagePolicyNotFoundError extends FleetNotFoundError {} +export class StreamNotFoundError extends FleetNotFoundError {} + export class FleetServerHostUnauthorizedError extends FleetUnauthorizedError {} export class FleetProxyUnauthorizedError extends FleetUnauthorizedError {} diff --git a/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts b/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts index 23cdc80b8d65..961f3c90fb54 100644 --- a/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts +++ b/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts @@ -8,6 +8,8 @@ import { adminTestUser } from '@kbn/test'; import { getSupertest, type createRoot, type HttpMethod } from '@kbn/core-test-helpers-kbn-server'; +import { FleetError } from '../../errors'; + type Root = ReturnType; export * from './docker_registry_helper'; @@ -18,7 +20,7 @@ export const waitForFleetSetup = async (root: Root) => { const resp = await statusApi.send(); const fleetStatus = resp.body?.status?.plugins?.fleet; if (fleetStatus?.meta?.error) { - throw new Error(`Setup failed: ${JSON.stringify(fleetStatus)}`); + throw new FleetError(`Setup failed: ${JSON.stringify(fleetStatus)}`); } return !fleetStatus || fleetStatus?.summary === 'Fleet is setting up'; diff --git a/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts b/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts index 3f571fdcb09c..efedf164f653 100644 --- a/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts +++ b/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts @@ -9,6 +9,7 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import { downloadSourceService } from '../../services'; import type { AgentPolicy } from '../../types'; +import { FleetError, DownloadSourceNotFound } from '../../errors'; export const getSourceUriForAgentPolicy = async ( soClient: SavedObjectsClientContract, @@ -17,12 +18,12 @@ export const getSourceUriForAgentPolicy = async ( const defaultDownloadSourceId = await downloadSourceService.getDefaultDownloadSourceId(soClient); if (!defaultDownloadSourceId) { - throw new Error('Default download source host is not setup'); + throw new FleetError('Default download source host is not setup'); } const downloadSourceId: string = agentPolicy.download_source_id || defaultDownloadSourceId; const downloadSource = await downloadSourceService.get(soClient, downloadSourceId); if (!downloadSource) { - throw new Error(`Download source host not found ${downloadSourceId}`); + throw new DownloadSourceNotFound(`Download source host not found ${downloadSourceId}`); } return { host: downloadSource.host, proxy_id: downloadSource.proxy_id }; }; diff --git a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts index aefcbfc5cd87..62f34559c79e 100644 --- a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts +++ b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts @@ -15,7 +15,7 @@ describe('upgrade handler', () => { it('should throw if upgrade version is higher than kibana version', () => { expect(() => checkKibanaVersion('8.5.0', '8.4.0')).toThrowError( - 'cannot upgrade agent to 8.5.0 because it is higher than the installed kibana version 8.4.0' + 'Cannot upgrade agent to 8.5.0 because it is higher than the installed kibana version 8.4.0' ); }); diff --git a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts index 547fda566a95..391c721e2ef9 100644 --- a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts +++ b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts @@ -19,7 +19,7 @@ import type { PostAgentUpgradeResponse } from '../../../common/types'; import type { PostAgentUpgradeRequestSchema, PostBulkAgentUpgradeRequestSchema } from '../../types'; import * as AgentService from '../../services/agents'; import { appContextService } from '../../services'; -import { defaultFleetErrorHandler } from '../../errors'; +import { defaultFleetErrorHandler, AgentRequestInvalidError } from '../../errors'; import { getRecentUpgradeInfoForAgent, isAgentUpgradeable, @@ -187,14 +187,15 @@ export const postBulkAgentsUpgradeHandler: RequestHandler< export const checkKibanaVersion = (version: string, kibanaVersion: string, force = false) => { // get version number only in case "-SNAPSHOT" is in it const kibanaVersionNumber = semverCoerce(kibanaVersion)?.version; - if (!kibanaVersionNumber) throw new Error(`kibanaVersion ${kibanaVersionNumber} is not valid`); + if (!kibanaVersionNumber) + throw new AgentRequestInvalidError(`KibanaVersion ${kibanaVersionNumber} is not valid`); const versionToUpgradeNumber = semverCoerce(version)?.version; if (!versionToUpgradeNumber) - throw new Error(`version to upgrade ${versionToUpgradeNumber} is not valid`); + throw new AgentRequestInvalidError(`Version to upgrade ${versionToUpgradeNumber} is not valid`); if (!force && semverGt(versionToUpgradeNumber, kibanaVersionNumber)) { - throw new Error( - `cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the installed kibana version ${kibanaVersionNumber}` + throw new AgentRequestInvalidError( + `Cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the installed kibana version ${kibanaVersionNumber}` ); } @@ -205,8 +206,8 @@ export const checkKibanaVersion = (version: string, kibanaVersion: string, force // When force is enabled, only the major and minor versions are checked if (force && !(kibanaMajorGt || kibanaMajorEqMinorGte)) { - throw new Error( - `cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the installed kibana version ${kibanaVersionNumber}` + throw new AgentRequestInvalidError( + `Cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the installed kibana version ${kibanaVersionNumber}` ); } }; @@ -228,8 +229,8 @@ const checkFleetServerVersion = ( } if (!force && semverGt(versionToUpgradeNumber, maxFleetServerVersion)) { - throw new Error( - `cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the latest fleet server version ${maxFleetServerVersion}` + throw new AgentRequestInvalidError( + `Cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the latest fleet server version ${maxFleetServerVersion}` ); } @@ -241,8 +242,8 @@ const checkFleetServerVersion = ( // When force is enabled, only the major and minor versions are checked if (force && !(fleetServerMajorGt || fleetServerMajorEqMinorGte)) { - throw new Error( - `cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the latest fleet server version ${maxFleetServerVersion}` + throw new AgentRequestInvalidError( + `Cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the latest fleet server version ${maxFleetServerVersion}` ); } }; diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index cbc560cc72dc..a16fd37c9ac1 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -6,7 +6,6 @@ */ import type { TypeOf } from '@kbn/config-schema'; -import Boom from '@hapi/boom'; import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import type { RequestHandler } from '@kbn/core/server'; @@ -43,7 +42,11 @@ import type { UpgradePackagePolicyResponse, } from '../../../common/types'; import { installationStatuses, inputsFormat } from '../../../common/constants'; -import { defaultFleetErrorHandler, PackagePolicyNotFoundError } from '../../errors'; +import { + defaultFleetErrorHandler, + PackagePolicyNotFoundError, + PackagePolicyRequestError, +} from '../../errors'; import { getInstallations, getPackageInfo } from '../../services/epm/packages'; import { PACKAGES_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../constants'; import { @@ -244,7 +247,7 @@ export const createPackagePolicyHandler: FleetRequestHandler< let newPackagePolicy: NewPackagePolicy; if (isSimplifiedCreatePackagePolicyRequest(newPolicy)) { if (!pkg) { - throw new Error('Package is required'); + throw new PackagePolicyRequestError('Package is required'); } const pkgInfo = await getPackageInfo({ savedObjectsClient: soClient, @@ -311,7 +314,7 @@ export const updatePackagePolicyHandler: FleetRequestHandler< const packagePolicy = await packagePolicyService.get(soClient, request.params.packagePolicyId); if (!packagePolicy) { - throw Boom.notFound('Package policy not found'); + throw new PackagePolicyNotFoundError('Package policy not found'); } if (limitedToPackages && limitedToPackages.length) { @@ -337,7 +340,7 @@ export const updatePackagePolicyHandler: FleetRequestHandler< isSimplifiedCreatePackagePolicyRequest(body as unknown as SimplifiedPackagePolicy) ) { if (!pkg) { - throw new Error('package is required'); + throw new PackagePolicyRequestError('Package is required'); } const pkgInfo = await getPackageInfo({ savedObjectsClient: soClient, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts index bbe00c49b414..c7c02d8a53fb 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts @@ -12,7 +12,7 @@ import { LICENCE_FOR_PER_POLICY_OUTPUT, outputType } from '../../../common/const import { policyHasFleetServer, policyHasSyntheticsIntegration } from '../../../common/services'; import { appContextService } from '..'; import { outputService } from '../output'; -import { OutputInvalidError, OutputLicenceError } from '../../errors'; +import { OutputInvalidError, OutputLicenceError, OutputNotFoundError } from '../../errors'; /** * Get the data output for a given agent policy @@ -28,7 +28,7 @@ export async function getDataOutputForAgentPolicy( agentPolicy.data_output_id || (await outputService.getDefaultDataOutputId(soClient)); if (!dataOutputId) { - throw new Error('No default data output found.'); + throw new OutputNotFoundError('No default data output found.'); } return outputService.get(soClient, dataOutputId); diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts index 4445ebbe8476..ec36c7575937 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts @@ -24,6 +24,7 @@ import type { RegistryDataStreamPrivileges, } from '../../../common/types'; import { PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES } from '../../constants'; +import { PackagePolicyRequestError } from '../../errors'; import type { PackagePolicy } from '../../types'; import { pkgToPkgKey } from '../epm/registry'; @@ -46,7 +47,7 @@ export function storedPackagePoliciesToAgentPermissions( ): FullAgentPolicyOutputPermissions | undefined { // I'm not sure what permissions to return for this case, so let's return the defaults if (!packagePolicies) { - throw new Error( + throw new PackagePolicyRequestError( 'storedPackagePoliciesToAgentPermissions should be called with a PackagePolicy' ); } @@ -57,7 +58,9 @@ export function storedPackagePoliciesToAgentPermissions( const permissionEntries = packagePolicies.map((packagePolicy) => { if (!packagePolicy.package) { - throw new Error(`No package for package policy ${packagePolicy.name ?? packagePolicy.id}`); + throw new PackagePolicyRequestError( + `No package for package policy ${packagePolicy.name ?? packagePolicy.id}` + ); } const pkg = packageInfoCache.get(pkgToPkgKey(packagePolicy.package))!; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts index b614b9c2dd9e..0108e9cd9772 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts @@ -16,6 +16,7 @@ import { getSourceUriForAgentPolicy } from '../../routes/agent/source_uri_utils' import { getFleetServerHostsForAgentPolicy } from '../fleet_server_host'; import { appContextService } from '../app_context'; import { bulkGetFleetProxies } from '../fleet_proxies'; +import { OutputNotFoundError } from '../../errors'; export async function fetchRelatedSavedObjects( soClient: SavedObjectsClientContract, @@ -27,7 +28,7 @@ export async function fetchRelatedSavedObjects( ]); if (!defaultDataOutputId) { - throw new Error('Default output is not setup'); + throw new OutputNotFoundError('Default output is not setup'); } const dataOutputId = agentPolicy.data_output_id || defaultDataOutputId; @@ -51,11 +52,11 @@ export async function fetchRelatedSavedObjects( const dataOutput = outputs.find((output) => output.id === dataOutputId); if (!dataOutput) { - throw new Error(`Data output not found ${dataOutputId}`); + throw new OutputNotFoundError(`Data output not found ${dataOutputId}`); } const monitoringOutput = outputs.find((output) => output.id === monitoringOutputId); if (!monitoringOutput) { - throw new Error(`Monitoring output not found ${monitoringOutputId}`); + throw new OutputNotFoundError(`Monitoring output not found ${monitoringOutputId}`); } const proxyIds = uniq( diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index 931168f545b5..3e97594ee959 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -8,6 +8,8 @@ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import { securityMock } from '@kbn/security-plugin/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { Logger } from '@kbn/core/server'; import { PackagePolicyRestrictionRelatedError, FleetUnauthorizedError } from '../errors'; import type { @@ -105,8 +107,13 @@ function getAgentPolicyCreateMock() { }); return soClient; } - +let mockedLogger: jest.Mocked; describe('agent policy', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + afterEach(() => { jest.resetAllMocks(); }); diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 568829fda978..b44e0616b7b6 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -71,6 +71,7 @@ import { AgentPolicyNotFoundError, PackagePolicyRestrictionRelatedError, FleetUnauthorizedError, + FleetError, } from '../errors'; import type { FullAgentConfigMap } from '../../common/types/models/agent_cm'; @@ -125,24 +126,24 @@ class AgentPolicyService { id, savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, }); + const logger = appContextService.getLogger(); + logger.debug(`Starting update of agent policy ${id}`); const existingAgentPolicy = await this.get(soClient, id, true); if (!existingAgentPolicy) { - throw new Error('Agent policy not found'); + throw new AgentPolicyNotFoundError('Agent policy not found'); } if ( existingAgentPolicy.status === agentPolicyStatuses.Inactive && agentPolicy.status !== agentPolicyStatuses.Active ) { - throw new Error( + throw new FleetError( `Agent policy ${id} cannot be updated because it is ${existingAgentPolicy.status}` ); } - const logger = appContextService.getLogger(); - if (options.removeProtection) { logger.warn(`Setting tamper protection for Agent Policy ${id} to false`); } @@ -166,7 +167,7 @@ class AgentPolicyService { if (options.bumpRevision || options.removeProtection) { await this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'updated', id); } - + logger.debug(`Agent policy ${id} update completed`); return (await this.get(soClient, id)) as AgentPolicy; } @@ -190,7 +191,7 @@ class AgentPolicyService { is_preconfigured: true, }; - if (!id) throw new Error('Missing ID'); + if (!id) throw new AgentPolicyNotFoundError('Missing ID'); return await this.ensureAgentPolicy(soClient, esClient, newAgentPolicy, id as string); } @@ -254,6 +255,7 @@ class AgentPolicyService { this.checkTamperProtectionLicense(agentPolicy); const logger = appContextService.getLogger(); + logger.debug(`Creating new agent policy`); if (agentPolicy?.is_protected) { logger.warn( @@ -282,7 +284,7 @@ class AgentPolicyService { await appContextService.getUninstallTokenService()?.generateTokenForPolicyId(newSo.id); await this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'created', newSo.id); - + logger.debug(`Created new agent policy with id ${newSo.id}`); return { id: newSo.id, ...newSo.attributes }; } @@ -320,7 +322,7 @@ class AgentPolicyService { } if (agentPolicySO.error) { - throw new Error(agentPolicySO.error.message); + throw new FleetError(agentPolicySO.error.message); } const agentPolicy = { id: agentPolicySO.id, ...agentPolicySO.attributes }; @@ -356,7 +358,7 @@ class AgentPolicyService { } else if (agentPolicySO.error.statusCode === 404) { throw new AgentPolicyNotFoundError(`Agent policy ${agentPolicySO.id} not found`); } else { - throw new Error(agentPolicySO.error.message); + throw new FleetError(agentPolicySO.error.message); } } @@ -498,6 +500,9 @@ class AgentPolicyService { authorizationHeader?: HTTPAuthorizationHeader | null; } ): Promise { + const logger = appContextService.getLogger(); + logger.debug(`Starting update of agent policy ${id}`); + if (agentPolicy.name) { await this.requireUniqueName(soClient, { id, @@ -508,14 +513,12 @@ class AgentPolicyService { const existingAgentPolicy = await this.get(soClient, id, true); if (!existingAgentPolicy) { - throw new Error('Agent policy not found'); + throw new AgentPolicyNotFoundError('Agent policy not found'); } this.checkTamperProtectionLicense(agentPolicy); await this.checkForValidUninstallToken(agentPolicy, id); - const logger = appContextService.getLogger(); - if (agentPolicy?.is_protected && !policyHasEndpointSecurity(existingAgentPolicy)) { logger.warn( 'Agent policy requires Elastic Defend integration to set tamper protection to true' @@ -558,10 +561,13 @@ class AgentPolicyService { newAgentPolicyProps: Pick, options?: { user?: AuthenticatedUser } ): Promise { + const logger = appContextService.getLogger(); + logger.debug(`Starting copy of agent policy ${id}`); + // Copy base agent policy const baseAgentPolicy = await this.get(soClient, id, true); if (!baseAgentPolicy) { - throw new Error('Agent policy not found'); + throw new AgentPolicyNotFoundError('Agent policy not found'); } const newAgentPolicy = await this.create( soClient, @@ -631,11 +637,11 @@ class AgentPolicyService { // Get updated agent policy with package policies and adjusted tamper protection const updatedAgentPolicy = await this.get(soClient, newAgentPolicy.id, true); if (!updatedAgentPolicy) { - throw new Error('Copied agent policy not found'); + throw new AgentPolicyNotFoundError('Copied agent policy not found'); } await this.deployPolicy(soClient, newAgentPolicy.id); - + logger.debug(`Completed copy of agent policy ${id}`); return updatedAgentPolicy; } @@ -799,6 +805,9 @@ class AgentPolicyService { id: string, options?: { force?: boolean; removeFleetServerDocuments?: boolean; user?: AuthenticatedUser } ): Promise { + const logger = appContextService.getLogger(); + logger.debug(`Deleting agent policy ${id}`); + auditLoggingService.writeCustomSoAuditLog({ action: 'delete', id, @@ -807,7 +816,7 @@ class AgentPolicyService { const agentPolicy = await this.get(soClient, id, false); if (!agentPolicy) { - throw new Error('Agent policy not found'); + throw new AgentPolicyNotFoundError('Agent policy not found'); } if (agentPolicy.is_managed && !options?.force) { @@ -822,7 +831,7 @@ class AgentPolicyService { }); if (total > 0) { - throw new Error('Cannot delete agent policy that is assigned to agent(s)'); + throw new FleetError('Cannot delete agent policy that is assigned to agent(s)'); } const packagePolicies = await packagePolicyService.findAllForAgentPolicy(soClient, id); @@ -860,7 +869,7 @@ class AgentPolicyService { if (options?.removeFleetServerDocuments) { await this.deleteFleetServerPoliciesForPolicyId(esClient, id); } - + logger.debug(`Deleted agent policy ${id}`); return { id, name: agentPolicy.name, @@ -954,7 +963,7 @@ class AgentPolicyService { return acc; }, [] as BulkResponseItem[]); - logger.debug( + logger.warn( `Failed to index documents during policy deployment: ${JSON.stringify(erroredDocuments)}` ); } @@ -1225,7 +1234,7 @@ class AgentPolicyService { ); if (uninstallTokenError) { - throw new Error( + throw new FleetError( `Cannot enable Agent Tamper Protection: ${uninstallTokenError.error.message}` ); } diff --git a/x-pack/plugins/fleet/server/services/agents/crud.ts b/x-pack/plugins/fleet/server/services/agents/crud.ts index 7ad50c8d962c..b8eb0f7d0ca1 100644 --- a/x-pack/plugins/fleet/server/services/agents/crud.ts +++ b/x-pack/plugins/fleet/server/services/agents/crud.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import Boom from '@hapi/boom'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { SortResults } from '@elastic/elasticsearch/lib/api/types'; import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; @@ -19,7 +18,12 @@ import type { AgentStatus, FleetServerAgent } from '../../../common/types'; import { SO_SEARCH_LIMIT } from '../../../common/constants'; import { isAgentUpgradeable } from '../../../common/services'; import { AGENTS_INDEX } from '../../constants'; -import { FleetError, isESClientError, AgentNotFoundError } from '../../errors'; +import { + FleetError, + isESClientError, + AgentNotFoundError, + FleetUnauthorizedError, +} from '../../errors'; import { auditLoggingService } from '../audit_logging'; @@ -548,10 +552,10 @@ export async function getAgentByAccessAPIKeyId( throw new AgentNotFoundError('Agent not found'); } if (agent.access_api_key_id !== accessAPIKeyId) { - throw new Error('Agent api key id is not matching'); + throw new FleetError('Agent api key id is not matching'); } if (!agent.active) { - throw Boom.forbidden('Agent inactive'); + throw new FleetUnauthorizedError('Agent inactive'); } return agent; diff --git a/x-pack/plugins/fleet/server/services/agents/reassign.ts b/x-pack/plugins/fleet/server/services/agents/reassign.ts index 86d368d39931..0a5c6f9b51ee 100644 --- a/x-pack/plugins/fleet/server/services/agents/reassign.ts +++ b/x-pack/plugins/fleet/server/services/agents/reassign.ts @@ -5,11 +5,14 @@ * 2.0. */ import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; -import Boom from '@hapi/boom'; import type { Agent } from '../../types'; import { agentPolicyService } from '../agent_policy'; -import { AgentReassignmentError, HostedAgentPolicyRestrictionRelatedError } from '../../errors'; +import { + AgentReassignmentError, + HostedAgentPolicyRestrictionRelatedError, + AgentPolicyNotFoundError, +} from '../../errors'; import { SO_SEARCH_LIMIT } from '../../constants'; @@ -33,7 +36,7 @@ export async function reassignAgent( ) { const newAgentPolicy = await agentPolicyService.get(soClient, newAgentPolicyId); if (!newAgentPolicy) { - throw Boom.notFound(`Agent policy not found: ${newAgentPolicyId}`); + throw new AgentPolicyNotFoundError(`Agent policy not found: ${newAgentPolicyId}`); } await reassignAgentIsAllowed(soClient, esClient, agentId, newAgentPolicyId); @@ -87,7 +90,7 @@ export async function reassignAgents( ): Promise<{ actionId: string }> { const newAgentPolicy = await agentPolicyService.get(soClient, newAgentPolicyId); if (!newAgentPolicy) { - throw Boom.notFound(`Agent policy not found: ${newAgentPolicyId}`); + throw new AgentPolicyNotFoundError(`Agent policy not found: ${newAgentPolicyId}`); } if (newAgentPolicy.is_managed) { throw new HostedAgentPolicyRestrictionRelatedError( diff --git a/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts b/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts index 84aa2226b485..d962279f0ca3 100644 --- a/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts @@ -185,7 +185,7 @@ describe('update_agent_tags', () => { await expect( updateAgentTags(soClient, esClient, { agentIds: ['agent1'] }, ['one'], []) - ).rejects.toThrowError('version conflict of 100 agents'); + ).rejects.toThrowError('Version conflict of 100 agents'); }); it('should write out error results on last retry with version conflicts', async () => { @@ -211,7 +211,7 @@ describe('update_agent_tags', () => { retryCount: MAX_RETRY_COUNT, } ) - ).rejects.toThrowError('version conflict of 100 agents'); + ).rejects.toThrowError('Version conflict of 100 agents'); const agentAction = esClient.create.mock.calls[0][0] as any; expect(agentAction?.body.agents.length).toEqual(100); @@ -243,7 +243,7 @@ describe('update_agent_tags', () => { retryCount: MAX_RETRY_COUNT, } ) - ).rejects.toThrowError('version conflict of 1 agents'); + ).rejects.toThrowError('Version conflict of 1 agents'); const agentAction = esClient.create.mock.calls[0][0] as any; expect(agentAction?.body.agents.length).toEqual(3); diff --git a/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts b/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts index d8208fd5f8d0..1c8db8451ccf 100644 --- a/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts +++ b/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts @@ -15,6 +15,8 @@ import { AGENTS_INDEX } from '../../constants'; import { appContextService } from '../app_context'; +import { FleetError } from '../../errors'; + import { ActionRunner } from './action_runner'; import { BulkActionTaskType } from './bulk_action_types'; @@ -124,7 +126,9 @@ export async function updateTagsBatch( conflicts: 'proceed', // relying on the task to retry in case of conflicts - retry only conflicted agents }); } catch (error) { - throw new Error('Caught error: ' + JSON.stringify(error).slice(0, 1000)); + throw new FleetError( + 'Caught error while batch updating tags: ' + JSON.stringify(error).slice(0, 1000) + ); } appContextService.getLogger().debug(JSON.stringify(res).slice(0, 1000)); @@ -203,7 +207,7 @@ export async function updateTagsBatch( .getLogger() .debug(`action conflict result wrote on ${versionConflictCount} agents`); } - throw new Error(`version conflict of ${versionConflictCount} agents`); + throw new FleetError(`Version conflict of ${versionConflictCount} agents`); } return { actionId, updated: res.updated, took: res.took }; diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts index bc58941e4c29..9200346961f1 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts @@ -6,6 +6,10 @@ */ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +import type { Logger } from '@kbn/core/server'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; import { ENROLLMENT_API_KEYS_INDEX } from '../../constants'; @@ -27,10 +31,20 @@ jest.mock('uuid', () => { const mockedAgentPolicyService = agentPolicyService as jest.Mocked; const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + +let mockedLogger: jest.Mocked; describe('enrollment api keys', () => { beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + afterEach(() => { jest.resetAllMocks(); }); diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts index ac7087c95296..360723ebcf22 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts @@ -16,13 +16,15 @@ import { toElasticsearchQuery, fromKueryExpression } from '@kbn/es-query'; import type { ESSearchResponse as SearchResponse } from '@kbn/es-types'; import type { EnrollmentAPIKey, FleetServerEnrollmentAPIKey } from '../../types'; -import { FleetError } from '../../errors'; +import { FleetError, EnrollmentKeyNameExistsError, EnrollmentKeyNotFoundError } from '../../errors'; import { ENROLLMENT_API_KEYS_INDEX } from '../../constants'; import { agentPolicyService } from '../agent_policy'; import { escapeSearchQueryPhrase } from '../saved_object'; import { auditLoggingService } from '../audit_logging'; +import { appContextService } from '../app_context'; + import { invalidateAPIKeys } from './security'; const uuidRegex = @@ -90,7 +92,7 @@ export async function getEnrollmentAPIKey( return esDocToEnrollmentApiKey(body); } catch (e) { if (e instanceof errors.ResponseError && e.statusCode === 404) { - throw Boom.notFound(`Enrollment api key ${id} not found`); + throw new EnrollmentKeyNotFoundError(`Enrollment api key ${id} not found`); } throw e; @@ -106,6 +108,9 @@ export async function deleteEnrollmentApiKey( id: string, forceDelete = false ) { + const logger = appContextService.getLogger(); + logger.debug(`Deleting enrollment API key ${id}`); + const enrollmentApiKey = await getEnrollmentAPIKey(esClient, id); auditLoggingService.writeCustomAuditLog({ @@ -132,6 +137,9 @@ export async function deleteEnrollmentApiKey( refresh: 'wait_for', }); } + logger.debug( + `Deleted enrollment API key ${enrollmentApiKey.id} [api_key_id=${enrollmentApiKey.api_key_id}` + ); } export async function deleteEnrollmentApiKeyForAgentPolicyId( @@ -169,6 +177,9 @@ export async function generateEnrollmentAPIKey( ): Promise { const id = uuidv4(); const { name: providedKeyName, forceRecreate } = data; + const logger = appContextService.getLogger(); + logger.debug(`Creating enrollment API key ${data}`); + if (data.agentPolicyId) { await validateAgentPolicyId(soClient, data.agentPolicyId); } @@ -199,7 +210,7 @@ export async function generateEnrollmentAPIKey( k.name?.replace(providedKeyName, '').trim().match(uuidRegex) ) ) { - throw new FleetError( + throw new EnrollmentKeyNameExistsError( i18n.translate('xpack.fleet.serverError.enrollmentKeyDuplicate', { defaultMessage: 'An enrollment key named {providedKeyName} already exists for agent policy {agentPolicyId}', @@ -217,6 +228,7 @@ export async function generateEnrollmentAPIKey( auditLoggingService.writeCustomAuditLog({ message: `User creating enrollment API key [name=${name}] [policy_id=${agentPolicyId}]`, }); + logger.debug(`Creating enrollment API key [name=${name}] [policy_id=${agentPolicyId}]`); const key = await esClient.security .createApiKey({ @@ -245,11 +257,11 @@ export async function generateEnrollmentAPIKey( }, }) .catch((err) => { - throw new Error(`Impossible to create an api key: ${err.message}`); + throw new FleetError(`Impossible to create an api key: ${err.message}`); }); if (!key) { - throw new Error( + throw new FleetError( i18n.translate('xpack.fleet.serverError.unableToCreateEnrollmentKey', { defaultMessage: 'Unable to create an enrollment api key', }) @@ -332,9 +344,9 @@ export async function getEnrollmentAPIKeyById(esClient: ElasticsearchClient, api const [enrollmentAPIKey] = res.hits.hits.map(esDocToEnrollmentApiKey); if (enrollmentAPIKey?.api_key_id !== apiKeyId) { - throw new Error( + throw new FleetError( i18n.translate('xpack.fleet.serverError.returnedIncorrectKey', { - defaultMessage: 'find enrollmentKeyById returned an incorrect key', + defaultMessage: 'Find enrollmentKeyById returned an incorrect key', }) ); } diff --git a/x-pack/plugins/fleet/server/services/download_source.test.ts b/x-pack/plugins/fleet/server/services/download_source.test.ts index 8b63d376340f..e244ec80077b 100644 --- a/x-pack/plugins/fleet/server/services/download_source.test.ts +++ b/x-pack/plugins/fleet/server/services/download_source.test.ts @@ -8,6 +8,9 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { securityMock } from '@kbn/security-plugin/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +import type { Logger } from '@kbn/core/server'; import type { DownloadSourceSOAttributes } from '../types'; @@ -132,9 +135,13 @@ function getMockedSoClient(options: { defaultDownloadSourceId?: string; sameName return soClient; } - +let mockedLogger: jest.Mocked; describe('Download Service', () => { beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + afterEach(() => { mockedAgentPolicyService.list.mockClear(); mockedAgentPolicyService.hasAPMIntegration.mockClear(); mockedAgentPolicyService.removeDefaultSourceFromAll.mockReset(); diff --git a/x-pack/plugins/fleet/server/services/download_source.ts b/x-pack/plugins/fleet/server/services/download_source.ts index f1719e2eb479..e679123f7e25 100644 --- a/x-pack/plugins/fleet/server/services/download_source.ts +++ b/x-pack/plugins/fleet/server/services/download_source.ts @@ -41,7 +41,7 @@ class DownloadSourceService { ); if (soResponse.error) { - throw new Error(soResponse.error.message); + throw new FleetError(soResponse.error.message); } return savedObjectToDownloadSource(soResponse); @@ -69,6 +69,9 @@ class DownloadSourceService { downloadSource: DownloadSourceBase, options?: { id?: string; overwrite?: boolean } ): Promise { + const logger = appContextService.getLogger(); + logger.debug(`Creating new download source`); + const data: DownloadSourceSOAttributes = downloadSource; await this.requireUniqueName(soClient, { @@ -100,6 +103,7 @@ class DownloadSourceService { overwrite: options?.overwrite ?? false, } ); + logger.debug(`Creating new download source ${options?.id}`); return savedObjectToDownloadSource(newSo); } @@ -108,6 +112,8 @@ class DownloadSourceService { id: string, newData: Partial ) { + const logger = appContextService.getLogger(); + logger.debug(`Updating download source ${id} with ${newData}`); const updateData: Partial = newData; if (updateData.proxy_id) { @@ -134,11 +140,16 @@ class DownloadSourceService { updateData ); if (soResponse.error) { - throw new Error(soResponse.error.message); + throw new FleetError(soResponse.error.message); + } else { + logger.debug(`Updated download source ${id}`); } } public async delete(soClient: SavedObjectsClientContract, id: string) { + const logger = appContextService.getLogger(); + logger.debug(`Deleting download source ${id}`); + const targetDS = await this.get(soClient, id); if (targetDS.is_default) { @@ -149,7 +160,7 @@ class DownloadSourceService { appContextService.getInternalUserESClient(), id ); - + logger.debug(`Deleted download source ${id}`); return soClient.delete(DOWNLOAD_SOURCE_SAVED_OBJECT_TYPE, id); } diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts index 9163b39575f8..0ab728affd75 100644 --- a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts @@ -5,9 +5,30 @@ * 2.0. */ +import { loggerMock } from '@kbn/logging-mocks'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; + +import type { Logger } from '@kbn/core/server'; + +import { appContextService } from '../..'; + import { compileTemplate } from './agent'; +jest.mock('../../app_context'); + +const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + +let mockedLogger: jest.Mocked; + describe('compileTemplate', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + it('should work', () => { const streamTemplate = ` input: log diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts index 0bc220a500fb..22d00451ce23 100644 --- a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts @@ -7,17 +7,21 @@ import Handlebars from 'handlebars'; import { safeLoad, safeDump } from 'js-yaml'; +import type { Logger } from '@kbn/core/server'; import type { PackagePolicyConfigRecord } from '../../../../common/types'; import { toCompiledSecretRef } from '../../secrets'; import { PackageInvalidArchiveError } from '../../../errors'; +import { appContextService } from '../..'; const handlebars = Handlebars.create(); export function compileTemplate(variables: PackagePolicyConfigRecord, templateStr: string) { - const { vars, yamlValues } = buildTemplateVariables(variables); + const logger = appContextService.getLogger(); + const { vars, yamlValues } = buildTemplateVariables(logger, variables); let compiledTemplate: string; try { + logger.debug(`Compiling agent template: ${templateStr}`); const template = handlebars.compile(templateStr, { noEscape: true }); compiledTemplate = template(vars); } catch (err) { @@ -65,12 +69,13 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) return yaml; } -function buildTemplateVariables(variables: PackagePolicyConfigRecord) { +function buildTemplateVariables(logger: Logger, variables: PackagePolicyConfigRecord) { const yamlValues: { [k: string]: any } = {}; const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => { // support variables with . like key.patterns const keyParts = key.split('.'); const lastKeyPart = keyParts.pop(); + logger.debug(`Building agent template variables`); if (!lastKeyPart || !isValidKey(lastKeyPart)) { throw new PackageInvalidArchiveError( diff --git a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts index 8b1fd141f300..db0b0d709e68 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts @@ -43,7 +43,7 @@ export const getArchiveFilelist = (keyArgs: SharedKey) => export const setArchiveFilelist = (keyArgs: SharedKey, paths: string[]) => { const logger = appContextService.getLogger(); - logger.debug(`setting file list to the cache for ${keyArgs.name}-${keyArgs.version}`); + logger.debug(`Setting file list to the cache for ${keyArgs.name}-${keyArgs.version}`); logger.trace(JSON.stringify(paths)); return archiveFilelistCache.set(sharedKey(keyArgs), paths); }; @@ -79,7 +79,7 @@ export const setPackageInfo = ({ }: SharedKey & { packageInfo: ArchivePackage | RegistryPackage }) => { const logger = appContextService.getLogger(); const key = sharedKey({ name, version }); - logger.debug(`setting package info to the cache for ${name}-${version}`); + logger.debug(`Setting package info to the cache for ${name}-${version}`); logger.trace(JSON.stringify(packageInfo)); return packageInfoCache.set(key, packageInfo); }; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index 330839b13dba..bf96318d8d41 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -6,7 +6,11 @@ */ import type { AssetParts } from '../../../../common/types'; -import { PackageInvalidArchiveError, PackageUnsupportedMediaTypeError } from '../../../errors'; +import { + PackageInvalidArchiveError, + PackageUnsupportedMediaTypeError, + PackageNotFoundError, +} from '../../../errors'; import { getArchiveEntry, @@ -149,7 +153,7 @@ export function getPathParts(path: string): AssetParts { export function getAsset(key: string) { const buffer = getArchiveEntry(key); - if (buffer === undefined) throw new Error(`Cannot find asset ${key}`); + if (buffer === undefined) throw new PackageNotFoundError(`Cannot find asset ${key}`); return buffer; } diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts index 1e7d7d24e2b8..ac72f56946d0 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts @@ -4,9 +4,16 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { loggerMock } from '@kbn/logging-mocks'; + +import type { Logger } from '@kbn/core/server'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; + import type { ArchivePackage } from '../../../../common/types'; import { PackageInvalidArchiveError } from '../../../errors'; +import { appContextService } from '../..'; + import { parseDefaultIngestPipeline, parseDataStreamElasticsearchEntry, @@ -21,7 +28,20 @@ import { parseAndVerifyReadme, } from './parse'; +jest.mock('../../app_context'); + +const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + +let mockedLogger: jest.Mocked; describe('parseDefaultIngestPipeline', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + it('Should return undefined for stream without any elasticsearch dir', () => { expect( parseDefaultIngestPipeline('pkg-1.0.0/data_stream/stream1/', [ diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts index e0111e196ddb..eac5c2ac43db 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts @@ -16,6 +16,8 @@ import { pick } from 'lodash'; import semverMajor from 'semver/functions/major'; import semverPrerelease from 'semver/functions/prerelease'; +import { appContextService } from '../..'; + import type { ArchivePackage, RegistryPolicyTemplate, @@ -198,7 +200,9 @@ export function parseAndVerifyArchive( topLevelDirOverride?: string ): ArchivePackage { // The top-level directory must match pkgName-pkgVersion, and no other top-level files or directories may be present + const logger = appContextService.getLogger(); const toplevelDir = topLevelDirOverride || paths[0].split('/')[0]; + paths.forEach((filePath) => { if (!filePath.startsWith(toplevelDir)) { throw new PackageInvalidArchiveError( @@ -210,6 +214,7 @@ export function parseAndVerifyArchive( // The package must contain a manifest file ... const manifestFile = path.posix.join(toplevelDir, MANIFEST_NAME); const manifestBuffer = assetsMap[manifestFile]; + logger.debug(`Verifying archive - checking manifest file and manifest buffer`); if (!paths.includes(manifestFile) || !manifestBuffer) { throw new PackageInvalidArchiveError( `Package at top-level directory ${toplevelDir} must contain a top-level ${MANIFEST_NAME} file.` @@ -219,6 +224,7 @@ export function parseAndVerifyArchive( // ... which must be valid YAML let manifest: ArchivePackage; try { + logger.debug(`Verifying archive - loading yaml`); manifest = yaml.safeLoad(manifestBuffer.toString()); } catch (error) { throw new PackageInvalidArchiveError( @@ -227,6 +233,7 @@ export function parseAndVerifyArchive( } // must have mandatory fields + logger.debug(`Verifying archive - verifying manifest content`); const reqGiven = pick(manifest, requiredArchivePackageProps); const requiredKeysMatch = Object.keys(reqGiven).toString() === requiredArchivePackageProps.toString(); @@ -246,13 +253,15 @@ export function parseAndVerifyArchive( const parsed: ArchivePackage = { ...reqGiven, ...optGiven }; // Package name and version from the manifest must match those from the toplevel directory + logger.debug(`Verifying archive - parsing manifest: ${parsed}`); const pkgKey = pkgToPkgKey({ name: parsed.name, version: parsed.version }); + if (!topLevelDirOverride && toplevelDir !== pkgKey) { throw new PackageInvalidArchiveError( `Name ${parsed.name} and version ${parsed.version} do not match top-level directory ${toplevelDir}` ); } - + logger.debug(`Parsing archive - parsing and verifying data streams`); const parsedDataStreams = parseAndVerifyDataStreams({ paths, pkgName: parsed.name, @@ -265,9 +274,11 @@ export function parseAndVerifyArchive( parsed.data_streams = parsedDataStreams; } + logger.debug(`Parsing archive - parsing and verifying policy templates`); parsed.policy_templates = parseAndVerifyPolicyTemplates(manifest); // add readme if exists + logger.debug(`Parsing archive - parsing and verifying Readme`); const readme = parseAndVerifyReadme(paths, parsed.name, parsed.version); if (readme) { parsed.readme = readme; @@ -283,6 +294,7 @@ export function parseAndVerifyArchive( // Ensure top-level variables are parsed as well if (manifest.vars) { + logger.debug(`Parsing archive - parsing and verifying top-level vars`); parsed.vars = parseAndVerifyVars(manifest.vars, 'manifest.yml'); } @@ -294,6 +306,7 @@ export function parseAndVerifyArchive( let tags: PackageSpecTags[]; try { tags = yaml.safeLoad(tagsBuffer.toString()); + logger.debug(`Parsing archive - parsing kibana/tags.yml file`); if (tags.length) { parsed.asset_tags = tags; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index d8a943fef534..5039891eef59 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -100,7 +100,6 @@ export async function _installPackage({ skipDataStreamRollover?: boolean; }): Promise { const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo; - try { // if some installation already exists if (installedPkg) { @@ -108,12 +107,16 @@ export async function _installPackage({ const hasExceededTimeout = Date.now() - Date.parse(installedPkg.attributes.install_started_at) < MAX_TIME_COMPLETE_INSTALL; + logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); // if the installation is currently running, don't try to install // instead, only return already installed assets if (isStatusInstalling && hasExceededTimeout) { // If this is a forced installation, ignore the timeout and restart the installation anyway + logger.debug(`Package install - Installation is running and has exceeded timeout`); + if (force) { + logger.debug(`Package install - Forced installation, restarting`); await restartInstallation({ savedObjectsClient, pkgName, @@ -131,6 +134,9 @@ export async function _installPackage({ } else { // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL // (it might be stuck) update the saved object and proceed + logger.debug( + `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` + ); await restartInstallation({ savedObjectsClient, pkgName, @@ -140,6 +146,7 @@ export async function _installPackage({ }); } } else { + logger.debug(`Package install - Create installation`); await createInstallation({ savedObjectsClient, packageInfo, @@ -148,7 +155,7 @@ export async function _installPackage({ verificationResult, }); } - + logger.debug(`Package install - Installing Kibana assets`); const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => installKibanaAssetsAndReferences({ savedObjectsClient, @@ -182,7 +189,7 @@ export async function _installPackage({ esReferences = await withPackageSpan('Install ILM policies', () => installILMPolicy(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) ); - + logger.debug(`Package install - Installing Data Stream ILM policies`); ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () => installIlmForDataStream( packageInfo, @@ -196,6 +203,7 @@ export async function _installPackage({ } // installs ml models + logger.debug(`Package install - installing ML models`); esReferences = await withPackageSpan('Install ML models', () => installMlModel(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) ); @@ -203,6 +211,9 @@ export async function _installPackage({ let indexTemplates: IndexTemplateEntry[] = []; if (packageInfo.type === 'integration') { + logger.debug( + `Package install - Installing index templates and pipelines, packageInfo.type ${packageInfo.type}` + ); const { installedTemplates, esReferences: templateEsReferences } = await installIndexTemplatesAndPipelines({ installedPkg: installedPkg ? installedPkg.attributes : undefined, @@ -221,6 +232,7 @@ export async function _installPackage({ // input packages create their data streams during package policy creation // we must use installed_es to infer which streams exist first then // we can install the new index templates + logger.debug(`Package install - packageInfo.type ${packageInfo.type}`); const dataStreamNames = installedPkg.attributes.installed_es .filter((ref) => ref.type === 'index_template') // index templates are named {type}-{dataset}, remove everything before first hyphen @@ -231,6 +243,9 @@ export async function _installPackage({ ); if (dataStreams.length) { + logger.debug( + `Package install - installing index templates and pipelines with datastreams length ${dataStreams.length}` + ); const { installedTemplates, esReferences: templateEsReferences } = await installIndexTemplatesAndPipelines({ installedPkg: installedPkg ? installedPkg.attributes : undefined, @@ -248,19 +263,21 @@ export async function _installPackage({ } try { + logger.debug(`Package install - Removing legacy templates`); await removeLegacyTemplates({ packageInfo, esClient, logger }); } catch (e) { logger.warn(`Error removing legacy templates: ${e.message}`); } // update current backing indices of each data stream + logger.debug(`Package install - Updating backing indices of each data stream`); await withPackageSpan('Update write indices', () => updateCurrentWriteIndices(esClient, logger, indexTemplates, { ignoreMappingUpdateErrors, skipDataStreamRollover, }) ); - + logger.debug(`Package install - Installing transforms`); ({ esReferences } = await withPackageSpan('Install transforms', () => installTransforms({ installablePackage: packageInfo, @@ -282,6 +299,9 @@ export async function _installPackage({ (installType === 'update' || installType === 'reupdate') && installedPkg ) { + logger.debug( + `Package install - installType ${installType} Deleting previous ingest pipelines` + ); esReferences = await withPackageSpan('Delete previous ingest pipelines', () => deletePreviousPipelines( esClient, @@ -294,6 +314,9 @@ export async function _installPackage({ } // pipelines from a different version may have installed during a failed update if (installType === 'rollback' && installedPkg) { + logger.debug( + `Package install - installType ${installType} Deleting previous ingest pipelines` + ); esReferences = await withPackageSpan('Delete previous ingest pipelines', () => deletePreviousPipelines( esClient, @@ -306,6 +329,7 @@ export async function _installPackage({ } const installedKibanaAssetsRefs = await kibanaAssetPromise; + logger.debug(`Package install - Updating archive entries`); const packageAssetResults = await withPackageSpan('Update archive entries', () => saveArchiveEntries({ savedObjectsClient, @@ -326,7 +350,7 @@ export async function _installPackage({ id: pkgName, savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, }); - + logger.debug(`Package install - Updating install status`); const updatedPackage = await withPackageSpan('Update install status', () => savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { version: pkgVersion, @@ -340,6 +364,7 @@ export async function _installPackage({ ), }) ); + logger.debug(`Package install - Install status ${updatedPackage?.attributes?.install_status}`); // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its // associated package policies after installation @@ -350,11 +375,13 @@ export async function _installPackage({ perPage: SO_SEARCH_LIMIT, kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`, }); - + logger.debug( + `Package install - Package is flagged with keep_policies_up_to_date, upgrading its associated package policies ${policyIdsToUpgrade}` + ); await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); }); } - + logger.debug(`Package install - Installation complete}`); return [...installedKibanaAssetsRefs, ...esReferences]; } catch (err) { if (SavedObjectsErrorHelpers.isConflictError(err)) { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index c58d14dd4732..bfb99d0f1798 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -186,7 +186,7 @@ describe('install', () => { expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { currentVersion: 'not_installed', dryRun: false, - errorMessage: 'Requires basic license', + errorMessage: 'Installation requires basic license', eventType: 'package-install', installType: 'install', newVersion: '1.3.0', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 0760bd94a56e..d4f9cd249177 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -516,6 +516,9 @@ async function installPackageCommon(options: { } = options; let { telemetryEvent } = options; const logger = appContextService.getLogger(); + logger.info( + `Install - Starting installation of ${pkgName}@${pkgVersion} from ${installSource}, paths: ${paths}` + ); // Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611 await Promise.resolve(); @@ -564,7 +567,8 @@ async function installPackageCommon(options: { } const elasticSubscription = getElasticSubscription(packageInfo); if (!licenseService.hasAtLeast(elasticSubscription)) { - const err = new Error(`Requires ${elasticSubscription} license`); + logger.error(`Installation requires ${elasticSubscription} license`); + const err = new FleetError(`Installation requires ${elasticSubscription} license`); sendEvent({ ...telemetryEvent, errorMessage: err.message, @@ -606,6 +610,7 @@ async function installPackageCommon(options: { skipDataStreamRollover, }) .then(async (assets) => { + logger.debug(`Removing old assets from previous versions of ${pkgName}`); await removeOldAssets({ soClient: savedObjectsClient, pkgName: packageInfo.name, @@ -759,7 +764,7 @@ export async function installPackage(args: InstallPackageParams): Promise .at(integrationPosition); if (!integration) { - throw new Error(`Index name ${index} does not seem to be a File storage index`); + throw new FleetError(`Index name ${index} does not seem to be a File storage index`); } response.direction = isDeliveryToHost ? 'to-host' : 'from-host'; @@ -69,7 +70,7 @@ export const parseFileStorageIndex = (index: string): ParsedFileStorageIndex => } } - throw new Error( + throw new FleetError( `Unable to parse index [${index}]. Does not match a known index pattern: [${fileStorageIndexPatterns.join( ' | ' )}]` diff --git a/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts b/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts index 89801b66b49c..730b368495cb 100644 --- a/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts +++ b/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts @@ -4,11 +4,16 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { loggerMock } from '@kbn/logging-mocks'; +import type { Logger } from '@kbn/core/server'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { FLEET_PROXY_SAVED_OBJECT_TYPE } from '../constants'; +import { appContextService } from './app_context'; + import { deleteFleetProxy } from './fleet_proxies'; import { listFleetServerHostsForProxyId, updateFleetServerHost } from './fleet_server_host'; import { outputService } from './output'; @@ -17,6 +22,7 @@ import { downloadSourceService } from './download_source'; jest.mock('./output'); jest.mock('./download_source'); jest.mock('./fleet_server_host'); +jest.mock('./app_context'); const mockedListFleetServerHostsForProxyId = listFleetServerHostsForProxyId as jest.MockedFunction< typeof listFleetServerHostsForProxyId @@ -35,8 +41,19 @@ const PROXY_IDS = { PRECONFIGURED: 'test-preconfigured', RELATED_PRECONFIGURED: 'test-related-preconfigured', }; +const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + +let mockedLogger: jest.Mocked; describe('Fleet proxies service', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + const soClientMock = savedObjectsClientMock.create(); const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); diff --git a/x-pack/plugins/fleet/server/services/fleet_proxies.ts b/x-pack/plugins/fleet/server/services/fleet_proxies.ts index cf45b90804c2..61aa07b8e061 100644 --- a/x-pack/plugins/fleet/server/services/fleet_proxies.ts +++ b/x-pack/plugins/fleet/server/services/fleet_proxies.ts @@ -24,6 +24,8 @@ import type { Output, } from '../types'; +import { appContextService } from './app_context'; + import { listFleetServerHostsForProxyId, updateFleetServerHost } from './fleet_server_host'; import { outputService } from './output'; import { downloadSourceService } from './download_source'; @@ -70,6 +72,9 @@ export async function createFleetProxy( data: NewFleetProxy, options?: { id?: string; overwrite?: boolean; fromPreconfiguration?: boolean } ): Promise { + const logger = appContextService.getLogger(); + logger.debug(`Creating fleet proxy ${data}`); + const res = await soClient.create( FLEET_PROXY_SAVED_OBJECT_TYPE, fleetProxyDataToSOAttribute(data), @@ -78,7 +83,7 @@ export async function createFleetProxy( overwrite: options?.overwrite, } ); - + logger.debug(`Created fleet proxy ${options?.id}`); return savedObjectToFleetProxy(res); } @@ -97,6 +102,9 @@ export async function deleteFleetProxy( id: string, options?: { fromPreconfiguration?: boolean } ) { + const logger = appContextService.getLogger(); + logger.debug(`Deleting fleet proxy ${id}`); + const fleetProxy = await getFleetProxy(soClient, id); if (fleetProxy.is_preconfigured && !options?.fromPreconfiguration) { @@ -120,6 +128,7 @@ export async function deleteFleetProxy( } await updateRelatedSavedObject(soClient, esClient, fleetServerHosts, outputs, downloadSources); + logger.debug(`Deleted fleet proxy ${id}`); return await soClient.delete(FLEET_PROXY_SAVED_OBJECT_TYPE, id); } @@ -130,6 +139,8 @@ export async function updateFleetProxy( data: Partial, options?: { fromPreconfiguration?: boolean } ) { + const logger = appContextService.getLogger(); + logger.debug(`Updating fleet proxy ${id}`); const originalItem = await getFleetProxy(soClient, id); if (data.is_preconfigured && !options?.fromPreconfiguration) { @@ -141,7 +152,7 @@ export async function updateFleetProxy( id, fleetProxyDataToSOAttribute(data) ); - + logger.debug(`Updated fleet proxy ${id}`); return { ...originalItem, ...data, diff --git a/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts b/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts index 40f65ca21c5e..f92261ffb9f8 100644 --- a/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts +++ b/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts @@ -6,6 +6,10 @@ */ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +import type { Logger } from '@kbn/core/server'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; import { GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, @@ -13,9 +17,25 @@ import { DEFAULT_FLEET_SERVER_HOST_ID, } from '../constants'; +import { appContextService } from './app_context'; + import { migrateSettingsToFleetServerHost } from './fleet_server_host'; +jest.mock('./app_context'); + +const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + +let mockedLogger: jest.Mocked; + describe('migrateSettingsToFleetServerHost', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + it('should not migrate settings if a default fleet server policy config exists', async () => { const soClient = savedObjectsClientMock.create(); soClient.find.mockImplementation(({ type }) => { diff --git a/x-pack/plugins/fleet/server/services/fleet_server_host.ts b/x-pack/plugins/fleet/server/services/fleet_server_host.ts index 156d7e478b02..66bed0b61977 100644 --- a/x-pack/plugins/fleet/server/services/fleet_server_host.ts +++ b/x-pack/plugins/fleet/server/services/fleet_server_host.ts @@ -26,7 +26,9 @@ import type { NewFleetServerHost, AgentPolicy, } from '../types'; -import { FleetServerHostUnauthorizedError } from '../errors'; +import { FleetServerHostUnauthorizedError, FleetServerHostNotFoundError } from '../errors'; + +import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; import { escapeSearchQueryPhrase } from './saved_object'; @@ -46,6 +48,7 @@ export async function createFleetServerHost( data: NewFleetServerHost, options?: { id?: string; overwrite?: boolean; fromPreconfiguration?: boolean } ): Promise { + const logger = appContextService.getLogger(); if (data.is_default) { const defaultItem = await getDefaultFleetServerHost(soClient); if (defaultItem && defaultItem.id !== options?.id) { @@ -61,13 +64,13 @@ export async function createFleetServerHost( if (data.host_urls) { data.host_urls = data.host_urls.map(normalizeHostsForAgents); } - + logger.debug(`Creating fleet server host with ${data}`); const res = await soClient.create( FLEET_SERVER_HOST_SAVED_OBJECT_TYPE, data, { id: options?.id, overwrite: options?.overwrite } ); - + logger.debug(`Created fleet server host ${options?.id}`); return savedObjectToFleetServerHost(res); } @@ -122,6 +125,9 @@ export async function deleteFleetServerHost( id: string, options?: { fromPreconfiguration?: boolean } ) { + const logger = appContextService.getLogger(); + logger.debug(`Deleting fleet server host ${id}`); + const fleetServerHost = await getFleetServerHost(soClient, id); if (fleetServerHost.is_preconfigured && !options?.fromPreconfiguration) { @@ -147,6 +153,9 @@ export async function updateFleetServerHost( data: Partial, options?: { fromPreconfiguration?: boolean } ) { + const logger = appContextService.getLogger(); + logger.debug(`Updating fleet server host ${id}`); + const originalItem = await getFleetServerHost(soClient, id); if (data.is_preconfigured && !options?.fromPreconfiguration) { @@ -174,7 +183,7 @@ export async function updateFleetServerHost( } await soClient.update(FLEET_SERVER_HOST_SAVED_OBJECT_TYPE, id, data); - + logger.debug(`Updated fleet server host ${id}`); return { ...originalItem, ...data, @@ -224,7 +233,7 @@ export async function getFleetServerHostsForAgentPolicy( const defaultFleetServerHost = await getDefaultFleetServerHost(soClient); if (!defaultFleetServerHost) { - throw new Error('Default Fleet Server host is not setup'); + throw new FleetServerHostNotFoundError('Default Fleet Server host is not setup'); } return defaultFleetServerHost; diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index f2d17b018727..7b838d9b9b0a 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -57,6 +57,7 @@ import { FleetEncryptedSavedObjectEncryptionKeyRequired, OutputInvalidError, OutputUnauthorizedError, + FleetError, } from '../errors'; import type { OutputType } from '../types'; @@ -436,6 +437,9 @@ class OutputService { secretHashes?: Record; } ): Promise { + const logger = appContextService.getLogger(); + logger.debug(`Creating new output`); + const data: OutputSOAttributes = { ...omit(output, ['ssl', 'secrets']) }; if (output.type === outputType.RemoteElasticsearch) { if (data.is_default) { @@ -604,7 +608,7 @@ class OutputService { overwrite: options?.overwrite || options?.fromPreconfiguration, id, }); - + logger.debug(`Created new output ${id}`); return outputSavedObjectToOutput(newSo); } @@ -694,7 +698,7 @@ class OutputService { }); if (outputSO.error) { - throw new Error(outputSO.error.message); + throw new FleetError(outputSO.error.message); } return outputSavedObjectToOutput(outputSO); @@ -707,6 +711,9 @@ class OutputService { fromPreconfiguration: false, } ) { + const logger = appContextService.getLogger(); + logger.debug(`Deleting output ${id}`); + const originalOutput = await this.get(soClient, id); if (originalOutput.is_preconfigured && !fromPreconfiguration) { @@ -741,7 +748,7 @@ class OutputService { esClient: appContextService.getInternalUserESClient(), output: originalOutput, }); - + logger.debug(`Deleted output ${id}`); return soDeleteResult; } @@ -757,6 +764,9 @@ class OutputService { fromPreconfiguration: false, } ) { + const logger = appContextService.getLogger(); + logger.debug(`Updating output ${id}`); + if (data.type === outputType.RemoteElasticsearch) { if (data.is_default) { throw new OutputInvalidError( @@ -998,18 +1008,17 @@ class OutputService { ); if (outputSO.error) { - throw new Error(outputSO.error.message); + throw new FleetError(outputSO.error.message); } if (secretsToDelete.length) { try { await deleteSecrets({ esClient, ids: secretsToDelete.map((s) => s.id) }); } catch (err) { - appContextService - .getLogger() - .warn(`Error cleaning up secrets for output ${id}: ${err.message}`); + logger.warn(`Error cleaning up secrets for output ${id}: ${err.message}`); } } + logger.debug(`Updated output ${id}`); } public async backfillAllOutputPresets( diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 983fdd7e8d29..d3d49a7f001f 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -80,6 +80,9 @@ import { HostedAgentPolicyRestrictionRelatedError, FleetUnauthorizedError, PackagePolicyNameExistsError, + AgentPolicyNotFoundError, + InputNotFoundError, + StreamNotFoundError, } from '../errors'; import { NewPackagePolicySchema, PackagePolicySchema, UpdatePackagePolicySchema } from '../types'; import type { @@ -171,6 +174,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const logger = appContextService.getLogger(); let secretReferences: PolicySecretReference[] | undefined; + logger.debug(`Creating new package policy`); + let enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyCreate', packagePolicy, @@ -300,6 +305,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } const createdPackagePolicy = { id: newSo.id, version: newSo.version, ...newSo.attributes }; + logger.debug(`Created new package policy with id ${newSo.id} and version ${newSo.version}`); return packagePolicyService.runExternalCallbacks( 'packagePolicyPostCreate', @@ -351,6 +357,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }> = []; const logger = appContextService.getLogger(); + logger.debug(`Starting bulk create of package policy`); const packagePoliciesWithIds = packagePolicies.map((p) => { if (!p.id) { @@ -420,7 +427,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (hasCreatedSO?.error && !hasFailed) { failedPolicies.push({ packagePolicy, - error: hasCreatedSO?.error ?? new Error('Failed to create package policy.'), + error: hasCreatedSO?.error ?? new FleetError('Failed to create package policy.'), }); } }); @@ -434,7 +441,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }); } } - + logger.debug(`Created new package policies`); return { created: newSos.map((newSo) => ({ id: newSo.id, @@ -498,7 +505,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } if (packagePolicySO.error) { - throw new Error(packagePolicySO.error.message); + throw new FleetError(packagePolicySO.error.message); } let experimentalFeatures: ExperimentalDataStreamFeature[] | undefined; @@ -587,7 +594,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } else if (so.error.statusCode === 404) { throw new PackagePolicyNotFoundError(`Package policy ${so.id} not found`); } else { - throw new Error(so.error.message); + throw new FleetError(so.error.message); } } @@ -689,12 +696,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { id, savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, }); + const logger = appContextService.getLogger(); let enrichedPackagePolicy: UpdatePackagePolicy; let secretReferences: PolicySecretReference[] | undefined; let secretsToDelete: PolicySecretReference[] | undefined; try { + logger.debug(`Starting update of package policy ${id}`); enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyUpdate', packagePolicyUpdate, @@ -702,7 +711,6 @@ class PackagePolicyClientImpl implements PackagePolicyClient { esClient ); } catch (error) { - const logger = appContextService.getLogger(); logger.error(`An error occurred executing "packagePolicyUpdate" callback: ${error}`); logger.error(error); if (error.apiPassThrough) { @@ -718,7 +726,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { throw new PackagePolicyRestrictionRelatedError(`Cannot update package policy ${id}`); } if (!oldPackagePolicy) { - throw new Error('Package policy not found'); + throw new PackagePolicyNotFoundError('Package policy not found'); } if ( @@ -774,6 +782,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { packagePolicy: restOfPackagePolicy, }); + logger.debug(`Updating SO with revision ${oldPackagePolicy.revision + 1}`); await soClient.update( SAVED_OBJECT_TYPE, id, @@ -825,6 +834,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } // Bump revision of associated agent policy + logger.debug(`Bumping revision of associated agent policy ${packagePolicy.policy_id}`); const bumpPromise = agentPolicyService.bumpRevision( soClient, esClient, @@ -845,6 +855,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { await Promise.all([bumpPromise, assetRemovePromise, deleteSecretsPromise]); sendUpdatePackagePolicyTelemetryEvent(soClient, [packagePolicyUpdate], [oldPackagePolicy]); + logger.debug(`Package policy ${id} update completed`); return newPolicy; } @@ -874,7 +885,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { ); if (!oldPackagePolicies || oldPackagePolicies.length === 0) { - throw new Error('Package policy not found'); + throw new PackagePolicyNotFoundError('Package policy not found'); } const packageInfos = await getPackageInfoForPackagePolicies(packagePolicyUpdates, soClient); @@ -892,7 +903,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const packagePolicy = { ...packagePolicyUpdate, name: packagePolicyUpdate.name.trim() }; const oldPackagePolicy = oldPackagePolicies.find((p) => p.id === id); if (!oldPackagePolicy) { - throw new Error('Package policy not found'); + throw new PackagePolicyNotFoundError('Package policy not found'); } let secretReferences: PolicySecretReference[] | undefined; @@ -1050,6 +1061,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const result: PostDeletePackagePoliciesResponse = []; const logger = appContextService.getLogger(); + logger.debug(`Deleting package policies ${ids}`); const packagePolicies = await this.getByIDs(soClient, ids, { ignoreMissing: true }); if (!packagePolicies) { @@ -1194,6 +1206,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { context, request ); + logger.debug(`Deleted package policies ${ids}`); } catch (error) { logger.error(`An error occurred executing "packagePolicyPostDelete" callback: ${error}`); logger.error(error); @@ -1924,7 +1937,9 @@ async function _compilePackagePolicyInput( const packageInput = packageInputs.find((pkgInput) => pkgInput.type === input.type); if (!packageInput) { - throw new Error(`Input template not found, unable to find input type ${input.type}`); + throw new InputNotFoundError( + `Input template not found, unable to find input type ${input.type}` + ); } if (!packageInput.template_path) { return undefined; @@ -1935,7 +1950,9 @@ async function _compilePackagePolicyInput( ); if (!pkgInputTemplate || !pkgInputTemplate.buffer) { - throw new Error(`Unable to load input template at /agent/input/${packageInput.template_path!}`); + throw new InputNotFoundError( + `Unable to load input template at /agent/input/${packageInput.template_path!}` + ); } return compileTemplate( @@ -2019,7 +2036,7 @@ async function _compilePackageStream( const packageDataStreams = getNormalizedDataStreams(pkgInfo); if (!packageDataStreams) { - throw new Error('Stream template not found, no data streams'); + throw new StreamNotFoundError('Stream template not found, no data streams'); } const packageDataStream = packageDataStreams.find( @@ -2027,7 +2044,7 @@ async function _compilePackageStream( ); if (!packageDataStream) { - throw new Error( + throw new StreamNotFoundError( `Stream template not found, unable to find dataset ${stream.data_stream.dataset}` ); } @@ -2038,11 +2055,15 @@ async function _compilePackageStream( (pkgStream) => pkgStream.input === input.type ); if (!streamFromPkg) { - throw new Error(`Stream template not found, unable to find stream for input ${input.type}`); + throw new StreamNotFoundError( + `Stream template not found, unable to find stream for input ${input.type}` + ); } if (!streamFromPkg.template_path) { - throw new Error(`Stream template path not found for dataset ${stream.data_stream.dataset}`); + throw new StreamNotFoundError( + `Stream template path not found for dataset ${stream.data_stream.dataset}` + ); } const datasetPath = packageDataStream.path; @@ -2054,7 +2075,7 @@ async function _compilePackageStream( ); if (!pkgStreamTemplate || !pkgStreamTemplate.buffer) { - throw new Error( + throw new StreamNotFoundError( `Unable to load stream template ${streamFromPkg.template_path} for dataset ${stream.data_stream.dataset}` ); } @@ -2484,7 +2505,7 @@ async function validateIsNotHostedPolicy( const agentPolicy = await agentPolicyService.get(soClient, id, false); if (!agentPolicy) { - throw new Error('Agent policy not found'); + throw new AgentPolicyNotFoundError('Agent policy not found'); } if (agentPolicy.is_managed && !force) { diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 73fc61cf0fab..ad1e1c0ddb8b 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -26,6 +26,8 @@ import type { PreconfigurationError } from '../../common/constants'; import { PRECONFIGURATION_LATEST_KEYWORD } from '../../common/constants'; import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE } from '../constants'; +import { FleetError } from '../errors'; + import { escapeSearchQueryPhrase } from './saved_object'; import { pkgToPkgKey } from './epm/registry'; import { getInstallation, getPackageInfo } from './epm/packages'; @@ -67,7 +69,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( .map(([, versions]) => versions.map((v) => pkgToPkgKey(v)).join(', ')) .join('; '); - throw new Error( + throw new FleetError( i18n.translate('xpack.fleet.preconfiguration.duplicatePackageError', { defaultMessage: 'Duplicate packages specified in configuration: {duplicateList}', values: { @@ -119,6 +121,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( await ensurePackagesCompletedInstall(soClient, esClient); // Create policies specified in Kibana config + logger.debug(`Creating preconfigured policies`); const preconfiguredPolicies = await Promise.allSettled( policies.map(async (preconfiguredAgentPolicy) => { if (preconfiguredAgentPolicy.id) { @@ -140,7 +143,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( !preconfiguredAgentPolicy.is_default && !preconfiguredAgentPolicy.is_default_fleet_server ) { - throw new Error( + throw new FleetError( i18n.translate('xpack.fleet.preconfiguration.missingIDError', { defaultMessage: '{agentPolicyName} is missing an `id` field. `id` is required, except for policies marked is_default or is_default_fleet_server.', @@ -221,7 +224,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( const rejectedPackage = rejectedPackages.find((rp) => rp.package?.name === pkg.name); if (rejectedPackage) { - throw new Error( + throw new FleetError( i18n.translate('xpack.fleet.preconfiguration.packageRejectedError', { defaultMessage: `[{agentPolicyName}] could not be added. [{pkgName}] could not be installed due to error: [{errorMessage}]`, values: { @@ -232,8 +235,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( }) ); } - - throw new Error( + throw new FleetError( i18n.translate('xpack.fleet.preconfiguration.packageMissingError', { defaultMessage: '[{agentPolicyName}] could not be added. [{pkgName}] is not installed, add [{pkgName}] to [{packagesConfigValue}] or remove it from [{packagePolicyName}].', @@ -257,7 +259,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( packagePolicy.name === installablePackagePolicy.name ); }); - + logger.debug(`Adding preconfigured package policies ${packagePoliciesToAdd}`); const s = apm.startSpan('Add preconfigured package policies', 'preconfiguration'); await addPreconfiguredPolicyPackages( soClient, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts b/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts index 8c6b9680318d..f622b1115e16 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.ts @@ -11,6 +11,8 @@ import { normalizeHostsForAgents } from '../../../common/services'; import type { FleetConfigType } from '../../config'; import { DEFAULT_FLEET_SERVER_HOST_ID } from '../../constants'; +import { FleetError } from '../../errors'; + import type { FleetServerHost } from '../../types'; import { appContextService } from '../app_context'; import { @@ -63,7 +65,7 @@ export function getPreconfiguredFleetServerHostFromConfig(config?: FleetConfigTy ]); if (fleetServerHosts.filter((fleetServerHost) => fleetServerHost.is_default).length > 1) { - throw new Error('Only one default Fleet Server host is allowed'); + throw new FleetError('Only one default Fleet Server host is allowed'); } return fleetServerHosts; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/reset_agent_policies.ts b/x-pack/plugins/fleet/server/services/preconfiguration/reset_agent_policies.ts index f32ec17e22ea..87adf58e4266 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/reset_agent_policies.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/reset_agent_policies.ts @@ -22,6 +22,7 @@ import { packagePolicyService } from '../package_policy'; import { getAgentsByKuery, forceUnenrollAgent } from '../agents'; import { listEnrollmentApiKeys, deleteEnrollmentApiKey } from '../api_keys'; import type { AgentPolicy } from '../../types'; +import { AgentPolicyInvalidError } from '../../errors'; export async function resetPreconfiguredAgentPolicies( soClient: SavedObjectsClientContract, @@ -135,7 +136,7 @@ async function _deleteExistingData( throw err; }); if (policy && !policy.is_preconfigured) { - throw new Error('Invalid policy'); + throw new AgentPolicyInvalidError(`Invalid policy ${agentPolicyId}`); } if (policy) { existingPolicies = [policy]; diff --git a/x-pack/plugins/fleet/server/services/security/message_signing_service.ts b/x-pack/plugins/fleet/server/services/security/message_signing_service.ts index b3d08e9a0b8e..741b9e33dec8 100644 --- a/x-pack/plugins/fleet/server/services/security/message_signing_service.ts +++ b/x-pack/plugins/fleet/server/services/security/message_signing_service.ts @@ -22,6 +22,7 @@ import { MessageSigningError } from '../../../common/errors'; import { MESSAGE_SIGNING_KEYS_SAVED_OBJECT_TYPE } from '../../constants'; import { appContextService } from '../app_context'; +import { SigningServiceNotFoundError } from '../../errors'; interface MessageSigningKeys { private_key: string; @@ -118,10 +119,10 @@ export class MessageSigningService implements MessageSigningServiceInterface { signer.end(); if (!serializedPrivateKey) { - throw new Error('unable to find private key'); + throw new SigningServiceNotFoundError('Unable to find private key'); } if (!passphrase) { - throw new Error('unable to find passphrase'); + throw new SigningServiceNotFoundError('Unable to find passphrase'); } const privateKey = Buffer.from(serializedPrivateKey, 'base64'); @@ -139,7 +140,7 @@ export class MessageSigningService implements MessageSigningServiceInterface { const { publicKey } = await this.generateKeyPair(); if (!publicKey) { - throw new Error('unable to find public key'); + throw new SigningServiceNotFoundError('Unable to find public key'); } return publicKey; diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 575da165d001..cc4be9bab0bc 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -125,7 +125,6 @@ async function createSetupSideEffects( esClient, getPreconfiguredOutputFromConfig(appContextService.getConfig()) ), - settingsService.settingsSetup(soClient), ]); @@ -228,7 +227,7 @@ async function createSetupSideEffects( stepSpan?.end(); stepSpan = apm.startSpan('Set up enrollment keys for preconfigured policies', 'preconfiguration'); - logger.debug('Setting up Fleet enrollment keys'); + logger.debug('Setting up Fleet enrollment keys for preconfigured policies'); await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient); stepSpan?.end(); diff --git a/x-pack/plugins/fleet/server/services/setup/clean_old_fleet_indices.tsx b/x-pack/plugins/fleet/server/services/setup/clean_old_fleet_indices.tsx index 856e3543cd96..cd3317c5eb23 100644 --- a/x-pack/plugins/fleet/server/services/setup/clean_old_fleet_indices.tsx +++ b/x-pack/plugins/fleet/server/services/setup/clean_old_fleet_indices.tsx @@ -28,6 +28,7 @@ const INDEX_TEMPLATE_TO_CLEAN = [ export async function cleanUpOldFileIndices(esClient: ElasticsearchClient, logger: Logger) { try { // Clean indices + logger.info('Cleaning old indices'); await pMap( INDICES_TO_CLEAN, async (indiceToClean) => { diff --git a/x-pack/test/fleet_api_integration/apis/enrollment_api_keys/crud.ts b/x-pack/test/fleet_api_integration/apis/enrollment_api_keys/crud.ts index 47c4d7ecd9f7..7d8cc253fa0e 100644 --- a/x-pack/test/fleet_api_integration/apis/enrollment_api_keys/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/enrollment_api_keys/crud.ts @@ -162,14 +162,14 @@ export default function (providerContext: FtrProviderContext) { .expect(400); }); - it('should return a 400 if the policy_id is not a valid policy', async () => { + it('should return a 404 if the policy_id does not exist', async () => { const { body: apiResponse } = await supertest .post(`/api/fleet/enrollment_api_keys`) .set('kbn-xsrf', 'xxx') .send({ policy_id: 'idonotexists', }) - .expect(400); + .expect(404); expect(apiResponse.message).to.be('Agent policy "idonotexists" not found'); }); @@ -223,11 +223,11 @@ export default function (providerContext: FtrProviderContext) { policy_id: 'policy1', name: 'something', }) - .expect(400); + .expect(409); expect(noSpacesDupe).to.eql({ - statusCode: 400, - error: 'Bad Request', + statusCode: 409, + error: 'Conflict', message: 'An enrollment key named something already exists for agent policy policy1', }); @@ -238,10 +238,10 @@ export default function (providerContext: FtrProviderContext) { policy_id: 'policy1', name: 'something else', }) - .expect(400); + .expect(409); expect(hasSpacesDupe).to.eql({ - statusCode: 400, - error: 'Bad Request', + statusCode: 409, + error: 'Conflict', message: 'An enrollment key named something else already exists for agent policy policy1', }); }); From 61a187224d982cd303a9f252a8dacc3bb42944ba Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:31:06 -0500 Subject: [PATCH 07/12] skip failing test suite (#173165) --- .../observability/observability_log_explorer/header_menu.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts b/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts index 0e7527bc7688..fe0c0c9fea19 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts @@ -22,7 +22,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'svlCommonNavigation', ]); - describe('Header menu', () => { + // Failing: See https://github.com/elastic/kibana/issues/173165 + describe.skip('Header menu', () => { before(async () => { await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); await esArchiver.load( From 95025129a791c7a2fd24d323a5bd240c6af495a6 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:40:43 -0500 Subject: [PATCH 08/12] skip failing test suite (#173165) --- .../observability/observability_log_explorer/header_menu.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts b/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts index fe0c0c9fea19..41a75282e22e 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/header_menu.ts @@ -22,6 +22,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'svlCommonNavigation', ]); + // Failing: See https://github.com/elastic/kibana/issues/173165 // Failing: See https://github.com/elastic/kibana/issues/173165 describe.skip('Header menu', () => { before(async () => { From 156c8f2323a65153397bbddd308bfd23901d50ba Mon Sep 17 00:00:00 2001 From: Adam Demjen Date: Tue, 12 Dec 2023 12:42:16 -0500 Subject: [PATCH 09/12] [Enterprise Search] Disallow creating pipeline with placeholder of model (#172988) ## Summary This PR adds validation to model selection when creating an inference pipeline. Pipelines cannot be created with a non-deployed placeholder of a model (e.g. E5, ELSER) as this would break other Search UIs and functionality. If such an item is selected, an error message is displayed and the Continue button is disabled. Note this feature is independent of the changing of the model selection component. Valid selection: Screenshot 2023-12-08 at 14 03 59 Invalid selection: Screenshot 2023-12-08 at 14 04 04 ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [x] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../ml_inference/configure_pipeline.tsx | 31 +++++++++++++------ .../ml_inference/ml_inference_logic.test.ts | 19 ++++++++++++ .../ml_inference/model_select.test.tsx | 17 ++++++++++ .../pipelines/ml_inference/model_select.tsx | 1 + .../pipelines/ml_inference/types.ts | 2 ++ .../pipelines/ml_inference/utils.ts | 8 +++++ 6 files changed, 69 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx index cc318831555a..52d4f38b4540 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx @@ -19,6 +19,7 @@ import { EuiTabbedContentTab, EuiTitle, EuiText, + EuiTextColor, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -130,15 +131,27 @@ export const ConfigurePipeline: React.FC = () => { )} - - - + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.titleSelectTrainedModel', + { defaultMessage: 'Select a trained ML Model' } + )} +
+ + {formErrors.modelStatus !== undefined && ( + <> + + +

+ {formErrors.modelStatus} +

+ + + )} + + ), diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts index a725371de024..7412d861d713 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts @@ -550,6 +550,25 @@ describe('MlInferenceLogic', () => { pipelineName: 'Name already used by another pipeline.', }); }); + it('has errors when non-deployed model is selected', () => { + MLInferenceLogic.actions.setInferencePipelineConfiguration({ + ...MLInferenceLogic.values.addInferencePipelineModal.configuration, + pipelineName: 'unit-test-pipeline', + modelID: 'unit-test-model', + existingPipeline: false, + fieldMappings: [ + { + sourceField: 'body', + targetField: 'ml.inference.body', + }, + ], + isModelPlaceholderSelected: true, + }); + + expect(MLInferenceLogic.values.formErrors).toEqual({ + modelStatus: 'Model must be deployed before use.', + }); + }); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.test.tsx index 15fb492fae56..9bd006f65883 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.test.tsx @@ -98,6 +98,23 @@ describe('ModelSelect', () => { }) ); }); + it('sets placeholder flag on selecting a placeholder item', () => { + setMockValues(DEFAULT_VALUES); + + const wrapper = shallow(); + expect(wrapper.find(EuiSelectable)).toHaveLength(1); + const selectable = wrapper.find(EuiSelectable); + selectable.simulate('change', [ + { modelId: 'model_1' }, + { modelId: 'model_2', isPlaceholder: true, checked: 'on' }, + ]); + expect(MOCK_ACTIONS.setInferencePipelineConfiguration).toHaveBeenCalledWith( + expect.objectContaining({ + modelID: 'model_2', + isModelPlaceholderSelected: true, + }) + ); + }); it('generates pipeline name on selecting an item', () => { setMockValues(DEFAULT_VALUES); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.tsx index 86c91c483702..ac3900b6ed66 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/model_select.tsx @@ -44,6 +44,7 @@ export const ModelSelect: React.FC = () => { ...configuration, inferenceConfig: undefined, modelID: selectedOption?.modelId ?? '', + isModelPlaceholderSelected: selectedOption?.isPlaceholder ?? false, fieldMappings: undefined, pipelineName: isPipelineNameUserSupplied ? pipelineName diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts index 3a645dcbba3b..87aed3eb4d71 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts @@ -13,6 +13,7 @@ export interface InferencePipelineConfiguration { existingPipeline?: boolean; inferenceConfig?: InferencePipelineInferenceConfig; isPipelineNameUserSupplied?: boolean; + isModelPlaceholderSelected?: boolean; modelID: string; pipelineName: string; fieldMappings?: FieldMapping[]; @@ -21,6 +22,7 @@ export interface InferencePipelineConfiguration { export interface AddInferencePipelineFormErrors { modelID?: string; + modelStatus?: string; fieldMappings?: string; pipelineName?: string; } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts index 02cf5a7463dd..5a01a3823a71 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts @@ -37,6 +37,12 @@ const PIPELINE_NAME_EXISTS_ERROR = i18n.translate( defaultMessage: 'Name already used by another pipeline.', } ); +const MODEL_NOT_DEPLOYED_ERROR = i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.modelNotDeployedError', + { + defaultMessage: 'Model must be deployed before use.', + } +); export const validateInferencePipelineConfiguration = ( config: InferencePipelineConfiguration @@ -55,6 +61,8 @@ export const validateInferencePipelineConfiguration = ( } if (config.modelID.trim().length === 0) { errors.modelID = FIELD_REQUIRED_ERROR; + } else if (config.isModelPlaceholderSelected) { + errors.modelStatus = MODEL_NOT_DEPLOYED_ERROR; } return errors; From 13434dc28e926b4ffb10570189b624230a5f723e Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Tue, 12 Dec 2023 11:22:02 -0700 Subject: [PATCH 10/12] Adds run without basepath configured to examples readme (#173086) Kibana crashes when running the examples with a basepath configured with too many redirects during auth. This PR warns about that in the examples/README Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- examples/README.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.asciidoc b/examples/README.asciidoc index d33c5e825ce1..3f2c58a17330 100644 --- a/examples/README.asciidoc +++ b/examples/README.asciidoc @@ -1,7 +1,7 @@ [[example-plugins]] == Example plugins -This folder contains example plugins. To run the plugins in this folder, use the `--run-examples` flag, via +This folder contains example plugins. To run the plugins in this folder, use the `--run-examples` flag (without a basepath), via [source,bash] ---- From c15c29490caca68e483a67e9f91f5575722f540d Mon Sep 17 00:00:00 2001 From: Ramon Butter Date: Tue, 12 Dec 2023 19:23:39 +0100 Subject: [PATCH 11/12] Set explicit resource requests in the QG (#173189) Kibana is fairly large, we should explicitly set in the gate what we are expecting in terms of resource for each individual step. --- .../pipelines/quality-gates/pipeline.kibana-tests.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml index 467df501bc9c..121785327591 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml @@ -10,6 +10,11 @@ # # Docs: https://docs.elastic.dev/serverless/qualitygates +agents: + cpu: 2 + ephemeralStorage: "20G" + memory: "8G" + env: TEAM_CHANNEL: "#kibana-mission-control" ENVIRONMENT: ${ENVIRONMENT?} From fe0ad5adad7610d73cd108d0a76eacfbdfbdc0d7 Mon Sep 17 00:00:00 2001 From: "Quynh Nguyen (Quinn)" <43350163+qn895@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:17:36 -0600 Subject: [PATCH 12/12] [ML] Fix View in Discover option in Anomaly explorer not handling multiple field values or values with quotation marks (#172897) ## Summary This PR addresses https://github.com/elastic/kibana/issues/172872 and https://github.com/elastic/kibana/issues/172461. Previously, clicking `View in Discover` option in Anomaly explorer for an anomaly with influencers containing quotation marks would lead to an error in Discover. Likewise, if the influencer has multiple field values, the link only brings over the first field value and not all. This PR fixes both issues. After: https://github.com/elastic/kibana/assets/43350163/d60055e6-04ec-4882-81ea-02c04f73d16b ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../components/anomalies_table/links_menu.tsx | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx b/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx index bf7df6091177..b81f632c9a62 100644 --- a/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx +++ b/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx @@ -34,6 +34,8 @@ import { formatHumanReadableDateTimeSeconds, timeFormatter } from '@kbn/ml-date- import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { CATEGORIZE_FIELD_TRIGGER } from '@kbn/ml-ui-actions'; +import { isDefined } from '@kbn/ml-is-defined'; +import { escapeQuotes } from '@kbn/es-query'; import { PLUGIN_ID } from '../../../../common/constants/app'; import { mlJobService } from '../../services/job_service'; import { findMessageField, getDataViewIdFromName } from '../../util/index_utils'; @@ -265,12 +267,19 @@ export const LinksMenuUI = (props: LinksMenuProps) => { if (record.influencers) { kqlQuery = record.influencers - .map( - (influencer) => - `"${influencer.influencer_field_name}":"${ - influencer.influencer_field_values[0] ?? '' - }"` - ) + .filter((influencer) => isDefined(influencer)) + .map((influencer) => { + const values = influencer.influencer_field_values; + + if (values.length > 0) { + const fieldName = escapeQuotes(influencer.influencer_field_name); + const escapedVals = values + .filter((value) => isDefined(value)) + .map((value) => `"${fieldName}":"${escapeQuotes(value)}"`); + // Ensure there's enclosing () if there are multiple field values, + return escapedVals.length > 1 ? `(${escapedVals.join(' OR ')})` : escapedVals[0]; + } + }) .join(' AND '); }