= ({ output, showBadge
iconType="error"
data-test-subj="outputHealthDegradedCallout"
>
-
+
{i18n.translate('xpack.fleet.output.calloutText', {
defaultMessage: 'Unable to connect to "{name}" at {host}.',
values: {
@@ -59,7 +59,7 @@ export const OutputHealth: React.FunctionComponent = ({ output, showBadge
host: output.hosts?.join(',') ?? '',
},
})}
-
{' '}
+
{i18n.translate('xpack.fleet.output.calloutPromptText', {
defaultMessage: 'Please check the details are correct.',
diff --git a/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts b/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts
index 219d101d6e1a..b8d1ba7e9bec 100644
--- a/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts
+++ b/x-pack/plugins/fleet/scripts/verify_test_packages/verify_test_packages.test.ts
@@ -5,9 +5,30 @@
* 2.0.
*/
+import { securityMock } from '@kbn/security-plugin/server/mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+
+import type { Logger } from '@kbn/core/server';
+
+import { appContextService } from '../../server/services/app_context';
+
import { verifyAllTestPackages } from './verify_test_packages';
+jest.mock('../../server/services/app_context');
+
+const mockedAppContextService = appContextService as jest.Mocked;
+mockedAppContextService.getSecuritySetup.mockImplementation(() => ({
+ ...securityMock.createSetup(),
+}));
+
+let mockedLogger: jest.Mocked;
+
describe('Test packages', () => {
+ beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+
test('All test packages should be valid (node scripts/verify_test_packages) ', async () => {
const { errors } = await verifyAllTestPackages();
expect(errors).toEqual([]);
diff --git a/x-pack/plugins/fleet/server/errors/handlers.ts b/x-pack/plugins/fleet/server/errors/handlers.ts
index 3bfe94537c58..c85bfeced9db 100644
--- a/x-pack/plugins/fleet/server/errors/handlers.ts
+++ b/x-pack/plugins/fleet/server/errors/handlers.ts
@@ -20,18 +20,14 @@ import { UninstallTokenError } from '../../common/errors';
import { appContextService } from '../services';
import {
- AgentNotFoundError,
- AgentActionNotFoundError,
AgentPolicyNameExistsError,
ConcurrentInstallOperationError,
FleetError,
- PackageNotFoundError,
PackageUnsupportedMediaTypeError,
RegistryConnectionError,
RegistryError,
RegistryResponseError,
PackageFailedVerificationError,
- PackagePolicyNotFoundError,
FleetUnauthorizedError,
PackagePolicyNameExistsError,
PackageOutdatedError,
@@ -41,6 +37,11 @@ import {
PackageESError,
KibanaSOReferenceError,
PackageAlreadyInstalledError,
+ AgentPolicyInvalidError,
+ EnrollmentKeyNameExistsError,
+ AgentRequestInvalidError,
+ PackagePolicyRequestError,
+ FleetNotFoundError,
} from '.';
type IngestErrorHandler = (
@@ -71,24 +72,31 @@ const getHTTPResponseCode = (error: FleetError): number => {
if (error instanceof KibanaSOReferenceError) {
return 400;
}
+ if (error instanceof AgentPolicyInvalidError) {
+ return 400;
+ }
+ if (error instanceof AgentRequestInvalidError) {
+ return 400;
+ }
+ if (error instanceof PackagePolicyRequestError) {
+ return 400;
+ }
// Unauthorized
if (error instanceof FleetUnauthorizedError) {
return 403;
}
// Not Found
- if (error instanceof PackageNotFoundError || error instanceof PackagePolicyNotFoundError) {
- return 404;
- }
- if (error instanceof AgentNotFoundError) {
- return 404;
- }
- if (error instanceof AgentActionNotFoundError) {
+ if (error instanceof FleetNotFoundError) {
return 404;
}
+
// Conflict
if (error instanceof AgentPolicyNameExistsError) {
return 409;
}
+ if (error instanceof EnrollmentKeyNameExistsError) {
+ return 409;
+ }
if (error instanceof ConcurrentInstallOperationError) {
return 409;
}
diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts
index 7f607f469277..ce7245672e62 100644
--- a/x-pack/plugins/fleet/server/errors/index.ts
+++ b/x-pack/plugins/fleet/server/errors/index.ts
@@ -28,7 +28,7 @@ export class RegistryResponseError extends RegistryError {
}
// Package errors
-export class PackageNotFoundError extends FleetError {}
+
export class PackageOutdatedError extends FleetError {}
export class PackageFailedVerificationError extends FleetError {
constructor(pkgName: string, pkgVersion: string) {
@@ -43,20 +43,24 @@ export class PackageInvalidArchiveError extends FleetError {}
export class PackageRemovalError extends FleetError {}
export class PackageESError extends FleetError {}
export class ConcurrentInstallOperationError extends FleetError {}
-export class BundledPackageLocationNotFoundError extends FleetError {}
+
export class KibanaSOReferenceError extends FleetError {}
export class PackageAlreadyInstalledError extends FleetError {}
export class AgentPolicyError extends FleetError {}
-export class AgentPolicyNotFoundError extends FleetError {}
-export class AgentNotFoundError extends FleetError {}
-export class AgentActionNotFoundError extends FleetError {}
+export class AgentRequestInvalidError extends FleetError {}
+export class AgentPolicyInvalidError extends FleetError {}
+
export class AgentPolicyNameExistsError extends AgentPolicyError {}
export class AgentReassignmentError extends FleetError {}
export class PackagePolicyIneligibleForUpgradeError extends FleetError {}
export class PackagePolicyValidationError extends FleetError {}
export class PackagePolicyNameExistsError extends FleetError {}
-export class PackagePolicyNotFoundError extends FleetError {}
+export class BundledPackageLocationNotFoundError extends FleetError {}
+
+export class PackagePolicyRequestError extends FleetError {}
+
+export class EnrollmentKeyNameExistsError extends FleetError {}
export class HostedAgentPolicyRestrictionRelatedError extends FleetError {
constructor(message = 'Cannot perform that action') {
super(
@@ -75,12 +79,27 @@ export class FleetEncryptedSavedObjectEncryptionKeyRequired extends FleetError {
export class FleetSetupError extends FleetError {}
export class GenerateServiceTokenError extends FleetError {}
export class FleetUnauthorizedError extends FleetError {}
+export class FleetNotFoundError extends FleetError {}
export class OutputUnauthorizedError extends FleetError {}
export class OutputInvalidError extends FleetError {}
export class OutputLicenceError extends FleetError {}
export class DownloadSourceError extends FleetError {}
+// Not found errors
+export class AgentNotFoundError extends FleetNotFoundError {}
+export class AgentPolicyNotFoundError extends FleetNotFoundError {}
+export class AgentActionNotFoundError extends FleetNotFoundError {}
+export class DownloadSourceNotFound extends FleetNotFoundError {}
+export class EnrollmentKeyNotFoundError extends FleetNotFoundError {}
+export class FleetServerHostNotFoundError extends FleetNotFoundError {}
+export class SigningServiceNotFoundError extends FleetNotFoundError {}
+export class InputNotFoundError extends FleetNotFoundError {}
+export class OutputNotFoundError extends FleetNotFoundError {}
+export class PackageNotFoundError extends FleetNotFoundError {}
+export class PackagePolicyNotFoundError extends FleetNotFoundError {}
+export class StreamNotFoundError extends FleetNotFoundError {}
+
export class FleetServerHostUnauthorizedError extends FleetUnauthorizedError {}
export class FleetProxyUnauthorizedError extends FleetUnauthorizedError {}
diff --git a/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts b/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts
index 23cdc80b8d65..961f3c90fb54 100644
--- a/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts
+++ b/x-pack/plugins/fleet/server/integration_tests/helpers/index.ts
@@ -8,6 +8,8 @@
import { adminTestUser } from '@kbn/test';
import { getSupertest, type createRoot, type HttpMethod } from '@kbn/core-test-helpers-kbn-server';
+import { FleetError } from '../../errors';
+
type Root = ReturnType;
export * from './docker_registry_helper';
@@ -18,7 +20,7 @@ export const waitForFleetSetup = async (root: Root) => {
const resp = await statusApi.send();
const fleetStatus = resp.body?.status?.plugins?.fleet;
if (fleetStatus?.meta?.error) {
- throw new Error(`Setup failed: ${JSON.stringify(fleetStatus)}`);
+ throw new FleetError(`Setup failed: ${JSON.stringify(fleetStatus)}`);
}
return !fleetStatus || fleetStatus?.summary === 'Fleet is setting up';
diff --git a/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts b/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts
index 3f571fdcb09c..efedf164f653 100644
--- a/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts
+++ b/x-pack/plugins/fleet/server/routes/agent/source_uri_utils.ts
@@ -9,6 +9,7 @@ import type { SavedObjectsClientContract } from '@kbn/core/server';
import { downloadSourceService } from '../../services';
import type { AgentPolicy } from '../../types';
+import { FleetError, DownloadSourceNotFound } from '../../errors';
export const getSourceUriForAgentPolicy = async (
soClient: SavedObjectsClientContract,
@@ -17,12 +18,12 @@ export const getSourceUriForAgentPolicy = async (
const defaultDownloadSourceId = await downloadSourceService.getDefaultDownloadSourceId(soClient);
if (!defaultDownloadSourceId) {
- throw new Error('Default download source host is not setup');
+ throw new FleetError('Default download source host is not setup');
}
const downloadSourceId: string = agentPolicy.download_source_id || defaultDownloadSourceId;
const downloadSource = await downloadSourceService.get(soClient, downloadSourceId);
if (!downloadSource) {
- throw new Error(`Download source host not found ${downloadSourceId}`);
+ throw new DownloadSourceNotFound(`Download source host not found ${downloadSourceId}`);
}
return { host: downloadSource.host, proxy_id: downloadSource.proxy_id };
};
diff --git a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts
index aefcbfc5cd87..62f34559c79e 100644
--- a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts
+++ b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.test.ts
@@ -15,7 +15,7 @@ describe('upgrade handler', () => {
it('should throw if upgrade version is higher than kibana version', () => {
expect(() => checkKibanaVersion('8.5.0', '8.4.0')).toThrowError(
- 'cannot upgrade agent to 8.5.0 because it is higher than the installed kibana version 8.4.0'
+ 'Cannot upgrade agent to 8.5.0 because it is higher than the installed kibana version 8.4.0'
);
});
diff --git a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts
index 547fda566a95..391c721e2ef9 100644
--- a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts
+++ b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts
@@ -19,7 +19,7 @@ import type { PostAgentUpgradeResponse } from '../../../common/types';
import type { PostAgentUpgradeRequestSchema, PostBulkAgentUpgradeRequestSchema } from '../../types';
import * as AgentService from '../../services/agents';
import { appContextService } from '../../services';
-import { defaultFleetErrorHandler } from '../../errors';
+import { defaultFleetErrorHandler, AgentRequestInvalidError } from '../../errors';
import {
getRecentUpgradeInfoForAgent,
isAgentUpgradeable,
@@ -187,14 +187,15 @@ export const postBulkAgentsUpgradeHandler: RequestHandler<
export const checkKibanaVersion = (version: string, kibanaVersion: string, force = false) => {
// get version number only in case "-SNAPSHOT" is in it
const kibanaVersionNumber = semverCoerce(kibanaVersion)?.version;
- if (!kibanaVersionNumber) throw new Error(`kibanaVersion ${kibanaVersionNumber} is not valid`);
+ if (!kibanaVersionNumber)
+ throw new AgentRequestInvalidError(`KibanaVersion ${kibanaVersionNumber} is not valid`);
const versionToUpgradeNumber = semverCoerce(version)?.version;
if (!versionToUpgradeNumber)
- throw new Error(`version to upgrade ${versionToUpgradeNumber} is not valid`);
+ throw new AgentRequestInvalidError(`Version to upgrade ${versionToUpgradeNumber} is not valid`);
if (!force && semverGt(versionToUpgradeNumber, kibanaVersionNumber)) {
- throw new Error(
- `cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the installed kibana version ${kibanaVersionNumber}`
+ throw new AgentRequestInvalidError(
+ `Cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the installed kibana version ${kibanaVersionNumber}`
);
}
@@ -205,8 +206,8 @@ export const checkKibanaVersion = (version: string, kibanaVersion: string, force
// When force is enabled, only the major and minor versions are checked
if (force && !(kibanaMajorGt || kibanaMajorEqMinorGte)) {
- throw new Error(
- `cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the installed kibana version ${kibanaVersionNumber}`
+ throw new AgentRequestInvalidError(
+ `Cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the installed kibana version ${kibanaVersionNumber}`
);
}
};
@@ -228,8 +229,8 @@ const checkFleetServerVersion = (
}
if (!force && semverGt(versionToUpgradeNumber, maxFleetServerVersion)) {
- throw new Error(
- `cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the latest fleet server version ${maxFleetServerVersion}`
+ throw new AgentRequestInvalidError(
+ `Cannot upgrade agent to ${versionToUpgradeNumber} because it is higher than the latest fleet server version ${maxFleetServerVersion}`
);
}
@@ -241,8 +242,8 @@ const checkFleetServerVersion = (
// When force is enabled, only the major and minor versions are checked
if (force && !(fleetServerMajorGt || fleetServerMajorEqMinorGte)) {
- throw new Error(
- `cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the latest fleet server version ${maxFleetServerVersion}`
+ throw new AgentRequestInvalidError(
+ `Cannot force upgrade agent to ${versionToUpgradeNumber} because it does not satisfy the major and minor of the latest fleet server version ${maxFleetServerVersion}`
);
}
};
diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts
new file mode 100644
index 000000000000..56d6d8c127bc
--- /dev/null
+++ b/x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts
@@ -0,0 +1,245 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
+import { httpServerMock } from '@kbn/core-http-server-mocks';
+import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
+import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
+import { Headers } from 'node-fetch';
+
+import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_packages';
+import { getFile, getInstallation } from '../../services/epm/packages/get';
+import type { FleetRequestHandlerContext } from '../..';
+import { appContextService } from '../../services';
+import { unpackBufferEntries, getArchiveEntry } from '../../services/epm/archive';
+import { getAsset } from '../../services/epm/archive/storage';
+
+import { getFileHandler } from './file_handler';
+
+jest.mock('../../services/app_context');
+jest.mock('../../services/epm/archive');
+jest.mock('../../services/epm/archive/storage');
+jest.mock('../../services/epm/packages/bundled_packages');
+jest.mock('../../services/epm/packages/get');
+
+const mockedGetBundledPackageByPkgKey = jest.mocked(getBundledPackageByPkgKey);
+const mockedGetInstallation = jest.mocked(getInstallation);
+const mockedGetFile = jest.mocked(getFile);
+const mockedGetArchiveEntry = jest.mocked(getArchiveEntry);
+const mockedUnpackBufferEntries = jest.mocked(unpackBufferEntries);
+const mockedGetAsset = jest.mocked(getAsset);
+
+function mockContext() {
+ const mockSavedObjectsClient = savedObjectsClientMock.create();
+ const mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+ return {
+ fleet: {
+ internalSOClient: async () => mockSavedObjectsClient,
+ },
+ core: {
+ savedObjects: {
+ client: mockSavedObjectsClient,
+ },
+ elasticsearch: {
+ client: {
+ asInternalUser: mockElasticsearchClient,
+ },
+ },
+ },
+ } as unknown as FleetRequestHandlerContext;
+}
+
+describe('getFileHandler', () => {
+ beforeEach(() => {
+ const logger = loggingSystemMock.createLogger();
+ jest.mocked(appContextService).getLogger.mockReturnValue(logger);
+ mockedGetBundledPackageByPkgKey.mockReset();
+ mockedUnpackBufferEntries.mockReset();
+ mockedGetFile.mockReset();
+ mockedGetInstallation.mockReset();
+ mockedGetArchiveEntry.mockReset();
+ mockedGetAsset.mockReset();
+ });
+
+ it('should return the file for bundled package and an existing file', async () => {
+ mockedGetBundledPackageByPkgKey.mockResolvedValue({
+ getBuffer: () => Promise.resolve(),
+ } as any);
+ const request = httpServerMock.createKibanaRequest({
+ params: {
+ pkgName: 'test',
+ pkgVersion: '1.0.0',
+ filePath: 'README.md',
+ },
+ });
+ const buffer = Buffer.from(`TEST`);
+ mockedUnpackBufferEntries.mockResolvedValue([
+ {
+ path: 'test-1.0.0/README.md',
+ buffer,
+ },
+ ]);
+ const response = httpServerMock.createResponseFactory();
+ const context = mockContext();
+ await getFileHandler(context, request, response);
+
+ expect(response.custom).toBeCalledWith(
+ expect.objectContaining({
+ statusCode: 200,
+ body: buffer,
+ headers: expect.objectContaining({
+ 'content-type': 'text/markdown; charset=utf-8',
+ }),
+ })
+ );
+ });
+
+ it('should a 404 for bundled package with a non existing file', async () => {
+ mockedGetBundledPackageByPkgKey.mockResolvedValue({
+ getBuffer: () => Promise.resolve(),
+ } as any);
+ const request = httpServerMock.createKibanaRequest({
+ params: {
+ pkgName: 'test',
+ pkgVersion: '1.0.0',
+ filePath: 'idonotexists.md',
+ },
+ });
+ mockedUnpackBufferEntries.mockResolvedValue([
+ {
+ path: 'test-1.0.0/README.md',
+ buffer: Buffer.from(`TEST`),
+ },
+ ]);
+ const response = httpServerMock.createResponseFactory();
+ const context = mockContext();
+ await getFileHandler(context, request, response);
+
+ expect(response.custom).toBeCalledWith(
+ expect.objectContaining({
+ statusCode: 404,
+ body: 'bundled package file not found: idonotexists.md',
+ })
+ );
+ });
+
+ it('should proxy registry 200 for non bundled and non installed package', async () => {
+ const request = httpServerMock.createKibanaRequest({
+ params: {
+ pkgName: 'test',
+ pkgVersion: '1.0.0',
+ filePath: 'idonotexists.md',
+ },
+ });
+ const response = httpServerMock.createResponseFactory();
+ const context = mockContext();
+
+ mockedGetFile.mockResolvedValue({
+ status: 200,
+ // @ts-expect-error
+ body: 'test',
+ headers: new Headers({
+ raw: '',
+ 'content-type': 'text/markdown',
+ }),
+ });
+
+ await getFileHandler(context, request, response);
+
+ expect(response.custom).toBeCalledWith(
+ expect.objectContaining({
+ statusCode: 200,
+ body: 'test',
+ headers: expect.objectContaining({
+ 'content-type': 'text/markdown',
+ }),
+ })
+ );
+ });
+
+ it('should proxy registry 404 for non bundled and non installed package', async () => {
+ const request = httpServerMock.createKibanaRequest({
+ params: {
+ pkgName: 'test',
+ pkgVersion: '1.0.0',
+ filePath: 'idonotexists.md',
+ },
+ });
+ const response = httpServerMock.createResponseFactory();
+ const context = mockContext();
+
+ mockedGetFile.mockResolvedValue({
+ status: 404,
+ // @ts-expect-error
+ body: 'not found',
+ headers: new Headers({
+ raw: '',
+ 'content-type': 'text',
+ }),
+ });
+
+ await getFileHandler(context, request, response);
+
+ expect(response.custom).toBeCalledWith(
+ expect.objectContaining({
+ statusCode: 404,
+ body: 'not found',
+ headers: expect.objectContaining({
+ 'content-type': 'text',
+ }),
+ })
+ );
+ });
+
+ it('should return the file from installation for installed package', async () => {
+ const request = httpServerMock.createKibanaRequest({
+ params: {
+ pkgName: 'test',
+ pkgVersion: '1.0.0',
+ filePath: 'README.md',
+ },
+ });
+ const response = httpServerMock.createResponseFactory();
+ const context = mockContext();
+
+ mockedGetInstallation.mockResolvedValue({ version: '1.0.0' } as any);
+ mockedGetArchiveEntry.mockReturnValue(Buffer.from('test'));
+
+ await getFileHandler(context, request, response);
+
+ expect(response.custom).toBeCalledWith(
+ expect.objectContaining({
+ statusCode: 200,
+ headers: expect.objectContaining({
+ 'content-type': 'text/markdown; charset=utf-8',
+ }),
+ })
+ );
+ });
+
+ it('should a 404 if the file from installation do not exists for installed package', async () => {
+ const request = httpServerMock.createKibanaRequest({
+ params: {
+ pkgName: 'test',
+ pkgVersion: '1.0.0',
+ filePath: 'README.md',
+ },
+ });
+ const response = httpServerMock.createResponseFactory();
+ const context = mockContext();
+
+ mockedGetInstallation.mockResolvedValue({ version: '1.0.0' } as any);
+ await getFileHandler(context, request, response);
+
+ expect(response.custom).toBeCalledWith(
+ expect.objectContaining({
+ statusCode: 404,
+ body: 'installed package file not found: README.md',
+ })
+ );
+ });
+});
diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts
new file mode 100644
index 000000000000..4b6b74628aa4
--- /dev/null
+++ b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import path from 'path';
+
+import type { TypeOf } from '@kbn/config-schema';
+import mime from 'mime-types';
+import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/core/server';
+
+import type { GetFileRequestSchema, FleetRequestHandler } from '../../types';
+import { getFile, getInstallation } from '../../services/epm/packages';
+import { defaultFleetErrorHandler } from '../../errors';
+import { getArchiveEntry } from '../../services/epm/archive';
+import { getAsset } from '../../services/epm/archive/storage';
+import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_packages';
+import { pkgToPkgKey } from '../../services/epm/registry';
+import { unpackBufferEntries } from '../../services/epm/archive';
+
+const CACHE_CONTROL_10_MINUTES_HEADER: HttpResponseOptions['headers'] = {
+ 'cache-control': 'max-age=600',
+};
+export const getFileHandler: FleetRequestHandler<
+ TypeOf
+> = async (context, request, response) => {
+ try {
+ const { pkgName, pkgVersion, filePath } = request.params;
+ const savedObjectsClient = (await context.fleet).internalSoClient;
+
+ const installation = await getInstallation({ savedObjectsClient, pkgName });
+ const useLocalFile = pkgVersion === installation?.version;
+ const assetPath = `${pkgName}-${pkgVersion}/${filePath}`;
+
+ if (useLocalFile) {
+ const fileBuffer = getArchiveEntry(assetPath);
+ // only pull local installation if we don't have it cached
+ const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath }));
+
+ // error, if neither is available
+ if (!fileBuffer && !storedAsset) {
+ return response.custom({
+ body: `installed package file not found: ${filePath}`,
+ statusCode: 404,
+ });
+ }
+
+ // if storedAsset is not available, fileBuffer *must* be
+ // b/c we error if we don't have at least one, and storedAsset is the least likely
+ const { buffer, contentType } = storedAsset
+ ? {
+ contentType: storedAsset.media_type,
+ buffer: storedAsset.data_utf8
+ ? Buffer.from(storedAsset.data_utf8, 'utf8')
+ : Buffer.from(storedAsset.data_base64, 'base64'),
+ }
+ : {
+ contentType: mime.contentType(path.extname(assetPath)),
+ buffer: fileBuffer,
+ };
+
+ if (!contentType) {
+ return response.custom({
+ body: `unknown content type for file: ${filePath}`,
+ statusCode: 400,
+ });
+ }
+
+ return response.custom({
+ body: buffer,
+ statusCode: 200,
+ headers: {
+ ...CACHE_CONTROL_10_MINUTES_HEADER,
+ 'content-type': contentType,
+ },
+ });
+ }
+
+ const bundledPackage = await getBundledPackageByPkgKey(
+ pkgToPkgKey({ name: pkgName, version: pkgVersion })
+ );
+ if (bundledPackage) {
+ const bufferEntries = await unpackBufferEntries(
+ await bundledPackage.getBuffer(),
+ 'application/zip'
+ );
+
+ const fileBuffer = bufferEntries.find((entry) => entry.path === assetPath)?.buffer;
+
+ if (!fileBuffer) {
+ return response.custom({
+ body: `bundled package file not found: ${filePath}`,
+ statusCode: 404,
+ });
+ }
+
+ // if storedAsset is not available, fileBuffer *must* be
+ // b/c we error if we don't have at least one, and storedAsset is the least likely
+ const { buffer, contentType } = {
+ contentType: mime.contentType(path.extname(assetPath)),
+ buffer: fileBuffer,
+ };
+
+ if (!contentType) {
+ return response.custom({
+ body: `unknown content type for file: ${filePath}`,
+ statusCode: 400,
+ });
+ }
+
+ return response.custom({
+ body: buffer,
+ statusCode: 200,
+ headers: {
+ ...CACHE_CONTROL_10_MINUTES_HEADER,
+ 'content-type': contentType,
+ },
+ });
+ } else {
+ const registryResponse = await getFile(pkgName, pkgVersion, filePath);
+ const headersToProxy: KnownHeaders[] = ['content-type'];
+ const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => {
+ const value = registryResponse.headers.get(knownHeader);
+ if (value !== null) {
+ headers[knownHeader] = value;
+ }
+ return headers;
+ }, {} as ResponseHeaders);
+
+ return response.custom({
+ body: registryResponse.body,
+ statusCode: registryResponse.status,
+ headers: { ...CACHE_CONTROL_10_MINUTES_HEADER, ...proxiedHeaders },
+ });
+ }
+ } catch (error) {
+ return defaultFleetErrorHandler({ error, response });
+ }
+};
diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts
index c03272119dd1..6fadeff5180c 100644
--- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts
+++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts
@@ -5,12 +5,9 @@
* 2.0.
*/
-import path from 'path';
-
import type { TypeOf } from '@kbn/config-schema';
-import mime from 'mime-types';
import semverValid from 'semver/functions/valid';
-import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/core/server';
+import type { HttpResponseOptions } from '@kbn/core/server';
import { pick } from 'lodash';
@@ -41,7 +38,6 @@ import type {
GetPackagesRequestSchema,
GetInstalledPackagesRequestSchema,
GetDataStreamsRequestSchema,
- GetFileRequestSchema,
GetInfoRequestSchema,
InstallPackageFromRegistryRequestSchema,
InstallPackageByUploadRequestSchema,
@@ -60,21 +56,17 @@ import {
getCategories,
getPackages,
getInstalledPackages,
- getFile,
getPackageInfo,
isBulkInstallError,
installPackage,
removeInstallation,
getLimitedPackages,
- getInstallation,
getBulkAssets,
getTemplateInputs,
} from '../../services/epm/packages';
import type { BulkInstallResponse } from '../../services/epm/packages';
import { defaultFleetErrorHandler, fleetErrorToResponseOptions, FleetError } from '../../errors';
import { appContextService, checkAllowedPackages } from '../../services';
-import { getArchiveEntry } from '../../services/epm/archive/cache';
-import { getAsset } from '../../services/epm/archive/storage';
import { getPackageUsageStats } from '../../services/epm/packages/get';
import { updatePackage } from '../../services/epm/packages/update';
import { getGpgKeyIdOrUndefined } from '../../services/epm/packages/package_verification';
@@ -206,80 +198,6 @@ export const getLimitedListHandler: FleetRequestHandler<
}
};
-export const getFileHandler: FleetRequestHandler<
- TypeOf
-> = async (context, request, response) => {
- try {
- const { pkgName, pkgVersion, filePath } = request.params;
- const savedObjectsClient = (await context.fleet).internalSoClient;
- const installation = await getInstallation({ savedObjectsClient, pkgName });
- const useLocalFile = pkgVersion === installation?.version;
-
- if (useLocalFile) {
- const assetPath = `${pkgName}-${pkgVersion}/${filePath}`;
- const fileBuffer = getArchiveEntry(assetPath);
- // only pull local installation if we don't have it cached
- const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath }));
-
- // error, if neither is available
- if (!fileBuffer && !storedAsset) {
- return response.custom({
- body: `installed package file not found: ${filePath}`,
- statusCode: 404,
- });
- }
-
- // if storedAsset is not available, fileBuffer *must* be
- // b/c we error if we don't have at least one, and storedAsset is the least likely
- const { buffer, contentType } = storedAsset
- ? {
- contentType: storedAsset.media_type,
- buffer: storedAsset.data_utf8
- ? Buffer.from(storedAsset.data_utf8, 'utf8')
- : Buffer.from(storedAsset.data_base64, 'base64'),
- }
- : {
- contentType: mime.contentType(path.extname(assetPath)),
- buffer: fileBuffer,
- };
-
- if (!contentType) {
- return response.custom({
- body: `unknown content type for file: ${filePath}`,
- statusCode: 400,
- });
- }
-
- return response.custom({
- body: buffer,
- statusCode: 200,
- headers: {
- ...CACHE_CONTROL_10_MINUTES_HEADER,
- 'content-type': contentType,
- },
- });
- } else {
- const registryResponse = await getFile(pkgName, pkgVersion, filePath);
- const headersToProxy: KnownHeaders[] = ['content-type'];
- const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => {
- const value = registryResponse.headers.get(knownHeader);
- if (value !== null) {
- headers[knownHeader] = value;
- }
- return headers;
- }, {} as ResponseHeaders);
-
- return response.custom({
- body: registryResponse.body,
- statusCode: registryResponse.status,
- headers: { ...CACHE_CONTROL_10_MINUTES_HEADER, ...proxiedHeaders },
- });
- }
- } catch (error) {
- return defaultFleetErrorHandler({ error, response });
- }
-};
-
export const getInfoHandler: FleetRequestHandler<
TypeOf,
TypeOf
diff --git a/x-pack/plugins/fleet/server/routes/epm/index.ts b/x-pack/plugins/fleet/server/routes/epm/index.ts
index 6e0000bf4ccb..5245381a409d 100644
--- a/x-pack/plugins/fleet/server/routes/epm/index.ts
+++ b/x-pack/plugins/fleet/server/routes/epm/index.ts
@@ -55,7 +55,6 @@ import {
getListHandler,
getInstalledListHandler,
getLimitedListHandler,
- getFileHandler,
getInfoHandler,
getBulkAssetsHandler,
installPackageFromRegistryHandler,
@@ -70,6 +69,7 @@ import {
createCustomIntegrationHandler,
getInputsHandler,
} from './handlers';
+import { getFileHandler } from './file_handler';
const MAX_FILE_SIZE_BYTES = 104857600; // 100MB
diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts
index cbc560cc72dc..a16fd37c9ac1 100644
--- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts
+++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts
@@ -6,7 +6,6 @@
*/
import type { TypeOf } from '@kbn/config-schema';
-import Boom from '@hapi/boom';
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import type { RequestHandler } from '@kbn/core/server';
@@ -43,7 +42,11 @@ import type {
UpgradePackagePolicyResponse,
} from '../../../common/types';
import { installationStatuses, inputsFormat } from '../../../common/constants';
-import { defaultFleetErrorHandler, PackagePolicyNotFoundError } from '../../errors';
+import {
+ defaultFleetErrorHandler,
+ PackagePolicyNotFoundError,
+ PackagePolicyRequestError,
+} from '../../errors';
import { getInstallations, getPackageInfo } from '../../services/epm/packages';
import { PACKAGES_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../constants';
import {
@@ -244,7 +247,7 @@ export const createPackagePolicyHandler: FleetRequestHandler<
let newPackagePolicy: NewPackagePolicy;
if (isSimplifiedCreatePackagePolicyRequest(newPolicy)) {
if (!pkg) {
- throw new Error('Package is required');
+ throw new PackagePolicyRequestError('Package is required');
}
const pkgInfo = await getPackageInfo({
savedObjectsClient: soClient,
@@ -311,7 +314,7 @@ export const updatePackagePolicyHandler: FleetRequestHandler<
const packagePolicy = await packagePolicyService.get(soClient, request.params.packagePolicyId);
if (!packagePolicy) {
- throw Boom.notFound('Package policy not found');
+ throw new PackagePolicyNotFoundError('Package policy not found');
}
if (limitedToPackages && limitedToPackages.length) {
@@ -337,7 +340,7 @@ export const updatePackagePolicyHandler: FleetRequestHandler<
isSimplifiedCreatePackagePolicyRequest(body as unknown as SimplifiedPackagePolicy)
) {
if (!pkg) {
- throw new Error('package is required');
+ throw new PackagePolicyRequestError('Package is required');
}
const pkgInfo = await getPackageInfo({
savedObjectsClient: soClient,
diff --git a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts
index bbe00c49b414..c7c02d8a53fb 100644
--- a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts
+++ b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts
@@ -12,7 +12,7 @@ import { LICENCE_FOR_PER_POLICY_OUTPUT, outputType } from '../../../common/const
import { policyHasFleetServer, policyHasSyntheticsIntegration } from '../../../common/services';
import { appContextService } from '..';
import { outputService } from '../output';
-import { OutputInvalidError, OutputLicenceError } from '../../errors';
+import { OutputInvalidError, OutputLicenceError, OutputNotFoundError } from '../../errors';
/**
* Get the data output for a given agent policy
@@ -28,7 +28,7 @@ export async function getDataOutputForAgentPolicy(
agentPolicy.data_output_id || (await outputService.getDefaultDataOutputId(soClient));
if (!dataOutputId) {
- throw new Error('No default data output found.');
+ throw new OutputNotFoundError('No default data output found.');
}
return outputService.get(soClient, dataOutputId);
diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts
index 4445ebbe8476..ec36c7575937 100644
--- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts
+++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts
@@ -24,6 +24,7 @@ import type {
RegistryDataStreamPrivileges,
} from '../../../common/types';
import { PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES } from '../../constants';
+import { PackagePolicyRequestError } from '../../errors';
import type { PackagePolicy } from '../../types';
import { pkgToPkgKey } from '../epm/registry';
@@ -46,7 +47,7 @@ export function storedPackagePoliciesToAgentPermissions(
): FullAgentPolicyOutputPermissions | undefined {
// I'm not sure what permissions to return for this case, so let's return the defaults
if (!packagePolicies) {
- throw new Error(
+ throw new PackagePolicyRequestError(
'storedPackagePoliciesToAgentPermissions should be called with a PackagePolicy'
);
}
@@ -57,7 +58,9 @@ export function storedPackagePoliciesToAgentPermissions(
const permissionEntries = packagePolicies.map((packagePolicy) => {
if (!packagePolicy.package) {
- throw new Error(`No package for package policy ${packagePolicy.name ?? packagePolicy.id}`);
+ throw new PackagePolicyRequestError(
+ `No package for package policy ${packagePolicy.name ?? packagePolicy.id}`
+ );
}
const pkg = packageInfoCache.get(pkgToPkgKey(packagePolicy.package))!;
diff --git a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts
index b614b9c2dd9e..0108e9cd9772 100644
--- a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts
+++ b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts
@@ -16,6 +16,7 @@ import { getSourceUriForAgentPolicy } from '../../routes/agent/source_uri_utils'
import { getFleetServerHostsForAgentPolicy } from '../fleet_server_host';
import { appContextService } from '../app_context';
import { bulkGetFleetProxies } from '../fleet_proxies';
+import { OutputNotFoundError } from '../../errors';
export async function fetchRelatedSavedObjects(
soClient: SavedObjectsClientContract,
@@ -27,7 +28,7 @@ export async function fetchRelatedSavedObjects(
]);
if (!defaultDataOutputId) {
- throw new Error('Default output is not setup');
+ throw new OutputNotFoundError('Default output is not setup');
}
const dataOutputId = agentPolicy.data_output_id || defaultDataOutputId;
@@ -51,11 +52,11 @@ export async function fetchRelatedSavedObjects(
const dataOutput = outputs.find((output) => output.id === dataOutputId);
if (!dataOutput) {
- throw new Error(`Data output not found ${dataOutputId}`);
+ throw new OutputNotFoundError(`Data output not found ${dataOutputId}`);
}
const monitoringOutput = outputs.find((output) => output.id === monitoringOutputId);
if (!monitoringOutput) {
- throw new Error(`Monitoring output not found ${monitoringOutputId}`);
+ throw new OutputNotFoundError(`Monitoring output not found ${monitoringOutputId}`);
}
const proxyIds = uniq(
diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts
index 931168f545b5..3e97594ee959 100644
--- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts
+++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts
@@ -8,6 +8,8 @@
import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks';
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import { securityMock } from '@kbn/security-plugin/server/mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+import type { Logger } from '@kbn/core/server';
import { PackagePolicyRestrictionRelatedError, FleetUnauthorizedError } from '../errors';
import type {
@@ -105,8 +107,13 @@ function getAgentPolicyCreateMock() {
});
return soClient;
}
-
+let mockedLogger: jest.Mocked;
describe('agent policy', () => {
+ beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+
afterEach(() => {
jest.resetAllMocks();
});
diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts
index 568829fda978..b44e0616b7b6 100644
--- a/x-pack/plugins/fleet/server/services/agent_policy.ts
+++ b/x-pack/plugins/fleet/server/services/agent_policy.ts
@@ -71,6 +71,7 @@ import {
AgentPolicyNotFoundError,
PackagePolicyRestrictionRelatedError,
FleetUnauthorizedError,
+ FleetError,
} from '../errors';
import type { FullAgentConfigMap } from '../../common/types/models/agent_cm';
@@ -125,24 +126,24 @@ class AgentPolicyService {
id,
savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE,
});
+ const logger = appContextService.getLogger();
+ logger.debug(`Starting update of agent policy ${id}`);
const existingAgentPolicy = await this.get(soClient, id, true);
if (!existingAgentPolicy) {
- throw new Error('Agent policy not found');
+ throw new AgentPolicyNotFoundError('Agent policy not found');
}
if (
existingAgentPolicy.status === agentPolicyStatuses.Inactive &&
agentPolicy.status !== agentPolicyStatuses.Active
) {
- throw new Error(
+ throw new FleetError(
`Agent policy ${id} cannot be updated because it is ${existingAgentPolicy.status}`
);
}
- const logger = appContextService.getLogger();
-
if (options.removeProtection) {
logger.warn(`Setting tamper protection for Agent Policy ${id} to false`);
}
@@ -166,7 +167,7 @@ class AgentPolicyService {
if (options.bumpRevision || options.removeProtection) {
await this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'updated', id);
}
-
+ logger.debug(`Agent policy ${id} update completed`);
return (await this.get(soClient, id)) as AgentPolicy;
}
@@ -190,7 +191,7 @@ class AgentPolicyService {
is_preconfigured: true,
};
- if (!id) throw new Error('Missing ID');
+ if (!id) throw new AgentPolicyNotFoundError('Missing ID');
return await this.ensureAgentPolicy(soClient, esClient, newAgentPolicy, id as string);
}
@@ -254,6 +255,7 @@ class AgentPolicyService {
this.checkTamperProtectionLicense(agentPolicy);
const logger = appContextService.getLogger();
+ logger.debug(`Creating new agent policy`);
if (agentPolicy?.is_protected) {
logger.warn(
@@ -282,7 +284,7 @@ class AgentPolicyService {
await appContextService.getUninstallTokenService()?.generateTokenForPolicyId(newSo.id);
await this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'created', newSo.id);
-
+ logger.debug(`Created new agent policy with id ${newSo.id}`);
return { id: newSo.id, ...newSo.attributes };
}
@@ -320,7 +322,7 @@ class AgentPolicyService {
}
if (agentPolicySO.error) {
- throw new Error(agentPolicySO.error.message);
+ throw new FleetError(agentPolicySO.error.message);
}
const agentPolicy = { id: agentPolicySO.id, ...agentPolicySO.attributes };
@@ -356,7 +358,7 @@ class AgentPolicyService {
} else if (agentPolicySO.error.statusCode === 404) {
throw new AgentPolicyNotFoundError(`Agent policy ${agentPolicySO.id} not found`);
} else {
- throw new Error(agentPolicySO.error.message);
+ throw new FleetError(agentPolicySO.error.message);
}
}
@@ -498,6 +500,9 @@ class AgentPolicyService {
authorizationHeader?: HTTPAuthorizationHeader | null;
}
): Promise {
+ const logger = appContextService.getLogger();
+ logger.debug(`Starting update of agent policy ${id}`);
+
if (agentPolicy.name) {
await this.requireUniqueName(soClient, {
id,
@@ -508,14 +513,12 @@ class AgentPolicyService {
const existingAgentPolicy = await this.get(soClient, id, true);
if (!existingAgentPolicy) {
- throw new Error('Agent policy not found');
+ throw new AgentPolicyNotFoundError('Agent policy not found');
}
this.checkTamperProtectionLicense(agentPolicy);
await this.checkForValidUninstallToken(agentPolicy, id);
- const logger = appContextService.getLogger();
-
if (agentPolicy?.is_protected && !policyHasEndpointSecurity(existingAgentPolicy)) {
logger.warn(
'Agent policy requires Elastic Defend integration to set tamper protection to true'
@@ -558,10 +561,13 @@ class AgentPolicyService {
newAgentPolicyProps: Pick,
options?: { user?: AuthenticatedUser }
): Promise {
+ const logger = appContextService.getLogger();
+ logger.debug(`Starting copy of agent policy ${id}`);
+
// Copy base agent policy
const baseAgentPolicy = await this.get(soClient, id, true);
if (!baseAgentPolicy) {
- throw new Error('Agent policy not found');
+ throw new AgentPolicyNotFoundError('Agent policy not found');
}
const newAgentPolicy = await this.create(
soClient,
@@ -631,11 +637,11 @@ class AgentPolicyService {
// Get updated agent policy with package policies and adjusted tamper protection
const updatedAgentPolicy = await this.get(soClient, newAgentPolicy.id, true);
if (!updatedAgentPolicy) {
- throw new Error('Copied agent policy not found');
+ throw new AgentPolicyNotFoundError('Copied agent policy not found');
}
await this.deployPolicy(soClient, newAgentPolicy.id);
-
+ logger.debug(`Completed copy of agent policy ${id}`);
return updatedAgentPolicy;
}
@@ -799,6 +805,9 @@ class AgentPolicyService {
id: string,
options?: { force?: boolean; removeFleetServerDocuments?: boolean; user?: AuthenticatedUser }
): Promise {
+ const logger = appContextService.getLogger();
+ logger.debug(`Deleting agent policy ${id}`);
+
auditLoggingService.writeCustomSoAuditLog({
action: 'delete',
id,
@@ -807,7 +816,7 @@ class AgentPolicyService {
const agentPolicy = await this.get(soClient, id, false);
if (!agentPolicy) {
- throw new Error('Agent policy not found');
+ throw new AgentPolicyNotFoundError('Agent policy not found');
}
if (agentPolicy.is_managed && !options?.force) {
@@ -822,7 +831,7 @@ class AgentPolicyService {
});
if (total > 0) {
- throw new Error('Cannot delete agent policy that is assigned to agent(s)');
+ throw new FleetError('Cannot delete agent policy that is assigned to agent(s)');
}
const packagePolicies = await packagePolicyService.findAllForAgentPolicy(soClient, id);
@@ -860,7 +869,7 @@ class AgentPolicyService {
if (options?.removeFleetServerDocuments) {
await this.deleteFleetServerPoliciesForPolicyId(esClient, id);
}
-
+ logger.debug(`Deleted agent policy ${id}`);
return {
id,
name: agentPolicy.name,
@@ -954,7 +963,7 @@ class AgentPolicyService {
return acc;
}, [] as BulkResponseItem[]);
- logger.debug(
+ logger.warn(
`Failed to index documents during policy deployment: ${JSON.stringify(erroredDocuments)}`
);
}
@@ -1225,7 +1234,7 @@ class AgentPolicyService {
);
if (uninstallTokenError) {
- throw new Error(
+ throw new FleetError(
`Cannot enable Agent Tamper Protection: ${uninstallTokenError.error.message}`
);
}
diff --git a/x-pack/plugins/fleet/server/services/agents/crud.ts b/x-pack/plugins/fleet/server/services/agents/crud.ts
index 7ad50c8d962c..b8eb0f7d0ca1 100644
--- a/x-pack/plugins/fleet/server/services/agents/crud.ts
+++ b/x-pack/plugins/fleet/server/services/agents/crud.ts
@@ -4,7 +4,6 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
-import Boom from '@hapi/boom';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { SortResults } from '@elastic/elasticsearch/lib/api/types';
import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server';
@@ -19,7 +18,12 @@ import type { AgentStatus, FleetServerAgent } from '../../../common/types';
import { SO_SEARCH_LIMIT } from '../../../common/constants';
import { isAgentUpgradeable } from '../../../common/services';
import { AGENTS_INDEX } from '../../constants';
-import { FleetError, isESClientError, AgentNotFoundError } from '../../errors';
+import {
+ FleetError,
+ isESClientError,
+ AgentNotFoundError,
+ FleetUnauthorizedError,
+} from '../../errors';
import { auditLoggingService } from '../audit_logging';
@@ -548,10 +552,10 @@ export async function getAgentByAccessAPIKeyId(
throw new AgentNotFoundError('Agent not found');
}
if (agent.access_api_key_id !== accessAPIKeyId) {
- throw new Error('Agent api key id is not matching');
+ throw new FleetError('Agent api key id is not matching');
}
if (!agent.active) {
- throw Boom.forbidden('Agent inactive');
+ throw new FleetUnauthorizedError('Agent inactive');
}
return agent;
diff --git a/x-pack/plugins/fleet/server/services/agents/reassign.ts b/x-pack/plugins/fleet/server/services/agents/reassign.ts
index 86d368d39931..0a5c6f9b51ee 100644
--- a/x-pack/plugins/fleet/server/services/agents/reassign.ts
+++ b/x-pack/plugins/fleet/server/services/agents/reassign.ts
@@ -5,11 +5,14 @@
* 2.0.
*/
import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server';
-import Boom from '@hapi/boom';
import type { Agent } from '../../types';
import { agentPolicyService } from '../agent_policy';
-import { AgentReassignmentError, HostedAgentPolicyRestrictionRelatedError } from '../../errors';
+import {
+ AgentReassignmentError,
+ HostedAgentPolicyRestrictionRelatedError,
+ AgentPolicyNotFoundError,
+} from '../../errors';
import { SO_SEARCH_LIMIT } from '../../constants';
@@ -33,7 +36,7 @@ export async function reassignAgent(
) {
const newAgentPolicy = await agentPolicyService.get(soClient, newAgentPolicyId);
if (!newAgentPolicy) {
- throw Boom.notFound(`Agent policy not found: ${newAgentPolicyId}`);
+ throw new AgentPolicyNotFoundError(`Agent policy not found: ${newAgentPolicyId}`);
}
await reassignAgentIsAllowed(soClient, esClient, agentId, newAgentPolicyId);
@@ -87,7 +90,7 @@ export async function reassignAgents(
): Promise<{ actionId: string }> {
const newAgentPolicy = await agentPolicyService.get(soClient, newAgentPolicyId);
if (!newAgentPolicy) {
- throw Boom.notFound(`Agent policy not found: ${newAgentPolicyId}`);
+ throw new AgentPolicyNotFoundError(`Agent policy not found: ${newAgentPolicyId}`);
}
if (newAgentPolicy.is_managed) {
throw new HostedAgentPolicyRestrictionRelatedError(
diff --git a/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts b/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts
index 84aa2226b485..d962279f0ca3 100644
--- a/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts
+++ b/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts
@@ -185,7 +185,7 @@ describe('update_agent_tags', () => {
await expect(
updateAgentTags(soClient, esClient, { agentIds: ['agent1'] }, ['one'], [])
- ).rejects.toThrowError('version conflict of 100 agents');
+ ).rejects.toThrowError('Version conflict of 100 agents');
});
it('should write out error results on last retry with version conflicts', async () => {
@@ -211,7 +211,7 @@ describe('update_agent_tags', () => {
retryCount: MAX_RETRY_COUNT,
}
)
- ).rejects.toThrowError('version conflict of 100 agents');
+ ).rejects.toThrowError('Version conflict of 100 agents');
const agentAction = esClient.create.mock.calls[0][0] as any;
expect(agentAction?.body.agents.length).toEqual(100);
@@ -243,7 +243,7 @@ describe('update_agent_tags', () => {
retryCount: MAX_RETRY_COUNT,
}
)
- ).rejects.toThrowError('version conflict of 1 agents');
+ ).rejects.toThrowError('Version conflict of 1 agents');
const agentAction = esClient.create.mock.calls[0][0] as any;
expect(agentAction?.body.agents.length).toEqual(3);
diff --git a/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts b/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts
index d8208fd5f8d0..1c8db8451ccf 100644
--- a/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts
+++ b/x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts
@@ -15,6 +15,8 @@ import { AGENTS_INDEX } from '../../constants';
import { appContextService } from '../app_context';
+import { FleetError } from '../../errors';
+
import { ActionRunner } from './action_runner';
import { BulkActionTaskType } from './bulk_action_types';
@@ -124,7 +126,9 @@ export async function updateTagsBatch(
conflicts: 'proceed', // relying on the task to retry in case of conflicts - retry only conflicted agents
});
} catch (error) {
- throw new Error('Caught error: ' + JSON.stringify(error).slice(0, 1000));
+ throw new FleetError(
+ 'Caught error while batch updating tags: ' + JSON.stringify(error).slice(0, 1000)
+ );
}
appContextService.getLogger().debug(JSON.stringify(res).slice(0, 1000));
@@ -203,7 +207,7 @@ export async function updateTagsBatch(
.getLogger()
.debug(`action conflict result wrote on ${versionConflictCount} agents`);
}
- throw new Error(`version conflict of ${versionConflictCount} agents`);
+ throw new FleetError(`Version conflict of ${versionConflictCount} agents`);
}
return { actionId, updated: res.updated, took: res.took };
diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts
index bc58941e4c29..9200346961f1 100644
--- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts
+++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts
@@ -6,6 +6,10 @@
*/
import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+
+import type { Logger } from '@kbn/core/server';
+import { securityMock } from '@kbn/security-plugin/server/mocks';
import { ENROLLMENT_API_KEYS_INDEX } from '../../constants';
@@ -27,10 +31,20 @@ jest.mock('uuid', () => {
const mockedAgentPolicyService = agentPolicyService as jest.Mocked;
const mockedAuditLoggingService = auditLoggingService as jest.Mocked;
+
const mockedAppContextService = appContextService as jest.Mocked;
+mockedAppContextService.getSecuritySetup.mockImplementation(() => ({
+ ...securityMock.createSetup(),
+}));
+
+let mockedLogger: jest.Mocked;
describe('enrollment api keys', () => {
beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+ afterEach(() => {
jest.resetAllMocks();
});
diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts
index ac7087c95296..360723ebcf22 100644
--- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts
+++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts
@@ -16,13 +16,15 @@ import { toElasticsearchQuery, fromKueryExpression } from '@kbn/es-query';
import type { ESSearchResponse as SearchResponse } from '@kbn/es-types';
import type { EnrollmentAPIKey, FleetServerEnrollmentAPIKey } from '../../types';
-import { FleetError } from '../../errors';
+import { FleetError, EnrollmentKeyNameExistsError, EnrollmentKeyNotFoundError } from '../../errors';
import { ENROLLMENT_API_KEYS_INDEX } from '../../constants';
import { agentPolicyService } from '../agent_policy';
import { escapeSearchQueryPhrase } from '../saved_object';
import { auditLoggingService } from '../audit_logging';
+import { appContextService } from '../app_context';
+
import { invalidateAPIKeys } from './security';
const uuidRegex =
@@ -90,7 +92,7 @@ export async function getEnrollmentAPIKey(
return esDocToEnrollmentApiKey(body);
} catch (e) {
if (e instanceof errors.ResponseError && e.statusCode === 404) {
- throw Boom.notFound(`Enrollment api key ${id} not found`);
+ throw new EnrollmentKeyNotFoundError(`Enrollment api key ${id} not found`);
}
throw e;
@@ -106,6 +108,9 @@ export async function deleteEnrollmentApiKey(
id: string,
forceDelete = false
) {
+ const logger = appContextService.getLogger();
+ logger.debug(`Deleting enrollment API key ${id}`);
+
const enrollmentApiKey = await getEnrollmentAPIKey(esClient, id);
auditLoggingService.writeCustomAuditLog({
@@ -132,6 +137,9 @@ export async function deleteEnrollmentApiKey(
refresh: 'wait_for',
});
}
+ logger.debug(
+ `Deleted enrollment API key ${enrollmentApiKey.id} [api_key_id=${enrollmentApiKey.api_key_id}`
+ );
}
export async function deleteEnrollmentApiKeyForAgentPolicyId(
@@ -169,6 +177,9 @@ export async function generateEnrollmentAPIKey(
): Promise {
const id = uuidv4();
const { name: providedKeyName, forceRecreate } = data;
+ const logger = appContextService.getLogger();
+ logger.debug(`Creating enrollment API key ${data}`);
+
if (data.agentPolicyId) {
await validateAgentPolicyId(soClient, data.agentPolicyId);
}
@@ -199,7 +210,7 @@ export async function generateEnrollmentAPIKey(
k.name?.replace(providedKeyName, '').trim().match(uuidRegex)
)
) {
- throw new FleetError(
+ throw new EnrollmentKeyNameExistsError(
i18n.translate('xpack.fleet.serverError.enrollmentKeyDuplicate', {
defaultMessage:
'An enrollment key named {providedKeyName} already exists for agent policy {agentPolicyId}',
@@ -217,6 +228,7 @@ export async function generateEnrollmentAPIKey(
auditLoggingService.writeCustomAuditLog({
message: `User creating enrollment API key [name=${name}] [policy_id=${agentPolicyId}]`,
});
+ logger.debug(`Creating enrollment API key [name=${name}] [policy_id=${agentPolicyId}]`);
const key = await esClient.security
.createApiKey({
@@ -245,11 +257,11 @@ export async function generateEnrollmentAPIKey(
},
})
.catch((err) => {
- throw new Error(`Impossible to create an api key: ${err.message}`);
+ throw new FleetError(`Impossible to create an api key: ${err.message}`);
});
if (!key) {
- throw new Error(
+ throw new FleetError(
i18n.translate('xpack.fleet.serverError.unableToCreateEnrollmentKey', {
defaultMessage: 'Unable to create an enrollment api key',
})
@@ -332,9 +344,9 @@ export async function getEnrollmentAPIKeyById(esClient: ElasticsearchClient, api
const [enrollmentAPIKey] = res.hits.hits.map(esDocToEnrollmentApiKey);
if (enrollmentAPIKey?.api_key_id !== apiKeyId) {
- throw new Error(
+ throw new FleetError(
i18n.translate('xpack.fleet.serverError.returnedIncorrectKey', {
- defaultMessage: 'find enrollmentKeyById returned an incorrect key',
+ defaultMessage: 'Find enrollmentKeyById returned an incorrect key',
})
);
}
diff --git a/x-pack/plugins/fleet/server/services/download_source.test.ts b/x-pack/plugins/fleet/server/services/download_source.test.ts
index 8b63d376340f..e244ec80077b 100644
--- a/x-pack/plugins/fleet/server/services/download_source.test.ts
+++ b/x-pack/plugins/fleet/server/services/download_source.test.ts
@@ -8,6 +8,9 @@
import { savedObjectsClientMock } from '@kbn/core/server/mocks';
import { securityMock } from '@kbn/security-plugin/server/mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+
+import type { Logger } from '@kbn/core/server';
import type { DownloadSourceSOAttributes } from '../types';
@@ -132,9 +135,13 @@ function getMockedSoClient(options: { defaultDownloadSourceId?: string; sameName
return soClient;
}
-
+let mockedLogger: jest.Mocked;
describe('Download Service', () => {
beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+ afterEach(() => {
mockedAgentPolicyService.list.mockClear();
mockedAgentPolicyService.hasAPMIntegration.mockClear();
mockedAgentPolicyService.removeDefaultSourceFromAll.mockReset();
diff --git a/x-pack/plugins/fleet/server/services/download_source.ts b/x-pack/plugins/fleet/server/services/download_source.ts
index f1719e2eb479..e679123f7e25 100644
--- a/x-pack/plugins/fleet/server/services/download_source.ts
+++ b/x-pack/plugins/fleet/server/services/download_source.ts
@@ -41,7 +41,7 @@ class DownloadSourceService {
);
if (soResponse.error) {
- throw new Error(soResponse.error.message);
+ throw new FleetError(soResponse.error.message);
}
return savedObjectToDownloadSource(soResponse);
@@ -69,6 +69,9 @@ class DownloadSourceService {
downloadSource: DownloadSourceBase,
options?: { id?: string; overwrite?: boolean }
): Promise {
+ const logger = appContextService.getLogger();
+ logger.debug(`Creating new download source`);
+
const data: DownloadSourceSOAttributes = downloadSource;
await this.requireUniqueName(soClient, {
@@ -100,6 +103,7 @@ class DownloadSourceService {
overwrite: options?.overwrite ?? false,
}
);
+ logger.debug(`Creating new download source ${options?.id}`);
return savedObjectToDownloadSource(newSo);
}
@@ -108,6 +112,8 @@ class DownloadSourceService {
id: string,
newData: Partial
) {
+ const logger = appContextService.getLogger();
+ logger.debug(`Updating download source ${id} with ${newData}`);
const updateData: Partial = newData;
if (updateData.proxy_id) {
@@ -134,11 +140,16 @@ class DownloadSourceService {
updateData
);
if (soResponse.error) {
- throw new Error(soResponse.error.message);
+ throw new FleetError(soResponse.error.message);
+ } else {
+ logger.debug(`Updated download source ${id}`);
}
}
public async delete(soClient: SavedObjectsClientContract, id: string) {
+ const logger = appContextService.getLogger();
+ logger.debug(`Deleting download source ${id}`);
+
const targetDS = await this.get(soClient, id);
if (targetDS.is_default) {
@@ -149,7 +160,7 @@ class DownloadSourceService {
appContextService.getInternalUserESClient(),
id
);
-
+ logger.debug(`Deleted download source ${id}`);
return soClient.delete(DOWNLOAD_SOURCE_SAVED_OBJECT_TYPE, id);
}
diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts
index 9163b39575f8..0ab728affd75 100644
--- a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts
@@ -5,9 +5,30 @@
* 2.0.
*/
+import { loggerMock } from '@kbn/logging-mocks';
+import { securityMock } from '@kbn/security-plugin/server/mocks';
+
+import type { Logger } from '@kbn/core/server';
+
+import { appContextService } from '../..';
+
import { compileTemplate } from './agent';
+jest.mock('../../app_context');
+
+const mockedAppContextService = appContextService as jest.Mocked;
+mockedAppContextService.getSecuritySetup.mockImplementation(() => ({
+ ...securityMock.createSetup(),
+}));
+
+let mockedLogger: jest.Mocked;
+
describe('compileTemplate', () => {
+ beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+
it('should work', () => {
const streamTemplate = `
input: log
diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts
index 0bc220a500fb..b58ab3e3fca0 100644
--- a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts
+++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts
@@ -7,15 +7,18 @@
import Handlebars from 'handlebars';
import { safeLoad, safeDump } from 'js-yaml';
+import type { Logger } from '@kbn/core/server';
import type { PackagePolicyConfigRecord } from '../../../../common/types';
import { toCompiledSecretRef } from '../../secrets';
import { PackageInvalidArchiveError } from '../../../errors';
+import { appContextService } from '../..';
const handlebars = Handlebars.create();
export function compileTemplate(variables: PackagePolicyConfigRecord, templateStr: string) {
- const { vars, yamlValues } = buildTemplateVariables(variables);
+ const logger = appContextService.getLogger();
+ const { vars, yamlValues } = buildTemplateVariables(logger, variables);
let compiledTemplate: string;
try {
const template = handlebars.compile(templateStr, { noEscape: true });
@@ -65,12 +68,13 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any)
return yaml;
}
-function buildTemplateVariables(variables: PackagePolicyConfigRecord) {
+function buildTemplateVariables(logger: Logger, variables: PackagePolicyConfigRecord) {
const yamlValues: { [k: string]: any } = {};
const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => {
// support variables with . like key.patterns
const keyParts = key.split('.');
const lastKeyPart = keyParts.pop();
+ logger.debug(`Building agent template variables`);
if (!lastKeyPart || !isValidKey(lastKeyPart)) {
throw new PackageInvalidArchiveError(
diff --git a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts
index 8b1fd141f300..db0b0d709e68 100644
--- a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts
+++ b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts
@@ -43,7 +43,7 @@ export const getArchiveFilelist = (keyArgs: SharedKey) =>
export const setArchiveFilelist = (keyArgs: SharedKey, paths: string[]) => {
const logger = appContextService.getLogger();
- logger.debug(`setting file list to the cache for ${keyArgs.name}-${keyArgs.version}`);
+ logger.debug(`Setting file list to the cache for ${keyArgs.name}-${keyArgs.version}`);
logger.trace(JSON.stringify(paths));
return archiveFilelistCache.set(sharedKey(keyArgs), paths);
};
@@ -79,7 +79,7 @@ export const setPackageInfo = ({
}: SharedKey & { packageInfo: ArchivePackage | RegistryPackage }) => {
const logger = appContextService.getLogger();
const key = sharedKey({ name, version });
- logger.debug(`setting package info to the cache for ${name}-${version}`);
+ logger.debug(`Setting package info to the cache for ${name}-${version}`);
logger.trace(JSON.stringify(packageInfo));
return packageInfoCache.set(key, packageInfo);
};
diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts
index 330839b13dba..bf96318d8d41 100644
--- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts
+++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts
@@ -6,7 +6,11 @@
*/
import type { AssetParts } from '../../../../common/types';
-import { PackageInvalidArchiveError, PackageUnsupportedMediaTypeError } from '../../../errors';
+import {
+ PackageInvalidArchiveError,
+ PackageUnsupportedMediaTypeError,
+ PackageNotFoundError,
+} from '../../../errors';
import {
getArchiveEntry,
@@ -149,7 +153,7 @@ export function getPathParts(path: string): AssetParts {
export function getAsset(key: string) {
const buffer = getArchiveEntry(key);
- if (buffer === undefined) throw new Error(`Cannot find asset ${key}`);
+ if (buffer === undefined) throw new PackageNotFoundError(`Cannot find asset ${key}`);
return buffer;
}
diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts
index 1e7d7d24e2b8..ac72f56946d0 100644
--- a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts
@@ -4,9 +4,16 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
+import { loggerMock } from '@kbn/logging-mocks';
+
+import type { Logger } from '@kbn/core/server';
+import { securityMock } from '@kbn/security-plugin/server/mocks';
+
import type { ArchivePackage } from '../../../../common/types';
import { PackageInvalidArchiveError } from '../../../errors';
+import { appContextService } from '../..';
+
import {
parseDefaultIngestPipeline,
parseDataStreamElasticsearchEntry,
@@ -21,7 +28,20 @@ import {
parseAndVerifyReadme,
} from './parse';
+jest.mock('../../app_context');
+
+const mockedAppContextService = appContextService as jest.Mocked;
+mockedAppContextService.getSecuritySetup.mockImplementation(() => ({
+ ...securityMock.createSetup(),
+}));
+
+let mockedLogger: jest.Mocked;
describe('parseDefaultIngestPipeline', () => {
+ beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+
it('Should return undefined for stream without any elasticsearch dir', () => {
expect(
parseDefaultIngestPipeline('pkg-1.0.0/data_stream/stream1/', [
diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts
index e0111e196ddb..eac5c2ac43db 100644
--- a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts
+++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts
@@ -16,6 +16,8 @@ import { pick } from 'lodash';
import semverMajor from 'semver/functions/major';
import semverPrerelease from 'semver/functions/prerelease';
+import { appContextService } from '../..';
+
import type {
ArchivePackage,
RegistryPolicyTemplate,
@@ -198,7 +200,9 @@ export function parseAndVerifyArchive(
topLevelDirOverride?: string
): ArchivePackage {
// The top-level directory must match pkgName-pkgVersion, and no other top-level files or directories may be present
+ const logger = appContextService.getLogger();
const toplevelDir = topLevelDirOverride || paths[0].split('/')[0];
+
paths.forEach((filePath) => {
if (!filePath.startsWith(toplevelDir)) {
throw new PackageInvalidArchiveError(
@@ -210,6 +214,7 @@ export function parseAndVerifyArchive(
// The package must contain a manifest file ...
const manifestFile = path.posix.join(toplevelDir, MANIFEST_NAME);
const manifestBuffer = assetsMap[manifestFile];
+ logger.debug(`Verifying archive - checking manifest file and manifest buffer`);
if (!paths.includes(manifestFile) || !manifestBuffer) {
throw new PackageInvalidArchiveError(
`Package at top-level directory ${toplevelDir} must contain a top-level ${MANIFEST_NAME} file.`
@@ -219,6 +224,7 @@ export function parseAndVerifyArchive(
// ... which must be valid YAML
let manifest: ArchivePackage;
try {
+ logger.debug(`Verifying archive - loading yaml`);
manifest = yaml.safeLoad(manifestBuffer.toString());
} catch (error) {
throw new PackageInvalidArchiveError(
@@ -227,6 +233,7 @@ export function parseAndVerifyArchive(
}
// must have mandatory fields
+ logger.debug(`Verifying archive - verifying manifest content`);
const reqGiven = pick(manifest, requiredArchivePackageProps);
const requiredKeysMatch =
Object.keys(reqGiven).toString() === requiredArchivePackageProps.toString();
@@ -246,13 +253,15 @@ export function parseAndVerifyArchive(
const parsed: ArchivePackage = { ...reqGiven, ...optGiven };
// Package name and version from the manifest must match those from the toplevel directory
+ logger.debug(`Verifying archive - parsing manifest: ${parsed}`);
const pkgKey = pkgToPkgKey({ name: parsed.name, version: parsed.version });
+
if (!topLevelDirOverride && toplevelDir !== pkgKey) {
throw new PackageInvalidArchiveError(
`Name ${parsed.name} and version ${parsed.version} do not match top-level directory ${toplevelDir}`
);
}
-
+ logger.debug(`Parsing archive - parsing and verifying data streams`);
const parsedDataStreams = parseAndVerifyDataStreams({
paths,
pkgName: parsed.name,
@@ -265,9 +274,11 @@ export function parseAndVerifyArchive(
parsed.data_streams = parsedDataStreams;
}
+ logger.debug(`Parsing archive - parsing and verifying policy templates`);
parsed.policy_templates = parseAndVerifyPolicyTemplates(manifest);
// add readme if exists
+ logger.debug(`Parsing archive - parsing and verifying Readme`);
const readme = parseAndVerifyReadme(paths, parsed.name, parsed.version);
if (readme) {
parsed.readme = readme;
@@ -283,6 +294,7 @@ export function parseAndVerifyArchive(
// Ensure top-level variables are parsed as well
if (manifest.vars) {
+ logger.debug(`Parsing archive - parsing and verifying top-level vars`);
parsed.vars = parseAndVerifyVars(manifest.vars, 'manifest.yml');
}
@@ -294,6 +306,7 @@ export function parseAndVerifyArchive(
let tags: PackageSpecTags[];
try {
tags = yaml.safeLoad(tagsBuffer.toString());
+ logger.debug(`Parsing archive - parsing kibana/tags.yml file`);
if (tags.length) {
parsed.asset_tags = tags;
}
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/retry.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/retry.ts
index 773ed7273d88..32d0c9e23fb7 100644
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/retry.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/retry.ts
@@ -17,7 +17,10 @@ const retryResponseStatuses = [
410, // Gone
];
+const retryMessages = ['no_shard_available_action_exception'];
+
const isRetryableError = (e: any, additionalResponseStatuses: number[] = []) =>
+ retryMessages.some((msg) => e.message.includes(msg)) ||
e instanceof EsErrors.NoLivingConnectionsError ||
e instanceof EsErrors.ConnectionError ||
e instanceof EsErrors.TimeoutError ||
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
index a621df33062a..670ba36ed6a8 100644
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
@@ -755,6 +755,84 @@ describe('EPM template', () => {
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});
+ it('tests processing object field with subobjects set to false (case B)', () => {
+ const objectFieldWithPropertyReversedLiteralYml = `
+- name: b.labels.*
+ type: object
+ object_type: keyword
+ subobjects: false
+ `;
+ const objectFieldWithPropertyReversedMapping = {
+ dynamic_templates: [
+ {
+ 'b.labels.*': {
+ path_match: 'b.labels.*',
+ match_mapping_type: 'string',
+ mapping: {
+ type: 'keyword',
+ },
+ },
+ },
+ ],
+ properties: {
+ b: {
+ type: 'object',
+ dynamic: true,
+ properties: {
+ labels: {
+ dynamic: true,
+ type: 'object',
+ subobjects: false,
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
+ });
+
+ it('tests processing object field with subobjects set to false (case D)', () => {
+ const objectFieldWithPropertyReversedLiteralYml = `
+- name: d.labels
+ type: object
+ object_type: keyword
+ subobjects: false
+ `;
+ const objectFieldWithPropertyReversedMapping = {
+ dynamic_templates: [
+ {
+ 'd.labels': {
+ path_match: 'd.labels.*',
+ match_mapping_type: 'string',
+ mapping: {
+ type: 'keyword',
+ },
+ },
+ },
+ ],
+ properties: {
+ d: {
+ type: 'object',
+ dynamic: true,
+ properties: {
+ labels: {
+ dynamic: true,
+ type: 'object',
+ subobjects: false,
+ },
+ },
+ },
+ },
+ };
+ const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
+ const processedFields = processFields(fields);
+ const mappings = generateMappings(processedFields);
+ expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
+ });
+
it('tests processing nested field with property', () => {
const nestedYaml = `
- name: a.b
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
index a26aa2c8e011..a60e9dc0c7ce 100644
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
@@ -222,9 +222,11 @@ function _generateMappings(
properties: IndexTemplateMappings['properties'];
hasNonDynamicTemplateMappings: boolean;
hasDynamicTemplateMappings: boolean;
+ subobjects?: boolean;
} {
let hasNonDynamicTemplateMappings = false;
let hasDynamicTemplateMappings = false;
+ let subobjects: boolean | undefined;
const props: Properties = {};
function addParentObjectAsStaticProperty(field: Field) {
@@ -237,6 +239,7 @@ function _generateMappings(
const fieldProps = {
type: 'object',
dynamic: true,
+ ...(field.subobjects !== undefined && { subobjects: field.subobjects }),
};
props[field.name] = fieldProps;
@@ -435,6 +438,17 @@ function _generateMappings(
);
}
+ // When a wildcard field specifies the subobjects setting,
+ // the parent intermediate object should set the subobjects
+ // setting.
+ //
+ // For example, if a wildcard field `foo.*` has subobjects,
+ // we should set subobjects on the intermediate object `foo`.
+ //
+ if (field.subobjects !== undefined && path.includes('*')) {
+ subobjects = field.subobjects;
+ }
+
if (dynProperties && matchingType) {
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
@@ -476,6 +490,9 @@ function _generateMappings(
} else {
return;
}
+ if (mappings.subobjects !== undefined) {
+ fieldProps.subobjects = mappings.subobjects;
+ }
break;
case 'group-nested':
fieldProps = {
@@ -583,6 +600,10 @@ function _generateMappings(
fieldProps.time_series_dimension = field.dimension;
}
+ if (field.subobjects !== undefined) {
+ fieldProps.subobjects = field.subobjects;
+ }
+
// Even if we don't add the property because it has a wildcard, notify
// the parent that there is some kind of property, so the intermediate object
// is still created.
@@ -601,7 +622,12 @@ function _generateMappings(
});
}
- return { properties: props, hasNonDynamicTemplateMappings, hasDynamicTemplateMappings };
+ return {
+ properties: props,
+ hasNonDynamicTemplateMappings,
+ hasDynamicTemplateMappings,
+ subobjects,
+ };
}
function generateDynamicAndEnabled(field: Field) {
diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts
index 437eed7c6419..495c0ddcaaf8 100644
--- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts
+++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts
@@ -40,6 +40,7 @@ export interface Field {
dimension?: boolean;
default_field?: boolean;
runtime?: boolean | string;
+ subobjects?: boolean;
// Fields specific of the aggregate_metric_double type
metrics?: string[];
diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts
index 23327a2253f8..27be17ef2a17 100644
--- a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts
+++ b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts
@@ -32,7 +32,11 @@ import type {
PackageSpecTags,
} from '../../../../types';
import { savedObjectTypes } from '../../packages';
-import { indexPatternTypes, getIndexPatternSavedObjects } from '../index_pattern/install';
+import {
+ indexPatternTypes,
+ getIndexPatternSavedObjects,
+ makeManagedIndexPatternsGlobal,
+} from '../index_pattern/install';
import { saveKibanaAssetsRefs } from '../../packages/install';
import { deleteKibanaSavedObjectsAssets } from '../../packages/remove';
import { KibanaSOReferenceError } from '../../../../errors';
@@ -114,12 +118,14 @@ export function createSavedObjectKibanaAsset(asset: ArchiveAsset): SavedObjectTo
}
export async function installKibanaAssets(options: {
+ savedObjectsClient: SavedObjectsClientContract;
savedObjectsImporter: SavedObjectsImporterContract;
logger: Logger;
pkgName: string;
kibanaAssets: Record;
}): Promise {
- const { kibanaAssets, savedObjectsImporter, logger } = options;
+ const { kibanaAssets, savedObjectsClient, savedObjectsImporter, logger } = options;
+
const assetsToInstall = Object.entries(kibanaAssets).flatMap(([assetType, assets]) => {
if (!validKibanaAssetTypes.has(assetType as KibanaAssetType)) {
return [];
@@ -152,6 +158,8 @@ export async function installKibanaAssets(options: {
managed: true,
});
+ await makeManagedIndexPatternsGlobal(savedObjectsClient);
+
const installedAssets = await installKibanaSavedObjects({
logger,
savedObjectsImporter,
@@ -196,6 +204,7 @@ export async function installKibanaAssetsAndReferences({
);
const importedAssets = await installKibanaAssets({
+ savedObjectsClient,
logger,
savedObjectsImporter,
pkgName,
diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts
new file mode 100644
index 000000000000..9e826d7f126a
--- /dev/null
+++ b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
+import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
+
+import { createAppContextStartContractMock } from '../../../../mocks';
+
+import { appContextService } from '../../../app_context';
+
+import { makeManagedIndexPatternsGlobal } from './install';
+
+describe('Fleet index patterns', () => {
+ let mockSoClient: jest.Mocked;
+ let mockContract: ReturnType;
+
+ beforeEach(() => {
+ mockSoClient = savedObjectsClientMock.create();
+ mockContract = createAppContextStartContractMock();
+ appContextService.start(mockContract);
+ });
+
+ afterEach(() => {
+ appContextService.stop();
+ });
+
+ describe('makeManagedIndexPatternsGlobal', () => {
+ it('should call updateObjectsSpaces with the correct params', async () => {
+ const result = await makeManagedIndexPatternsGlobal(mockSoClient);
+
+ for (const pattern of ['logs-*', 'metrics-*']) {
+ expect(mockSoClient.updateObjectsSpaces).toHaveBeenCalledWith(
+ [{ id: pattern, type: 'index-pattern' }],
+ ['*'],
+ []
+ );
+ }
+
+ expect(result).toHaveLength(2);
+ });
+
+ it('handles errors from updateObjectsSpaces', async () => {
+ mockSoClient.updateObjectsSpaces.mockRejectedValue(new Error('foo'));
+
+ const result = await makeManagedIndexPatternsGlobal(mockSoClient);
+
+ expect(result).toHaveLength(0);
+ });
+ });
+});
diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts
index 91e603e80aee..6f1390f5e17c 100644
--- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts
+++ b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts
@@ -28,6 +28,30 @@ export function getIndexPatternSavedObjects() {
}));
}
+export async function makeManagedIndexPatternsGlobal(
+ savedObjectsClient: SavedObjectsClientContract
+) {
+ const logger = appContextService.getLogger();
+
+ const results = [];
+
+ for (const indexPatternType of indexPatternTypes) {
+ try {
+ const result = await savedObjectsClient.updateObjectsSpaces(
+ [{ id: `${indexPatternType}-*`, type: INDEX_PATTERN_SAVED_OBJECT_TYPE }],
+ ['*'],
+ []
+ );
+
+ results.push(result);
+ } catch (error) {
+ logger.error(`Error making managed index patterns global: ${error.message}`);
+ }
+ }
+
+ return results;
+}
+
export async function removeUnusedIndexPatterns(savedObjectsClient: SavedObjectsClientContract) {
const logger = appContextService.getLogger();
// get all user installed packages
diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
index 4007ad7545ec..3eb689dfa1a2 100644
--- a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
+++ b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
@@ -10,6 +10,7 @@ import type { PackageClient, PackageService } from './package_service';
const createClientMock = (): jest.Mocked => ({
getInstallation: jest.fn(),
ensureInstalledPackage: jest.fn(),
+ installPackage: jest.fn(),
fetchFindLatestPackage: jest.fn(),
readBundledPackage: jest.fn(),
getPackage: jest.fn(),
diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts
index eee1afb37dca..a535af9636d1 100644
--- a/x-pack/plugins/fleet/server/services/epm/package_service.ts
+++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts
@@ -8,34 +8,40 @@
/* eslint-disable max-classes-per-file */
import type {
- KibanaRequest,
ElasticsearchClient,
- SavedObjectsClientContract,
+ KibanaRequest,
Logger,
+ SavedObjectsClientContract,
} from '@kbn/core/server';
+import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common';
+
import { HTTPAuthorizationHeader } from '../../../common/http_authorization_header';
import type { PackageList } from '../../../common';
import type {
+ ArchivePackage,
+ BundledPackage,
CategoryId,
EsAssetReference,
InstallablePackage,
Installation,
RegistryPackage,
- ArchivePackage,
- BundledPackage,
} from '../../types';
import type { FleetAuthzRouteConfig } from '../security/types';
-import { checkSuperuser, getAuthzFromRequest, doesNotHaveRequiredFleetAuthz } from '../security';
-import { FleetUnauthorizedError, FleetError } from '../../errors';
+import { checkSuperuser, doesNotHaveRequiredFleetAuthz, getAuthzFromRequest } from '../security';
+import { FleetError, FleetUnauthorizedError } from '../../errors';
import { INSTALL_PACKAGES_AUTHZ, READ_PACKAGE_INFO_AUTHZ } from '../../routes/epm';
-import { installTransforms, isTransform } from './elasticsearch/transform/install';
+import type { InstallResult } from '../../../common';
+
import type { FetchFindLatestPackageOptions } from './registry';
+import * as Registry from './registry';
import { fetchFindLatestPackageOrThrow, getPackage } from './registry';
-import { ensureInstalledPackage, getInstallation, getPackages } from './packages';
+
+import { installTransforms, isTransform } from './elasticsearch/transform/install';
+import { ensureInstalledPackage, getInstallation, getPackages, installPackage } from './packages';
import { generatePackageInfoFromArchiveBuffer } from './archive';
export type InstalledAssetType = EsAssetReference;
@@ -52,8 +58,16 @@ export interface PackageClient {
pkgName: string;
pkgVersion?: string;
spaceId?: string;
+ force?: boolean;
}): Promise;
+ installPackage(options: {
+ pkgName: string;
+ pkgVersion?: string;
+ spaceId?: string;
+ force?: boolean;
+ }): Promise;
+
fetchFindLatestPackage(
packageName: string,
options?: FetchFindLatestPackageOptions
@@ -151,6 +165,7 @@ class PackageClientImpl implements PackageClient {
pkgName: string;
pkgVersion?: string;
spaceId?: string;
+ force?: boolean;
}): Promise {
await this.#runPreflight(INSTALL_PACKAGES_AUTHZ);
@@ -160,6 +175,32 @@ class PackageClientImpl implements PackageClient {
savedObjectsClient: this.internalSoClient,
});
}
+ public async installPackage(options: {
+ pkgName: string;
+ pkgVersion?: string;
+ spaceId?: string;
+ force?: boolean;
+ }): Promise {
+ await this.#runPreflight(INSTALL_PACKAGES_AUTHZ);
+
+ const { pkgName, pkgVersion, spaceId = DEFAULT_SPACE_ID, force = false } = options;
+
+ // If pkgVersion isn't specified, find the latest package version
+ const pkgKeyProps = pkgVersion
+ ? { name: pkgName, version: pkgVersion }
+ : await Registry.fetchFindLatestPackageOrThrow(pkgName, { prerelease: true });
+ const pkgkey = Registry.pkgToPkgKey(pkgKeyProps);
+
+ return await installPackage({
+ force,
+ pkgkey,
+ spaceId,
+ installSource: 'registry',
+ esClient: this.internalEsClient,
+ savedObjectsClient: this.internalSoClient,
+ neverIgnoreVerificationError: !force,
+ });
+ }
public async fetchFindLatestPackage(
packageName: string,
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
index d8a943fef534..8f45e8eee6b1 100644
--- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
+++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
@@ -100,7 +100,6 @@ export async function _installPackage({
skipDataStreamRollover?: boolean;
}): Promise {
const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo;
-
try {
// if some installation already exists
if (installedPkg) {
@@ -108,12 +107,16 @@ export async function _installPackage({
const hasExceededTimeout =
Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
MAX_TIME_COMPLETE_INSTALL;
+ logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`);
// if the installation is currently running, don't try to install
// instead, only return already installed assets
if (isStatusInstalling && hasExceededTimeout) {
// If this is a forced installation, ignore the timeout and restart the installation anyway
+ logger.debug(`Package install - Installation is running and has exceeded timeout`);
+
if (force) {
+ logger.debug(`Package install - Forced installation, restarting`);
await restartInstallation({
savedObjectsClient,
pkgName,
@@ -131,6 +134,9 @@ export async function _installPackage({
} else {
// if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL
// (it might be stuck) update the saved object and proceed
+ logger.debug(
+ `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting`
+ );
await restartInstallation({
savedObjectsClient,
pkgName,
@@ -140,6 +146,7 @@ export async function _installPackage({
});
}
} else {
+ logger.debug(`Package install - Create installation`);
await createInstallation({
savedObjectsClient,
packageInfo,
@@ -148,7 +155,7 @@ export async function _installPackage({
verificationResult,
});
}
-
+ logger.debug(`Package install - Installing Kibana assets`);
const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () =>
installKibanaAssetsAndReferences({
savedObjectsClient,
@@ -182,7 +189,7 @@ export async function _installPackage({
esReferences = await withPackageSpan('Install ILM policies', () =>
installILMPolicy(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences)
);
-
+ logger.debug(`Package install - Installing Data Stream ILM policies`);
({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () =>
installIlmForDataStream(
packageInfo,
@@ -196,6 +203,7 @@ export async function _installPackage({
}
// installs ml models
+ logger.debug(`Package install - installing ML models`);
esReferences = await withPackageSpan('Install ML models', () =>
installMlModel(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences)
);
@@ -203,6 +211,9 @@ export async function _installPackage({
let indexTemplates: IndexTemplateEntry[] = [];
if (packageInfo.type === 'integration') {
+ logger.debug(
+ `Package install - Installing index templates and pipelines, packageInfo.type ${packageInfo.type}`
+ );
const { installedTemplates, esReferences: templateEsReferences } =
await installIndexTemplatesAndPipelines({
installedPkg: installedPkg ? installedPkg.attributes : undefined,
@@ -221,6 +232,7 @@ export async function _installPackage({
// input packages create their data streams during package policy creation
// we must use installed_es to infer which streams exist first then
// we can install the new index templates
+ logger.debug(`Package install - packageInfo.type ${packageInfo.type}`);
const dataStreamNames = installedPkg.attributes.installed_es
.filter((ref) => ref.type === 'index_template')
// index templates are named {type}-{dataset}, remove everything before first hyphen
@@ -231,6 +243,9 @@ export async function _installPackage({
);
if (dataStreams.length) {
+ logger.debug(
+ `Package install - installing index templates and pipelines with datastreams length ${dataStreams.length}`
+ );
const { installedTemplates, esReferences: templateEsReferences } =
await installIndexTemplatesAndPipelines({
installedPkg: installedPkg ? installedPkg.attributes : undefined,
@@ -248,19 +263,21 @@ export async function _installPackage({
}
try {
+ logger.debug(`Package install - Removing legacy templates`);
await removeLegacyTemplates({ packageInfo, esClient, logger });
} catch (e) {
logger.warn(`Error removing legacy templates: ${e.message}`);
}
// update current backing indices of each data stream
+ logger.debug(`Package install - Updating backing indices of each data stream`);
await withPackageSpan('Update write indices', () =>
updateCurrentWriteIndices(esClient, logger, indexTemplates, {
ignoreMappingUpdateErrors,
skipDataStreamRollover,
})
);
-
+ logger.debug(`Package install - Installing transforms`);
({ esReferences } = await withPackageSpan('Install transforms', () =>
installTransforms({
installablePackage: packageInfo,
@@ -282,6 +299,9 @@ export async function _installPackage({
(installType === 'update' || installType === 'reupdate') &&
installedPkg
) {
+ logger.debug(
+ `Package install - installType ${installType} Deleting previous ingest pipelines`
+ );
esReferences = await withPackageSpan('Delete previous ingest pipelines', () =>
deletePreviousPipelines(
esClient,
@@ -294,6 +314,9 @@ export async function _installPackage({
}
// pipelines from a different version may have installed during a failed update
if (installType === 'rollback' && installedPkg) {
+ logger.debug(
+ `Package install - installType ${installType} Deleting previous ingest pipelines`
+ );
esReferences = await withPackageSpan('Delete previous ingest pipelines', () =>
deletePreviousPipelines(
esClient,
@@ -306,6 +329,7 @@ export async function _installPackage({
}
const installedKibanaAssetsRefs = await kibanaAssetPromise;
+ logger.debug(`Package install - Updating archive entries`);
const packageAssetResults = await withPackageSpan('Update archive entries', () =>
saveArchiveEntries({
savedObjectsClient,
@@ -326,7 +350,7 @@ export async function _installPackage({
id: pkgName,
savedObjectType: PACKAGES_SAVED_OBJECT_TYPE,
});
-
+ logger.debug(`Package install - Updating install status`);
const updatedPackage = await withPackageSpan('Update install status', () =>
savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
version: pkgVersion,
@@ -340,6 +364,7 @@ export async function _installPackage({
),
})
);
+ logger.debug(`Package install - Install status ${updatedPackage?.attributes?.install_status}`);
// If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its
// associated package policies after installation
@@ -350,11 +375,13 @@ export async function _installPackage({
perPage: SO_SEARCH_LIMIT,
kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`,
});
-
+ logger.debug(
+ `Package install - Package is flagged with keep_policies_up_to_date, upgrading its associated package policies ${policyIdsToUpgrade}`
+ );
await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items);
});
}
-
+ logger.debug(`Package install - Installation complete`);
return [...installedKibanaAssetsRefs, ...esReferences];
} catch (err) {
if (SavedObjectsErrorHelpers.isConflictError(err)) {
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts
index c58d14dd4732..bfb99d0f1798 100644
--- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts
@@ -186,7 +186,7 @@ describe('install', () => {
expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, {
currentVersion: 'not_installed',
dryRun: false,
- errorMessage: 'Requires basic license',
+ errorMessage: 'Installation requires basic license',
eventType: 'package-install',
installType: 'install',
newVersion: '1.3.0',
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts
index 0760bd94a56e..33f1b3a6ab73 100644
--- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts
+++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts
@@ -516,6 +516,7 @@ async function installPackageCommon(options: {
} = options;
let { telemetryEvent } = options;
const logger = appContextService.getLogger();
+ logger.info(`Install - Starting installation of ${pkgName}@${pkgVersion} from ${installSource} `);
// Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611
await Promise.resolve();
@@ -564,7 +565,8 @@ async function installPackageCommon(options: {
}
const elasticSubscription = getElasticSubscription(packageInfo);
if (!licenseService.hasAtLeast(elasticSubscription)) {
- const err = new Error(`Requires ${elasticSubscription} license`);
+ logger.error(`Installation requires ${elasticSubscription} license`);
+ const err = new FleetError(`Installation requires ${elasticSubscription} license`);
sendEvent({
...telemetryEvent,
errorMessage: err.message,
@@ -606,6 +608,7 @@ async function installPackageCommon(options: {
skipDataStreamRollover,
})
.then(async (assets) => {
+ logger.debug(`Removing old assets from previous versions of ${pkgName}`);
await removeOldAssets({
soClient: savedObjectsClient,
pkgName: packageInfo.name,
@@ -759,7 +762,7 @@ export async function installPackage(args: InstallPackageParams): Promise
.at(integrationPosition);
if (!integration) {
- throw new Error(`Index name ${index} does not seem to be a File storage index`);
+ throw new FleetError(`Index name ${index} does not seem to be a File storage index`);
}
response.direction = isDeliveryToHost ? 'to-host' : 'from-host';
@@ -69,7 +70,7 @@ export const parseFileStorageIndex = (index: string): ParsedFileStorageIndex =>
}
}
- throw new Error(
+ throw new FleetError(
`Unable to parse index [${index}]. Does not match a known index pattern: [${fileStorageIndexPatterns.join(
' | '
)}]`
diff --git a/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts b/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts
index 89801b66b49c..730b368495cb 100644
--- a/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts
+++ b/x-pack/plugins/fleet/server/services/fleet_proxies.test.ts
@@ -4,11 +4,16 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
+import { loggerMock } from '@kbn/logging-mocks';
+import type { Logger } from '@kbn/core/server';
+import { securityMock } from '@kbn/security-plugin/server/mocks';
import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';
import { FLEET_PROXY_SAVED_OBJECT_TYPE } from '../constants';
+import { appContextService } from './app_context';
+
import { deleteFleetProxy } from './fleet_proxies';
import { listFleetServerHostsForProxyId, updateFleetServerHost } from './fleet_server_host';
import { outputService } from './output';
@@ -17,6 +22,7 @@ import { downloadSourceService } from './download_source';
jest.mock('./output');
jest.mock('./download_source');
jest.mock('./fleet_server_host');
+jest.mock('./app_context');
const mockedListFleetServerHostsForProxyId = listFleetServerHostsForProxyId as jest.MockedFunction<
typeof listFleetServerHostsForProxyId
@@ -35,8 +41,19 @@ const PROXY_IDS = {
PRECONFIGURED: 'test-preconfigured',
RELATED_PRECONFIGURED: 'test-related-preconfigured',
};
+const mockedAppContextService = appContextService as jest.Mocked;
+mockedAppContextService.getSecuritySetup.mockImplementation(() => ({
+ ...securityMock.createSetup(),
+}));
+
+let mockedLogger: jest.Mocked;
describe('Fleet proxies service', () => {
+ beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+
const soClientMock = savedObjectsClientMock.create();
const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
diff --git a/x-pack/plugins/fleet/server/services/fleet_proxies.ts b/x-pack/plugins/fleet/server/services/fleet_proxies.ts
index cf45b90804c2..61aa07b8e061 100644
--- a/x-pack/plugins/fleet/server/services/fleet_proxies.ts
+++ b/x-pack/plugins/fleet/server/services/fleet_proxies.ts
@@ -24,6 +24,8 @@ import type {
Output,
} from '../types';
+import { appContextService } from './app_context';
+
import { listFleetServerHostsForProxyId, updateFleetServerHost } from './fleet_server_host';
import { outputService } from './output';
import { downloadSourceService } from './download_source';
@@ -70,6 +72,9 @@ export async function createFleetProxy(
data: NewFleetProxy,
options?: { id?: string; overwrite?: boolean; fromPreconfiguration?: boolean }
): Promise {
+ const logger = appContextService.getLogger();
+ logger.debug(`Creating fleet proxy ${data}`);
+
const res = await soClient.create(
FLEET_PROXY_SAVED_OBJECT_TYPE,
fleetProxyDataToSOAttribute(data),
@@ -78,7 +83,7 @@ export async function createFleetProxy(
overwrite: options?.overwrite,
}
);
-
+ logger.debug(`Created fleet proxy ${options?.id}`);
return savedObjectToFleetProxy(res);
}
@@ -97,6 +102,9 @@ export async function deleteFleetProxy(
id: string,
options?: { fromPreconfiguration?: boolean }
) {
+ const logger = appContextService.getLogger();
+ logger.debug(`Deleting fleet proxy ${id}`);
+
const fleetProxy = await getFleetProxy(soClient, id);
if (fleetProxy.is_preconfigured && !options?.fromPreconfiguration) {
@@ -120,6 +128,7 @@ export async function deleteFleetProxy(
}
await updateRelatedSavedObject(soClient, esClient, fleetServerHosts, outputs, downloadSources);
+ logger.debug(`Deleted fleet proxy ${id}`);
return await soClient.delete(FLEET_PROXY_SAVED_OBJECT_TYPE, id);
}
@@ -130,6 +139,8 @@ export async function updateFleetProxy(
data: Partial,
options?: { fromPreconfiguration?: boolean }
) {
+ const logger = appContextService.getLogger();
+ logger.debug(`Updating fleet proxy ${id}`);
const originalItem = await getFleetProxy(soClient, id);
if (data.is_preconfigured && !options?.fromPreconfiguration) {
@@ -141,7 +152,7 @@ export async function updateFleetProxy(
id,
fleetProxyDataToSOAttribute(data)
);
-
+ logger.debug(`Updated fleet proxy ${id}`);
return {
...originalItem,
...data,
diff --git a/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts b/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts
index 40f65ca21c5e..f92261ffb9f8 100644
--- a/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts
+++ b/x-pack/plugins/fleet/server/services/fleet_server_host.test.ts
@@ -6,6 +6,10 @@
*/
import { savedObjectsClientMock } from '@kbn/core/server/mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+
+import type { Logger } from '@kbn/core/server';
+import { securityMock } from '@kbn/security-plugin/server/mocks';
import {
GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
@@ -13,9 +17,25 @@ import {
DEFAULT_FLEET_SERVER_HOST_ID,
} from '../constants';
+import { appContextService } from './app_context';
+
import { migrateSettingsToFleetServerHost } from './fleet_server_host';
+jest.mock('./app_context');
+
+const mockedAppContextService = appContextService as jest.Mocked;
+mockedAppContextService.getSecuritySetup.mockImplementation(() => ({
+ ...securityMock.createSetup(),
+}));
+
+let mockedLogger: jest.Mocked;
+
describe('migrateSettingsToFleetServerHost', () => {
+ beforeEach(() => {
+ mockedLogger = loggerMock.create();
+ mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+ });
+
it('should not migrate settings if a default fleet server policy config exists', async () => {
const soClient = savedObjectsClientMock.create();
soClient.find.mockImplementation(({ type }) => {
diff --git a/x-pack/plugins/fleet/server/services/fleet_server_host.ts b/x-pack/plugins/fleet/server/services/fleet_server_host.ts
index 156d7e478b02..66bed0b61977 100644
--- a/x-pack/plugins/fleet/server/services/fleet_server_host.ts
+++ b/x-pack/plugins/fleet/server/services/fleet_server_host.ts
@@ -26,7 +26,9 @@ import type {
NewFleetServerHost,
AgentPolicy,
} from '../types';
-import { FleetServerHostUnauthorizedError } from '../errors';
+import { FleetServerHostUnauthorizedError, FleetServerHostNotFoundError } from '../errors';
+
+import { appContextService } from './app_context';
import { agentPolicyService } from './agent_policy';
import { escapeSearchQueryPhrase } from './saved_object';
@@ -46,6 +48,7 @@ export async function createFleetServerHost(
data: NewFleetServerHost,
options?: { id?: string; overwrite?: boolean; fromPreconfiguration?: boolean }
): Promise {
+ const logger = appContextService.getLogger();
if (data.is_default) {
const defaultItem = await getDefaultFleetServerHost(soClient);
if (defaultItem && defaultItem.id !== options?.id) {
@@ -61,13 +64,13 @@ export async function createFleetServerHost(
if (data.host_urls) {
data.host_urls = data.host_urls.map(normalizeHostsForAgents);
}
-
+ logger.debug(`Creating fleet server host with ${data}`);
const res = await soClient.create(
FLEET_SERVER_HOST_SAVED_OBJECT_TYPE,
data,
{ id: options?.id, overwrite: options?.overwrite }
);
-
+ logger.debug(`Created fleet server host ${options?.id}`);
return savedObjectToFleetServerHost(res);
}
@@ -122,6 +125,9 @@ export async function deleteFleetServerHost(
id: string,
options?: { fromPreconfiguration?: boolean }
) {
+ const logger = appContextService.getLogger();
+ logger.debug(`Deleting fleet server host ${id}`);
+
const fleetServerHost = await getFleetServerHost(soClient, id);
if (fleetServerHost.is_preconfigured && !options?.fromPreconfiguration) {
@@ -147,6 +153,9 @@ export async function updateFleetServerHost(
data: Partial,
options?: { fromPreconfiguration?: boolean }
) {
+ const logger = appContextService.getLogger();
+ logger.debug(`Updating fleet server host ${id}`);
+
const originalItem = await getFleetServerHost(soClient, id);
if (data.is_preconfigured && !options?.fromPreconfiguration) {
@@ -174,7 +183,7 @@ export async function updateFleetServerHost(
}
await soClient.update(FLEET_SERVER_HOST_SAVED_OBJECT_TYPE, id, data);
-
+ logger.debug(`Updated fleet server host ${id}`);
return {
...originalItem,
...data,
@@ -224,7 +233,7 @@ export async function getFleetServerHostsForAgentPolicy(
const defaultFleetServerHost = await getDefaultFleetServerHost(soClient);
if (!defaultFleetServerHost) {
- throw new Error('Default Fleet Server host is not setup');
+ throw new FleetServerHostNotFoundError('Default Fleet Server host is not setup');
}
return defaultFleetServerHost;
diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts
index 6e7df5c7d4e5..c17cf9d3af21 100644
--- a/x-pack/plugins/fleet/server/services/output.test.ts
+++ b/x-pack/plugins/fleet/server/services/output.test.ts
@@ -1841,4 +1841,66 @@ describe('Output Service', () => {
});
});
});
+
+ describe('backfillAllOutputPresets', () => {
+ it('should update non-preconfigured output', async () => {
+ const soClient = getMockedSoClient({});
+
+ soClient.find.mockResolvedValue({
+ saved_objects: [
+ {
+ ...mockOutputSO('non-preconfigured-output', {
+ is_preconfigured: false,
+ type: 'elasticsearch',
+ }),
+ score: 0,
+ },
+ ],
+ total: 1,
+ per_page: 1,
+ page: 1,
+ });
+
+ soClient.get.mockResolvedValue({
+ ...mockOutputSO('non-preconfigured-output', {
+ is_preconfigured: false,
+ type: 'elasticsearch',
+ }),
+ });
+
+ const promise = outputService.backfillAllOutputPresets(soClient, esClientMock);
+
+ await expect(promise).resolves.not.toThrow();
+ });
+
+ it('should update preconfigured output', async () => {
+ const soClient = getMockedSoClient({});
+
+ soClient.find.mockResolvedValue({
+ saved_objects: [
+ {
+ ...mockOutputSO('preconfigured-output', {
+ is_preconfigured: true,
+ type: 'elasticsearch',
+ }),
+ score: 0,
+ },
+ ],
+ total: 1,
+ per_page: 1,
+ page: 1,
+ });
+
+ soClient.get.mockResolvedValue({
+ ...mockOutputSO('preconfigured-output', {
+ is_preconfigured: true,
+ type: 'elasticsearch',
+ }),
+ });
+
+ const promise = outputService.backfillAllOutputPresets(soClient, esClientMock);
+
+ await expect(promise).resolves.not.toThrow();
+ });
+ });
});
diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts
index 32784a94f672..7b838d9b9b0a 100644
--- a/x-pack/plugins/fleet/server/services/output.ts
+++ b/x-pack/plugins/fleet/server/services/output.ts
@@ -57,6 +57,7 @@ import {
FleetEncryptedSavedObjectEncryptionKeyRequired,
OutputInvalidError,
OutputUnauthorizedError,
+ FleetError,
} from '../errors';
import type { OutputType } from '../types';
@@ -436,6 +437,9 @@ class OutputService {
secretHashes?: Record;
}
): Promise