diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 87bf8c7104871..27fd348a6843c 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -338,6 +338,7 @@ "installed_es.version", "installed_kibana", "installed_kibana_space_id", + "installed_misc", "internal", "keep_policies_up_to_date", "latest_executed_state", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 7b315efcbcd4f..16fb3a95904dc 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1143,6 +1143,10 @@ "installed_kibana_space_id": { "type": "keyword" }, + "installed_misc": { + "dynamic": false, + "properties": {} + }, "internal": { "type": "boolean" }, diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index a8b31ffdd90fa..70f7807e4c5ca 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -94,7 +94,7 @@ describe('checking migration metadata changes on all registered SO types', () => "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9", "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88", "entity-engine-status": "0738aa1a06d3361911740f8f166071ea43a00927", - "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd", + "epm-packages": "a8071c1e2b9ddeb3a26acd9ef02b5155791bffc4", "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1", "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582", "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88", diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 3188a40846deb..13b65e6afe779 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -44,10 +44,18 @@ export type AgentAssetType = typeof agentAssetTypes; export type DocAssetType = 'doc' | 'notice' | 'license'; export type AssetType = | KibanaAssetType + | KibanaMiscAssetTypes | ElasticsearchAssetType | ValueOf | DocAssetType; +/* + Enum mapping of asset types living under the Kibana service/folder and that are not saved objects. +*/ +export enum KibanaMiscAssetTypes { + knowledgeBaseEntry = 'knowledge_base_entry', +} + /* Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased) */ @@ -566,6 +574,7 @@ export interface InstallFailedAttempt { export enum INSTALL_STATES { CREATE_RESTART_INSTALLATION = 'create_restart_installation', INSTALL_KIBANA_ASSETS = 'install_kibana_assets', + INSTALL_KNOWLEDGE_BASE_ASSETS = 'install_knowledge_base_assets', INSTALL_ILM_POLICIES = 'install_ilm_policies', INSTALL_ML_MODEL = 'install_ml_model', INSTALL_INDEX_TEMPLATE_PIPELINES = 'install_index_template_pipelines', @@ -597,6 +606,7 @@ export interface Installation { installed_kibana: KibanaAssetReference[]; additional_spaces_installed_kibana?: Record; installed_es: EsAssetReference[]; + installed_misc?: MiscAssetReference[]; package_assets?: PackageAssetReference[]; es_index_patterns: Record; name: string; @@ -652,7 +662,7 @@ export type InstallFailed = T & { status: InstallationStatus['InstallFailed']; }; -export type AssetReference = KibanaAssetReference | EsAssetReference; +export type AssetReference = KibanaAssetReference | EsAssetReference | MiscAssetReference; export interface KibanaAssetReference { id: string; @@ -665,6 +675,15 @@ export interface EsAssetReference { deferred?: boolean; } +export interface KnowledgeBaseMiscAssetReference { + id: string; + type: KibanaMiscAssetTypes.knowledgeBaseEntry; + system?: boolean; +} + +// polymorphic type, even if only one subtype for now +export type MiscAssetReference = KnowledgeBaseMiscAssetReference; + export interface PackageAssetReference { id: string; type: typeof ASSETS_SAVED_OBJECT_TYPE; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 0eb6c86df01e2..bcca7a267aee6 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -827,6 +827,10 @@ export const getSavedObjectTypes = ( dynamic: false, properties: {}, }, + installed_misc: { + dynamic: false, + properties: {}, + }, installed_kibana_space_id: { type: 'keyword' }, package_assets: { dynamic: false, @@ -888,6 +892,19 @@ export const getSavedObjectTypes = ( }, ], }, + '4': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + installed_misc: { + dynamic: false, + properties: {}, + }, + }, + }, + ], + }, }, migrations: { '7.14.0': migrateInstallationToV7140, diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.test.ts index ab2bb44621b0e..fe4bfe344598f 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.test.ts @@ -44,6 +44,17 @@ describe('getPathParts', () => { type: 'fields', }, }, + { + path: 'knowledgebase-1.0.1/kibana/knowledge_base_entry/foo/manifest.yml', + assetParts: { + dataset: 'foo', + file: 'manifest.yml', + path: 'knowledgebase-1.0.1/kibana/knowledge_base_entry/foo/manifest.yml', + pkgkey: 'knowledgebase-1.0.1', + service: 'kibana', + type: 'knowledge_base_entry', + }, + }, ]; test('testPathParts', () => { for (const value of testPaths) { diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index 5943f8f838fcb..d7c379be6a8c5 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -99,6 +99,13 @@ export function getPathParts(path: string): AssetParts { [pkgkey, service, type, file] = path.replace(`data_stream/${dataset}/`, '').split('/'); } + // if it's a knowledge base entry + if (type === 'knowledge_base_entry') { + // there is an additional depth level + dataset = file; + file = path.split('/')[4]; + } + // To support the NOTICE asset at the root level if (service === 'NOTICE.txt') { file = service; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index c7d2e4eacb32a..2b76c690205f8 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -1715,6 +1715,31 @@ describe('EPM template', () => { expect(mappings).toEqual(runtimeFieldMapping); }); + it('tests processing semantic_text fields', () => { + const textWithRuntimeFieldsLiteralYml = ` +- name: sem_without_inference_id + type: semantic_text +- name: sem_with_inference_id + type: semantic_text + inference_id: .model_id +`; + const semanticFieldMapping = { + properties: { + sem_without_inference_id: { + type: 'semantic_text', + }, + sem_with_inference_id: { + inference_id: '.model_id', + type: 'semantic_text', + }, + }, + }; + const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields, true); + expect(mappings).toEqual(semanticFieldMapping); + }); + it('tests unexpected type for field as dynamic template fails', () => { const textWithRuntimeFieldsLiteralYml = ` - name: labels.* diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 3709975c57a5e..ae636f09bb37b 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -543,6 +543,13 @@ function _generateMappings( fieldProps.fields = generateMultiFields(field.multi_fields); } break; + case 'semantic_text': + fieldProps.type = 'semantic_text'; + if (field.inference_id) { + fieldProps.inference_id = field.inference_id; + } + fieldProps.path = field.path; + break; case 'object': fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' }; break; diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts index 9381403f3f10d..166fecf0c878f 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts @@ -11,7 +11,12 @@ import path from 'path'; import globby from 'globby'; import { load } from 'js-yaml'; -import { getField, processFields, processFieldsWithWildcard } from './field'; +import { + getField, + processFields, + processFieldsWithWildcard, + filterForKnowledgeBaseEntryAssets, +} from './field'; import type { Field, Fields } from './field'; // Add our own serialiser to just do JSON.stringify @@ -813,3 +818,25 @@ describe('processFields', () => { }); }); }); + +describe('filterForKnowledgeBaseEntryAssets', () => { + it('returns true for assets within the given knowledge base entry folder', () => { + expect( + filterForKnowledgeBaseEntryAssets('foo')( + '/kb-1.0/kibana/knowledge_base_entry/foo/manifest.yml' + ) + ).toBe(true); + }); + it('returns false for assets within another knowledge base entry folder', () => { + expect( + filterForKnowledgeBaseEntryAssets('bar')( + '/kb-1.0/kibana/knowledge_base_entry/foo/manifest.yml' + ) + ).toBe(false); + }); + it('returns false for assets outside of a knowledge base entry folder', () => { + expect( + filterForKnowledgeBaseEntryAssets('foo')('/kb-1.0/kibana/dashboard/some_dashboard.json') + ).toBe(false); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index a3ebf58d02e3b..060909b3b469a 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -47,6 +47,9 @@ export interface Field { metrics?: string[]; default_metric?: string; + // Fields specific to the semantic_text type + inference_id?: string; + // Meta fields metric_type?: string; unit?: string; @@ -296,6 +299,12 @@ export const filterForTransformAssets = (transformName: string) => { }; }; +export const filterForKnowledgeBaseEntryAssets = (knowledgeBaseEntryName: string) => { + return function isTransformAssets(path: string) { + return path.includes(`/knowledge_base_entry/${knowledgeBaseEntryName}`); + }; +}; + function combineFilter(...filters: Array<(path: string) => boolean>) { return function filterAsset(path: string) { return filters.every((filter) => filter(path)); @@ -354,3 +363,26 @@ export const loadTransformFieldsFromYaml = ( return acc; }, []); }; + +export const loadKnowledgeBaseEntryFieldsFromYaml = ( + packageInstallContext: PackageInstallContext, + knowledgeBaseEntryName: string +): Field[] => { + // Fetch all field definition files + const fieldDefinitionFiles = getAssetsDataFromAssetsMap( + packageInstallContext.packageInfo, + packageInstallContext.assetsMap, + combineFilter(isFields, filterForKnowledgeBaseEntryAssets(knowledgeBaseEntryName)) + ); + return fieldDefinitionFiles.reduce((acc, file) => { + // Make sure it is defined as it is optional. Should never happen. + if (file.buffer) { + const tmpFields = load(file.buffer.toString()); + // load() returns undefined for empty files, we don't want that + if (tmpFields) { + acc = acc.concat(tmpFields); + } + } + return acc; + }, []); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/consts.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/consts.ts new file mode 100644 index 0000000000000..b3b0e622a9cab --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/consts.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const knowledgeBaseEntrySavedObjectType = 'knowledge_base_entry'; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/install.test.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/install.test.ts new file mode 100644 index 0000000000000..866db3236077d --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/install.test.ts @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { times } from 'lodash'; +import type { SavedObject } from '@kbn/core/server'; +import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; +import type { MockedLogger } from '@kbn/logging-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { Installation } from '../../../../types'; +import { KibanaMiscAssetTypes } from '../../../../types'; +import type { PackageInstallContext } from '../../../../../common/types'; +import { auditLoggingService } from '../../../audit_logging'; +import { installKibanaKnowledgeBaseEntries } from './install'; +import { removeKnowledgeBaseEntries } from './remove'; +import { knowledgeBaseEntrySavedObjectType } from './consts'; +import { parseKnowledgeBaseEntries } from './parse_entries'; +import { updateMiscAssetReferences } from '../../packages/misc_assets_reference'; + +jest.mock('../../../audit_logging'); +jest.mock('./remove'); +jest.mock('./parse_entries'); +jest.mock('../../packages/misc_assets_reference'); + +const removeKnowledgeBaseEntriesMock = removeKnowledgeBaseEntries as jest.MockedFn< + typeof removeKnowledgeBaseEntries +>; +const parseKnowledgeBaseEntriesMock = parseKnowledgeBaseEntries as jest.MockedFn< + typeof parseKnowledgeBaseEntries +>; +const updateMiscAssetReferencesMock = updateMiscAssetReferences as jest.MockedFn< + typeof updateMiscAssetReferences +>; + +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +const createInstallContext = (assets: Record): PackageInstallContext => { + const installContext: PackageInstallContext = { + packageInfo: { + format_version: '3.4.0', + name: 'test-pkg', + title: 'Test Pkg', + description: 'Some desc', + version: '0.0.1', + owner: { github: 'owner', type: 'elastic' }, + }, + assetsMap: new Map(), + paths: [], + }; + + Object.entries(assets).forEach(([assetPath, assetContent]) => { + installContext.paths.push(assetPath); + installContext.assetsMap.set(assetPath, Buffer.from(assetContent)); + }); + + return installContext; +}; + +export const createContentFile = (length = 5) => { + return times(5) + .map(() => '{}') + .join('\n'); +}; + +const createInstalledPkgObject = (parts: Partial = {}): SavedObject => { + return { + id: 'id', + type: 'installation', + references: [], + attributes: { + installed_kibana: [], + installed_es: [], + installed_misc: [], + es_index_patterns: {}, + name: 'my-pkg', + version: '0.0.1', + install_version: '0.0.1', + install_started_at: 'now', + install_source: 'custom', + install_status: 'installed', + verification_status: 'unknown', + ...parts, + }, + }; +}; + +describe('installKibanaKnowledgeBaseEntries', () => { + let esClient: ReturnType; + let savedObjectsClient: ReturnType; + let logger: MockedLogger; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + savedObjectsClient = savedObjectsClientMock.create(); + logger = loggerMock.create(); + + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + + removeKnowledgeBaseEntriesMock.mockReset(); + parseKnowledgeBaseEntriesMock.mockReset(); + updateMiscAssetReferencesMock.mockReset(); + + parseKnowledgeBaseEntriesMock.mockReturnValue([]); + }); + + it('does not call removeKnowledgeBaseEntries when the package is not installed', async () => { + const packageInstallContext = createInstallContext({}); + + await installKibanaKnowledgeBaseEntries({ + packageInstallContext, + savedObjectsClient, + esClient, + logger, + installedPkg: undefined, + }); + + expect(removeKnowledgeBaseEntriesMock).not.toHaveBeenCalled(); + }); + + it('calls removeKnowledgeBaseEntries when the package is installed and has content entries', async () => { + const packageInstallContext = createInstallContext({}); + const installedPkg = createInstalledPkgObject({ + installed_misc: [ + { type: KibanaMiscAssetTypes.knowledgeBaseEntry, id: 'foo' }, + { type: KibanaMiscAssetTypes.knowledgeBaseEntry, id: 'bar' }, + ], + }); + + await installKibanaKnowledgeBaseEntries({ + packageInstallContext, + savedObjectsClient, + esClient, + logger, + installedPkg, + }); + + expect(removeKnowledgeBaseEntriesMock).toHaveBeenCalledTimes(1); + expect(removeKnowledgeBaseEntriesMock).toHaveBeenCalledWith({ + installedObjects: installedPkg.attributes.installed_misc, + packageName: packageInstallContext.packageInfo.name, + savedObjectsClient, + esClient, + }); + }); + + it('does not call underlying methods when no knowledge base entries are present', async () => { + parseKnowledgeBaseEntriesMock.mockReturnValue([]); + + const packageInstallContext = createInstallContext({}); + const installedPkg = createInstalledPkgObject({}); + + await installKibanaKnowledgeBaseEntries({ + packageInstallContext, + savedObjectsClient, + esClient, + logger, + installedPkg, + }); + + expect(updateMiscAssetReferencesMock).not.toHaveBeenCalled(); + expect(esClient.indices.create).not.toHaveBeenCalled(); + expect(savedObjectsClient.create).not.toHaveBeenCalled(); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).not.toHaveBeenCalled(); + }); + + it('installs the knowledge base entries from the package', async () => { + parseKnowledgeBaseEntriesMock.mockReturnValue([ + { + name: 'foo', + folderPath: 'test-pkg/kibana/knowledge_base_entry/foo', + manifest: { + title: 'foo', + description: 'desc', + index: { + system: true, + }, + retrieval: { + syntactic_fields: [], + semantic_fields: [], + }, + }, + fields: [], + contentFilePaths: [ + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-1.ndjson', + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-2.ndjson', + ], + }, + ]); + + const packageInstallContext = createInstallContext({ + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-1.ndjson': createContentFile(), + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-2.ndjson': createContentFile(), + }); + const installedPkg = createInstalledPkgObject({}); + + await installKibanaKnowledgeBaseEntries({ + packageInstallContext, + savedObjectsClient, + esClient, + logger, + installedPkg, + }); + + expect(esClient.indices.create).toHaveBeenCalledTimes(1); + expect(esClient.indices.create).toHaveBeenCalledWith({ + index: '.kibana-test-pkg_foo', + mappings: { + dynamic: false, + properties: {}, + }, + }); + + expect(savedObjectsClient.create).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.create).toHaveBeenCalledWith( + knowledgeBaseEntrySavedObjectType, + { + name: 'foo', + type: 'index', + description: 'desc', + }, + { id: 'entry_test-pkg_foo' } + ); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledTimes(1); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'create', + id: 'entry_test-pkg_foo', + savedObjectType: knowledgeBaseEntrySavedObjectType, + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/install.ts new file mode 100644 index 0000000000000..39c36be1d7662 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/install.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { + ElasticsearchClient, + SavedObject, + SavedObjectsClientContract, + Logger, +} from '@kbn/core/server'; +import { auditLoggingService } from '../../../audit_logging'; +import { getAssetFromAssetsMap } from '../../archive'; +import { KibanaMiscAssetTypes } from '../../../../types'; +import type { Installation, KnowledgeBaseMiscAssetReference } from '../../../../types'; +import type { MiscAssetReference, PackageInstallContext } from '../../../../../common/types'; +import { updateMiscAssetReferences } from '../../packages/misc_assets_reference'; +import { parseKnowledgeBaseEntries, type KnowledgeBaseEntryInfo } from './parse_entries'; +import { generateMappings } from '../../elasticsearch/template/template'; +import { getSavedObjectId, getIndexName } from './utils'; +import { knowledgeBaseEntrySavedObjectType } from './consts'; +import { removeKnowledgeBaseEntries } from './remove'; + +interface InstallKibanaKnowledgeBaseEntriesOptions { + packageInstallContext: PackageInstallContext; + savedObjectsClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; + logger: Logger; + installedPkg?: SavedObject; +} + +export async function installKibanaKnowledgeBaseEntries( + options: InstallKibanaKnowledgeBaseEntriesOptions +): Promise { + const { packageInstallContext, installedPkg, savedObjectsClient, esClient } = options; + + if (installedPkg?.attributes.installed_misc?.length) { + await removeKnowledgeBaseEntries({ + installedObjects: installedPkg.attributes.installed_misc, + packageName: packageInstallContext.packageInfo.name, + savedObjectsClient, + esClient, + }); + } + + const entries = parseKnowledgeBaseEntries(packageInstallContext); + if (entries.length === 0) { + return []; + } + + const references: KnowledgeBaseMiscAssetReference[] = entries.map((entry) => { + return { + id: entry.name, + type: KibanaMiscAssetTypes.knowledgeBaseEntry, + system: entry.manifest.index?.system ?? true, + }; + }); + + await updateMiscAssetReferences( + savedObjectsClient, + packageInstallContext.packageInfo.name, + installedPkg?.attributes.installed_misc ?? [], + { + assetsToAdd: references, + } + ); + + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + await installKibanaKnowledgeBaseEntry(entry, options); + } + + return references; +} + +async function installKibanaKnowledgeBaseEntry( + entry: KnowledgeBaseEntryInfo, + options: InstallKibanaKnowledgeBaseEntriesOptions +) { + const { packageInstallContext, esClient, savedObjectsClient } = options; + + // create the index + const indexName = getIndexName({ + system: entry.manifest.index?.system ?? false, + entryName: entry.name, + packageName: packageInstallContext.packageInfo.name, + }); + const { properties: mappingProperties } = generateMappings(entry.fields); + await esClient.indices.create({ + index: indexName, + mappings: { + dynamic: false, + properties: mappingProperties, + }, + }); + + // populate the index + for (const contentFilePath of entry.contentFilePaths) { + await indexContentFile({ + indexName, + esClient, + contentBuffer: getAssetFromAssetsMap(packageInstallContext.assetsMap, contentFilePath), + }); + } + + // create the saved object entry + const savedObjectId = getSavedObjectId({ + entryName: entry.name, + packageName: packageInstallContext.packageInfo.name, + }); + auditLoggingService.writeCustomSoAuditLog({ + action: 'create', + id: savedObjectId, + savedObjectType: knowledgeBaseEntrySavedObjectType, + }); + await savedObjectsClient.create( + knowledgeBaseEntrySavedObjectType, + { + // TODO: update the props to the full set once the KB PR has been merged + name: entry.manifest.title, + type: 'index', + description: entry.manifest.description, + }, + { id: savedObjectId } + ); +} + +const indexContentFile = async ({ + indexName, + contentBuffer, + esClient, +}: { + indexName: string; + contentBuffer: Buffer; + esClient: ElasticsearchClient; +}) => { + const fileContent = contentBuffer.toString('utf-8'); + const lines = fileContent.split('\n'); + + const documents = lines + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + return JSON.parse(line); + }); + + const operations = documents.reduce((ops, document) => { + ops!.push(...[{ index: { _index: indexName } }, document]); + return ops; + }, [] as BulkRequest['operations']); + + await esClient.bulk({ + refresh: false, + operations, + }); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/parse_entries.test.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/parse_entries.test.ts new file mode 100644 index 0000000000000..48aede8a5839f --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/parse_entries.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { times } from 'lodash'; +import type { PackageInstallContext } from '../../../../../common/types'; +import { parseKnowledgeBaseEntries } from './parse_entries'; + +export const createManifest = ({ + title, + description = 'description', + systemIndex = true, + syntacticFields = ['field_1', 'field_2'], +}: { + title: string; + description?: string; + systemIndex?: boolean; + syntacticFields?: string[]; +}) => { + return ` + title: ${title} + description: ${description} + index: + system: ${systemIndex} + retrieval: + syntactic_fields: [${syntacticFields.join(',')}] + semantic_fields: [] + `; +}; + +export const createFieldsFile = () => { + return ` + - name: content_title + type: text + description: The title of the document + - name: content_body + type: semantic_text + inference_id: kibana-elser2 + description: The content of the document + `; +}; + +export const createContentFile = (length = 5) => { + return times(5) + .map(() => '{}') + .join('\n'); +}; + +describe('parseKnowledgeBaseEntries', () => { + const createInstallContext = (assets: Record): PackageInstallContext => { + const installContext: PackageInstallContext = { + packageInfo: { + format_version: '3.4.0', + name: 'test-pkg', + title: 'Test Pkg', + description: 'Some desc', + version: '0.0.1', + owner: { github: 'owner', type: 'elastic' }, + }, + assetsMap: new Map(), + paths: [], + }; + + Object.entries(assets).forEach(([assetPath, assetContent]) => { + installContext.paths.push(assetPath); + installContext.assetsMap.set(assetPath, Buffer.from(assetContent)); + }); + + return installContext; + }; + + it('parses a single entry package', async () => { + const assets: Record = { + 'test-pkg/kibana/knowledge_base_entry/foo/manifest.yml': createManifest({ title: 'foo' }), + 'test-pkg/kibana/knowledge_base_entry/foo/fields/fields.yml': createFieldsFile(), + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-1.ndjson': createContentFile(), + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-2.ndjson': createContentFile(), + }; + const installContext = createInstallContext(assets); + + const parsed = parseKnowledgeBaseEntries(installContext); + + const entryInfo = parsed[0]; + + expect(entryInfo.name).toEqual('foo'); + expect(entryInfo.manifest).toEqual({ + title: 'foo', + description: 'description', + index: { + system: true, + }, + retrieval: { + semantic_fields: [], + syntactic_fields: ['field_1', 'field_2'], + }, + }); + expect(entryInfo.folderPath).toEqual('test-pkg/kibana/knowledge_base_entry/foo'); + expect(entryInfo.fields.map((field) => field.name)).toEqual(['content_title', 'content_body']); + expect(entryInfo.contentFilePaths).toEqual([ + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-1.ndjson', + 'test-pkg/kibana/knowledge_base_entry/foo/content/content-2.ndjson', + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/parse_entries.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/parse_entries.ts new file mode 100644 index 0000000000000..48765ccd4fe9b --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/parse_entries.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { load } from 'js-yaml'; +import { getAssetFromAssetsMap, getPathParts } from '../../archive'; +import { KibanaMiscAssetTypes } from '../../../../types'; +import type { PackageInstallContext } from '../../../../../common/types'; +import { + loadKnowledgeBaseEntryFieldsFromYaml, + processFields, + type Fields, +} from '../../fields/field'; + +export const parseKnowledgeBaseEntries = ( + packageInstallContext: PackageInstallContext +): KnowledgeBaseEntryInfo[] => { + const manifestPaths = packageInstallContext.paths.filter(isKnowledgeBaseManifest); + return manifestPaths.map((manifestPath) => { + return getEntryInfoFromManifestPath(manifestPath, packageInstallContext); + }); +}; + +interface KnowledgeBaseEntryManifest { + title: string; + description: string; + index: { + system: boolean; + }; + retrieval: { + syntactic_fields: string[]; + semantic_fields: string[]; + }; +} + +export interface KnowledgeBaseEntryInfo { + name: string; + folderPath: string; + manifest: KnowledgeBaseEntryManifest; + fields: Fields; + contentFilePaths: string[]; +} + +const getEntryInfoFromManifestPath = ( + manifestPath: string, + packageInstallContext: PackageInstallContext +): KnowledgeBaseEntryInfo => { + const entryInfo = entryPathsFromManifestPath(manifestPath); + const rawFields = loadKnowledgeBaseEntryFieldsFromYaml( + packageInstallContext, + entryInfo.entryName + ); + const fields = processFields(rawFields); + const contentFilePaths = packageInstallContext.paths.filter((path) => + path.includes(entryInfo.contentFolderPath) + ); + + const manifest: KnowledgeBaseEntryManifest = load( + getAssetFromAssetsMap(packageInstallContext.assetsMap, manifestPath).toString() + ); + + return { + name: entryInfo.entryName, + folderPath: entryInfo.rootFolderPath, + fields, + manifest, + contentFilePaths, + }; +}; + +const entryPathsFromManifestPath = (manifestPath: string) => { + const splits = manifestPath.split('/'); + const rootFolderPath = splits.slice(0, splits.length - 1).join('/'); + const entryName = splits[splits.length - 2]; + const contentFolderPath = [rootFolderPath, 'content'].join('/'); + + return { + entryName, + rootFolderPath, + contentFolderPath, + }; +}; + +const isKnowledgeBaseManifest = (path: string): boolean => { + // does not support an additional level + const pathParts = getPathParts(path); + return ( + pathParts.service === 'kibana' && + pathParts.type === KibanaMiscAssetTypes.knowledgeBaseEntry && + pathParts.file === 'manifest.yml' + ); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/remove.test.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/remove.test.ts new file mode 100644 index 0000000000000..edb23f3686968 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/remove.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { KibanaMiscAssetTypes, type MiscAssetReference } from '../../../../types'; +import { auditLoggingService } from '../../../audit_logging'; +import { knowledgeBaseEntrySavedObjectType } from './consts'; +import { removeKnowledgeBaseEntries } from './remove'; + +jest.mock('../../../audit_logging'); + +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +describe('removeKnowledgeBaseEntries', () => { + let esClient: ReturnType; + let savedObjectsClient: ReturnType; + + const installedAssets: MiscAssetReference[] = [ + { + type: KibanaMiscAssetTypes.knowledgeBaseEntry, + id: 'kb-entry-1', + }, + { + type: KibanaMiscAssetTypes.knowledgeBaseEntry, + id: 'kb-entry-2', + }, + ]; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + savedObjectsClient = savedObjectsClientMock.create(); + + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + }); + + it('calls esClient.indices.delete with the right parameters', async () => { + await removeKnowledgeBaseEntries({ + installedObjects: installedAssets, + packageName: 'my-package', + esClient, + savedObjectsClient, + }); + + expect(esClient.indices.delete).toHaveBeenCalledTimes(1); + expect(esClient.indices.delete).toHaveBeenCalledWith( + { index: ['.kibana-my-package_kb-entry-1', '.kibana-my-package_kb-entry-2'] }, + { ignore: [404] } + ); + }); + + it('calls auditLoggingService.writeCustomSoAuditLog once per entry', async () => { + await removeKnowledgeBaseEntries({ + installedObjects: installedAssets, + packageName: 'my-package', + esClient, + savedObjectsClient, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledTimes(2); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'delete', + id: 'entry_my-package_kb-entry-1', + savedObjectType: knowledgeBaseEntrySavedObjectType, + }); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'delete', + id: 'entry_my-package_kb-entry-2', + savedObjectType: knowledgeBaseEntrySavedObjectType, + }); + }); + + it('calls soClient.bulkDelete with the right parameters', async () => { + await removeKnowledgeBaseEntries({ + installedObjects: installedAssets, + packageName: 'my-package', + esClient, + savedObjectsClient, + }); + + expect(savedObjectsClient.bulkDelete).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.bulkDelete).toHaveBeenCalledWith( + [ + { id: 'entry_my-package_kb-entry-1', type: knowledgeBaseEntrySavedObjectType }, + { id: 'entry_my-package_kb-entry-2', type: knowledgeBaseEntrySavedObjectType }, + ], + { force: true } + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/remove.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/remove.ts new file mode 100644 index 0000000000000..20ece79b1c90d --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/remove.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; +import type { MiscAssetReference } from '../../../../types'; +import { auditLoggingService } from '../../../audit_logging'; +import { getSavedObjectId, getIndexName, isKnowledgeBaseEntryReference } from './utils'; +import { knowledgeBaseEntrySavedObjectType } from './consts'; + +export const removeKnowledgeBaseEntries = async ({ + installedObjects, + savedObjectsClient, + esClient, + packageName, +}: { + installedObjects: MiscAssetReference[]; + esClient: ElasticsearchClient; + savedObjectsClient: SavedObjectsClientContract; + packageName: string; +}) => { + const knowledgeBaseEntryAssets = installedObjects.filter(isKnowledgeBaseEntryReference); + + const indicesToDelete = knowledgeBaseEntryAssets.map((entry) => { + return getIndexName({ + system: entry.system ?? true, + entryName: entry.id, + packageName, + }); + }); + + const savedObjectsToDelete = knowledgeBaseEntryAssets.map((entry) => { + return { + id: getSavedObjectId({ + entryName: entry.id, + packageName, + }), + type: knowledgeBaseEntrySavedObjectType, + }; + }); + + await esClient.indices.delete( + { + index: indicesToDelete, + }, + { ignore: [404] } + ); + + savedObjectsToDelete.forEach((asset) => { + auditLoggingService.writeCustomSoAuditLog({ + action: 'delete', + id: asset.id, + savedObjectType: knowledgeBaseEntrySavedObjectType, + }); + }); + + await savedObjectsClient.bulkDelete(savedObjectsToDelete, { force: true }); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/utils.test.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/utils.test.ts new file mode 100644 index 0000000000000..39945764ff0e3 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/utils.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + KibanaMiscAssetTypes, + KibanaSavedObjectType, + ElasticsearchAssetType, +} from '../../../../types'; +import { getIndexName, getSavedObjectId, isKnowledgeBaseEntryReference } from './utils'; + +describe('getIndexName', () => { + it('returns the correct index name for system index', () => { + expect( + getIndexName({ + packageName: 'kibana-kb', + entryName: 'kibana-8.15', + system: true, + }) + ).toEqual('.kibana-kibana-kb_kibana-8.15'); + }); + it('returns the correct index name for non-system index', () => { + expect( + getIndexName({ + packageName: 'es-kb', + entryName: 'es-8.15', + system: false, + }) + ).toEqual('es-kb_es-8.15'); + }); +}); + +describe('getSavedObjectId', () => { + it('returns the expected Id', () => { + expect( + getSavedObjectId({ + packageName: 'kibana-kb', + entryName: 'kibana-8.15', + }) + ).toEqual('entry_kibana-kb_kibana-8.15'); + }); +}); + +describe('isKnowledgeBaseEntryReference', () => { + it('returns true for knowledgeBaseEntry references', () => { + expect( + isKnowledgeBaseEntryReference({ + type: KibanaMiscAssetTypes.knowledgeBaseEntry, + id: 'some-id', + }) + ).toBe(true); + }); + + it('returns false for Kibana asset references', () => { + expect( + isKnowledgeBaseEntryReference({ + type: KibanaSavedObjectType.map, + id: 'some-id', + }) + ).toBe(false); + }); + + it('returns false for ES asset references', () => { + expect( + isKnowledgeBaseEntryReference({ + type: ElasticsearchAssetType.ingestPipeline, + id: 'some-id', + }) + ).toBe(false); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/utils.ts b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/utils.ts new file mode 100644 index 0000000000000..13b1841716572 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/utils.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + KibanaMiscAssetTypes, + type AssetReference, + type KnowledgeBaseMiscAssetReference, +} from '../../../../types'; + +export const getIndexName = ({ + entryName, + packageName, + system, +}: { + packageName: string; + system: boolean; + entryName: string; +}): string => { + const prefix = system ? '.kibana-' : ''; + return `${prefix}${packageName}_${entryName}`; +}; + +export const getSavedObjectId = ({ + entryName, + packageName, +}: { + packageName: string; + entryName: string; +}): string => { + return `entry_${packageName}_${entryName}`; +}; + +export function isKnowledgeBaseEntryReference( + reference: AssetReference +): reference is KnowledgeBaseMiscAssetReference { + return reference.type === KibanaMiscAssetTypes.knowledgeBaseEntry; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 65f1a75f76f84..f91b14977cbcd 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -1113,6 +1113,7 @@ export async function createInstallation(options: { installed_kibana: [], installed_kibana_space_id: options.spaceId, installed_es: [], + installed_misc: [], package_assets: [], es_index_patterns: toSaveESIndexPatterns, name: pkgName, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts index 174076a9e9b1b..df087ed32cba9 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts @@ -27,6 +27,7 @@ import { saveArchiveEntriesFromAssetsMap } from '../../archive/storage'; jest.mock('../../elasticsearch/template/template'); jest.mock('../../kibana/assets/install'); jest.mock('../../kibana/index_pattern/install'); +jest.mock('../../kibana/knowledge_base/install'); jest.mock('../get'); jest.mock('../install_index_template_pipeline'); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts index 1f10d40feba38..ec6179dd467c8 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts @@ -26,6 +26,7 @@ import type { PackageVerificationResult, EsAssetReference, KibanaAssetReference, + MiscAssetReference, IndexTemplateEntry, AssetReference, } from '../../../../types'; @@ -35,6 +36,7 @@ import { appContextService } from '../../..'; import { stepCreateRestartInstallation, stepInstallKibanaAssets, + stepInstallKnowledgeBaseAssets, stepInstallILMPolicies, stepInstallMlModel, stepInstallIndexTemplatePipelines, @@ -48,6 +50,7 @@ import { updateLatestExecutedState, cleanupLatestExecutedState, cleanUpKibanaAssetsStep, + cleanUpKnowledgeBaseAssetsStep, cleanupILMPoliciesStep, cleanUpMlModelStep, cleanupIndexTemplatePipelinesStep, @@ -77,6 +80,7 @@ export interface InstallContext extends StateContext { indexTemplates?: IndexTemplateEntry[]; packageAssetRefs?: PackageAssetReference[]; // output values + miscReferences?: MiscAssetReference[]; esReferences?: EsAssetReference[]; kibanaAssetPromise?: Promise; } @@ -92,6 +96,12 @@ const statesDefinition: StateMachineStates = { install_kibana_assets: { onPreTransition: cleanUpKibanaAssetsStep, onTransition: stepInstallKibanaAssets, + nextState: INSTALL_STATES.INSTALL_KNOWLEDGE_BASE_ASSETS, + onPostTransition: updateLatestExecutedState, + }, + install_knowledge_base_assets: { + onPreTransition: cleanUpKnowledgeBaseAssetsStep, + onTransition: stepInstallKnowledgeBaseAssets, nextState: INSTALL_STATES.INSTALL_ILM_POLICIES, onPostTransition: updateLatestExecutedState, }, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts index c34c4f566715b..d8f54581fcd03 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts @@ -18,3 +18,4 @@ export * from './step_save_archive_entries'; export * from './step_save_system_object'; export * from './step_resolve_kibana_promise'; export * from './update_latest_executed_state'; +export * from './step_install_knowledge_base_assets'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_knowledge_base_assets.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_knowledge_base_assets.test.ts new file mode 100644 index 0000000000000..4a6f540514e25 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_knowledge_base_assets.test.ts @@ -0,0 +1,301 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installKibanaKnowledgeBaseEntries } from '../../../kibana/knowledge_base/install'; +import { removeKnowledgeBaseEntries } from '../../../kibana/knowledge_base/remove'; +import { KibanaMiscAssetTypes, type Installation } from '../../../../../types'; +import { + stepInstallKnowledgeBaseAssets, + cleanUpKnowledgeBaseAssetsStep, +} from './step_install_knowledge_base_assets'; + +jest.mock('../../../kibana/knowledge_base/install'); +jest.mock('../../../kibana/knowledge_base/remove'); + +const mockedInstallKibanaKnowledgeBaseEntries = + installKibanaKnowledgeBaseEntries as jest.MockedFunction< + typeof installKibanaKnowledgeBaseEntries + >; +const mockedRemoveKnowledgeBaseEntries = removeKnowledgeBaseEntries as jest.MockedFunction< + typeof removeKnowledgeBaseEntries +>; + +let soClient: jest.Mocked; +let esClient: jest.Mocked; + +const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), +}; + +describe('stepInstallKnowledgeBaseAssets', () => { + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + + soClient.update.mockImplementation(async (type, id, attributes) => { + return { id, attributes } as any; + }); + soClient.get.mockImplementation(async (type, id) => { + return { id, attributes: {} } as any; + }); + }); + + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + + it('Should call installKibanaAssetsAndReferences', async () => { + const installationPromise = stepInstallKnowledgeBaseAssets({ + savedObjectsClient: soClient, + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + await expect(installationPromise).resolves.not.toThrowError(); + expect(mockedInstallKibanaKnowledgeBaseEntries).toBeCalledTimes(1); + }); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + + it('Should correctly rethrow errors', async () => { + // force errors from this function + mockedInstallKibanaKnowledgeBaseEntries.mockImplementation(async () => { + throw new Error('mocked async error A: should be caught'); + }); + + const installationPromise = stepInstallKnowledgeBaseAssets({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + await expect(installationPromise).rejects.toThrowErrorMatchingInlineSnapshot( + `"mocked async error A: should be caught"` + ); + }); +}); + +describe('cleanUpKibanaAssetsStep', () => { + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + mockedRemoveKnowledgeBaseEntries.mockReset(); + }); + const installedMisc = [{ type: KibanaMiscAssetTypes.knowledgeBaseEntry, id: 'kb-1' }]; + + it('should clean up kibana assets previously installed', async () => { + await cleanUpKnowledgeBaseAssetsStep({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + installed_misc: installedMisc as any, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + retryFromLastState: true, + initialState: 'install_knowledge_base_assets' as any, + }); + + expect(mockedRemoveKnowledgeBaseEntries).toBeCalledWith({ + installedObjects: installedMisc, + savedObjectsClient: soClient, + esClient, + packageName: packageInstallContext.packageInfo.name, + }); + }); + + it('should not clean up assets if force is passed', async () => { + await cleanUpKnowledgeBaseAssetsStep({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + installed_misc: installedMisc as any, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + force: true, + retryFromLastState: true, + initialState: 'install_kibana_assets' as any, + }); + + expect(mockedRemoveKnowledgeBaseEntries).not.toBeCalled(); + }); + + it('should not clean up assets if retryFromLastState is not passed', async () => { + await cleanUpKnowledgeBaseAssetsStep({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + installed_misc: installedMisc as any, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + initialState: 'install_kibana_assets' as any, + }); + + expect(mockedRemoveKnowledgeBaseEntries).not.toBeCalled(); + }); + + it('should not clean up assets if initialState != install_kibana_assets', async () => { + await cleanUpKnowledgeBaseAssetsStep({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + installed_misc: installedMisc as any, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + retryFromLastState: true, + initialState: 'create_restart_install' as any, + }); + + expect(mockedRemoveKnowledgeBaseEntries).not.toBeCalled(); + }); + + it('should not clean up assets if attributes are not present', async () => { + await cleanUpKnowledgeBaseAssetsStep({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + retryFromLastState: true, + initialState: 'install_kibana_assets' as any, + }); + + expect(mockedRemoveKnowledgeBaseEntries).not.toBeCalled(); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_knowledge_base_assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_knowledge_base_assets.ts new file mode 100644 index 0000000000000..25bbbc6e188cb --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_knowledge_base_assets.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installKibanaKnowledgeBaseEntries } from '../../../kibana/knowledge_base/install'; +import { removeKnowledgeBaseEntries } from '../../../kibana/knowledge_base/remove'; +import { withPackageSpan } from '../../utils'; +import type { InstallContext } from '../_state_machine_package_install'; +import { INSTALL_STATES } from '../../../../../../common/types'; + +export async function stepInstallKnowledgeBaseAssets(context: InstallContext) { + const { savedObjectsClient, logger, installedPkg, packageInstallContext, esClient } = context; + + const { miscReferences = [] } = context; + const references = await withPackageSpan('Install knowledge base assets', () => + installKibanaKnowledgeBaseEntries({ + savedObjectsClient, + packageInstallContext, + installedPkg, + logger, + esClient, + }) + ); + + return { miscReferences: [...miscReferences, references] }; +} + +export async function cleanUpKnowledgeBaseAssetsStep(context: InstallContext) { + const { + logger, + installedPkg, + retryFromLastState, + force, + initialState, + esClient, + savedObjectsClient, + } = context; + + // In case of retry clean up previous installed knowledge base assets + if ( + !force && + retryFromLastState && + initialState === INSTALL_STATES.INSTALL_KNOWLEDGE_BASE_ASSETS && + installedPkg?.attributes?.installed_misc && + installedPkg.attributes.installed_misc?.length > 0 + ) { + const { installed_misc: installedObjects = [] } = installedPkg.attributes; + logger.debug('Retry transition - clean up knowledge base assets first'); + + await withPackageSpan('Retry transition - clean up knowledge base assets first', async () => { + await removeKnowledgeBaseEntries({ + installedObjects, + savedObjectsClient, + esClient, + packageName: installedPkg.attributes.name, + }); + }); + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/misc_assets_reference.ts b/x-pack/plugins/fleet/server/services/epm/packages/misc_assets_reference.ts new file mode 100644 index 0000000000000..e95a3c32ed790 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/misc_assets_reference.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pRetry from 'p-retry'; +import { uniqBy } from 'lodash'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { MiscAssetReference, Installation } from '../../../types'; +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; +import { auditLoggingService } from '../../audit_logging'; + +/** + * Utility function for updating the installed_misc field of a package + */ +export const updateMiscAssetReferences = async ( + savedObjectsClient: SavedObjectsClientContract, + pkgName: string, + currentAssets: MiscAssetReference[], + { + assetsToAdd = [], + assetsToRemove = [], + refresh = false, + }: { + assetsToAdd?: MiscAssetReference[]; + assetsToRemove?: MiscAssetReference[]; + /** + * Whether or not the update should force a refresh on the SO index. + * Defaults to `false` for faster updates, should only be `wait_for` if the update needs to be queried back from ES + * immediately. + */ + refresh?: 'wait_for' | false; + } +): Promise => { + const withAssetsRemoved = currentAssets.filter(({ type, id }) => { + if ( + assetsToRemove.some( + ({ type: removeType, id: removeId }) => removeType === type && removeId === id + ) + ) { + return false; + } + return true; + }); + + const deduplicatedAssets = uniqBy( + [...withAssetsRemoved, ...assetsToAdd], + ({ type, id }) => `${type}-${id}` + ); + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + + const { + attributes: { installed_misc: updatedAssets }, + } = + // Because Kibana assets are installed in parallel with ES assets with refresh: false, we almost always run into an + // issue that causes a conflict error due to this issue: https://github.com/elastic/kibana/issues/126240. This is safe + // to retry constantly until it succeeds to optimize this critical user journey path as much as possible. + await pRetry( + () => + savedObjectsClient.update( + PACKAGES_SAVED_OBJECT_TYPE, + pkgName, + { + installed_misc: deduplicatedAssets, + }, + { + refresh, + } + ), + // Use a lower number of retries for ES assets since they're installed in serial and can only conflict with + // the single Kibana update call. + { retries: 5 } + ); + + return updatedAssets ?? []; +}; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts index 2886774a32d12..2f0dba8808c9b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts @@ -21,6 +21,9 @@ jest.mock('../..', () => { warn: jest.fn(), }), getInternalUserSOClientWithoutSpaceExtension: jest.fn(), + getSavedObjects: jest.fn().mockReturnValue({ + createInternalRepository: jest.fn(), + }), }, packagePolicyService: { list: jest.fn().mockImplementation((soClient, params) => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index ac3f5def5d09c..dbd57de12a9db 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -37,6 +37,7 @@ import type { } from '../../../types'; import { deletePipeline } from '../elasticsearch/ingest_pipeline'; import { removeUnusedIndexPatterns } from '../kibana/index_pattern/install'; +import { removeKnowledgeBaseEntries } from '../kibana/knowledge_base/remove'; import { deleteTransforms } from '../elasticsearch/transform/remove'; import { deleteMlModel } from '../elasticsearch/ml_model'; import { packagePolicyService, appContextService } from '../..'; @@ -94,7 +95,11 @@ export async function removeInstallation(options: { } // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users - const installedAssets = [...installation.installed_kibana, ...installation.installed_es]; + const installedAssets = [ + ...installation.installed_kibana, + ...installation.installed_es, + ...(installation.installed_misc ?? []), + ]; await deleteAssets(installation, esClient); // Delete the manager saved object with references to the asset objects @@ -311,6 +316,7 @@ async function deleteAssets( { installed_es: installedEs, installed_kibana: installedKibana, + installed_misc: installedMisc = [], installed_kibana_space_id: spaceId = DEFAULT_SPACE_ID, additional_spaces_installed_kibana: installedInAdditionalSpacesKibana = {}, name, @@ -332,12 +338,22 @@ async function deleteAssets( esClient ); + const savedObjectsClient = new SavedObjectsClient( + appContextService.getSavedObjects().createInternalRepository() + ); + // delete the other asset types try { const packageInfo = await Registry.fetchInfo(name, version); await Promise.all([ ...deleteESAssets(otherAssets, esClient), deleteKibanaAssets({ installedObjects: installedKibana, spaceId, packageInfo }), + removeKnowledgeBaseEntries({ + installedObjects: installedMisc, + packageName: packageInfo.name, + esClient, + savedObjectsClient, + }), Object.entries(installedInAdditionalSpacesKibana).map(([additionalSpaceId, kibanaAssets]) => deleteKibanaAssets({ installedObjects: kibanaAssets, diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 786db010c8eed..28b11e2049071 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -50,6 +50,8 @@ export type { AssetReference, EsAssetReference, KibanaAssetReference, + MiscAssetReference, + KnowledgeBaseMiscAssetReference, RegistryPackage, BundledPackage, InstallablePackage, @@ -102,7 +104,12 @@ export type { TemplateAgentPolicyInput, NewPackagePolicyInput, } from '../../common/types'; -export { ElasticsearchAssetType, KibanaAssetType, KibanaSavedObjectType } from '../../common/types'; +export { + ElasticsearchAssetType, + KibanaAssetType, + KibanaSavedObjectType, + KibanaMiscAssetTypes, +} from '../../common/types'; export { dataTypes } from '../../common/constants'; export type AgentPolicyUpdateHandler = ( diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index fc8225e9df02d..4d8819ec7dc5a 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -660,6 +660,7 @@ const expectAssetsInstalled = ({ type: 'ml_model', }, ], + installed_misc: [], package_assets: [ { id: '333a22a1-e639-5af5-ae62-907ffc83d603', diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 17d54786245af..c41506f955f36 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -455,6 +455,7 @@ export default function (providerContext: FtrProviderContext) { type: 'component_template', }, ], + installed_misc: [], es_index_patterns: { test_logs: 'logs-all_assets.test_logs-*', test_metrics: 'metrics-all_assets.test_metrics-*',