[fleet] Add support for knowledge base installation

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -338,6 +338,7 @@
     "installed_es.version",
     "installed_kibana",
     "installed_kibana_space_id",
+    "installed_misc",
     "internal",
     "keep_policies_up_to_date",
     "latest_executed_state",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1143,6 +1143,10 @@
       "installed_kibana_space_id": {
         "type": "keyword"
       },
+      "installed_misc": {
+        "dynamic": false,
+        "properties": {}
+      },
       "internal": {
         "type": "boolean"
       },

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -94,7 +94,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
         "entity-engine-status": "0738aa1a06d3361911740f8f166071ea43a00927",
-        "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
+        "epm-packages": "a8071c1e2b9ddeb3a26acd9ef02b5155791bffc4",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",
         "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88",

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -44,10 +44,18 @@ export type AgentAssetType = typeof agentAssetTypes;
 export type DocAssetType = 'doc' | 'notice' | 'license';
 export type AssetType =
   | KibanaAssetType
+  | KibanaMiscAssetTypes
   | ElasticsearchAssetType
   | ValueOf<AgentAssetType>
   | DocAssetType;
 
+/*
+  Enum mapping of asset types living under the Kibana service/folder and that are not saved objects.
+*/
+export enum KibanaMiscAssetTypes {
+  knowledgeBaseEntry = 'knowledge_base_entry',
+}
+
 /*
   Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased)
 */
@@ -566,6 +574,7 @@ export interface InstallFailedAttempt {
 export enum INSTALL_STATES {
   CREATE_RESTART_INSTALLATION = 'create_restart_installation',
   INSTALL_KIBANA_ASSETS = 'install_kibana_assets',
+  INSTALL_KNOWLEDGE_BASE_ASSETS = 'install_knowledge_base_assets',
   INSTALL_ILM_POLICIES = 'install_ilm_policies',
   INSTALL_ML_MODEL = 'install_ml_model',
   INSTALL_INDEX_TEMPLATE_PIPELINES = 'install_index_template_pipelines',
@@ -597,6 +606,7 @@ export interface Installation {
   installed_kibana: KibanaAssetReference[];
   additional_spaces_installed_kibana?: Record<string, KibanaAssetReference[]>;
   installed_es: EsAssetReference[];
+  installed_misc?: MiscAssetReference[];
   package_assets?: PackageAssetReference[];
   es_index_patterns: Record<string, string>;
   name: string;
@@ -652,7 +662,7 @@ export type InstallFailed<T = {}> = T & {
   status: InstallationStatus['InstallFailed'];
 };
 
-export type AssetReference = KibanaAssetReference | EsAssetReference;
+export type AssetReference = KibanaAssetReference | EsAssetReference | MiscAssetReference;
 
 export interface KibanaAssetReference {
   id: string;
@@ -665,6 +675,15 @@ export interface EsAssetReference {
   deferred?: boolean;
 }
 
+export interface KnowledgeBaseMiscAssetReference {
+  id: string;
+  type: KibanaMiscAssetTypes.knowledgeBaseEntry;
+  system?: boolean;
+}
+
+// polymorphic type, even if only one subtype for now
+export type MiscAssetReference = KnowledgeBaseMiscAssetReference;
+
 export interface PackageAssetReference {
   id: string;
   type: typeof ASSETS_SAVED_OBJECT_TYPE;

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -827,6 +827,10 @@ export const getSavedObjectTypes = (
             dynamic: false,
             properties: {},
           },
+          installed_misc: {
+            dynamic: false,
+            properties: {},
+          },
           installed_kibana_space_id: { type: 'keyword' },
           package_assets: {
             dynamic: false,
@@ -888,6 +892,19 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '4': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                installed_misc: {
+                  dynamic: false,
+                  properties: {},
+                },
+              },
+            },
+          ],
+        },
       },
       migrations: {
         '7.14.0': migrateInstallationToV7140,

x-pack/plugins/fleet/server/services/epm/archive/index.test.ts

@@ -44,6 +44,17 @@ describe('getPathParts', () => {
         type: 'fields',
       },
     },
+    {
+      path: 'knowledgebase-1.0.1/kibana/knowledge_base_entry/foo/manifest.yml',
+      assetParts: {
+        dataset: 'foo',
+        file: 'manifest.yml',
+        path: 'knowledgebase-1.0.1/kibana/knowledge_base_entry/foo/manifest.yml',
+        pkgkey: 'knowledgebase-1.0.1',
+        service: 'kibana',
+        type: 'knowledge_base_entry',
+      },
+    },
   ];
   test('testPathParts', () => {
     for (const value of testPaths) {

x-pack/plugins/fleet/server/services/epm/archive/index.ts

@@ -99,6 +99,13 @@ export function getPathParts(path: string): AssetParts {
     [pkgkey, service, type, file] = path.replace(`data_stream/${dataset}/`, '').split('/');
   }
 
+  // if it's a knowledge base entry
+  if (type === 'knowledge_base_entry') {
+    // there is an additional depth level
+    dataset = file;
+    file = path.split('/')[4];
+  }
+
   // To support the NOTICE asset at the root level
   if (service === 'NOTICE.txt') {
     file = service;

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -1715,6 +1715,31 @@ describe('EPM template', () => {
     expect(mappings).toEqual(runtimeFieldMapping);
   });
 
+  it('tests processing semantic_text fields', () => {
+    const textWithRuntimeFieldsLiteralYml = `
+- name: sem_without_inference_id
+  type: semantic_text
+- name: sem_with_inference_id
+  type: semantic_text
+  inference_id: .model_id
+`;
+    const semanticFieldMapping = {
+      properties: {
+        sem_without_inference_id: {
+          type: 'semantic_text',
+        },
+        sem_with_inference_id: {
+          inference_id: '.model_id',
+          type: 'semantic_text',
+        },
+      },
+    };
+    const fields: Field[] = load(textWithRuntimeFieldsLiteralYml);
+    const processedFields = processFields(fields);
+    const mappings = generateMappings(processedFields, true);
+    expect(mappings).toEqual(semanticFieldMapping);
+  });
+
   it('tests unexpected type for field as dynamic template fails', () => {
     const textWithRuntimeFieldsLiteralYml = `
 - name: labels.*

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -543,6 +543,13 @@ function _generateMappings(
               fieldProps.fields = generateMultiFields(field.multi_fields);
             }
             break;
+          case 'semantic_text':
+            fieldProps.type = 'semantic_text';
+            if (field.inference_id) {
+              fieldProps.inference_id = field.inference_id;
+            }
+            fieldProps.path = field.path;
+            break;
           case 'object':
             fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' };
             break;

x-pack/plugins/fleet/server/services/epm/fields/field.test.ts

@@ -11,7 +11,12 @@ import path from 'path';
 import globby from 'globby';
 import { load } from 'js-yaml';
 
-import { getField, processFields, processFieldsWithWildcard } from './field';
+import {
+  getField,
+  processFields,
+  processFieldsWithWildcard,
+  filterForKnowledgeBaseEntryAssets,
+} from './field';
 import type { Field, Fields } from './field';
 
 // Add our own serialiser to just do JSON.stringify
@@ -813,3 +818,25 @@ describe('processFields', () => {
     });
   });
 });
+
+describe('filterForKnowledgeBaseEntryAssets', () => {
+  it('returns true for assets within the given knowledge base entry folder', () => {
+    expect(
+      filterForKnowledgeBaseEntryAssets('foo')(
+        '/kb-1.0/kibana/knowledge_base_entry/foo/manifest.yml'
+      )
+    ).toBe(true);
+  });
+  it('returns false for assets within another knowledge base entry folder', () => {
+    expect(
+      filterForKnowledgeBaseEntryAssets('bar')(
+        '/kb-1.0/kibana/knowledge_base_entry/foo/manifest.yml'
+      )
+    ).toBe(false);
+  });
+  it('returns false for assets outside of a knowledge base entry folder', () => {
+    expect(
+      filterForKnowledgeBaseEntryAssets('foo')('/kb-1.0/kibana/dashboard/some_dashboard.json')
+    ).toBe(false);
+  });
+});

x-pack/plugins/fleet/server/services/epm/fields/field.ts

@@ -47,6 +47,9 @@ export interface Field {
   metrics?: string[];
   default_metric?: string;
 
+  // Fields specific to the semantic_text type
+  inference_id?: string;
+
   // Meta fields
   metric_type?: string;
   unit?: string;
@@ -296,6 +299,12 @@ export const filterForTransformAssets = (transformName: string) => {
   };
 };
 
+export const filterForKnowledgeBaseEntryAssets = (knowledgeBaseEntryName: string) => {
+  return function isTransformAssets(path: string) {
+    return path.includes(`/knowledge_base_entry/${knowledgeBaseEntryName}`);
+  };
+};
+
 function combineFilter(...filters: Array<(path: string) => boolean>) {
   return function filterAsset(path: string) {
     return filters.every((filter) => filter(path));
@@ -354,3 +363,26 @@ export const loadTransformFieldsFromYaml = (
     return acc;
   }, []);
 };
+
+export const loadKnowledgeBaseEntryFieldsFromYaml = (
+  packageInstallContext: PackageInstallContext,
+  knowledgeBaseEntryName: string
+): Field[] => {
+  // Fetch all field definition files
+  const fieldDefinitionFiles = getAssetsDataFromAssetsMap(
+    packageInstallContext.packageInfo,
+    packageInstallContext.assetsMap,
+    combineFilter(isFields, filterForKnowledgeBaseEntryAssets(knowledgeBaseEntryName))
+  );
+  return fieldDefinitionFiles.reduce<Field[]>((acc, file) => {
+    // Make sure it is defined as it is optional. Should never happen.
+    if (file.buffer) {
+      const tmpFields = load(file.buffer.toString());
+      // load() returns undefined for empty files, we don't want that
+      if (tmpFields) {
+        acc = acc.concat(tmpFields);
+      }
+    }
+    return acc;
+  }, []);
+};

x-pack/plugins/fleet/server/services/epm/kibana/knowledge_base/consts.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const knowledgeBaseEntrySavedObjectType = 'knowledge_base_entry';