From 76e3984205e5c78358bf54448ffdb1cbcff40a5a Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 30 Sep 2024 09:47:24 +0200 Subject: [PATCH 01/97] Create knowledge base registry plugin --- .../knowledge_base_registry/README.md | 4 ++ .../knowledge_base_registry/jest.config.js | 23 +++++++++ .../knowledge_base_registry/kibana.jsonc | 15 ++++++ .../knowledge_base_registry/public/index.ts | 26 ++++++++++ .../knowledge_base_registry/public/plugin.tsx | 48 +++++++++++++++++++ .../knowledge_base_registry/public/types.ts | 18 +++++++ .../knowledge_base_registry/server/config.ts | 14 ++++++ .../knowledge_base_registry/server/index.ts | 26 ++++++++++ .../knowledge_base_registry/server/plugin.ts | 48 +++++++++++++++++++ .../knowledge_base_registry/server/types.ts | 16 +++++++ .../knowledge_base_registry/tsconfig.json | 18 +++++++ 11 files changed, 256 insertions(+) create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/README.md create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/kibana.jsonc create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/public/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/public/plugin.tsx create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/public/types.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/types.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/README.md b/x-pack/plugins/ai_infra/knowledge_base_registry/README.md new file mode 100644 index 000000000000..db52c35a8f8f --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/README.md @@ -0,0 +1,4 @@ +# Knowledge base registry plugin + +This plugin contains the registry for the knowledge base. + diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js b/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js new file mode 100644 index 000000000000..2a08e50cc7e3 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: [ + '/x-pack/plugins/ai_infra/knowledge_base_registry/public', + '/x-pack/plugins/ai_infra/knowledge_base_registry/server', + '/x-pack/plugins/ai_infra/knowledge_base_registry/common', + ], + setupFiles: [], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/ai_infra/knowledge_base_registry/{public,server,common}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/kibana.jsonc b/x-pack/plugins/ai_infra/knowledge_base_registry/kibana.jsonc new file mode 100644 index 000000000000..77151b0879c7 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/kibana.jsonc @@ -0,0 +1,15 @@ +{ + "type": "plugin", + "id": "@kbn/knowledge-base-registry-plugin", + "owner": "@elastic/appex-ai-infra", + "plugin": { + "id": "knowledgeBaseRegistry", + "server": true, + "browser": true, + "configPath": ["xpack", "knowledgeBaseRegistry"], + "requiredPlugins": [], + "requiredBundles": [], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/public/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/public/index.ts new file mode 100644 index 000000000000..0a674f4704d6 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/public/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { KnowledgeBaseRegistryPlugin } from './plugin'; +import type { + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies, + PublicPluginConfig, +} from './types'; + +export type { KnowledgeBaseRegistrySetupContract, KnowledgeBaseRegistryStartContract }; + +export const plugin: PluginInitializer< + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies +> = (pluginInitializerContext: PluginInitializerContext) => + new KnowledgeBaseRegistryPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/public/plugin.tsx b/x-pack/plugins/ai_infra/knowledge_base_registry/public/plugin.tsx new file mode 100644 index 000000000000..7897b7bcc2ff --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/public/plugin.tsx @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; +import type { Logger } from '@kbn/logging'; +import type { + PublicPluginConfig, + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies, +} from './types'; + +export class KnowledgeBaseRegistryPlugin + implements + Plugin< + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup< + KnowledgeBaseRegistryStartDependencies, + KnowledgeBaseRegistryStartContract + >, + pluginsSetup: KnowledgeBaseRegistrySetupDependencies + ): KnowledgeBaseRegistrySetupContract { + return {}; + } + + start( + coreStart: CoreStart, + pluginsStart: KnowledgeBaseRegistryStartDependencies + ): KnowledgeBaseRegistryStartContract { + return {}; + } +} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/public/types.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/public/types.ts new file mode 100644 index 000000000000..eb99333d37e8 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/public/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface PublicPluginConfig {} + +export interface KnowledgeBaseRegistrySetupDependencies {} + +export interface KnowledgeBaseRegistryStartDependencies {} + +export interface KnowledgeBaseRegistrySetupContract {} + +export interface KnowledgeBaseRegistryStartContract {} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts new file mode 100644 index 000000000000..3c7b72c75dde --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; + +export const config = schema.object({ + enabled: schema.boolean({ defaultValue: true }), +}); + +export type KnowledgeBaseRegistryConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts new file mode 100644 index 000000000000..6644a292892c --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server'; +import type { KnowledgeBaseRegistryConfig } from './config'; +import type { + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies, +} from './types'; +import { KnowledgeBaseRegistryPlugin } from './plugin'; + +export type { KnowledgeBaseRegistrySetupContract, KnowledgeBaseRegistryStartContract }; + +export const plugin: PluginInitializer< + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new KnowledgeBaseRegistryPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts new file mode 100644 index 000000000000..30b97774ba63 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import type { KnowledgeBaseRegistryConfig } from './config'; +import type { + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies, +} from './types'; + +export class KnowledgeBaseRegistryPlugin + implements + Plugin< + KnowledgeBaseRegistrySetupContract, + KnowledgeBaseRegistryStartContract, + KnowledgeBaseRegistrySetupDependencies, + KnowledgeBaseRegistryStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup< + KnowledgeBaseRegistryStartDependencies, + KnowledgeBaseRegistryStartContract + >, + pluginsSetup: KnowledgeBaseRegistrySetupDependencies + ): KnowledgeBaseRegistrySetupContract { + return {}; + } + + start( + core: CoreStart, + pluginsStart: KnowledgeBaseRegistryStartDependencies + ): KnowledgeBaseRegistryStartContract { + return {}; + } +} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/types.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/types.ts new file mode 100644 index 000000000000..8ce82731f7b6 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/types.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface KnowledgeBaseRegistrySetupDependencies {} + +export interface KnowledgeBaseRegistryStartDependencies {} + +export interface KnowledgeBaseRegistrySetupContract {} + +export interface KnowledgeBaseRegistryStartContract {} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json b/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json new file mode 100644 index 000000000000..5ef51802f41a --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + "scripts/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [] +} From 89e033c2ff603559f661655ab234fff8a90f7ef1 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 30 Sep 2024 10:11:43 +0200 Subject: [PATCH 02/97] run bootstrap --- package.json | 1 + tsconfig.base.json | 2 ++ yarn.lock | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/package.json b/package.json index cad364998e3c..76585083a60a 100644 --- a/package.json +++ b/package.json @@ -592,6 +592,7 @@ "@kbn/kibana-react-plugin": "link:src/plugins/kibana_react", "@kbn/kibana-usage-collection-plugin": "link:src/plugins/kibana_usage_collection", "@kbn/kibana-utils-plugin": "link:src/plugins/kibana_utils", + "@kbn/knowledge-base-registry-plugin": "link:x-pack/plugins/ai_infra/knowledge_base_registry", "@kbn/kubernetes-security-plugin": "link:x-pack/plugins/kubernetes_security", "@kbn/langchain": "link:x-pack/packages/kbn-langchain", "@kbn/language-documentation": "link:packages/kbn-language-documentation", diff --git a/tsconfig.base.json b/tsconfig.base.json index 7a66911a4bee..f6fb08fc7661 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1098,6 +1098,8 @@ "@kbn/kibana-usage-collection-plugin/*": ["src/plugins/kibana_usage_collection/*"], "@kbn/kibana-utils-plugin": ["src/plugins/kibana_utils"], "@kbn/kibana-utils-plugin/*": ["src/plugins/kibana_utils/*"], + "@kbn/knowledge-base-registry-plugin": ["x-pack/plugins/ai_infra/knowledge_base_registry"], + "@kbn/knowledge-base-registry-plugin/*": ["x-pack/plugins/ai_infra/knowledge_base_registry/*"], "@kbn/kubernetes-security-plugin": ["x-pack/plugins/kubernetes_security"], "@kbn/kubernetes-security-plugin/*": ["x-pack/plugins/kubernetes_security/*"], "@kbn/langchain": ["x-pack/packages/kbn-langchain"], diff --git a/yarn.lock b/yarn.lock index a3d54dbd3f1b..3d2c2585d613 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5467,6 +5467,10 @@ version "0.0.0" uid "" +"@kbn/knowledge-base-registry-plugin@link:x-pack/plugins/ai_infra/knowledge_base_registry": + version "0.0.0" + uid "" + "@kbn/kubernetes-security-plugin@link:x-pack/plugins/kubernetes_security": version "0.0.0" uid "" From b3c67ee9a4f87af64598a95206d40618c657654d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:23:33 +0000 Subject: [PATCH 03/97] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 1 + docs/developer/plugin-list.asciidoc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7f8f48e63814..81756bbeb99d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -552,6 +552,7 @@ src/plugins/kibana_overview @elastic/appex-sharedux src/plugins/kibana_react @elastic/appex-sharedux src/plugins/kibana_usage_collection @elastic/kibana-core src/plugins/kibana_utils @elastic/appex-sharedux +x-pack/plugins/ai_infra/knowledge_base_registry @elastic/appex-ai-infra x-pack/plugins/kubernetes_security @elastic/kibana-cloud-security-posture x-pack/packages/kbn-langchain @elastic/security-generative-ai packages/kbn-language-documentation @elastic/kibana-esql diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 55a2a19040ae..4ae90c4d1857 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -664,6 +664,10 @@ the infrastructure monitoring use-case within Kibana. |undefined +|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/knowledge_base_registry/README.md[knowledgeBaseRegistry] +|This plugin contains the registry for the knowledge base. + + |{kib-repo}blob/{branch}/x-pack/plugins/kubernetes_security/README.md[kubernetesSecurity] |This plugin provides interactive visualizations of your Kubernetes workload and session data. From 3dc368ea1054a8d0664e319a9f2b2ac3bd1e10ec Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:24:17 +0000 Subject: [PATCH 04/97] [CI] Auto-commit changed files from 'node scripts/notice' --- .../plugins/ai_infra/knowledge_base_registry/tsconfig.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json b/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json index 5ef51802f41a..cd604a32b114 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json @@ -14,5 +14,9 @@ ".storybook/**/*" ], "exclude": ["target/**/*", ".storybook/**/*.js"], - "kbn_references": [] + "kbn_references": [ + "@kbn/core", + "@kbn/logging", + "@kbn/config-schema", + ] } From 54430dd31d3166d0021c248e1ed19e3d9243fbfd Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 30 Sep 2024 13:17:12 +0200 Subject: [PATCH 05/97] Add knowledge_base_entry SO type --- .../knowledge_base_registry/common/consts.ts | 8 ++++ .../common/saved_objects.ts | 35 ++++++++++++++++ .../knowledge_base_registry/server/plugin.ts | 2 + .../server/saved_objects/index.ts | 8 ++++ .../saved_objects/knowledge_base_entry.ts | 41 +++++++++++++++++++ 5 files changed, 94 insertions(+) create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts new file mode 100644 index 000000000000..49700d894a01 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const knowledgeBaseEntryTypeName = 'knowledge_base_entry'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts new file mode 100644 index 000000000000..35a0efbd7e16 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Interface describing the raw attributes of the KB Entry SO type. + * Contains more fields than the mappings, which only list + * indexed fields. + */ +export interface KnowledgeBaseEntryAttributes { + name: string; + description?: string; + source: KnowledgeBaseEntrySource; + created_by: KnowledgeBaseEntryCreatedBy; +} + +interface KnowledgeBaseEntryIndexSource { + type: 'index'; + index_name: string; + syntactic_fields: string[]; + semantic_fields: string[]; +} + +type KnowledgeBaseEntrySource = KnowledgeBaseEntryIndexSource; + +interface KnowledgeBaseEntryCreatedByPackage { + type: 'package'; + packageName: string; + version: string; +} + +type KnowledgeBaseEntryCreatedBy = KnowledgeBaseEntryCreatedByPackage; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts index 30b97774ba63..f49d424f2440 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts @@ -14,6 +14,7 @@ import type { KnowledgeBaseRegistrySetupDependencies, KnowledgeBaseRegistryStartDependencies, } from './types'; +import { knowledgeBaseEntrySavedObjectType } from './saved_objects'; export class KnowledgeBaseRegistryPlugin implements @@ -36,6 +37,7 @@ export class KnowledgeBaseRegistryPlugin >, pluginsSetup: KnowledgeBaseRegistrySetupDependencies ): KnowledgeBaseRegistrySetupContract { + coreSetup.savedObjects.registerType(knowledgeBaseEntrySavedObjectType); return {}; } diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts new file mode 100644 index 000000000000..bc42be92f8d2 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { knowledgeBaseEntrySavedObjectType } from './knowledge_base_entry'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts new file mode 100644 index 000000000000..33c5394645c9 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsType } from '@kbn/core/server'; +import { knowledgeBaseEntryTypeName } from '../../common/consts'; +import type { KnowledgeBaseEntryAttributes } from '../../common/saved_objects'; + +export const knowledgeBaseEntrySavedObjectType: SavedObjectsType = { + name: knowledgeBaseEntryTypeName, + hidden: false, + namespaceType: 'multiple', + mappings: { + dynamic: false, + properties: { + name: { type: 'keyword' }, + source: { + type: 'object', + dynamic: false, + properties: { + type: { type: 'keyword' }, + }, + }, + created_by: { + type: 'object', + dynamic: false, + properties: { + type: { type: 'keyword' }, + }, + }, + }, + }, + management: { + importableAndExportable: false, + visibleInManagement: false, + }, + modelVersions: {}, +}; From 7694e75f806df87b787941f95a298b4c2e8599cb Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 1 Oct 2024 15:14:18 +0200 Subject: [PATCH 06/97] make this empty plugin work, maybe? --- packages/kbn-optimizer/limits.yml | 1 + x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index da5ba0500eeb..1c88bd529c0a 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -94,6 +94,7 @@ pageLoadAssetSize: kibanaReact: 74422 kibanaUsageCollection: 16463 kibanaUtils: 79713 + knowledgeBaseRegistry: 22500 kubernetesSecurity: 77234 lens: 57135 licenseManagement: 41817 diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js b/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js index 2a08e50cc7e3..d233032aebc2 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js @@ -7,7 +7,7 @@ module.exports = { preset: '@kbn/test', - rootDir: '../../..', + rootDir: '../../../..', roots: [ '/x-pack/plugins/ai_infra/knowledge_base_registry/public', '/x-pack/plugins/ai_infra/knowledge_base_registry/server', From e7c482f5053516fc1d124c1d4d59cf6350337421 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 1 Oct 2024 15:54:50 +0200 Subject: [PATCH 07/97] fix SO meta --- .../server/saved_objects/knowledge_base_entry.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts index 33c5394645c9..91845b48c2a5 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts @@ -35,7 +35,6 @@ export const knowledgeBaseEntrySavedObjectType: SavedObjectsType Date: Tue, 1 Oct 2024 16:16:53 +0200 Subject: [PATCH 08/97] move attributes to server folder --- .../common/saved_objects.ts | 20 ++++--------------- .../saved_objects/knowledge_base_entry.ts | 19 ++++++++++++++++-- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts index 35a0efbd7e16..10b682f6b748 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts @@ -5,31 +5,19 @@ * 2.0. */ -/** - * Interface describing the raw attributes of the KB Entry SO type. - * Contains more fields than the mappings, which only list - * indexed fields. - */ -export interface KnowledgeBaseEntryAttributes { - name: string; - description?: string; - source: KnowledgeBaseEntrySource; - created_by: KnowledgeBaseEntryCreatedBy; -} - -interface KnowledgeBaseEntryIndexSource { +export interface KnowledgeBaseEntryIndexSource { type: 'index'; index_name: string; syntactic_fields: string[]; semantic_fields: string[]; } -type KnowledgeBaseEntrySource = KnowledgeBaseEntryIndexSource; +export type KnowledgeBaseEntrySource = KnowledgeBaseEntryIndexSource; -interface KnowledgeBaseEntryCreatedByPackage { +interface KnowledgeBaseEntryInstalledByPackage { type: 'package'; packageName: string; version: string; } -type KnowledgeBaseEntryCreatedBy = KnowledgeBaseEntryCreatedByPackage; +export type KnowledgeBaseEntryInstalledBy = KnowledgeBaseEntryInstalledByPackage; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts index 91845b48c2a5..fedb6a36e1a2 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts @@ -7,7 +7,22 @@ import type { SavedObjectsType } from '@kbn/core/server'; import { knowledgeBaseEntryTypeName } from '../../common/consts'; -import type { KnowledgeBaseEntryAttributes } from '../../common/saved_objects'; +import type { + KnowledgeBaseEntrySource, + KnowledgeBaseEntryInstalledBy, +} from '../../common/saved_objects'; + +/** + * Interface describing the raw attributes of the KB Entry SO type. + * Contains more fields than the mappings, which only list + * indexed fields. + */ +export interface KnowledgeBaseEntryAttributes { + name: string; + description?: string; + source: KnowledgeBaseEntrySource; + installed_by: KnowledgeBaseEntryInstalledBy; +} export const knowledgeBaseEntrySavedObjectType: SavedObjectsType = { name: knowledgeBaseEntryTypeName, @@ -24,7 +39,7 @@ export const knowledgeBaseEntrySavedObjectType: SavedObjectsType Date: Tue, 1 Oct 2024 16:30:42 +0200 Subject: [PATCH 09/97] some tsdoc --- .../common/saved_objects.ts | 16 +++++++++++++--- .../knowledge_base_registry/server/config.ts | 4 +--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts index 10b682f6b748..be48a43813b0 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts @@ -5,6 +5,13 @@ * 2.0. */ +/** + * Represents the source of the KB entry, where the data it resolved from + * + * Only possible subtype for now is `index` + */ +export type KnowledgeBaseEntrySource = KnowledgeBaseEntryIndexSource; + export interface KnowledgeBaseEntryIndexSource { type: 'index'; index_name: string; @@ -12,12 +19,15 @@ export interface KnowledgeBaseEntryIndexSource { semantic_fields: string[]; } -export type KnowledgeBaseEntrySource = KnowledgeBaseEntryIndexSource; +/** + * Represents how the package was installed + * + * Only possible subtype for now is `package` + */ +export type KnowledgeBaseEntryInstalledBy = KnowledgeBaseEntryInstalledByPackage; interface KnowledgeBaseEntryInstalledByPackage { type: 'package'; packageName: string; version: string; } - -export type KnowledgeBaseEntryInstalledBy = KnowledgeBaseEntryInstalledByPackage; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts index 3c7b72c75dde..a2e18568065e 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts @@ -7,8 +7,6 @@ import { schema, type TypeOf } from '@kbn/config-schema'; -export const config = schema.object({ - enabled: schema.boolean({ defaultValue: true }), -}); +export const config = schema.object({}); export type KnowledgeBaseRegistryConfig = TypeOf; From b3097dfe07778acc7cda60e1db1386d14df93d70 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:12:03 +0000 Subject: [PATCH 10/97] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../current_fields.json | 7 +++++ .../current_mappings.json | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 87bf8c710487..5217a65150d0 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -726,6 +726,13 @@ "use_space_awareness_migration_status" ], "inventory-view": [], + "knowledge_base_entry": [ + "installed_by", + "installed_by.type", + "name", + "source", + "source.type" + ], "kql-telemetry": [], "legacy-url-alias": [ "disabled", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 7b315efcbcd4..eccbd1e7b31e 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2399,6 +2399,32 @@ "dynamic": false, "properties": {} }, + "knowledge_base_entry": { + "dynamic": false, + "properties": { + "installed_by": { + "dynamic": false, + "properties": { + "type": { + "type": "keyword" + } + }, + "type": "object" + }, + "name": { + "type": "keyword" + }, + "source": { + "dynamic": false, + "properties": { + "type": { + "type": "keyword" + } + }, + "type": "object" + } + } + }, "kql-telemetry": { "dynamic": false, "properties": {} From dac2b4084eac45b84203b8cba20e50d05bcb214e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:50:27 +0000 Subject: [PATCH 11/97] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index a8b31ffdd90f..a696c5bee1b6 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -126,6 +126,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", "ingest_manager_settings": "e794576a05d19dd5306a1e23cbb82c09bffabd65", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", + "knowledge_base_entry": "5a9b5821eb2c22ed66e6f8298c45d519ac13de5c", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", "legacy-url-alias": "9b8cca3fbb2da46fd12823d3cd38fdf1c9f24bc8", "lens": "5cfa2c52b979b4f8df56dd13c477e152183468b9", From 65cc3e719b14445d012f634750097637c928bad9 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 2 Oct 2024 14:15:30 +0200 Subject: [PATCH 12/97] add new type to list --- .../saved_objects/migrations/group3/type_registrations.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index e95a82e63d0f..ce65c320cb54 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -93,6 +93,7 @@ const previouslyRegisteredTypes = [ 'ingest-package-policies', 'ingest_manager_settings', 'inventory-view', + 'knowledge_base_entry', 'kql-telemetry', 'legacy-url-alias', 'lens', From febae38f8df4603588cb0b4ee95fb33e433fb332 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 3 Oct 2024 15:26:42 +0200 Subject: [PATCH 13/97] installer: WIP --- .../knowledge_base_registry/common/consts.ts | 2 +- .../common/saved_objects.ts | 31 ++--- .../server/dao/product_doc_install/index.ts | 8 ++ .../product_doc_install_service.ts | 35 +++++ .../dao/product_doc_install/so_to_dto.ts | 26 ++++ .../knowledge_base_registry/server/plugin.ts | 39 +++++- .../server/saved_objects/index.ts | 5 +- .../saved_objects/knowledge_base_entry.ts | 55 -------- .../saved_objects/product_doc_install.ts | 46 +++++++ .../services/package_installer/index.ts | 8 ++ .../package_installer/package_installer.ts | 127 ++++++++++++++++++ .../package_installer/steps/create_index.ts | 39 ++++++ .../services/package_installer/steps/index.ts | 9 ++ .../package_installer/steps/populate_index.ts | 86 ++++++++++++ .../utils/archive_accessors.ts | 41 ++++++ .../package_installer/utils/download.ts | 23 ++++ .../services/package_installer/utils/index.ts | 11 ++ .../utils/validate_artifact_archive.ts | 25 ++++ .../package_installer/utils/zip_archive.ts | 90 +++++++++++++ 19 files changed, 623 insertions(+), 83 deletions(-) create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/so_to_dto.ts delete mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/download.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/validate_artifact_archive.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/zip_archive.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts index 49700d894a01..3ff0b31f48ed 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts @@ -5,4 +5,4 @@ * 2.0. */ -export const knowledgeBaseEntryTypeName = 'knowledge_base_entry'; +export const knowledgeBaseProductDocInstallTypeName = 'ai_kb_product_doc_installation'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts index be48a43813b0..0c0fb4732d4b 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts @@ -5,29 +5,14 @@ * 2.0. */ -/** - * Represents the source of the KB entry, where the data it resolved from - * - * Only possible subtype for now is `index` - */ -export type KnowledgeBaseEntrySource = KnowledgeBaseEntryIndexSource; - -export interface KnowledgeBaseEntryIndexSource { - type: 'index'; - index_name: string; - syntactic_fields: string[]; - semantic_fields: string[]; -} - -/** - * Represents how the package was installed - * - * Only possible subtype for now is `package` - */ -export type KnowledgeBaseEntryInstalledBy = KnowledgeBaseEntryInstalledByPackage; +export type ProductDocInstallStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; -interface KnowledgeBaseEntryInstalledByPackage { - type: 'package'; +export interface ProducDocInstallDTO { + id: string; packageName: string; - version: string; + packageVersion: string; + productName: string; + installationStatus: ProductDocInstallStatus; + lastInstallationDate: Date | undefined; + indexName: string; } diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/index.ts new file mode 100644 index 000000000000..d55cb303b190 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ProductDocInstallClient } from './product_doc_install_service'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts new file mode 100644 index 000000000000..f1709a058509 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract, SavedObject } from '@kbn/core/server'; +import type { ProducDocInstallDTO } from '../../../common/saved_objects'; +import { knowledgeBaseProductDocInstallTypeName as typeName } from '../../../common/consts'; +import type { KnowledgeBaseProductDocInstallAttributes } from '../../saved_objects'; +import { soToDto } from './so_to_dto'; + +export class ProductDocInstallClient { + private soClient: SavedObjectsClientContract; + + constructor({ soClient }: { soClient: SavedObjectsClientContract }) { + this.soClient = soClient; + } + + async getByProductName(productName: string): Promise { + const response = await this.soClient.find({ + type: typeName, + perPage: 1, + filter: `${typeName}.product_name: "${productName}"`, + }); + if (response.saved_objects.length === 1) { + return soToDto(response.saved_objects[0]); + } + if (response.saved_objects.length > 1) { + throw new Error(`Found multiple records for product name : ${productName}`); + } + return undefined; + } +} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/so_to_dto.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/so_to_dto.ts new file mode 100644 index 000000000000..2c8a8f61ccb0 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/so_to_dto.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject } from '@kbn/core/server'; +import type { ProducDocInstallDTO } from '../../../common/saved_objects'; +import type { KnowledgeBaseProductDocInstallAttributes } from '../../saved_objects'; + +export const soToDto = ( + so: SavedObject +): ProducDocInstallDTO => { + return { + id: so.id, + packageName: so.attributes.package_name, + packageVersion: so.attributes.package_version, + productName: so.attributes.product_name, + installationStatus: so.attributes.installation_status, + indexName: so.attributes.index_name, + lastInstallationDate: so.attributes.last_installation_date + ? new Date(so.attributes.last_installation_date) + : undefined, + }; +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts index f49d424f2440..373a38598c66 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts @@ -5,8 +5,12 @@ * 2.0. */ -import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import Path from 'path'; import type { Logger } from '@kbn/logging'; +import { getDataPath } from '@kbn/utils'; +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import { SavedObjectsClient } from '@kbn/core/server'; +import { knowledgeBaseProductDocInstallTypeName } from '../common/consts'; import type { KnowledgeBaseRegistryConfig } from './config'; import type { KnowledgeBaseRegistrySetupContract, @@ -14,7 +18,9 @@ import type { KnowledgeBaseRegistrySetupDependencies, KnowledgeBaseRegistryStartDependencies, } from './types'; -import { knowledgeBaseEntrySavedObjectType } from './saved_objects'; +import { knowledgeBaseProductDocInstallSavedObjectType } from './saved_objects'; +import { PackageInstaller } from './services/package_installer'; +import { ProductDocInstallClient } from './dao/product_doc_install'; export class KnowledgeBaseRegistryPlugin implements @@ -37,7 +43,8 @@ export class KnowledgeBaseRegistryPlugin >, pluginsSetup: KnowledgeBaseRegistrySetupDependencies ): KnowledgeBaseRegistrySetupContract { - coreSetup.savedObjects.registerType(knowledgeBaseEntrySavedObjectType); + coreSetup.savedObjects.registerType(knowledgeBaseProductDocInstallSavedObjectType); + return {}; } @@ -45,6 +52,32 @@ export class KnowledgeBaseRegistryPlugin core: CoreStart, pluginsStart: KnowledgeBaseRegistryStartDependencies ): KnowledgeBaseRegistryStartContract { + const soClient = new SavedObjectsClient( + core.savedObjects.createInternalRepository([knowledgeBaseProductDocInstallTypeName]) + ); + const productDocClient = new ProductDocInstallClient({ soClient }); + + const packageInstaller = new PackageInstaller({ + esClient: core.elasticsearch.client.asInternalUser, + productDocClient, + artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'), + logger: this.logger.get('package-installer'), + }); + + delay(10) + .then(() => { + console.log('*** test installating package'); + return packageInstaller.installPackage({ + productName: 'Kibana', + productVersion: '8.15', + }); + }) + .catch((e) => { + console.log('*** ERROR', e); + }); + return {}; } } + +const delay = (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds * 1000)); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts index bc42be92f8d2..3715eab90a00 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts @@ -5,4 +5,7 @@ * 2.0. */ -export { knowledgeBaseEntrySavedObjectType } from './knowledge_base_entry'; +export { + knowledgeBaseProductDocInstallSavedObjectType, + type KnowledgeBaseProductDocInstallAttributes, +} from './product_doc_install'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts deleted file mode 100644 index fedb6a36e1a2..000000000000 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/knowledge_base_entry.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SavedObjectsType } from '@kbn/core/server'; -import { knowledgeBaseEntryTypeName } from '../../common/consts'; -import type { - KnowledgeBaseEntrySource, - KnowledgeBaseEntryInstalledBy, -} from '../../common/saved_objects'; - -/** - * Interface describing the raw attributes of the KB Entry SO type. - * Contains more fields than the mappings, which only list - * indexed fields. - */ -export interface KnowledgeBaseEntryAttributes { - name: string; - description?: string; - source: KnowledgeBaseEntrySource; - installed_by: KnowledgeBaseEntryInstalledBy; -} - -export const knowledgeBaseEntrySavedObjectType: SavedObjectsType = { - name: knowledgeBaseEntryTypeName, - hidden: false, - namespaceType: 'multiple', - mappings: { - dynamic: false, - properties: { - name: { type: 'keyword' }, - source: { - type: 'object', - dynamic: false, - properties: { - type: { type: 'keyword' }, - }, - }, - installed_by: { - type: 'object', - dynamic: false, - properties: { - type: { type: 'keyword' }, - }, - }, - }, - }, - management: { - importableAndExportable: false, - }, - modelVersions: {}, -}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts new file mode 100644 index 000000000000..5cbd1cdea335 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsType } from '@kbn/core/server'; +import { knowledgeBaseProductDocInstallTypeName } from '../../common/consts'; +import type { ProductDocInstallStatus } from '../../common/saved_objects'; + +/** + * Interface describing the raw attributes of the KB Entry SO type. + * Contains more fields than the mappings, which only list + * indexed fields. + */ +export interface KnowledgeBaseProductDocInstallAttributes { + package_name: string; + package_version: string; + product_name: string; + installation_status: ProductDocInstallStatus; + last_installation_date: number | undefined; + index_name: string; +} + +export const knowledgeBaseProductDocInstallSavedObjectType: SavedObjectsType = + { + name: knowledgeBaseProductDocInstallTypeName, + hidden: true, + namespaceType: 'multiple', + mappings: { + dynamic: false, + properties: { + package_name: { type: 'keyword' }, + package_version: { type: 'keyword' }, + product_name: { type: 'keyword' }, + installation_status: { type: 'keyword' }, + last_installation_date: { type: 'integer' }, + index_name: { type: 'keyword' }, + }, + }, + management: { + importableAndExportable: false, + }, + modelVersions: {}, + }; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/index.ts new file mode 100644 index 000000000000..a9edb7c38fda --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { PackageInstaller } from './package_installer'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts new file mode 100644 index 000000000000..434343a11cdc --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fetch from 'node-fetch'; +import Fs from 'fs'; +import Path from 'path'; +import { getDataPath } from '@kbn/utils'; +import { Logger } from '@kbn/logging'; +import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; +import type { ProductDocInstallClient } from '../../dao/product_doc_install'; +import { + downloadToDisk, + openZipArchive, + validateArtifactArchive, + loadManifestFile, + loadMappingFile, +} from './utils'; +import { createIndex, populateIndex } from './steps'; + +const ARTIFACT_BUCKET_URL = 'http://34.120.162.240'; + +export class PackageInstaller { + private readonly logger: Logger; + private readonly artifactsFolder: string; + private readonly esClient: ElasticsearchClient; + private readonly productDocClient: ProductDocInstallClient; + + constructor({ + artifactsFolder, + logger, + esClient, + productDocClient, + }: { + artifactsFolder: string; + logger: Logger; + esClient: ElasticsearchClient; + productDocClient: ProductDocInstallClient; + }) { + this.esClient = esClient; + this.productDocClient = productDocClient; + this.artifactsFolder = artifactsFolder; + this.logger = logger; + } + + async installPackage({ + productName, + productVersion, + }: { + productName: string; + productVersion: string; + }) { + // TODO: uninstall previous/current if present + await this.uninstallPackage({ + productName, + productVersion, + }); + // TODO: ensure elser is installed + + const artifactFileName = getArtifactName({ productName, productVersion }); + const artifactUrl = `${ARTIFACT_BUCKET_URL}/${artifactFileName}`; + const artifactPath = `${this.artifactsFolder}/${artifactFileName}`; + + console.log(`*** downloading ${artifactUrl} to ${artifactPath}`); + + await downloadToDisk(artifactUrl, artifactPath); + + const zipArchive = await openZipArchive(artifactPath); + try { + const manifest = await loadManifestFile(zipArchive); + const mappings = await loadMappingFile(zipArchive); + + // TODO: move helper to package + const indexName = `.kibana-ai-kb-${manifest.productName}-${productVersion}`.toLowerCase(); + + await createIndex({ + indexName, + mappings, + esClient: this.esClient, + log: this.logger, + }); + + await populateIndex({ + indexName, + archive: zipArchive, + esClient: this.esClient, + log: this.logger, + }); + + // TODO: update the product_doc_install SO + } finally { + zipArchive.close(); + } + } + + async uninstallPackage({ + productName, + productVersion, + }: { + productName: string; + productVersion: string; + }) { + // TODO + + const indexName = `.kibana-ai-kb-${productName}-${productVersion}`.toLowerCase(); + await this.esClient.indices.delete( + { + index: indexName, + }, + { ignore: [404] } + ); + } +} + +// need to be factorized with the script +const getArtifactName = ({ + productName, + productVersion, +}: { + productName: string; + productVersion: string; +}): string => { + return `kibana-kb-${productName}-${productVersion}.zip`.toLowerCase(); +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts new file mode 100644 index 000000000000..f0013a7f599d --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; + +export const createIndex = async ({ + esClient, + indexName, + mappings, + log, +}: { + esClient: ElasticsearchClient; + indexName: string; + mappings: MappingTypeMapping; + log: Logger; +}) => { + log.debug(`Creating index ${indexName}`); + + // TODO: extract / do it right + mappings.properties.ai_questions_answered.inference_id = 'default-elser'; + mappings.properties.ai_subtitle.inference_id = 'default-elser'; + mappings.properties.ai_summary.inference_id = 'default-elser'; + mappings.properties.content_body.inference_id = 'default-elser'; + + await esClient.indices.create({ + index: indexName, + mappings, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }, + }); +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts new file mode 100644 index 000000000000..bcada62f48c7 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { createIndex } from './create_index'; +export { populateIndex } from './populate_index'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts new file mode 100644 index 000000000000..438a4348c10e --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { ZipArchive } from '../utils/zip_archive'; + +// TODO: factorize with utils/validate_artifact_archive +const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/; + +export const populateIndex = async ({ + esClient, + indexName, + archive, + log, +}: { + esClient: ElasticsearchClient; + indexName: string; + archive: ZipArchive; + log: Logger; +}) => { + log.debug(`Starting populating index ${indexName}`); + + const contentEntries = archive + .getEntryPaths() + .filter((entryPath) => contentFileRegexp.test(entryPath)); + + for (let i = 0; i < contentEntries.length; i++) { + const entryPath = contentEntries[i]; + log.debug(`Indexing content for entry ${entryPath}`); + const contentBuffer = await archive.getEntryContent(entryPath); + await indexContentFile({ indexName, esClient, contentBuffer }); + } + + log.debug(`Done populating index ${indexName}`); +}; + +const indexContentFile = async ({ + indexName, + contentBuffer, + esClient, +}: { + indexName: string; + contentBuffer: Buffer; + esClient: ElasticsearchClient; +}) => { + const fileContent = contentBuffer.toString('utf-8'); + const lines = fileContent.split('\n'); + + const documents = lines + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + return JSON.parse(line); + }) + .map((doc) => rewriteInferenceId(doc, 'default-elser')); + + const operations = documents.reduce((ops, document) => { + ops!.push(...[{ index: { _index: indexName } }, document]); + return ops; + }, [] as BulkRequest['operations']); + + const response = await esClient.bulk({ + refresh: false, + operations, + }); + + if (response.errors) { + const error = response.items.find((item) => item.index!.error)!.index!.error!; + throw new Error(`Error indexing documents: ${JSON.stringify(error)}`); + } +}; + +// TODO: extract / do it right. +const rewriteInferenceId = (document: Record, inferenceId: string) => { + document.ai_questions_answered.inference.inference_id = inferenceId; + document.ai_subtitle.inference.inference_id = inferenceId; + document.ai_summary.inference.inference_id = inferenceId; + document.content_body.inference.inference_id = inferenceId; + return document; +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts new file mode 100644 index 000000000000..a4cd018292e9 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ZipArchive } from './zip_archive'; + +// TODO: factorize with script +export interface ArtifactManifest { + formatVersion: string; + productName: string; + productVersion: string; +} + +const manifestEntryPath = 'manifest.json'; +const mappingsEntryPath = 'mappings.json'; + +export const loadManifestFile = async (archive: ZipArchive): Promise => { + const manifest = await parseEntryContent(manifestEntryPath, archive); + // TODO: schema validation + return manifest; +}; + +export const loadMappingFile = async (archive: ZipArchive): Promise => { + return await parseEntryContent(mappingsEntryPath, archive); +}; + +const parseEntryContent = async (entryPath: string, archive: ZipArchive): Promise => { + if (!archive.hasEntry(entryPath)) { + throw new Error(`Could not load archive file: "${entryPath}" not found in archive`); + } + try { + const buffer = await archive.getEntryContent(entryPath); + return JSON.parse(buffer.toString('utf-8')); + } catch (e) { + throw new Error(`Could not parse archive file: ${e}`); + } +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/download.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/download.ts new file mode 100644 index 000000000000..ea5357792ef5 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/download.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createWriteStream } from 'fs'; +import { mkdir } from 'fs/promises'; +import Path from 'path'; +import fetch from 'node-fetch'; + +export const downloadToDisk = async (fileUrl: string, filePath: string) => { + const dirPath = Path.dirname(filePath); + await mkdir(dirPath, { recursive: true }); + const res = await fetch(fileUrl); + const fileStream = createWriteStream(filePath); + await new Promise((resolve, reject) => { + res.body.pipe(fileStream); + res.body.on('error', reject); + fileStream.on('finish', resolve); + }); +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts new file mode 100644 index 000000000000..db68bc13f365 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { downloadToDisk } from './download'; +export { openZipArchive, type ZipArchive } from './zip_archive'; +export { validateArtifactArchive } from './validate_artifact_archive'; +export { loadManifestFile, loadMappingFile } from './archive_accessors'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/validate_artifact_archive.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/validate_artifact_archive.ts new file mode 100644 index 000000000000..75e42f4cf066 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/validate_artifact_archive.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ZipArchive } from './zip_archive'; + +type ValidationResult = { valid: true } | { valid: false; error: string }; + +const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/; + +export const validateArtifactArchive = (archive: ZipArchive): ValidationResult => { + if (!archive.hasEntry('manifest.json')) { + return { valid: false, error: 'Manifest file not found' }; + } + if (!archive.hasEntry('mappings.json')) { + return { valid: false, error: 'Mapping file not found' }; + } + if (!archive.getEntryPaths().some((entryPath) => contentFileRegexp.test(entryPath))) { + return { valid: false, error: 'No content files were found' }; + } + return { valid: true }; +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/zip_archive.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/zip_archive.ts new file mode 100644 index 000000000000..fee6d886e7b1 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/zip_archive.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buffer } from 'stream/consumers'; +import yauzl from 'yauzl'; + +export interface ZipArchive { + hasEntry(entryPath: string): boolean; + getEntryPaths(): string[]; + getEntryContent(entryPath: string): Promise; + close(): void; +} + +export const openZipArchive = async (archivePath: string): Promise => { + return new Promise((resolve, reject) => { + const entries: yauzl.Entry[] = []; + yauzl.open(archivePath, { lazyEntries: true, autoClose: false }, (err, zipFile) => { + if (err || !zipFile) { + return reject(err ?? 'No zip file'); + } + + zipFile!.on('entry', function (entry) { + entries.push(entry); + zipFile.readEntry(); + }); + + zipFile.on('end', () => { + const archive = new ZipArchiveImpl(entries, zipFile); + resolve(archive); + }); + + zipFile.on('close', () => {}); + + zipFile.readEntry(); + }); + }); +}; + +class ZipArchiveImpl implements ZipArchive { + private readonly zipFile: yauzl.ZipFile; + private readonly entries: Map; + + constructor(entries: yauzl.Entry[], zipFile: yauzl.ZipFile) { + this.zipFile = zipFile; + this.entries = new Map(entries.map((entry) => [entry.fileName, entry])); + } + + hasEntry(entryPath: string) { + return this.entries.has(entryPath); + } + + getEntryPaths() { + return [...this.entries.keys()]; + } + + getEntryContent(entryPath: string) { + const foundEntry = this.entries.get(entryPath); + if (!foundEntry) { + throw new Error(`Entry ${entryPath} not found in archive`); + } + return getZipEntryContent(this.zipFile, foundEntry); + } + + close() { + this.zipFile.close(); + } +} + +const getZipEntryContent = async (zipFile: yauzl.ZipFile, entry: yauzl.Entry): Promise => { + return new Promise((resolve, reject) => { + zipFile.openReadStream(entry, function (err, readStream) { + if (err) { + return reject(err); + } else { + buffer(readStream!).then( + (result) => { + resolve(result); + }, + (error) => { + reject(error); + } + ); + } + }); + }); +}; From 1f831e1403828969659d2308990fc8c098df7e40 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 3 Oct 2024 16:07:12 +0200 Subject: [PATCH 14/97] add plugin config --- .../knowledge_base_registry/server/config.ts | 13 +++++++++++-- .../knowledge_base_registry/server/index.ts | 2 ++ .../knowledge_base_registry/server/plugin.ts | 3 ++- .../services/package_installer/package_installer.ts | 8 +++++--- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts index a2e18568065e..fa461c8cf488 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts @@ -6,7 +6,16 @@ */ import { schema, type TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from '@kbn/core/server'; -export const config = schema.object({}); +const configSchema = schema.object({ + // TODO: use elastic.co subdomain once available + artifactRepositoryUrl: schema.string({ defaultValue: 'http://34.120.162.240' }), +}); -export type KnowledgeBaseRegistryConfig = TypeOf; +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: {}, +}; + +export type KnowledgeBaseRegistryConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts index 6644a292892c..ffad4020c419 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts @@ -15,6 +15,8 @@ import type { } from './types'; import { KnowledgeBaseRegistryPlugin } from './plugin'; +export { config } from './config'; + export type { KnowledgeBaseRegistrySetupContract, KnowledgeBaseRegistryStartContract }; export const plugin: PluginInitializer< diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts index 373a38598c66..b3f0013881b2 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts @@ -33,7 +33,7 @@ export class KnowledgeBaseRegistryPlugin { logger: Logger; - constructor(context: PluginInitializerContext) { + constructor(private readonly context: PluginInitializerContext) { this.logger = context.logger.get(); } setup( @@ -61,6 +61,7 @@ export class KnowledgeBaseRegistryPlugin esClient: core.elasticsearch.client.asInternalUser, productDocClient, artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'), + artifactRepositoryUrl: this.context.config.get().artifactRepositoryUrl, logger: this.logger.get('package-installer'), }); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts index 434343a11cdc..8812b115c92b 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts @@ -21,28 +21,30 @@ import { } from './utils'; import { createIndex, populateIndex } from './steps'; -const ARTIFACT_BUCKET_URL = 'http://34.120.162.240'; - export class PackageInstaller { private readonly logger: Logger; private readonly artifactsFolder: string; private readonly esClient: ElasticsearchClient; private readonly productDocClient: ProductDocInstallClient; + private readonly artifactRepositoryUrl: string; constructor({ artifactsFolder, logger, esClient, productDocClient, + artifactRepositoryUrl, }: { artifactsFolder: string; logger: Logger; esClient: ElasticsearchClient; productDocClient: ProductDocInstallClient; + artifactRepositoryUrl: string; }) { this.esClient = esClient; this.productDocClient = productDocClient; this.artifactsFolder = artifactsFolder; + this.artifactRepositoryUrl = artifactRepositoryUrl; this.logger = logger; } @@ -61,7 +63,7 @@ export class PackageInstaller { // TODO: ensure elser is installed const artifactFileName = getArtifactName({ productName, productVersion }); - const artifactUrl = `${ARTIFACT_BUCKET_URL}/${artifactFileName}`; + const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`; const artifactPath = `${this.artifactsFolder}/${artifactFileName}`; console.log(`*** downloading ${artifactUrl} to ${artifactPath}`); From 5292ad2f9bdf597ccbc482947cc36bd09c25ab4b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 4 Oct 2024 09:58:03 +0200 Subject: [PATCH 15/97] add inference endpoint installation logic --- .../knowledge_base_registry/common/consts.ts | 2 + .../knowledge_base_registry/server/plugin.ts | 8 ++++ .../inference_endpoint/endpoint_manager.ts | 41 +++++++++++++++++++ .../services/inference_endpoint/index.ts | 8 ++++ .../utils/get_model_install_status.ts | 35 ++++++++++++++++ .../inference_endpoint/utils/index.ts | 10 +++++ .../inference_endpoint/utils/install_elser.ts | 32 +++++++++++++++ .../utils/wait_until_model_deployed.ts | 40 ++++++++++++++++++ .../package_installer/package_installer.ts | 39 +++++++++++++----- 9 files changed, 204 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/endpoint_manager.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/get_model_install_status.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/index.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/install_elser.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/wait_until_model_deployed.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts index 3ff0b31f48ed..f23f149b4572 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts @@ -6,3 +6,5 @@ */ export const knowledgeBaseProductDocInstallTypeName = 'ai_kb_product_doc_installation'; + +export const internalElserInferenceId = 'kibana-internal-elser2'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts index b3f0013881b2..f3cc0b7e62b9 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts @@ -20,6 +20,7 @@ import type { } from './types'; import { knowledgeBaseProductDocInstallSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; +import { InferenceEndpointManager } from './services/inference_endpoint'; import { ProductDocInstallClient } from './dao/product_doc_install'; export class KnowledgeBaseRegistryPlugin @@ -57,9 +58,15 @@ export class KnowledgeBaseRegistryPlugin ); const productDocClient = new ProductDocInstallClient({ soClient }); + const endpointManager = new InferenceEndpointManager({ + esClient: core.elasticsearch.client.asInternalUser, + logger: this.logger.get('endpoint-manager'), + }); + const packageInstaller = new PackageInstaller({ esClient: core.elasticsearch.client.asInternalUser, productDocClient, + endpointManager, artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'), artifactRepositoryUrl: this.context.config.get().artifactRepositoryUrl, logger: this.logger.get('package-installer'), @@ -67,6 +74,7 @@ export class KnowledgeBaseRegistryPlugin delay(10) .then(() => { + return; console.log('*** test installating package'); return packageInstaller.installPackage({ productName: 'Kibana', diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/endpoint_manager.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/endpoint_manager.ts new file mode 100644 index 000000000000..4f7467501d61 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/endpoint_manager.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { internalElserInferenceId } from '../../../common/consts'; +import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils'; + +export class InferenceEndpointManager { + private readonly log: Logger; + private readonly esClient: ElasticsearchClient; + + constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) { + this.log = logger; + this.esClient = esClient; + } + + async ensureInternalElserInstalled() { + const { installed } = await getModelInstallStatus({ + inferenceId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + if (!installed) { + await installElser({ + inferenceId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + } + + await waitUntilModelDeployed({ + modelId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + } +} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/index.ts new file mode 100644 index 000000000000..e4098ff58fe5 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { InferenceEndpointManager } from './endpoint_manager'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/get_model_install_status.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/get_model_install_status.ts new file mode 100644 index 000000000000..49da426a1925 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/get_model_install_status.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceTaskType } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const getModelInstallStatus = async ({ + inferenceId, + taskType = 'sparse_embedding', + client, + log, +}: { + inferenceId: string; + taskType?: InferenceTaskType; + client: ElasticsearchClient; + log: Logger; +}) => { + const getInferenceRes = await client.inference.get( + { + task_type: taskType, + inference_id: inferenceId, + }, + { ignore: [404] } + ); + + const installed = (getInferenceRes.endpoints ?? []).some( + (endpoint) => endpoint.inference_id === inferenceId + ); + + return { installed }; +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/index.ts new file mode 100644 index 000000000000..089997557f30 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { waitUntilModelDeployed } from './wait_until_model_deployed'; +export { getModelInstallStatus } from './get_model_install_status'; +export { installElser } from './install_elser'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/install_elser.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/install_elser.ts new file mode 100644 index 000000000000..d1e054042b52 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/install_elser.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const installElser = async ({ + inferenceId, + client, + log, +}: { + inferenceId: string; + client: ElasticsearchClient; + log: Logger; +}) => { + await client.inference.put({ + task_type: 'sparse_embedding', + inference_id: inferenceId, + inference_config: { + service: 'elser', + service_settings: { + num_allocations: 1, + num_threads: 1, + model_id: '.elser_model_2', + }, + task_settings: {}, + }, + }); +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/wait_until_model_deployed.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/wait_until_model_deployed.ts new file mode 100644 index 000000000000..53f022174fef --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/wait_until_model_deployed.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const waitUntilModelDeployed = async ({ + modelId, + client, + log, + maxRetries = 20, + delay = 2000, +}: { + modelId: string; + client: ElasticsearchClient; + log: Logger; + maxRetries?: number; + delay?: number; +}) => { + for (let i = 0; i < maxRetries; i++) { + const statsRes = await client.ml.getTrainedModelsStats({ + model_id: modelId, + }); + const deploymentStats = statsRes.trained_model_stats[0]?.deployment_stats; + // @ts-expect-error deploymentStats.nodes not defined as array even if it is. + if (!deploymentStats || deploymentStats.nodes.length === 0) { + log.debug(`ML model [${modelId}] was not deployed - attempt ${i + 1} of ${maxRetries}`); + await sleep(delay); + continue; + } + return; + } + + throw new Error(`Timeout waiting for ML model ${modelId} to be deployed`); +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts index 8812b115c92b..b9c0422a9899 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts @@ -12,6 +12,7 @@ import { getDataPath } from '@kbn/utils'; import { Logger } from '@kbn/logging'; import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import type { ProductDocInstallClient } from '../../dao/product_doc_install'; +import type { InferenceEndpointManager } from '../inference_endpoint'; import { downloadToDisk, openZipArchive, @@ -21,11 +22,21 @@ import { } from './utils'; import { createIndex, populateIndex } from './steps'; +interface PackageInstallerOpts { + artifactsFolder: string; + logger: Logger; + esClient: ElasticsearchClient; + productDocClient: ProductDocInstallClient; + endpointManager: InferenceEndpointManager; + artifactRepositoryUrl: string; +} + export class PackageInstaller { private readonly logger: Logger; private readonly artifactsFolder: string; private readonly esClient: ElasticsearchClient; private readonly productDocClient: ProductDocInstallClient; + private readonly endpointManager: InferenceEndpointManager; private readonly artifactRepositoryUrl: string; constructor({ @@ -33,17 +44,13 @@ export class PackageInstaller { logger, esClient, productDocClient, + endpointManager, artifactRepositoryUrl, - }: { - artifactsFolder: string; - logger: Logger; - esClient: ElasticsearchClient; - productDocClient: ProductDocInstallClient; - artifactRepositoryUrl: string; - }) { + }: PackageInstallerOpts) { this.esClient = esClient; this.productDocClient = productDocClient; this.artifactsFolder = artifactsFolder; + this.endpointManager = endpointManager; this.artifactRepositoryUrl = artifactRepositoryUrl; this.logger = logger; } @@ -60,7 +67,8 @@ export class PackageInstaller { productName, productVersion, }); - // TODO: ensure elser is installed + + await this.endpointManager.ensureInternalElserInstalled(); const artifactFileName = getArtifactName({ productName, productVersion }); const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`; @@ -75,8 +83,7 @@ export class PackageInstaller { const manifest = await loadManifestFile(zipArchive); const mappings = await loadMappingFile(zipArchive); - // TODO: move helper to package - const indexName = `.kibana-ai-kb-${manifest.productName}-${productVersion}`.toLowerCase(); + const indexName = getIndexName({ productName, productVersion }); await createIndex({ indexName, @@ -107,7 +114,7 @@ export class PackageInstaller { }) { // TODO - const indexName = `.kibana-ai-kb-${productName}-${productVersion}`.toLowerCase(); + const indexName = getIndexName({ productName, productVersion }); await this.esClient.indices.delete( { index: indexName, @@ -127,3 +134,13 @@ const getArtifactName = ({ }): string => { return `kibana-kb-${productName}-${productVersion}.zip`.toLowerCase(); }; + +const getIndexName = ({ + productName, + productVersion, +}: { + productName: string; + productVersion: string; +}): string => { + return `.kibana-ai-kb-${productName}-${productVersion}`.toLowerCase(); +}; From 0689dbc1ed93e82d7ef772e724eae4d977ef8b0c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 4 Oct 2024 14:53:14 +0200 Subject: [PATCH 16/97] reports install status --- .../knowledge_base_registry/common/consts.ts | 2 +- .../common/saved_objects.ts | 10 +-- .../index.ts | 0 .../model_conversion.ts} | 10 +-- .../product_doc_install_service.ts | 80 +++++++++++++++++++ .../product_doc_install_service.ts | 35 -------- .../knowledge_base_registry/server/plugin.ts | 7 +- .../saved_objects/product_doc_install.ts | 21 +++-- .../package_installer/package_installer.ts | 68 ++++++++++------ .../package_installer/steps/create_index.ts | 9 ++- .../package_installer/steps/populate_index.ts | 3 +- 11 files changed, 156 insertions(+), 89 deletions(-) rename x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/{product_doc_install => doc_install_status}/index.ts (100%) rename x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/{product_doc_install/so_to_dto.ts => doc_install_status/model_conversion.ts} (73%) create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts delete mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts index f23f149b4572..bc83730d6389 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts @@ -5,6 +5,6 @@ * 2.0. */ -export const knowledgeBaseProductDocInstallTypeName = 'ai_kb_product_doc_installation'; +export const productDocInstallStatusSavedObjectTypeName = 'kb_product_doc_install_status'; export const internalElserInferenceId = 'kibana-internal-elser2'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts index 0c0fb4732d4b..17e2baffa1cc 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts @@ -5,14 +5,14 @@ * 2.0. */ -export type ProductDocInstallStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; +export type InstallationStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; -export interface ProducDocInstallDTO { +export interface ProductDocInstallStatus { id: string; - packageName: string; - packageVersion: string; productName: string; - installationStatus: ProductDocInstallStatus; + productVersion: string; + installationStatus: InstallationStatus; lastInstallationDate: Date | undefined; + lastInstallationFailureReason: string | undefined; indexName: string; } diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/index.ts rename to x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/so_to_dto.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/model_conversion.ts similarity index 73% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/so_to_dto.ts rename to x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/model_conversion.ts index 2c8a8f61ccb0..f917cdb7d883 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/so_to_dto.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/model_conversion.ts @@ -6,21 +6,21 @@ */ import type { SavedObject } from '@kbn/core/server'; -import type { ProducDocInstallDTO } from '../../../common/saved_objects'; +import type { ProductDocInstallStatus } from '../../../common/saved_objects'; import type { KnowledgeBaseProductDocInstallAttributes } from '../../saved_objects'; -export const soToDto = ( +export const soToModel = ( so: SavedObject -): ProducDocInstallDTO => { +): ProductDocInstallStatus => { return { id: so.id, - packageName: so.attributes.package_name, - packageVersion: so.attributes.package_version, productName: so.attributes.product_name, + productVersion: so.attributes.product_version, installationStatus: so.attributes.installation_status, indexName: so.attributes.index_name, lastInstallationDate: so.attributes.last_installation_date ? new Date(so.attributes.last_installation_date) : undefined, + lastInstallationFailureReason: so.attributes.last_installation_failure_reason, }; }; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts new file mode 100644 index 000000000000..ea2475748610 --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract, SavedObject } from '@kbn/core/server'; +import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; +import type { InstallationStatus, ProductDocInstallStatus } from '../../../common/saved_objects'; +import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; +import type { KnowledgeBaseProductDocInstallAttributes as TypeAttributes } from '../../saved_objects'; +import { soToModel } from './model_conversion'; + +export class ProductDocInstallClient { + private soClient: SavedObjectsClientContract; + + constructor({ soClient }: { soClient: SavedObjectsClientContract }) { + this.soClient = soClient; + } + + /* + async getForProduct(productName: string): Promise { + const objectId = getObjectIdFromProductName(productName); + try { + const object = await this.soClient.get(typeName, objectId); + return soToModel(object); + } catch (e) { + // TODO + } + } + */ + + async setInstallationStarted(fields: { productName: string; productVersion: string }) { + const { productName, productVersion } = fields; + const objectId = getObjectIdFromProductName(productName); + const attributes = { + product_name: productName, + product_version: productVersion, + installation_status: 'installing' as const, + last_installation_failure_reason: '', + }; + await this.soClient.update(typeName, objectId, attributes, { + upsert: attributes, + }); + } + + async setInstallationSuccessful(productName: string, indexName: string) { + const objectId = getObjectIdFromProductName(productName); + await this.soClient.update(typeName, objectId, { + installation_status: 'installed', + index_name: indexName, + }); + } + + async setInstallationFailed(productName: string, failureReason: string) { + const objectId = getObjectIdFromProductName(productName); + await this.soClient.update(typeName, objectId, { + installation_status: 'error', + last_installation_failure_reason: failureReason, + }); + } + + async setUninstalled(productName: string) { + const objectId = getObjectIdFromProductName(productName); + try { + await this.soClient.update(typeName, objectId, { + installation_status: 'uninstalled', + last_installation_failure_reason: '', + }); + } catch (e) { + if (!SavedObjectsErrorHelpers.isNotFoundError(e)) { + throw e; + } + } + } +} + +const getObjectIdFromProductName = (productName: string) => + `kb-product-doc-${productName}-status`.toLowerCase(); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts deleted file mode 100644 index f1709a058509..000000000000 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/product_doc_install/product_doc_install_service.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SavedObjectsClientContract, SavedObject } from '@kbn/core/server'; -import type { ProducDocInstallDTO } from '../../../common/saved_objects'; -import { knowledgeBaseProductDocInstallTypeName as typeName } from '../../../common/consts'; -import type { KnowledgeBaseProductDocInstallAttributes } from '../../saved_objects'; -import { soToDto } from './so_to_dto'; - -export class ProductDocInstallClient { - private soClient: SavedObjectsClientContract; - - constructor({ soClient }: { soClient: SavedObjectsClientContract }) { - this.soClient = soClient; - } - - async getByProductName(productName: string): Promise { - const response = await this.soClient.find({ - type: typeName, - perPage: 1, - filter: `${typeName}.product_name: "${productName}"`, - }); - if (response.saved_objects.length === 1) { - return soToDto(response.saved_objects[0]); - } - if (response.saved_objects.length > 1) { - throw new Error(`Found multiple records for product name : ${productName}`); - } - return undefined; - } -} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts index f3cc0b7e62b9..b323d3205b1a 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts @@ -10,7 +10,7 @@ import type { Logger } from '@kbn/logging'; import { getDataPath } from '@kbn/utils'; import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; import { SavedObjectsClient } from '@kbn/core/server'; -import { knowledgeBaseProductDocInstallTypeName } from '../common/consts'; +import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; import type { KnowledgeBaseRegistryConfig } from './config'; import type { KnowledgeBaseRegistrySetupContract, @@ -21,7 +21,7 @@ import type { import { knowledgeBaseProductDocInstallSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; import { InferenceEndpointManager } from './services/inference_endpoint'; -import { ProductDocInstallClient } from './dao/product_doc_install'; +import { ProductDocInstallClient } from './dao/doc_install_status'; export class KnowledgeBaseRegistryPlugin implements @@ -54,7 +54,7 @@ export class KnowledgeBaseRegistryPlugin pluginsStart: KnowledgeBaseRegistryStartDependencies ): KnowledgeBaseRegistryStartContract { const soClient = new SavedObjectsClient( - core.savedObjects.createInternalRepository([knowledgeBaseProductDocInstallTypeName]) + core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName]) ); const productDocClient = new ProductDocInstallClient({ soClient }); @@ -74,7 +74,6 @@ export class KnowledgeBaseRegistryPlugin delay(10) .then(() => { - return; console.log('*** test installating package'); return packageInstaller.installPackage({ productName: 'Kibana', diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts index 5cbd1cdea335..b84fa3dd6515 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts @@ -6,8 +6,8 @@ */ import type { SavedObjectsType } from '@kbn/core/server'; -import { knowledgeBaseProductDocInstallTypeName } from '../../common/consts'; -import type { ProductDocInstallStatus } from '../../common/saved_objects'; +import { productDocInstallStatusSavedObjectTypeName } from '../../common/consts'; +import type { InstallationStatus } from '../../common/saved_objects'; /** * Interface describing the raw attributes of the KB Entry SO type. @@ -15,25 +15,24 @@ import type { ProductDocInstallStatus } from '../../common/saved_objects'; * indexed fields. */ export interface KnowledgeBaseProductDocInstallAttributes { - package_name: string; - package_version: string; product_name: string; - installation_status: ProductDocInstallStatus; - last_installation_date: number | undefined; - index_name: string; + product_version: string; + installation_status: InstallationStatus; + last_installation_date?: number; + last_installation_failure_reason?: string; + index_name?: string; } export const knowledgeBaseProductDocInstallSavedObjectType: SavedObjectsType = { - name: knowledgeBaseProductDocInstallTypeName, + name: productDocInstallStatusSavedObjectTypeName, hidden: true, - namespaceType: 'multiple', + namespaceType: 'agnostic', mappings: { dynamic: false, properties: { - package_name: { type: 'keyword' }, - package_version: { type: 'keyword' }, product_name: { type: 'keyword' }, + product_version: { type: 'keyword' }, installation_status: { type: 'keyword' }, last_installation_date: { type: 'integer' }, index_name: { type: 'keyword' }, diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts index b9c0422a9899..452f5e8426fb 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts @@ -5,13 +5,9 @@ * 2.0. */ -import fetch from 'node-fetch'; -import Fs from 'fs'; -import Path from 'path'; -import { getDataPath } from '@kbn/utils'; import { Logger } from '@kbn/logging'; -import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; -import type { ProductDocInstallClient } from '../../dao/product_doc_install'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { ProductDocInstallClient } from '../../dao/doc_install_status'; import type { InferenceEndpointManager } from '../inference_endpoint'; import { downloadToDisk, @@ -19,6 +15,7 @@ import { validateArtifactArchive, loadManifestFile, loadMappingFile, + type ZipArchive, } from './utils'; import { createIndex, populateIndex } from './steps'; @@ -32,7 +29,7 @@ interface PackageInstallerOpts { } export class PackageInstaller { - private readonly logger: Logger; + private readonly log: Logger; private readonly artifactsFolder: string; private readonly esClient: ElasticsearchClient; private readonly productDocClient: ProductDocInstallClient; @@ -52,7 +49,7 @@ export class PackageInstaller { this.artifactsFolder = artifactsFolder; this.endpointManager = endpointManager; this.artifactRepositoryUrl = artifactRepositoryUrl; - this.logger = logger; + this.log = logger; } async installPackage({ @@ -62,24 +59,35 @@ export class PackageInstaller { productName: string; productVersion: string; }) { - // TODO: uninstall previous/current if present + this.log.info( + `Starting pkg installation for product [${productName}] and version [${productVersion}]` + ); + await this.uninstallPackage({ productName, productVersion, }); - await this.endpointManager.ensureInternalElserInstalled(); + let zipArchive: ZipArchive | undefined; + try { + await this.productDocClient.setInstallationStarted({ + productName, + productVersion, + }); - const artifactFileName = getArtifactName({ productName, productVersion }); - const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`; - const artifactPath = `${this.artifactsFolder}/${artifactFileName}`; + await this.endpointManager.ensureInternalElserInstalled(); - console.log(`*** downloading ${artifactUrl} to ${artifactPath}`); + const artifactFileName = getArtifactName({ productName, productVersion }); + const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`; + const artifactPath = `${this.artifactsFolder}/${artifactFileName}`; - await downloadToDisk(artifactUrl, artifactPath); + this.log.debug(`Downloading from [${artifactUrl}] to [${artifactPath}]`); + await downloadToDisk(artifactUrl, artifactPath); + + zipArchive = await openZipArchive(artifactPath); + + validateArtifactArchive(zipArchive); - const zipArchive = await openZipArchive(artifactPath); - try { const manifest = await loadManifestFile(zipArchive); const mappings = await loadMappingFile(zipArchive); @@ -89,19 +97,31 @@ export class PackageInstaller { indexName, mappings, esClient: this.esClient, - log: this.logger, + log: this.log, }); await populateIndex({ indexName, archive: zipArchive, esClient: this.esClient, - log: this.logger, + log: this.log, }); - - // TODO: update the product_doc_install SO + await this.productDocClient.setInstallationSuccessful(productName, indexName); + + this.log.info( + `Pkg installation successful for product [${productName}] and version [${productVersion}]` + ); + } catch (e) { + this.log.error( + `Error during pkg installation of product [${productName}]/[${productVersion}] : ${e.message}` + ); + + await this.productDocClient.setInstallationFailed(productName, e.message); + throw e; } finally { - zipArchive.close(); + if (zipArchive) { + zipArchive.close(); + } } } @@ -112,7 +132,7 @@ export class PackageInstaller { productName: string; productVersion: string; }) { - // TODO + // TODO: retrieve entry to check installed version instead const indexName = getIndexName({ productName, productVersion }); await this.esClient.indices.delete( @@ -121,6 +141,8 @@ export class PackageInstaller { }, { ignore: [404] } ); + + await this.productDocClient.setUninstalled(productName); } } diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts index f0013a7f599d..14d0705d7ccf 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts @@ -8,6 +8,7 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { internalElserInferenceId } from '../../../../common/consts'; export const createIndex = async ({ esClient, @@ -23,10 +24,10 @@ export const createIndex = async ({ log.debug(`Creating index ${indexName}`); // TODO: extract / do it right - mappings.properties.ai_questions_answered.inference_id = 'default-elser'; - mappings.properties.ai_subtitle.inference_id = 'default-elser'; - mappings.properties.ai_summary.inference_id = 'default-elser'; - mappings.properties.content_body.inference_id = 'default-elser'; + mappings.properties.ai_questions_answered.inference_id = internalElserInferenceId; + mappings.properties.ai_subtitle.inference_id = internalElserInferenceId; + mappings.properties.ai_summary.inference_id = internalElserInferenceId; + mappings.properties.content_body.inference_id = internalElserInferenceId; await esClient.indices.create({ index: indexName, diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts index 438a4348c10e..7462040a5857 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts @@ -8,6 +8,7 @@ import type { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; +import { internalElserInferenceId } from '../../../../common/consts'; import type { ZipArchive } from '../utils/zip_archive'; // TODO: factorize with utils/validate_artifact_archive @@ -58,7 +59,7 @@ const indexContentFile = async ({ .map((line) => { return JSON.parse(line); }) - .map((doc) => rewriteInferenceId(doc, 'default-elser')); + .map((doc) => rewriteInferenceId(doc, internalElserInferenceId)); const operations = documents.reduce((ops, document) => { ops!.push(...[{ index: { _index: indexName } }, document]); From a9f085a227a2ed84fc55cfc0cf84304711d3a910 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 7 Oct 2024 09:33:09 +0200 Subject: [PATCH 17/97] Add bucket content parsing --- .../knowledge_base_registry/common/consts.ts | 7 ++ .../common/saved_objects.ts | 6 +- .../product_doc_install_service.ts | 12 ++-- .../knowledge_base_registry/server/plugin.ts | 11 ++- .../saved_objects/product_doc_install.ts | 4 +- .../package_installer/package_installer.ts | 43 ++++++++++-- .../package_installer/steps/create_index.ts | 22 ++++-- .../package_installer/steps/populate_index.ts | 12 ++-- .../utils/fetch_artifact_versions.ts | 67 +++++++++++++++++++ .../services/package_installer/utils/index.ts | 1 + .../package_installer/utils/semver.ts | 26 +++++++ 11 files changed, 179 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/fetch_artifact_versions.ts create mode 100644 x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/semver.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts index bc83730d6389..149b2c202887 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts @@ -8,3 +8,10 @@ export const productDocInstallStatusSavedObjectTypeName = 'kb_product_doc_install_status'; export const internalElserInferenceId = 'kibana-internal-elser2'; + +export enum DocumentationProduct { + kibana = 'kibana', + elasticsearch = 'elasticsearch', + observability = 'observability', + security = 'security', +} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts index 17e2baffa1cc..f1cbf9149921 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts @@ -5,11 +5,15 @@ * 2.0. */ +import { DocumentationProduct } from './consts'; + +export type ProductName = keyof typeof DocumentationProduct; + export type InstallationStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; export interface ProductDocInstallStatus { id: string; - productName: string; + productName: ProductName; productVersion: string; installationStatus: InstallationStatus; lastInstallationDate: Date | undefined; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts index ea2475748610..bdf50badb67f 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts @@ -7,7 +7,7 @@ import type { SavedObjectsClientContract, SavedObject } from '@kbn/core/server'; import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; -import type { InstallationStatus, ProductDocInstallStatus } from '../../../common/saved_objects'; +import type { InstallationStatus, ProductName, ProductDocInstallStatus } from '../../../common/saved_objects'; import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; import type { KnowledgeBaseProductDocInstallAttributes as TypeAttributes } from '../../saved_objects'; import { soToModel } from './model_conversion'; @@ -31,7 +31,7 @@ export class ProductDocInstallClient { } */ - async setInstallationStarted(fields: { productName: string; productVersion: string }) { + async setInstallationStarted(fields: { productName: ProductName; productVersion: string }) { const { productName, productVersion } = fields; const objectId = getObjectIdFromProductName(productName); const attributes = { @@ -45,7 +45,7 @@ export class ProductDocInstallClient { }); } - async setInstallationSuccessful(productName: string, indexName: string) { + async setInstallationSuccessful(productName: ProductName, indexName: string) { const objectId = getObjectIdFromProductName(productName); await this.soClient.update(typeName, objectId, { installation_status: 'installed', @@ -53,7 +53,7 @@ export class ProductDocInstallClient { }); } - async setInstallationFailed(productName: string, failureReason: string) { + async setInstallationFailed(productName: ProductName, failureReason: string) { const objectId = getObjectIdFromProductName(productName); await this.soClient.update(typeName, objectId, { installation_status: 'error', @@ -61,7 +61,7 @@ export class ProductDocInstallClient { }); } - async setUninstalled(productName: string) { + async setUninstalled(productName: ProductName) { const objectId = getObjectIdFromProductName(productName); try { await this.soClient.update(typeName, objectId, { @@ -76,5 +76,5 @@ export class ProductDocInstallClient { } } -const getObjectIdFromProductName = (productName: string) => +const getObjectIdFromProductName = (productName: ProductName) => `kb-product-doc-${productName}-status`.toLowerCase(); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts index b323d3205b1a..a875fcbf99b7 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts @@ -67,18 +67,17 @@ export class KnowledgeBaseRegistryPlugin esClient: core.elasticsearch.client.asInternalUser, productDocClient, endpointManager, + kibanaVersion: this.context.env.packageInfo.version, artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'), artifactRepositoryUrl: this.context.config.get().artifactRepositoryUrl, logger: this.logger.get('package-installer'), }); delay(10) - .then(() => { - console.log('*** test installating package'); - return packageInstaller.installPackage({ - productName: 'Kibana', - productVersion: '8.15', - }); + .then(async () => { + console.log('*** test installating packages'); + + return packageInstaller.installAll({}); }) .catch((e) => { console.log('*** ERROR', e); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts index b84fa3dd6515..95dbb772c9d7 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts @@ -7,7 +7,7 @@ import type { SavedObjectsType } from '@kbn/core/server'; import { productDocInstallStatusSavedObjectTypeName } from '../../common/consts'; -import type { InstallationStatus } from '../../common/saved_objects'; +import type { InstallationStatus, ProductName } from '../../common/saved_objects'; /** * Interface describing the raw attributes of the KB Entry SO type. @@ -15,7 +15,7 @@ import type { InstallationStatus } from '../../common/saved_objects'; * indexed fields. */ export interface KnowledgeBaseProductDocInstallAttributes { - product_name: string; + product_name: ProductName; product_version: string; installation_status: InstallationStatus; last_installation_date?: number; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts index 452f5e8426fb..c8da5c1f945d 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts @@ -5,8 +5,10 @@ * 2.0. */ -import { Logger } from '@kbn/logging'; +import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; +import { DocumentationProduct } from '../../../common/consts'; +import type { ProductName } from '../../../common/saved_objects'; import type { ProductDocInstallClient } from '../../dao/doc_install_status'; import type { InferenceEndpointManager } from '../inference_endpoint'; import { @@ -15,8 +17,10 @@ import { validateArtifactArchive, loadManifestFile, loadMappingFile, + fetchArtifactVersions, type ZipArchive, } from './utils'; +import { majorMinor, latestVersion } from './utils/semver'; import { createIndex, populateIndex } from './steps'; interface PackageInstallerOpts { @@ -26,6 +30,7 @@ interface PackageInstallerOpts { productDocClient: ProductDocInstallClient; endpointManager: InferenceEndpointManager; artifactRepositoryUrl: string; + kibanaVersion: string; } export class PackageInstaller { @@ -35,6 +40,7 @@ export class PackageInstaller { private readonly productDocClient: ProductDocInstallClient; private readonly endpointManager: InferenceEndpointManager; private readonly artifactRepositoryUrl: string; + private readonly currentVersion: string; constructor({ artifactsFolder, @@ -43,24 +49,48 @@ export class PackageInstaller { productDocClient, endpointManager, artifactRepositoryUrl, + kibanaVersion, }: PackageInstallerOpts) { this.esClient = esClient; this.productDocClient = productDocClient; this.artifactsFolder = artifactsFolder; this.endpointManager = endpointManager; this.artifactRepositoryUrl = artifactRepositoryUrl; + this.currentVersion = majorMinor(kibanaVersion); this.log = logger; } + async installAll({}: {}) { + const artifactVersions = await fetchArtifactVersions({ + artifactRepositoryUrl: this.artifactRepositoryUrl, + }); + const allProducts = Object.values(DocumentationProduct) as ProductName[]; + for (const productName of allProducts) { + const availableVersions = artifactVersions[productName]; + if (!availableVersions || !availableVersions.length) { + this.log.warn(`No version found for product [${productName}]`); + continue; + } + const selectedVersion = availableVersions.includes(this.currentVersion) + ? this.currentVersion + : latestVersion(availableVersions); + + await this.installPackage({ + productName, + productVersion: selectedVersion, + }); + } + } + async installPackage({ productName, productVersion, }: { - productName: string; + productName: ProductName; productVersion: string; }) { this.log.info( - `Starting pkg installation for product [${productName}] and version [${productVersion}]` + `Starting installing documentation for product [${productName}] and version [${productVersion}]` ); await this.uninstallPackage({ @@ -109,11 +139,11 @@ export class PackageInstaller { await this.productDocClient.setInstallationSuccessful(productName, indexName); this.log.info( - `Pkg installation successful for product [${productName}] and version [${productVersion}]` + `Documentation installation successful for product [${productName}] and version [${productVersion}]` ); } catch (e) { this.log.error( - `Error during pkg installation of product [${productName}]/[${productVersion}] : ${e.message}` + `Error during documentation installation of product [${productName}]/[${productVersion}] : ${e.message}` ); await this.productDocClient.setInstallationFailed(productName, e.message); @@ -129,7 +159,7 @@ export class PackageInstaller { productName, productVersion, }: { - productName: string; + productName: ProductName; productVersion: string; }) { // TODO: retrieve entry to check installed version instead @@ -164,5 +194,6 @@ const getIndexName = ({ productName: string; productVersion: string; }): string => { + // TODO: '.kibana-product-doc-*' return `.kibana-ai-kb-${productName}-${productVersion}`.toLowerCase(); }; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts index 14d0705d7ccf..decd62e556ba 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts @@ -7,7 +7,7 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; -import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { MappingTypeMapping, MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import { internalElserInferenceId } from '../../../../common/consts'; export const createIndex = async ({ @@ -23,11 +23,7 @@ export const createIndex = async ({ }) => { log.debug(`Creating index ${indexName}`); - // TODO: extract / do it right - mappings.properties.ai_questions_answered.inference_id = internalElserInferenceId; - mappings.properties.ai_subtitle.inference_id = internalElserInferenceId; - mappings.properties.ai_summary.inference_id = internalElserInferenceId; - mappings.properties.content_body.inference_id = internalElserInferenceId; + overrideInferenceId(mappings, internalElserInferenceId); await esClient.indices.create({ index: indexName, @@ -38,3 +34,17 @@ export const createIndex = async ({ }, }); }; + +const overrideInferenceId = (mappings: MappingTypeMapping, inferenceId: string) => { + const recursiveOverride = (current: MappingTypeMapping | MappingProperty) => { + if ('type' in current && current.type === 'semantic_text') { + current.inference_id = inferenceId; + } + if ('properties' in current && current.properties) { + for (const prop of Object.values(current.properties)) { + recursiveOverride(prop); + } + } + }; + recursiveOverride(mappings); +}; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts index 7462040a5857..1583074d85ca 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts @@ -77,11 +77,13 @@ const indexContentFile = async ({ } }; -// TODO: extract / do it right. +// TODO: extract const rewriteInferenceId = (document: Record, inferenceId: string) => { - document.ai_questions_answered.inference.inference_id = inferenceId; - document.ai_subtitle.inference.inference_id = inferenceId; - document.ai_summary.inference.inference_id = inferenceId; - document.content_body.inference.inference_id = inferenceId; + // we don't need to handle nested fields, we don't have any and won't. + Object.values(document).forEach((field) => { + if (field.inference) { + field.inference.inference_id = inferenceId; + } + }); return document; }; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/fetch_artifact_versions.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/fetch_artifact_versions.ts new file mode 100644 index 000000000000..7a29c786fcac --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/fetch_artifact_versions.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fetch from 'node-fetch'; +import { parseString } from 'xml2js'; +import type { ProductName } from '../../../../common/saved_objects'; +import { DocumentationProduct } from '../../../../common/consts'; + +type ArtifactAvailableVersions = Record; + +// TODO: extract to common package +// kibana-kb-elasticsearch-8.15.zip +const artifactNameRegexp = /^kibana-kb-([a-zA-Z]+)-([0-9]+\.[0-9]+)\.zip$/; + +export const fetchArtifactVersions = async ({ + artifactRepositoryUrl, +}: { + artifactRepositoryUrl: string; +}): Promise => { + const res = await fetch(`${artifactRepositoryUrl}?max-keys=1000`); + const xml = await res.text(); + return new Promise((resolve, reject) => { + parseString(xml, (err, result: ListBucketResponse) => { + if (err) { + reject(err); + } + + // 6 artifacts per minor stack version means we have a few decades before facing this problem + if (result.ListBucketResult.IsTruncated?.includes('true')) { + throw new Error('bucket content is truncated, cannot retrieve all versions'); + } + + const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); + + const record: ArtifactAvailableVersions = {} as ArtifactAvailableVersions; + allowedProductNames.forEach((product) => { + record[product] = []; + }); + + result.ListBucketResult.Contents.forEach((contentEntry) => { + const artifactName = contentEntry.Key[0]; + const match = artifactNameRegexp.exec(artifactName); + if (match) { + const productName = match[1].toLowerCase() as ProductName; + const productVersion = match[2].toLowerCase(); + if (allowedProductNames.includes(productName)) { + record[productName]!.push(productVersion); + } + } + }); + + resolve(record); + }); + }); +}; + +interface ListBucketResponse { + ListBucketResult: { + Name: string[]; + IsTruncated: string[]; + Contents: Array<{ Key: string[] }>; + }; +} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts index db68bc13f365..5af3e4568bbb 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts @@ -9,3 +9,4 @@ export { downloadToDisk } from './download'; export { openZipArchive, type ZipArchive } from './zip_archive'; export { validateArtifactArchive } from './validate_artifact_archive'; export { loadManifestFile, loadMappingFile } from './archive_accessors'; +export { fetchArtifactVersions } from './fetch_artifact_versions'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/semver.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/semver.ts new file mode 100644 index 000000000000..6b405f2ceb4e --- /dev/null +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/semver.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Semver from 'semver'; + +export const latestVersion = (versions: string[]): string => { + let latest: string = versions[0]; + for (let i = 1; 1 < versions.length; i++) { + if (Semver.gt(versions[i], latest)) { + latest = versions[i]; + } + } + return latest; +}; + +export const majorMinor = (version: string): string => { + const parsed = Semver.parse(version); + if (!parsed) { + throw new Error(`Not a valid semver version: [${version}]`); + } + return `${parsed.major}.${parsed.minor}`; +}; From 8bbd7e8a9930f945effe528373c854ef51839a63 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 7 Oct 2024 14:30:28 +0200 Subject: [PATCH 18/97] create new package --- package.json | 1 + tsconfig.base.json | 2 ++ .../ai-infra/product-doc-common/README.md | 3 +++ .../ai-infra/product-doc-common/index.ts | 10 ++++++++++ .../ai-infra/product-doc-common/jest.config.js | 12 ++++++++++++ .../ai-infra/product-doc-common/kibana.jsonc | 5 +++++ .../ai-infra/product-doc-common/package.json | 6 ++++++ .../ai-infra/product-doc-common/tsconfig.json | 17 +++++++++++++++++ yarn.lock | 4 ++++ 9 files changed, 60 insertions(+) create mode 100644 x-pack/packages/ai-infra/product-doc-common/README.md create mode 100644 x-pack/packages/ai-infra/product-doc-common/index.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/jest.config.js create mode 100644 x-pack/packages/ai-infra/product-doc-common/kibana.jsonc create mode 100644 x-pack/packages/ai-infra/product-doc-common/package.json create mode 100644 x-pack/packages/ai-infra/product-doc-common/tsconfig.json diff --git a/package.json b/package.json index 41c9579b2b8a..2ee3bb8fbb47 100644 --- a/package.json +++ b/package.json @@ -708,6 +708,7 @@ "@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel", "@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing", "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util", + "@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common", "@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access", "@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling", "@kbn/profiling-utils": "link:packages/kbn-profiling-utils", diff --git a/tsconfig.base.json b/tsconfig.base.json index d8aa59eef235..6bef7cb6a541 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1364,6 +1364,8 @@ "@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"], "@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"], "@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"], + "@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"], + "@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"], "@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"], "@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/observability_solution/profiling_data_access/*"], "@kbn/profiling-plugin": ["x-pack/plugins/observability_solution/profiling"], diff --git a/x-pack/packages/ai-infra/product-doc-common/README.md b/x-pack/packages/ai-infra/product-doc-common/README.md new file mode 100644 index 000000000000..6a309c5de351 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/README.md @@ -0,0 +1,3 @@ +# @kbn/product-doc-common + +Empty package generated by @kbn/generate diff --git a/x-pack/packages/ai-infra/product-doc-common/index.ts b/x-pack/packages/ai-infra/product-doc-common/index.ts new file mode 100644 index 000000000000..7210d7181a9e --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function foo() { + return 'hello world'; +} diff --git a/x-pack/packages/ai-infra/product-doc-common/jest.config.js b/x-pack/packages/ai-infra/product-doc-common/jest.config.js new file mode 100644 index 000000000000..e6cae43806c8 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../../..', + roots: ['/x-pack/packages/ai-infra/product-doc-common'], +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc b/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc new file mode 100644 index 000000000000..16336c1fc8e2 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/product-doc-common", + "owner": "@elastic/appex-ai-infra" +} diff --git a/x-pack/packages/ai-infra/product-doc-common/package.json b/x-pack/packages/ai-infra/product-doc-common/package.json new file mode 100644 index 000000000000..5dfd6eed6cfc --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/product-doc-common", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0" +} \ No newline at end of file diff --git a/x-pack/packages/ai-infra/product-doc-common/tsconfig.json b/x-pack/packages/ai-infra/product-doc-common/tsconfig.json new file mode 100644 index 000000000000..0d78dace105e --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [] +} diff --git a/yarn.lock b/yarn.lock index ea107a99ecb8..26ea644fb6b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6011,6 +6011,10 @@ version "0.0.0" uid "" +"@kbn/product-doc-common@link:x-pack/packages/ai-infra/product-doc-common": + version "0.0.0" + uid "" + "@kbn/profiling-data-access-plugin@link:x-pack/plugins/observability_solution/profiling_data_access": version "0.0.0" uid "" From 8e9680826bfe7bc9d4fc736a0a6ef9f8d24c811c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 7 Oct 2024 15:56:07 +0200 Subject: [PATCH 19/97] factorize reusable bits into common package --- .../src/artifact/manifest.ts | 6 +-- .../src/tasks/create_artifact.ts | 2 +- .../ai-infra/product-doc-common/index.ts | 7 ++-- .../product-doc-common/src/artifact.ts | 39 +++++++++++++++++++ .../src/artifact_content.ts | 12 ++++++ .../src/manifest.ts} | 10 ++--- .../product-doc-common/src/product.ts | 15 +++++++ .../knowledge_base_registry/common/consts.ts | 7 ---- .../common/saved_objects.ts | 4 +- .../knowledge_base_registry/server/config.ts | 5 ++- .../package_installer/package_installer.ts | 23 ++++------- .../fetch_artifact_versions.ts | 18 +++------ .../services/package_installer/steps/index.ts | 2 + .../package_installer/steps/populate_index.ts | 9 +---- .../validate_artifact_archive.ts | 7 ++-- .../utils/archive_accessors.ts | 8 +--- .../services/package_installer/utils/index.ts | 2 - 17 files changed, 99 insertions(+), 77 deletions(-) create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts rename x-pack/packages/ai-infra/{product-doc-artifact-builder/src/artifact/artifact_name.ts => product-doc-common/src/manifest.ts} (64%) create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/product.ts rename x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/{utils => steps}/fetch_artifact_versions.ts (72%) rename x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/{utils => steps}/validate_artifact_archive.ts (78%) diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts index cbebcdc22981..a6916f6fb9af 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts @@ -5,11 +5,7 @@ * 2.0. */ -export interface ArtifactManifest { - formatVersion: string; - productName: string; - productVersion: string; -} +import type { ArtifactManifest } from '@kbn/product-doc-common'; export const getArtifactManifest = ({ productName, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts index 343099876585..e289b894a639 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts @@ -8,9 +8,9 @@ import Path from 'path'; import AdmZip from 'adm-zip'; import type { ToolingLog } from '@kbn/tooling-log'; +import { getArtifactName } from '@kbn/product-doc-common'; import { getArtifactMappings } from '../artifact/mappings'; import { getArtifactManifest } from '../artifact/manifest'; -import { getArtifactName } from '../artifact/artifact_name'; export const createArtifact = async ({ productName, diff --git a/x-pack/packages/ai-infra/product-doc-common/index.ts b/x-pack/packages/ai-infra/product-doc-common/index.ts index 7210d7181a9e..4cd58111a72e 100644 --- a/x-pack/packages/ai-infra/product-doc-common/index.ts +++ b/x-pack/packages/ai-infra/product-doc-common/index.ts @@ -5,6 +5,7 @@ * 2.0. */ -export function foo() { - return 'hello world'; -} +export { getArtifactName, parseArtifactName } from './src/artifact'; +export { type ArtifactManifest } from './src/manifest'; +export { DocumentationProduct, type ProductName } from './src/product'; +export { isArtifactContentFilePath } from './src/artifact_content'; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts new file mode 100644 index 000000000000..1348ed4e8c84 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type ProductName, DocumentationProduct } from './product'; + +// kibana-kb-elasticsearch-8.15.zip +const artifactNameRegexp = /^kibana-kb-([a-zA-Z]+)-([0-9]+\.[0-9]+)(\.zip)?$/; +const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); + +export const getArtifactName = ({ + productName, + productVersion, + excludeExtension = false, +}: { + productName: string; + productVersion: string; + excludeExtension?: boolean; +}): string => { + const ext = excludeExtension ? '' : '.zip'; + return `kibana-kb-${productName}-${productVersion}${ext}`.toLowerCase(); +}; + +export const parseArtifactName = (artifactName: string) => { + const match = artifactNameRegexp.exec(artifactName); + if (match) { + const productName = match[1].toLowerCase() as ProductName; + const productVersion = match[2].toLowerCase(); + if (allowedProductNames.includes(productName)) { + return { + productName, + productVersion, + }; + } + } +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts new file mode 100644 index 000000000000..757e6664bb58 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/; + +export const isArtifactContentFilePath = (path: string): boolean => { + return contentFileRegexp.test(path); +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts similarity index 64% rename from x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts rename to x-pack/packages/ai-infra/product-doc-common/src/manifest.ts index 678b17088c7b..6649791d0c5f 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts @@ -5,12 +5,8 @@ * 2.0. */ -export const getArtifactName = ({ - productName, - productVersion, -}: { +export interface ArtifactManifest { + formatVersion: string; productName: string; productVersion: string; -}): string => { - return `kibana-kb-${productName}-${productVersion}.zip`.toLowerCase(); -}; +} diff --git a/x-pack/packages/ai-infra/product-doc-common/src/product.ts b/x-pack/packages/ai-infra/product-doc-common/src/product.ts new file mode 100644 index 000000000000..417033f5083e --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/product.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum DocumentationProduct { + kibana = 'kibana', + elasticsearch = 'elasticsearch', + observability = 'observability', + security = 'security', +} + +export type ProductName = keyof typeof DocumentationProduct; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts index 149b2c202887..bc83730d6389 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts @@ -8,10 +8,3 @@ export const productDocInstallStatusSavedObjectTypeName = 'kb_product_doc_install_status'; export const internalElserInferenceId = 'kibana-internal-elser2'; - -export enum DocumentationProduct { - kibana = 'kibana', - elasticsearch = 'elasticsearch', - observability = 'observability', - security = 'security', -} diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts index f1cbf9149921..5e66adca99e8 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts @@ -5,9 +5,7 @@ * 2.0. */ -import { DocumentationProduct } from './consts'; - -export type ProductName = keyof typeof DocumentationProduct; +import type { ProductName } from '@kbn/product-doc-common'; export type InstallationStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts index fa461c8cf488..849ccbd0e49a 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts @@ -9,8 +9,9 @@ import { schema, type TypeOf } from '@kbn/config-schema'; import type { PluginConfigDescriptor } from '@kbn/core/server'; const configSchema = schema.object({ - // TODO: use elastic.co subdomain once available - artifactRepositoryUrl: schema.string({ defaultValue: 'http://34.120.162.240' }), + artifactRepositoryUrl: schema.string({ + defaultValue: 'https://kibana-knowledge-base-artifacts.elastic.co', + }), }); export const config: PluginConfigDescriptor = { diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts index c8da5c1f945d..7d1f37605406 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts @@ -7,21 +7,23 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; -import { DocumentationProduct } from '../../../common/consts'; -import type { ProductName } from '../../../common/saved_objects'; +import { getArtifactName, DocumentationProduct, type ProductName } from '@kbn/product-doc-common'; import type { ProductDocInstallClient } from '../../dao/doc_install_status'; import type { InferenceEndpointManager } from '../inference_endpoint'; import { downloadToDisk, openZipArchive, - validateArtifactArchive, loadManifestFile, loadMappingFile, - fetchArtifactVersions, type ZipArchive, } from './utils'; import { majorMinor, latestVersion } from './utils/semver'; -import { createIndex, populateIndex } from './steps'; +import { + validateArtifactArchive, + fetchArtifactVersions, + createIndex, + populateIndex, +} from './steps'; interface PackageInstallerOpts { artifactsFolder: string; @@ -176,17 +178,6 @@ export class PackageInstaller { } } -// need to be factorized with the script -const getArtifactName = ({ - productName, - productVersion, -}: { - productName: string; - productVersion: string; -}): string => { - return `kibana-kb-${productName}-${productVersion}.zip`.toLowerCase(); -}; - const getIndexName = ({ productName, productVersion, diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/fetch_artifact_versions.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/fetch_artifact_versions.ts similarity index 72% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/fetch_artifact_versions.ts rename to x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/fetch_artifact_versions.ts index 7a29c786fcac..c3e32ca39e21 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/fetch_artifact_versions.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/fetch_artifact_versions.ts @@ -7,15 +7,10 @@ import fetch from 'node-fetch'; import { parseString } from 'xml2js'; -import type { ProductName } from '../../../../common/saved_objects'; -import { DocumentationProduct } from '../../../../common/consts'; +import { type ProductName, DocumentationProduct, parseArtifactName } from '@kbn/product-doc-common'; type ArtifactAvailableVersions = Record; -// TODO: extract to common package -// kibana-kb-elasticsearch-8.15.zip -const artifactNameRegexp = /^kibana-kb-([a-zA-Z]+)-([0-9]+\.[0-9]+)\.zip$/; - export const fetchArtifactVersions = async ({ artifactRepositoryUrl, }: { @@ -43,13 +38,10 @@ export const fetchArtifactVersions = async ({ result.ListBucketResult.Contents.forEach((contentEntry) => { const artifactName = contentEntry.Key[0]; - const match = artifactNameRegexp.exec(artifactName); - if (match) { - const productName = match[1].toLowerCase() as ProductName; - const productVersion = match[2].toLowerCase(); - if (allowedProductNames.includes(productName)) { - record[productName]!.push(productVersion); - } + const parsed = parseArtifactName(artifactName); + if (parsed) { + const { productName, productVersion } = parsed; + record[productName]!.push(productVersion); } }); diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts index bcada62f48c7..3c84fc9cccf1 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts @@ -7,3 +7,5 @@ export { createIndex } from './create_index'; export { populateIndex } from './populate_index'; +export { validateArtifactArchive } from './validate_artifact_archive'; +export { fetchArtifactVersions } from './fetch_artifact_versions'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts index 1583074d85ca..234b886b6f52 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts @@ -8,12 +8,10 @@ import type { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; +import { isArtifactContentFilePath } from '@kbn/product-doc-common'; import { internalElserInferenceId } from '../../../../common/consts'; import type { ZipArchive } from '../utils/zip_archive'; -// TODO: factorize with utils/validate_artifact_archive -const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/; - export const populateIndex = async ({ esClient, indexName, @@ -27,9 +25,7 @@ export const populateIndex = async ({ }) => { log.debug(`Starting populating index ${indexName}`); - const contentEntries = archive - .getEntryPaths() - .filter((entryPath) => contentFileRegexp.test(entryPath)); + const contentEntries = archive.getEntryPaths().filter(isArtifactContentFilePath); for (let i = 0; i < contentEntries.length; i++) { const entryPath = contentEntries[i]; @@ -77,7 +73,6 @@ const indexContentFile = async ({ } }; -// TODO: extract const rewriteInferenceId = (document: Record, inferenceId: string) => { // we don't need to handle nested fields, we don't have any and won't. Object.values(document).forEach((field) => { diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/validate_artifact_archive.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/validate_artifact_archive.ts similarity index 78% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/validate_artifact_archive.ts rename to x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/validate_artifact_archive.ts index 75e42f4cf066..471d7c080c48 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/validate_artifact_archive.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/validate_artifact_archive.ts @@ -5,12 +5,11 @@ * 2.0. */ -import type { ZipArchive } from './zip_archive'; +import { isArtifactContentFilePath } from '@kbn/product-doc-common'; +import type { ZipArchive } from '../utils/zip_archive'; type ValidationResult = { valid: true } | { valid: false; error: string }; -const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/; - export const validateArtifactArchive = (archive: ZipArchive): ValidationResult => { if (!archive.hasEntry('manifest.json')) { return { valid: false, error: 'Manifest file not found' }; @@ -18,7 +17,7 @@ export const validateArtifactArchive = (archive: ZipArchive): ValidationResult = if (!archive.hasEntry('mappings.json')) { return { valid: false, error: 'Mapping file not found' }; } - if (!archive.getEntryPaths().some((entryPath) => contentFileRegexp.test(entryPath))) { + if (!archive.getEntryPaths().some(isArtifactContentFilePath)) { return { valid: false, error: 'No content files were found' }; } return { valid: true }; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts index a4cd018292e9..d5e7b87181b3 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts @@ -6,15 +6,9 @@ */ import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ArtifactManifest } from '@kbn/product-doc-common'; import type { ZipArchive } from './zip_archive'; -// TODO: factorize with script -export interface ArtifactManifest { - formatVersion: string; - productName: string; - productVersion: string; -} - const manifestEntryPath = 'manifest.json'; const mappingsEntryPath = 'mappings.json'; diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts index 5af3e4568bbb..a612a8c6e9f4 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts +++ b/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts @@ -7,6 +7,4 @@ export { downloadToDisk } from './download'; export { openZipArchive, type ZipArchive } from './zip_archive'; -export { validateArtifactArchive } from './validate_artifact_archive'; export { loadManifestFile, loadMappingFile } from './archive_accessors'; -export { fetchArtifactVersions } from './fetch_artifact_versions'; From 8c0da9d533d2703c60758b93ce154cf5862096cc Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 7 Oct 2024 16:27:30 +0200 Subject: [PATCH 20/97] rename plugin --- .github/CODEOWNERS | 8 ++++---- package.json | 2 +- packages/kbn-optimizer/limits.yml | 2 +- tsconfig.base.json | 4 ++-- .../README.md | 0 .../common/consts.ts | 0 .../common/saved_objects.ts | 0 .../jest.config.js | 8 ++++---- .../kibana.jsonc | 6 +++--- .../public/index.ts | 0 .../public/plugin.tsx | 0 .../public/types.ts | 0 .../server/config.ts | 0 .../server/dao/doc_install_status/index.ts | 0 .../server/dao/doc_install_status/model_conversion.ts | 0 .../dao/doc_install_status/product_doc_install_service.ts | 0 .../server/index.ts | 0 .../server/plugin.ts | 0 .../server/saved_objects/index.ts | 0 .../server/saved_objects/product_doc_install.ts | 0 .../services/inference_endpoint/endpoint_manager.ts | 0 .../server/services/inference_endpoint/index.ts | 0 .../inference_endpoint/utils/get_model_install_status.ts | 0 .../server/services/inference_endpoint/utils/index.ts | 0 .../services/inference_endpoint/utils/install_elser.ts | 0 .../inference_endpoint/utils/wait_until_model_deployed.ts | 0 .../server/services/package_installer/index.ts | 0 .../services/package_installer/package_installer.ts | 0 .../services/package_installer/steps/create_index.ts | 0 .../package_installer/steps/fetch_artifact_versions.ts | 0 .../server/services/package_installer/steps/index.ts | 0 .../services/package_installer/steps/populate_index.ts | 0 .../package_installer/steps/validate_artifact_archive.ts | 0 .../services/package_installer/utils/archive_accessors.ts | 0 .../server/services/package_installer/utils/download.ts | 0 .../server/services/package_installer/utils/index.ts | 0 .../server/services/package_installer/utils/semver.ts | 0 .../services/package_installer/utils/zip_archive.ts | 0 .../server/types.ts | 0 .../tsconfig.json | 0 yarn.lock | 2 +- 41 files changed, 16 insertions(+), 16 deletions(-) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/README.md (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/common/consts.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/common/saved_objects.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/jest.config.js (58%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/kibana.jsonc (62%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/public/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/public/plugin.tsx (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/public/types.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/config.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/dao/doc_install_status/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/dao/doc_install_status/model_conversion.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/dao/doc_install_status/product_doc_install_service.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/plugin.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/saved_objects/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/saved_objects/product_doc_install.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/inference_endpoint/endpoint_manager.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/inference_endpoint/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/inference_endpoint/utils/get_model_install_status.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/inference_endpoint/utils/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/inference_endpoint/utils/install_elser.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/inference_endpoint/utils/wait_until_model_deployed.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/package_installer.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/steps/create_index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/steps/fetch_artifact_versions.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/steps/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/steps/populate_index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/steps/validate_artifact_archive.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/utils/archive_accessors.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/utils/download.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/utils/index.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/utils/semver.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/services/package_installer/utils/zip_archive.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/server/types.ts (100%) rename x-pack/plugins/ai_infra/{knowledge_base_registry => product_doc_base}/tsconfig.json (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 01fc37ae3269..91c85fe4c4bb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -552,7 +552,7 @@ src/plugins/kibana_overview @elastic/appex-sharedux src/plugins/kibana_react @elastic/appex-sharedux src/plugins/kibana_usage_collection @elastic/kibana-core src/plugins/kibana_utils @elastic/appex-sharedux -x-pack/plugins/ai_infra/knowledge_base_registry @elastic/appex-ai-infra +x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra x-pack/plugins/kubernetes_security @elastic/kibana-cloud-security-posture x-pack/packages/kbn-langchain @elastic/security-generative-ai packages/kbn-language-documentation @elastic/kibana-esql @@ -924,9 +924,9 @@ packages/kbn-test-eui-helpers @elastic/kibana-visualizations x-pack/test/licensing_plugin/plugins/test_feature_usage @elastic/kibana-security packages/kbn-test-jest-helpers @elastic/kibana-operations @elastic/appex-qa packages/kbn-test-subj-selector @elastic/kibana-operations @elastic/appex-qa -x-pack/test_serverless -test -x-pack/test +x-pack/test_serverless +test +x-pack/test x-pack/performance @elastic/appex-qa x-pack/examples/testing_embedded_lens @elastic/kibana-visualizations x-pack/examples/third_party_lens_navigation_prompt @elastic/kibana-visualizations diff --git a/package.json b/package.json index 2ee3bb8fbb47..702b60f097a8 100644 --- a/package.json +++ b/package.json @@ -594,7 +594,7 @@ "@kbn/kibana-react-plugin": "link:src/plugins/kibana_react", "@kbn/kibana-usage-collection-plugin": "link:src/plugins/kibana_usage_collection", "@kbn/kibana-utils-plugin": "link:src/plugins/kibana_utils", - "@kbn/knowledge-base-registry-plugin": "link:x-pack/plugins/ai_infra/knowledge_base_registry", + "@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base", "@kbn/kubernetes-security-plugin": "link:x-pack/plugins/kubernetes_security", "@kbn/langchain": "link:x-pack/packages/kbn-langchain", "@kbn/language-documentation": "link:packages/kbn-language-documentation", diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index cd33379acef0..369284743247 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -93,7 +93,6 @@ pageLoadAssetSize: kibanaReact: 74422 kibanaUsageCollection: 16463 kibanaUtils: 79713 - knowledgeBaseRegistry: 22500 kubernetesSecurity: 77234 lens: 57135 licenseManagement: 41817 @@ -125,6 +124,7 @@ pageLoadAssetSize: painlessLab: 179748 presentationPanel: 55463 presentationUtil: 58834 + productDocBase: 22500 profiling: 36694 remoteClusters: 51327 reporting: 58600 diff --git a/tsconfig.base.json b/tsconfig.base.json index 6bef7cb6a541..8ab37eeac090 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1098,8 +1098,8 @@ "@kbn/kibana-usage-collection-plugin/*": ["src/plugins/kibana_usage_collection/*"], "@kbn/kibana-utils-plugin": ["src/plugins/kibana_utils"], "@kbn/kibana-utils-plugin/*": ["src/plugins/kibana_utils/*"], - "@kbn/knowledge-base-registry-plugin": ["x-pack/plugins/ai_infra/knowledge_base_registry"], - "@kbn/knowledge-base-registry-plugin/*": ["x-pack/plugins/ai_infra/knowledge_base_registry/*"], + "@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"], + "@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"], "@kbn/kubernetes-security-plugin": ["x-pack/plugins/kubernetes_security"], "@kbn/kubernetes-security-plugin/*": ["x-pack/plugins/kubernetes_security/*"], "@kbn/langchain": ["x-pack/packages/kbn-langchain"], diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/README.md b/x-pack/plugins/ai_infra/product_doc_base/README.md similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/README.md rename to x-pack/plugins/ai_infra/product_doc_base/README.md diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/common/consts.ts rename to x-pack/plugins/ai_infra/product_doc_base/common/consts.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts b/x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/common/saved_objects.ts rename to x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js b/x-pack/plugins/ai_infra/product_doc_base/jest.config.js similarity index 58% rename from x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js rename to x-pack/plugins/ai_infra/product_doc_base/jest.config.js index d233032aebc2..fc06be251a6f 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/jest.config.js +++ b/x-pack/plugins/ai_infra/product_doc_base/jest.config.js @@ -9,14 +9,14 @@ module.exports = { preset: '@kbn/test', rootDir: '../../../..', roots: [ - '/x-pack/plugins/ai_infra/knowledge_base_registry/public', - '/x-pack/plugins/ai_infra/knowledge_base_registry/server', - '/x-pack/plugins/ai_infra/knowledge_base_registry/common', + '/x-pack/plugins/ai_infra/product_doc_base/public', + '/x-pack/plugins/ai_infra/product_doc_base/server', + '/x-pack/plugins/ai_infra/product_doc_base/common', ], setupFiles: [], collectCoverage: true, collectCoverageFrom: [ - '/x-pack/plugins/ai_infra/knowledge_base_registry/{public,server,common}/**/*.{js,ts,tsx}', + '/x-pack/plugins/ai_infra/product_doc_base/{public,server,common}/**/*.{js,ts,tsx}', ], coverageReporters: ['html'], diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/kibana.jsonc b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc similarity index 62% rename from x-pack/plugins/ai_infra/knowledge_base_registry/kibana.jsonc rename to x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc index 77151b0879c7..4e5ebdd3c068 100644 --- a/x-pack/plugins/ai_infra/knowledge_base_registry/kibana.jsonc +++ b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc @@ -1,12 +1,12 @@ { "type": "plugin", - "id": "@kbn/knowledge-base-registry-plugin", + "id": "@kbn/product-doc-base-plugin", "owner": "@elastic/appex-ai-infra", "plugin": { - "id": "knowledgeBaseRegistry", + "id": "productDocBase", "server": true, "browser": true, - "configPath": ["xpack", "knowledgeBaseRegistry"], + "configPath": ["xpack", "productDocBase"], "requiredPlugins": [], "requiredBundles": [], "optionalPlugins": [], diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/public/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/public/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/public/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/public/plugin.tsx b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/public/plugin.tsx rename to x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/public/types.ts b/x-pack/plugins/ai_infra/product_doc_base/public/types.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/public/types.ts rename to x-pack/plugins/ai_infra/product_doc_base/public/types.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts b/x-pack/plugins/ai_infra/product_doc_base/server/config.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/config.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/config.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/model_conversion.ts b/x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/model_conversion.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/model_conversion.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/model_conversion.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/product_doc_install_service.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/dao/doc_install_status/product_doc_install_service.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/product_doc_install_service.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/plugin.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/saved_objects/product_doc_install.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/endpoint_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/endpoint_manager.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/get_model_install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/get_model_install_status.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/install_elser.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/install_elser.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/wait_until_model_deployed.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/inference_endpoint/utils/wait_until_model_deployed.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/package_installer.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/create_index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/fetch_artifact_versions.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/fetch_artifact_versions.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/populate_index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/validate_artifact_archive.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/steps/validate_artifact_archive.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/archive_accessors.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/download.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/download.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/semver.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/semver.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/zip_archive.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/services/package_installer/utils/zip_archive.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/server/types.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/types.ts diff --git a/x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json similarity index 100% rename from x-pack/plugins/ai_infra/knowledge_base_registry/tsconfig.json rename to x-pack/plugins/ai_infra/product_doc_base/tsconfig.json diff --git a/yarn.lock b/yarn.lock index 26ea644fb6b5..4a68f02255c5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5479,7 +5479,7 @@ version "0.0.0" uid "" -"@kbn/knowledge-base-registry-plugin@link:x-pack/plugins/ai_infra/knowledge_base_registry": +"@kbn/product-doc-base-plugin@link:x-pack/plugins/ai_infra/product_doc_base": version "0.0.0" uid "" From 7ba6ee3d2d8d9916120515a731787dd63e681d3c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 7 Oct 2024 16:30:38 +0200 Subject: [PATCH 21/97] commit autochange --- package.json | 2 +- tsconfig.base.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 702b60f097a8..ba68c36183d8 100644 --- a/package.json +++ b/package.json @@ -594,7 +594,6 @@ "@kbn/kibana-react-plugin": "link:src/plugins/kibana_react", "@kbn/kibana-usage-collection-plugin": "link:src/plugins/kibana_usage_collection", "@kbn/kibana-utils-plugin": "link:src/plugins/kibana_utils", - "@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base", "@kbn/kubernetes-security-plugin": "link:x-pack/plugins/kubernetes_security", "@kbn/langchain": "link:x-pack/packages/kbn-langchain", "@kbn/language-documentation": "link:packages/kbn-language-documentation", @@ -708,6 +707,7 @@ "@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel", "@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing", "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util", + "@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base", "@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common", "@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access", "@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling", diff --git a/tsconfig.base.json b/tsconfig.base.json index 8ab37eeac090..f28510bb6544 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1098,8 +1098,6 @@ "@kbn/kibana-usage-collection-plugin/*": ["src/plugins/kibana_usage_collection/*"], "@kbn/kibana-utils-plugin": ["src/plugins/kibana_utils"], "@kbn/kibana-utils-plugin/*": ["src/plugins/kibana_utils/*"], - "@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"], - "@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"], "@kbn/kubernetes-security-plugin": ["x-pack/plugins/kubernetes_security"], "@kbn/kubernetes-security-plugin/*": ["x-pack/plugins/kubernetes_security/*"], "@kbn/langchain": ["x-pack/packages/kbn-langchain"], @@ -1364,6 +1362,8 @@ "@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"], "@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"], "@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"], + "@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"], + "@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"], "@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"], "@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"], "@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"], From 3f16f2707a24c14a6f264bdd01c76d5932498025 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:44:20 +0000 Subject: [PATCH 22/97] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- .github/CODEOWNERS | 9 +++++---- docs/developer/plugin-list.asciidoc | 8 ++++---- x-pack/packages/ai-infra/product-doc-common/package.json | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 91c85fe4c4bb..6279ba176f09 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -552,7 +552,6 @@ src/plugins/kibana_overview @elastic/appex-sharedux src/plugins/kibana_react @elastic/appex-sharedux src/plugins/kibana_usage_collection @elastic/kibana-core src/plugins/kibana_utils @elastic/appex-sharedux -x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra x-pack/plugins/kubernetes_security @elastic/kibana-cloud-security-posture x-pack/packages/kbn-langchain @elastic/security-generative-ai packages/kbn-language-documentation @elastic/kibana-esql @@ -685,6 +684,8 @@ src/plugins/presentation_panel @elastic/kibana-presentation packages/presentation/presentation_publishing @elastic/kibana-presentation src/plugins/presentation_util @elastic/kibana-presentation x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra +x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra +x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra x-pack/plugins/observability_solution/profiling_data_access @elastic/obs-ux-infra_services-team x-pack/plugins/observability_solution/profiling @elastic/obs-ux-infra_services-team packages/kbn-profiling-utils @elastic/obs-ux-infra_services-team @@ -924,9 +925,9 @@ packages/kbn-test-eui-helpers @elastic/kibana-visualizations x-pack/test/licensing_plugin/plugins/test_feature_usage @elastic/kibana-security packages/kbn-test-jest-helpers @elastic/kibana-operations @elastic/appex-qa packages/kbn-test-subj-selector @elastic/kibana-operations @elastic/appex-qa -x-pack/test_serverless -test -x-pack/test +x-pack/test_serverless +test +x-pack/test x-pack/performance @elastic/appex-qa x-pack/examples/testing_embedded_lens @elastic/kibana-visualizations x-pack/examples/third_party_lens_navigation_prompt @elastic/kibana-visualizations diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 8dd98ebc841d..71713d716198 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -665,10 +665,6 @@ the infrastructure monitoring use-case within Kibana. |undefined -|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/knowledge_base_registry/README.md[knowledgeBaseRegistry] -|This plugin contains the registry for the knowledge base. - - |{kib-repo}blob/{branch}/x-pack/plugins/kubernetes_security/README.md[kubernetesSecurity] |This plugin provides interactive visualizations of your Kubernetes workload and session data. @@ -771,6 +767,10 @@ Elastic. |This plugin helps users learn how to use the Painless scripting language. +|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase] +|This plugin contains the registry for the knowledge base. + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling] |Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views. diff --git a/x-pack/packages/ai-infra/product-doc-common/package.json b/x-pack/packages/ai-infra/product-doc-common/package.json index 5dfd6eed6cfc..839d411a2efb 100644 --- a/x-pack/packages/ai-infra/product-doc-common/package.json +++ b/x-pack/packages/ai-infra/product-doc-common/package.json @@ -2,5 +2,5 @@ "name": "@kbn/product-doc-common", "private": true, "version": "1.0.0", - "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0" + "license": "Elastic License 2.0" } \ No newline at end of file From af0ee262f09e919b78ccfaadac0eb155de97ca99 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:44:50 +0000 Subject: [PATCH 23/97] [CI] Auto-commit changed files from 'node scripts/notice' --- .../ai-infra/product-doc-artifact-builder/tsconfig.json | 1 + x-pack/plugins/ai_infra/product_doc_base/tsconfig.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json b/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json index 508d4c715d0a..68ff27852c4d 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json @@ -16,5 +16,6 @@ "kbn_references": [ "@kbn/tooling-log", "@kbn/repo-info", + "@kbn/product-doc-common", ] } diff --git a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json index cd604a32b114..4f44448828fd 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json +++ b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json @@ -18,5 +18,8 @@ "@kbn/core", "@kbn/logging", "@kbn/config-schema", + "@kbn/product-doc-common", + "@kbn/core-saved-objects-server", + "@kbn/utils", ] } From 96e5df5f48043895bb181264d69824f30c9b772c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 7 Oct 2024 16:50:34 +0200 Subject: [PATCH 24/97] move things --- .../ai_infra/product_doc_base/common/saved_objects.ts | 2 +- x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts | 8 ++++---- .../server/{dao => services}/doc_install_status/index.ts | 0 .../doc_install_status/model_conversion.ts | 0 .../doc_install_status/product_doc_install_service.ts | 6 +++--- .../services/package_installer/package_installer.ts | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) rename x-pack/plugins/ai_infra/product_doc_base/server/{dao => services}/doc_install_status/index.ts (100%) rename x-pack/plugins/ai_infra/product_doc_base/server/{dao => services}/doc_install_status/model_conversion.ts (100%) rename x-pack/plugins/ai_infra/product_doc_base/server/{dao => services}/doc_install_status/product_doc_install_service.ts (91%) diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts b/x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts index 5e66adca99e8..5a7da36adb89 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts @@ -16,5 +16,5 @@ export interface ProductDocInstallStatus { installationStatus: InstallationStatus; lastInstallationDate: Date | undefined; lastInstallationFailureReason: string | undefined; - indexName: string; + indexName?: string; } diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index a875fcbf99b7..34c501a187c4 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -21,7 +21,7 @@ import type { import { knowledgeBaseProductDocInstallSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; import { InferenceEndpointManager } from './services/inference_endpoint'; -import { ProductDocInstallClient } from './dao/doc_install_status'; +import { ProductDocInstallClient } from './services/doc_install_status'; export class KnowledgeBaseRegistryPlugin implements @@ -73,14 +73,14 @@ export class KnowledgeBaseRegistryPlugin logger: this.logger.get('package-installer'), }); + // TODO: remove delay(10) .then(async () => { - console.log('*** test installating packages'); - + this.logger.info('*** test installating packages'); return packageInstaller.installAll({}); }) .catch((e) => { - console.log('*** ERROR', e); + this.logger.error('*** ERROR', e); }); return {}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts similarity index 100% rename from x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/index.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/model_conversion.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts similarity index 100% rename from x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/model_conversion.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts similarity index 91% rename from x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/product_doc_install_service.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts index bdf50badb67f..c5d6beb3cb0a 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/dao/doc_install_status/product_doc_install_service.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts @@ -5,12 +5,12 @@ * 2.0. */ -import type { SavedObjectsClientContract, SavedObject } from '@kbn/core/server'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; -import type { InstallationStatus, ProductName, ProductDocInstallStatus } from '../../../common/saved_objects'; +import { ProductName } from '@kbn/product-doc-common'; import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; import type { KnowledgeBaseProductDocInstallAttributes as TypeAttributes } from '../../saved_objects'; -import { soToModel } from './model_conversion'; +// import { soToModel } from './model_conversion'; export class ProductDocInstallClient { private soClient: SavedObjectsClientContract; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts index 7d1f37605406..f3d20eacd9dd 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -8,7 +8,7 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; import { getArtifactName, DocumentationProduct, type ProductName } from '@kbn/product-doc-common'; -import type { ProductDocInstallClient } from '../../dao/doc_install_status'; +import type { ProductDocInstallClient } from '../doc_install_status'; import type { InferenceEndpointManager } from '../inference_endpoint'; import { downloadToDisk, From 3ecaa5b8e0ce2dfd3b485f59194ddb4b60a1259f Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:04:23 +0000 Subject: [PATCH 25/97] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- yarn.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4a68f02255c5..70c49f120573 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5479,10 +5479,6 @@ version "0.0.0" uid "" -"@kbn/product-doc-base-plugin@link:x-pack/plugins/ai_infra/product_doc_base": - version "0.0.0" - uid "" - "@kbn/kubernetes-security-plugin@link:x-pack/plugins/kubernetes_security": version "0.0.0" uid "" @@ -6011,6 +6007,10 @@ version "0.0.0" uid "" +"@kbn/product-doc-base-plugin@link:x-pack/plugins/ai_infra/product_doc_base": + version "0.0.0" + uid "" + "@kbn/product-doc-common@link:x-pack/packages/ai-infra/product-doc-common": version "0.0.0" uid "" From 0369210dc6abe76464cc0582881be81e8cc839ef Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 8 Oct 2024 15:32:27 +0200 Subject: [PATCH 26/97] add first version of search --- .../src/tasks/create_chunk_files.ts | 2 +- .../src/tasks/extract_documentation.ts | 2 +- .../src/tasks/index.ts | 1 - .../ai-infra/product-doc-common/index.ts | 6 +++ .../product-doc-common/src/documents.ts | 23 ++++++++ .../product-doc-common/src/indices.ts | 15 ++++++ .../product_doc_base/server/config.ts | 4 +- .../ai_infra/product_doc_base/server/index.ts | 23 ++++---- .../product_doc_base/server/plugin.ts | 54 +++++++++++-------- .../inference_endpoint/utils/install_elser.ts | 2 +- .../package_installer/package_installer.ts | 37 ++++--------- .../server/services/search/index.ts | 9 ++++ .../server/services/search/perform_search.ts} | 25 +++++---- .../server/services/search/search_service.ts | 37 +++++++++++++ .../server/services/search/types.ts | 27 ++++++++++ .../utils/get_indices_for_product_names.ts | 21 ++++++++ .../server/services/search/utils/index.ts | 9 ++++ .../services/search/utils/map_result.ts | 19 +++++++ .../ai_infra/product_doc_base/server/types.ts | 12 +++-- 19 files changed, 248 insertions(+), 80 deletions(-) create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/documents.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/indices.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts rename x-pack/{packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts => plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts} (80%) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts index 8b0e7323c288..73cf8f010922 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts @@ -10,7 +10,7 @@ import Fs from 'fs/promises'; import type { Client } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; -const fileSizeLimit = 250_000; +const fileSizeLimit = 500_000; export const createChunkFiles = async ({ index, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts index f1dd051394bb..d553fef023bc 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts @@ -43,7 +43,7 @@ const convertHit = (hit: SearchHit): ExtractedDocument => { return { content_title: source.content_title, content_body: source.content_body, - product_name: source.product_name, + product_name: source.product_name, // TODO: lowercase / convert root_type: 'documentation', slug: source.slug, url: source.url, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts index 0c6343136232..bba81d70f7bb 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts @@ -10,7 +10,6 @@ export { indexDocuments } from './index_documents'; export { createTargetIndex } from './create_index'; export { installElser } from './install_elser'; export { createChunkFiles } from './create_chunk_files'; -export { performSemanticSearch } from './perform_semantic_search'; export { checkConnectivity } from './check_connectivity'; export { createArtifact } from './create_artifact'; export { cleanupFolders } from './cleanup_folders'; diff --git a/x-pack/packages/ai-infra/product-doc-common/index.ts b/x-pack/packages/ai-infra/product-doc-common/index.ts index 4cd58111a72e..1a9673713899 100644 --- a/x-pack/packages/ai-infra/product-doc-common/index.ts +++ b/x-pack/packages/ai-infra/product-doc-common/index.ts @@ -9,3 +9,9 @@ export { getArtifactName, parseArtifactName } from './src/artifact'; export { type ArtifactManifest } from './src/manifest'; export { DocumentationProduct, type ProductName } from './src/product'; export { isArtifactContentFilePath } from './src/artifact_content'; +export { + productDocIndexPrefix, + productDocIndexPattern, + getProductDocIndexName, +} from './src/indices'; +export type { ProductDocumentationAttributes } from './src/documents'; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts new file mode 100644 index 000000000000..fcb1ac323190 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from './product'; + +// TODO: proper structure (e.g. semantic field +export interface ProductDocumentationAttributes { + content_title: string; + content_body: string; + product_name: ProductName; + root_type: string; + slug: string; + url: string; + version: string; + ai_subtitle: string; + ai_summary: string; + ai_questions_answered: string[]; + ai_tags: string[]; +} diff --git a/x-pack/packages/ai-infra/product-doc-common/src/indices.ts b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts new file mode 100644 index 000000000000..3c19bf0d198a --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from './product'; + +export const productDocIndexPrefix = '.kibana-ai-product-doc'; +export const productDocIndexPattern = `${productDocIndexPrefix}-*`; + +export const getProductDocIndexName = (productName: ProductName): string => { + return `${productDocIndexPrefix}-${productName}`; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/config.ts b/x-pack/plugins/ai_infra/product_doc_base/server/config.ts index 849ccbd0e49a..bd0892d58270 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/config.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/config.ts @@ -14,9 +14,9 @@ const configSchema = schema.object({ }), }); -export const config: PluginConfigDescriptor = { +export const config: PluginConfigDescriptor = { schema: configSchema, exposeToBrowser: {}, }; -export type KnowledgeBaseRegistryConfig = TypeOf; +export type ProductDocBaseConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts index ffad4020c419..af357202b613 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts @@ -6,23 +6,24 @@ */ import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server'; -import type { KnowledgeBaseRegistryConfig } from './config'; +import type { ProductDocBaseConfig } from './config'; import type { - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies, + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies, } from './types'; import { KnowledgeBaseRegistryPlugin } from './plugin'; export { config } from './config'; -export type { KnowledgeBaseRegistrySetupContract, KnowledgeBaseRegistryStartContract }; +export type { ProductDocBaseSetupContract, ProductDocBaseStartContract }; +export type { SearchApi as ProductDocSearchApi } from './services/search/types'; export const plugin: PluginInitializer< - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies -> = async (pluginInitializerContext: PluginInitializerContext) => + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => new KnowledgeBaseRegistryPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index 34c501a187c4..dca831f2ec74 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -11,39 +11,37 @@ import { getDataPath } from '@kbn/utils'; import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; import { SavedObjectsClient } from '@kbn/core/server'; import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; -import type { KnowledgeBaseRegistryConfig } from './config'; +import type { ProductDocBaseConfig } from './config'; import type { - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies, + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies, } from './types'; import { knowledgeBaseProductDocInstallSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; import { InferenceEndpointManager } from './services/inference_endpoint'; import { ProductDocInstallClient } from './services/doc_install_status'; +import { SearchService } from './services/search'; export class KnowledgeBaseRegistryPlugin implements Plugin< - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies > { logger: Logger; - constructor(private readonly context: PluginInitializerContext) { + constructor(private readonly context: PluginInitializerContext) { this.logger = context.logger.get(); } setup( - coreSetup: CoreSetup< - KnowledgeBaseRegistryStartDependencies, - KnowledgeBaseRegistryStartContract - >, - pluginsSetup: KnowledgeBaseRegistrySetupDependencies - ): KnowledgeBaseRegistrySetupContract { + coreSetup: CoreSetup, + pluginsSetup: ProductDocBaseSetupDependencies + ): ProductDocBaseSetupContract { coreSetup.savedObjects.registerType(knowledgeBaseProductDocInstallSavedObjectType); return {}; @@ -51,8 +49,8 @@ export class KnowledgeBaseRegistryPlugin start( core: CoreStart, - pluginsStart: KnowledgeBaseRegistryStartDependencies - ): KnowledgeBaseRegistryStartContract { + pluginsStart: ProductDocBaseStartDependencies + ): ProductDocBaseStartContract { const soClient = new SavedObjectsClient( core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName]) ); @@ -73,17 +71,29 @@ export class KnowledgeBaseRegistryPlugin logger: this.logger.get('package-installer'), }); + const searchService = new SearchService({ + esClient: core.elasticsearch.client.asInternalUser, + logger: this.logger.get('search-service'), + }); + // TODO: remove delay(10) .then(async () => { - this.logger.info('*** test installating packages'); - return packageInstaller.installAll({}); + this.logger.info('*** test installing packages'); + // await packageInstaller.installAll({}); + + const results = await searchService.search({ + query: 'How to create a space in Kibana?', + products: ['kibana'], + }); + console.log(JSON.stringify(results.results.map((result) => result.title))); }) .catch((e) => { this.logger.error('*** ERROR', e); }); - - return {}; + return { + search: searchService.search.bind(searchService), + }; } } diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts index d1e054042b52..8b75cd5b0c41 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts @@ -20,7 +20,7 @@ export const installElser = async ({ task_type: 'sparse_embedding', inference_id: inferenceId, inference_config: { - service: 'elser', + service: 'elasticsearch', service_settings: { num_allocations: 1, num_threads: 1, diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts index f3d20eacd9dd..d7757277cdb3 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -7,7 +7,12 @@ import type { Logger } from '@kbn/logging'; import type { ElasticsearchClient } from '@kbn/core/server'; -import { getArtifactName, DocumentationProduct, type ProductName } from '@kbn/product-doc-common'; +import { + getArtifactName, + getProductDocIndexName, + DocumentationProduct, + type ProductName, +} from '@kbn/product-doc-common'; import type { ProductDocInstallClient } from '../doc_install_status'; import type { InferenceEndpointManager } from '../inference_endpoint'; import { @@ -95,10 +100,7 @@ export class PackageInstaller { `Starting installing documentation for product [${productName}] and version [${productVersion}]` ); - await this.uninstallPackage({ - productName, - productVersion, - }); + await this.uninstallPackage({ productName }); let zipArchive: ZipArchive | undefined; try { @@ -123,7 +125,7 @@ export class PackageInstaller { const manifest = await loadManifestFile(zipArchive); const mappings = await loadMappingFile(zipArchive); - const indexName = getIndexName({ productName, productVersion }); + const indexName = getProductDocIndexName(productName); await createIndex({ indexName, @@ -157,16 +159,8 @@ export class PackageInstaller { } } - async uninstallPackage({ - productName, - productVersion, - }: { - productName: ProductName; - productVersion: string; - }) { - // TODO: retrieve entry to check installed version instead - - const indexName = getIndexName({ productName, productVersion }); + async uninstallPackage({ productName }: { productName: ProductName }) { + const indexName = getProductDocIndexName(productName); await this.esClient.indices.delete( { index: indexName, @@ -177,14 +171,3 @@ export class PackageInstaller { await this.productDocClient.setUninstalled(productName); } } - -const getIndexName = ({ - productName, - productVersion, -}: { - productName: string; - productVersion: string; -}): string => { - // TODO: '.kibana-product-doc-*' - return `.kibana-ai-kb-${productName}-${productVersion}`.toLowerCase(); -}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts new file mode 100644 index 000000000000..3e5ac95ae4ed --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { SearchService } from './search_service'; +export type { DocSearchOptions, DocSearchResult, DocSearchResponse, SearchApi } from './types'; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts similarity index 80% rename from x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts index 373a6b875542..40c70ef785fb 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts @@ -5,29 +5,34 @@ * 2.0. */ -import type { Client } from '@elastic/elasticsearch'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; // https://search-labs.elastic.co/search-labs/blog/elser-rag-search-for-relevance -export const performSemanticSearch = async ({ +export const performSearch = async ({ searchQuery, + size, index, client, }: { searchQuery: string; - index: string; - client: Client; + size: number; + index: string | string[]; + client: ElasticsearchClient; }) => { - const results = await client.search({ + const results = await client.search({ index, - size: 3, + size, query: { bool: { - filter: { - bool: { - must: [{ term: { version: '8.15' } }], + filter: [ + { + bool: { + must: [{ term: { version: '8.15' } }], + }, }, - }, + ], should: [ { multi_match: { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts new file mode 100644 index 000000000000..a0b1e4fd4a83 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { getIndicesForProductNames, mapResult } from './utils'; +import { performSearch } from './perform_search'; +import type { DocSearchOptions, DocSearchResponse } from './types'; + +export class SearchService { + private readonly log: Logger; + private readonly esClient: ElasticsearchClient; + + constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) { + this.log = logger; + this.esClient = esClient; + } + + async search(options: DocSearchOptions): Promise { + const { query, max = 3, products } = options; + this.log.debug(`performing search - query=[${query}]`); + const results = await performSearch({ + searchQuery: query, + size: max, + index: getIndicesForProductNames(products), + client: this.esClient, + }); + + return { + results: results.map(mapResult), + }; + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts new file mode 100644 index 000000000000..fb474bbf4dea --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; + +export interface DocSearchOptions { + query: string; + max?: number; + products?: ProductName[]; +} + +export interface DocSearchResult { + title: string; + content: string; + url: string; + productName: ProductName; +} + +export interface DocSearchResponse { + results: DocSearchResult[]; +} + +export type SearchApi = (options: DocSearchOptions) => Promise; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts new file mode 100644 index 000000000000..e97ed9cea361 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + productDocIndexPattern, + getProductDocIndexName, + type ProductName, +} from '@kbn/product-doc-common'; + +export const getIndicesForProductNames = ( + productNames: ProductName[] | undefined +): string | string[] => { + if (!productNames || !productNames.length) { + return productDocIndexPattern; + } + return productNames.map(getProductDocIndexName); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts new file mode 100644 index 000000000000..1a6a2eaa24a9 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getIndicesForProductNames } from './get_indices_for_product_names'; +export { mapResult } from './map_result'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts new file mode 100644 index 000000000000..3c0f7f1cf585 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; +import type { DocSearchResult } from '../types'; + +export const mapResult = (docHit: SearchHit): DocSearchResult => { + return { + title: docHit._source!.content_title, + content: docHit._source!.content_body, + url: docHit._source!.url, + productName: docHit._source!.product_name, + }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts index 8ce82731f7b6..6a56752d2f88 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts @@ -5,12 +5,16 @@ * 2.0. */ +import type { SearchApi } from './services/search'; + /* eslint-disable @typescript-eslint/no-empty-interface*/ -export interface KnowledgeBaseRegistrySetupDependencies {} +export interface ProductDocBaseSetupDependencies {} -export interface KnowledgeBaseRegistryStartDependencies {} +export interface ProductDocBaseStartDependencies {} -export interface KnowledgeBaseRegistrySetupContract {} +export interface ProductDocBaseSetupContract {} -export interface KnowledgeBaseRegistryStartContract {} +export interface ProductDocBaseStartContract { + search: SearchApi; +} From 7cfb398886c11b7b4391a6e912ba43d4da85ccec Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 8 Oct 2024 16:25:36 +0200 Subject: [PATCH 27/97] tweaks in artifact builder --- .../src/artifact/product_name.ts | 35 ++++++++++++++++--- .../src/build_artifacts.ts | 3 +- .../src/command.ts | 8 ++--- .../src/tasks/extract_documentation.ts | 10 +++--- .../product-doc-artifact-builder/src/types.ts | 4 ++- .../product-doc-common/src/artifact.ts | 6 ++-- 6 files changed, 49 insertions(+), 17 deletions(-) diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts index cfcc141323f4..8164b1213054 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts @@ -5,7 +5,34 @@ * 2.0. */ -/** - * The allowed product names, as found in the source's cluster - */ -export const sourceProductNames = ['Kibana', 'Elasticsearch', 'Security', 'Observability']; +import type { ProductName } from '@kbn/product-doc-common'; + +export const getSourceProductName = (productName: ProductName) => { + switch (productName) { + case 'elasticsearch': + return 'Elasticsearch'; + case 'observability': + return 'Observability'; + case 'security': + return 'Security'; + case 'kibana': + return 'Kibana'; + default: + throw new Error(`Unknown product name: ${productName}`); + } +}; + +export const getProductNameFromSource = (source: string): ProductName => { + switch (source) { + case 'Elasticsearch': + return 'elasticsearch'; + case 'Observability': + return 'observability'; + case 'Security': + return 'security'; + case 'Kibana': + return 'kibana'; + default: + throw new Error(`Unknown source product name: ${source}`); + } +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts index bbde3310f8e3..230a2a38cc69 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts @@ -8,6 +8,7 @@ import Path from 'path'; import { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; +import type { ProductName } from '@kbn/product-doc-common'; import { // checkConnectivity, createTargetIndex, @@ -93,7 +94,7 @@ const buildArtifact = async ({ sourceClient, log, }: { - productName: string; + productName: ProductName; stackVersion: string; buildFolder: string; targetFolder: string; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts index 49af1d158db8..e8d0d9486e33 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts @@ -6,19 +6,19 @@ */ import Path from 'path'; -import { REPO_ROOT } from '@kbn/repo-info'; import yargs from 'yargs'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { DocumentationProduct } from '@kbn/product-doc-common'; import type { TaskConfig } from './types'; import { buildArtifacts } from './build_artifacts'; -import { sourceProductNames } from './artifact/product_name'; function options(y: yargs.Argv) { return y .option('productName', { describe: 'name of products to generate documentation for', array: true, - choices: sourceProductNames, - default: ['Kibana'], + choices: Object.values(DocumentationProduct), + default: [DocumentationProduct.kibana], }) .option('stackVersion', { describe: 'The stack version to generate documentation for', diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts index d553fef023bc..d5b321ccfafb 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts @@ -8,6 +8,8 @@ import type { Client } from '@elastic/elasticsearch'; import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import type { ToolingLog } from '@kbn/tooling-log'; +import type { ProductName } from '@kbn/product-doc-common'; +import { getSourceProductName, getProductNameFromSource } from '../artifact/product_name'; /** the list of fields to import from the source cluster */ const fields = [ @@ -27,7 +29,7 @@ const fields = [ export interface ExtractedDocument { content_title: string; content_body: string; - product_name: string; + product_name: ProductName; root_type: string; slug: string; url: string; @@ -43,7 +45,7 @@ const convertHit = (hit: SearchHit): ExtractedDocument => { return { content_title: source.content_title, content_body: source.content_body, - product_name: source.product_name, // TODO: lowercase / convert + product_name: getProductNameFromSource(source.product_name), root_type: 'documentation', slug: source.slug, url: source.url, @@ -65,7 +67,7 @@ export const extractDocumentation = async ({ client: Client; index: string; stackVersion: string; - productName: string; + productName: ProductName; log: ToolingLog; }) => { log.info(`Starting to extract documents from source cluster`); @@ -76,7 +78,7 @@ export const extractDocumentation = async ({ query: { bool: { must: [ - { term: { product_name: productName } }, + { term: { product_name: getSourceProductName(productName) } }, { term: { version: stackVersion } }, { exists: { field: 'ai_fields.ai_summary' } }, ], diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts index d2acfb577450..1eb4a4348d21 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts @@ -5,8 +5,10 @@ * 2.0. */ +import type { ProductName } from '@kbn/product-doc-common'; + export interface TaskConfig { - productNames: string[]; + productNames: ProductName[]; stackVersion: string; buildFolder: string; targetFolder: string; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts index 1348ed4e8c84..d252a04424b6 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts @@ -7,8 +7,8 @@ import { type ProductName, DocumentationProduct } from './product'; -// kibana-kb-elasticsearch-8.15.zip -const artifactNameRegexp = /^kibana-kb-([a-zA-Z]+)-([0-9]+\.[0-9]+)(\.zip)?$/; +// kb-product-doc-elasticsearch-8.15.zip +const artifactNameRegexp = /^kb-product-doc-([a-zA-Z]+)-([0-9]+\.[0-9]+)(\.zip)?$/; const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); export const getArtifactName = ({ @@ -21,7 +21,7 @@ export const getArtifactName = ({ excludeExtension?: boolean; }): string => { const ext = excludeExtension ? '' : '.zip'; - return `kibana-kb-${productName}-${productVersion}${ext}`.toLowerCase(); + return `kb-product-doc-${productName}-${productVersion}${ext}`.toLowerCase(); }; export const parseArtifactName = (artifactName: string) => { From 258ffc94bd0e440458dc0bf44c5e00dd6514f2c1 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 9 Oct 2024 14:33:57 +0200 Subject: [PATCH 28/97] add package upgrade check --- .../{saved_objects.ts => install_status.ts} | 0 .../product_doc_base/server/plugin.ts | 17 +++--- .../saved_objects/product_doc_install.ts | 2 +- .../product_doc_install_service.ts | 29 +++++++++- .../package_installer/package_installer.ts | 54 +++++++++++++++++-- .../services/search/utils/map_result.ts | 2 +- 6 files changed, 90 insertions(+), 14 deletions(-) rename x-pack/plugins/ai_infra/product_doc_base/common/{saved_objects.ts => install_status.ts} (100%) diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts similarity index 100% rename from x-pack/plugins/ai_infra/product_doc_base/common/saved_objects.ts rename to x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index dca831f2ec74..bb64231c1ed7 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -76,17 +76,22 @@ export class KnowledgeBaseRegistryPlugin logger: this.logger.get('search-service'), }); + // TODO: see if we should be using taskManager for that. + packageInstaller.ensureUpToDate({}).catch((err) => { + this.logger.error(`Error checking if product documentation is up to date: ${err.message}`); + }); + // TODO: remove delay(10) .then(async () => { - this.logger.info('*** test installing packages'); + // this.logger.info('*** test installing packages'); // await packageInstaller.installAll({}); - const results = await searchService.search({ - query: 'How to create a space in Kibana?', - products: ['kibana'], - }); - console.log(JSON.stringify(results.results.map((result) => result.title))); + // const results = await searchService.search({ + // query: 'How to create a space in Kibana?', + // products: ['kibana'], + // ); + // console.log(JSON.stringify(results.results[0])); }) .catch((e) => { this.logger.error('*** ERROR', e); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts index 95dbb772c9d7..9c4fff582be7 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts @@ -7,7 +7,7 @@ import type { SavedObjectsType } from '@kbn/core/server'; import { productDocInstallStatusSavedObjectTypeName } from '../../common/consts'; -import type { InstallationStatus, ProductName } from '../../common/saved_objects'; +import type { InstallationStatus } from '../../common/install_status'; /** * Interface describing the raw attributes of the KB Entry SO type. diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts index c5d6beb3cb0a..d28902bf4ab5 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts @@ -7,11 +7,17 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; -import { ProductName } from '@kbn/product-doc-common'; +import { ProductName, DocumentationProduct } from '@kbn/product-doc-common'; +import { InstallationStatus } from '../../../common/install_status'; import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; import type { KnowledgeBaseProductDocInstallAttributes as TypeAttributes } from '../../saved_objects'; // import { soToModel } from './model_conversion'; +interface ProductInstallState { + status: InstallationStatus; + version?: string; +} + export class ProductDocInstallClient { private soClient: SavedObjectsClientContract; @@ -31,6 +37,27 @@ export class ProductDocInstallClient { } */ + async getInstallationStatus() { + const response = await this.soClient.find({ + type: typeName, + perPage: 100, + }); + + const installStatus = Object.values(DocumentationProduct).reduce((memo, product) => { + memo[product] = { status: 'uninstalled' }; + return memo; + }, {} as Record); + + response.saved_objects.forEach(({ attributes }) => { + installStatus[attributes.product_name as ProductName] = { + status: attributes.installation_status, + version: attributes.product_version, + }; + }); + + return installStatus; + } + async setInstallationStarted(fields: { productName: ProductName; productVersion: string }) { const { productName, productVersion } = fields; const objectId = getObjectIdFromProductName(productName); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts index d7757277cdb3..ef5ca27cad57 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -67,20 +67,58 @@ export class PackageInstaller { this.log = logger; } + /** + * Make sure that the currently installed doc packages are up to date. + * Will not upgrade products that are not already installed + */ + async ensureUpToDate({}: {}) { + const repositoryVersions = await fetchArtifactVersions({ + artifactRepositoryUrl: this.artifactRepositoryUrl, + }); + + const installStatuses = await this.productDocClient.getInstallationStatus(); + + const toUpdate: Array<{ + productName: ProductName; + productVersion: string; + }> = []; + Object.entries(installStatuses).forEach(([productName, productState]) => { + if (productState.status === 'uninstalled') { + return; + } + const availableVersions = repositoryVersions[productName as ProductName]; + if (!availableVersions || !availableVersions.length) { + return; + } + const selectedVersion = selectVersion(this.currentVersion, availableVersions); + if (productState.version !== selectedVersion) { + toUpdate.push({ + productName: productName as ProductName, + productVersion: selectedVersion, + }); + } + }); + + for (const { productName, productVersion } of toUpdate) { + await this.installPackage({ + productName, + productVersion, + }); + } + } + async installAll({}: {}) { - const artifactVersions = await fetchArtifactVersions({ + const repositoryVersions = await fetchArtifactVersions({ artifactRepositoryUrl: this.artifactRepositoryUrl, }); const allProducts = Object.values(DocumentationProduct) as ProductName[]; for (const productName of allProducts) { - const availableVersions = artifactVersions[productName]; + const availableVersions = repositoryVersions[productName]; if (!availableVersions || !availableVersions.length) { this.log.warn(`No version found for product [${productName}]`); continue; } - const selectedVersion = availableVersions.includes(this.currentVersion) - ? this.currentVersion - : latestVersion(availableVersions); + const selectedVersion = selectVersion(this.currentVersion, availableVersions); await this.installPackage({ productName, @@ -171,3 +209,9 @@ export class PackageInstaller { await this.productDocClient.setUninstalled(productName); } } + +const selectVersion = (currentVersion: string, availableVersions: string[]): string => { + return availableVersions.includes(currentVersion) + ? currentVersion + : latestVersion(availableVersions); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts index 3c0f7f1cf585..f4f66b211182 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts @@ -12,7 +12,7 @@ import type { DocSearchResult } from '../types'; export const mapResult = (docHit: SearchHit): DocSearchResult => { return { title: docHit._source!.content_title, - content: docHit._source!.content_body, + content: docHit._source!.content_body.text, url: docHit._source!.url, productName: docHit._source!.product_name, }; From 63e7597370dd1d66cecb63cae2c2723c7974f13e Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 9 Oct 2024 16:07:44 +0200 Subject: [PATCH 29/97] add llmTasks plugin --- package.json | 1 + tsconfig.base.json | 2 + x-pack/plugins/ai_infra/llm_tasks/README.md | 4 + .../plugins/ai_infra/llm_tasks/jest.config.js | 23 ++++++ .../plugins/ai_infra/llm_tasks/kibana.jsonc | 15 ++++ .../ai_infra/llm_tasks/public/index.ts | 26 +++++++ .../ai_infra/llm_tasks/public/plugin.tsx | 48 ++++++++++++ .../ai_infra/llm_tasks/public/types.ts | 18 +++++ .../ai_infra/llm_tasks/server/config.ts | 18 +++++ .../ai_infra/llm_tasks/server/index.ts | 28 +++++++ .../ai_infra/llm_tasks/server/plugin.ts | 51 ++++++++++++ .../ai_infra/llm_tasks/server/tasks/index.ts | 8 ++ .../extract_relevant_chunks.ts | 77 +++++++++++++++++++ .../tasks/retrieve_documentation/index.ts | 13 ++++ .../retrieve_documentation.ts | 45 +++++++++++ .../tasks/retrieve_documentation/types.ts | 37 +++++++++ .../ai_infra/llm_tasks/server/types.ts | 25 ++++++ .../plugins/ai_infra/llm_tasks/tsconfig.json | 25 ++++++ .../ai_infra/product_doc_base/server/index.ts | 2 +- yarn.lock | 4 + 20 files changed, 469 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/ai_infra/llm_tasks/README.md create mode 100644 x-pack/plugins/ai_infra/llm_tasks/jest.config.js create mode 100644 x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc create mode 100644 x-pack/plugins/ai_infra/llm_tasks/public/index.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx create mode 100644 x-pack/plugins/ai_infra/llm_tasks/public/types.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/config.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/index.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/types.ts create mode 100644 x-pack/plugins/ai_infra/llm_tasks/tsconfig.json diff --git a/package.json b/package.json index ba68c36183d8..e31cafe33709 100644 --- a/package.json +++ b/package.json @@ -607,6 +607,7 @@ "@kbn/licensing-plugin": "link:x-pack/plugins/licensing", "@kbn/links-plugin": "link:src/plugins/links", "@kbn/lists-plugin": "link:x-pack/plugins/lists", + "@kbn/llm-tasks-plugin": "link:x-pack/plugins/ai_infra/llm_tasks", "@kbn/locator-examples-plugin": "link:examples/locator_examples", "@kbn/locator-explorer-plugin": "link:examples/locator_explorer", "@kbn/logging": "link:packages/kbn-logging", diff --git a/tsconfig.base.json b/tsconfig.base.json index f28510bb6544..6fc0d69f1aec 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1128,6 +1128,8 @@ "@kbn/lint-ts-projects-cli/*": ["packages/kbn-lint-ts-projects-cli/*"], "@kbn/lists-plugin": ["x-pack/plugins/lists"], "@kbn/lists-plugin/*": ["x-pack/plugins/lists/*"], + "@kbn/llm-tasks-plugin": ["x-pack/plugins/ai_infra/llm_tasks"], + "@kbn/llm-tasks-plugin/*": ["x-pack/plugins/ai_infra/llm_tasks/*"], "@kbn/locator-examples-plugin": ["examples/locator_examples"], "@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"], "@kbn/locator-explorer-plugin": ["examples/locator_explorer"], diff --git a/x-pack/plugins/ai_infra/llm_tasks/README.md b/x-pack/plugins/ai_infra/llm_tasks/README.md new file mode 100644 index 000000000000..a7a37a4e0d62 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/README.md @@ -0,0 +1,4 @@ +# LLM Tasks plugin + +This plugin contains various LLM tasks + diff --git a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js new file mode 100644 index 000000000000..aaf90e7fafaf --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: [ + '/x-pack/plugins/ai_infra/llm_tasks/public', + '/x-pack/plugins/ai_infra/llm_tasks/server', + '/x-pack/plugins/ai_infra/llm_tasks/common', + ], + setupFiles: [], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/ai_infra/llm_tasks/{public,server,common}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc new file mode 100644 index 000000000000..a6b45924c613 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc @@ -0,0 +1,15 @@ +{ + "type": "plugin", + "id": "@kbn/llm-tasks-plugin", + "owner": "@elastic/appex-ai-infra", + "plugin": { + "id": "llmTasks", + "server": true, + "browser": true, + "configPath": ["xpack", "llmTasks"], + "requiredPlugins": ["inference", "productDocBase"], + "requiredBundles": [], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/public/index.ts b/x-pack/plugins/ai_infra/llm_tasks/public/index.ts new file mode 100644 index 000000000000..229b2bd3c01b --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/public/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { KnowledgeBaseRegistryPlugin } from './plugin'; +import type { + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, + PublicPluginConfig, +} from './types'; + +export type { LlmTasksPluginSetup, LlmTasksPluginStart }; + +export const plugin: PluginInitializer< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies +> = (pluginInitializerContext: PluginInitializerContext) => + new KnowledgeBaseRegistryPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx b/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx new file mode 100644 index 000000000000..53297e3d0663 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; +import type { Logger } from '@kbn/logging'; +import type { + PublicPluginConfig, + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; + +export class KnowledgeBaseRegistryPlugin + implements + Plugin< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup< + PluginStartDependencies, + LlmTasksPluginStart + >, + pluginsSetup: PluginSetupDependencies + ): LlmTasksPluginSetup { + return {}; + } + + start( + coreStart: CoreStart, + pluginsStart: PluginStartDependencies + ): LlmTasksPluginStart { + return {}; + } +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/public/types.ts b/x-pack/plugins/ai_infra/llm_tasks/public/types.ts new file mode 100644 index 000000000000..feeb691ba509 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/public/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface PublicPluginConfig {} + +export interface PluginSetupDependencies {} + +export interface PluginStartDependencies {} + +export interface LlmTasksPluginSetup {} + +export interface LlmTasksPluginStart {} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/config.ts b/x-pack/plugins/ai_infra/llm_tasks/server/config.ts new file mode 100644 index 000000000000..c509af8bda64 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from '@kbn/core/server'; + +const configSchema = schema.object({}); + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: {}, +}; + +export type LlmTasksConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/index.ts new file mode 100644 index 000000000000..1b18426dc2c3 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server'; +import type { LlmTasksConfig } from './config'; +import type { + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { LlmTasksPlugin } from './plugin'; + +export { config } from './config'; + +export type { LlmTasksPluginSetup, LlmTasksPluginStart }; + +export const plugin: PluginInitializer< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new LlmTasksPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts new file mode 100644 index 000000000000..489d71a66889 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import type { LlmTasksConfig } from './config'; +import type { + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { retrieveDocumentation } from './tasks'; + +export class LlmTasksPlugin + implements + Plugin< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + setupDependencies: PluginSetupDependencies + ): LlmTasksPluginSetup { + return {}; + } + + start(core: CoreStart, startDependencies: PluginStartDependencies): LlmTasksPluginStart { + const { inference, productDocBase } = startDependencies; + return { + retrieveDocumentation: (options) => { + return retrieveDocumentation({ + outputAPI: inference.getClient({ request: options.request }).output, + searchDocAPI: productDocBase.search, + })(options); + }, + }; + } +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts new file mode 100644 index 000000000000..41d391182344 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { retrieveDocumentation } from './retrieve_documentation'; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts new file mode 100644 index 000000000000..58f0ec1e57d2 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { lastValueFrom } from 'rxjs'; +import type { ToolSchema } from '@kbn/inference-plugin/common'; +import type { FunctionCallingMode } from '@kbn/inference-plugin/common/chat_complete'; +import type { OutputAPI } from '@kbn/inference-plugin/common/output'; +import { withoutOutputUpdateEvents } from '@kbn/inference-plugin/common/output/without_output_update_events'; + +const extractRelevantChunksSchema = { + type: 'object', + properties: { + useful: { + type: 'boolean', + description: `Whether the provided document has any useful information related to the user's query`, + }, + chunks: { + type: 'array', + items: { + type: 'string', + }, + description: `The chunks of text of the document that are relevant to the user's query. Can be empty`, + }, + }, + required: ['useful'], +} as const satisfies ToolSchema; + +interface ExtractRelevantChunksResponse { + chunks: string[]; +} + +export const extractRelevantChunks = async ({ + searchTerm, + documentContent, + connectorId, + outputAPI, + functionCalling, +}: { + searchTerm: string; + documentContent: string; + outputAPI: OutputAPI; + connectorId: string; + functionCalling?: FunctionCallingMode; +}): Promise => { + const result = await lastValueFrom( + outputAPI('extract_relevant_chunks', { + connectorId, + functionCalling, + system: `You are an Elastic assistant in charge of helping answering the user question + + Given a search query and a document, your current task will be to extract + the parts of the document that could be useful in answering the question. + + - If multiple parts are useful, return them all. + - If you think nothing in the document could help answering the question, return an empty list. + `, + input: ` + ## Search query + + ${searchTerm} + + ## Document + + ${documentContent} + `, + schema: extractRelevantChunksSchema, + }).pipe(withoutOutputUpdateEvents()) + ); + + return { + chunks: result.output.chunks ?? [], + }; +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts new file mode 100644 index 000000000000..22bf0745bd77 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { retrieveDocumentation } from './retrieve_documentation'; +export type { + RetrieveDocumentationAPI, + RetrieveDocumentationResult, + RetrieveDocumentationParams, +} from './types'; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts new file mode 100644 index 000000000000..04a650078b02 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { OutputAPI } from '@kbn/inference-plugin/common/output'; +import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server'; +import type { RetrieveDocumentationAPI } from './types'; +import { extractRelevantChunks } from './extract_relevant_chunks'; + +export const retrieveDocumentation = + ({ + outputAPI, + searchDocAPI, + }: { + outputAPI: OutputAPI; + searchDocAPI: ProductDocSearchAPI; + }): RetrieveDocumentationAPI => + async ({ searchTerm, connectorId, functionCalling }) => { + const searchResults = await searchDocAPI({ query: searchTerm, max: 3 }); + + const processedDocuments = await Promise.all( + searchResults.results.map(async (document) => { + const { chunks } = await extractRelevantChunks({ + searchTerm, + documentContent: document.content, + outputAPI, + connectorId, + functionCalling, + }); + + return { + title: document.title, + url: document.url, + chunks, + }; + }) + ); + + return { + documents: processedDocuments.filter((doc) => doc.chunks.length > 0), + }; + }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts new file mode 100644 index 000000000000..8283b5f6ceec --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRequest } from '@kbn/core/server'; +import type { InferenceServerStart } from '@kbn/inference-plugin/server'; +import type { OutputAPI } from '@kbn/inference-plugin/common/output'; +import type { ChatCompleteAPI } from '@kbn/inference-plugin/common/chat_complete'; +import type { FunctionCallingMode } from '@kbn/inference-plugin/common/chat_complete'; +import type { + ProductDocBaseStartContract, + ProductDocSearchAPI, +} from '@kbn/product-doc-base-plugin/server'; + +export interface RetrieveDocumentationParams { + request: KibanaRequest; + connectorId: string; + searchTerm: string; + functionCalling?: FunctionCallingMode; +} + +export interface DocumentRelevantChunks { + title: string; + url: string; + chunks: string[]; +} + +export interface RetrieveDocumentationResult { + documents: DocumentRelevantChunks[]; +} + +export type RetrieveDocumentationAPI = ( + options: RetrieveDocumentationParams +) => Promise; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts new file mode 100644 index 000000000000..df7872d6cdce --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceServerStart } from '@kbn/inference-plugin/server'; +import type { ProductDocBaseStartContract } from '@kbn/product-doc-base-plugin/server'; +import type { RetrieveDocumentationAPI } from './tasks/retrieve_documentation'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface PluginSetupDependencies {} + +export interface PluginStartDependencies { + inference: InferenceServerStart; + productDocBase: ProductDocBaseStartContract; +} + +export interface LlmTasksPluginSetup {} + +export interface LlmTasksPluginStart { + retrieveDocumentation: RetrieveDocumentationAPI; +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json new file mode 100644 index 000000000000..4f44448828fd --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json @@ -0,0 +1,25 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + "scripts/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [ + "@kbn/core", + "@kbn/logging", + "@kbn/config-schema", + "@kbn/product-doc-common", + "@kbn/core-saved-objects-server", + "@kbn/utils", + ] +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts index af357202b613..9d08b297573c 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts @@ -18,7 +18,7 @@ import { KnowledgeBaseRegistryPlugin } from './plugin'; export { config } from './config'; export type { ProductDocBaseSetupContract, ProductDocBaseStartContract }; -export type { SearchApi as ProductDocSearchApi } from './services/search/types'; +export type { SearchApi as ProductDocSearchAPI } from './services/search/types'; export const plugin: PluginInitializer< ProductDocBaseSetupContract, diff --git a/yarn.lock b/yarn.lock index 70c49f120573..c1c0f6b0ee3b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5539,6 +5539,10 @@ version "0.0.0" uid "" +"@kbn/llm-tasks-plugin@link:x-pack/plugins/ai_infra/llm_tasks": + version "0.0.0" + uid "" + "@kbn/locator-examples-plugin@link:examples/locator_examples": version "0.0.0" uid "" From b039828f4235950ce48ebdec937d5532964b0036 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 11 Oct 2024 14:14:18 +0200 Subject: [PATCH 30/97] Add product documentation installation from the o11y assistant config --- .../product-doc-common/src/indices.ts | 2 +- .../retrieve_documentation.ts | 35 +++- .../tasks/retrieve_documentation/types.ts | 10 +- .../common/http_api/installation.ts | 26 +++ .../product_doc_base/common/install_status.ts | 5 + .../ai_infra/product_doc_base/public/index.ts | 18 +- .../product_doc_base/public/plugin.tsx | 41 ++--- .../public/services/installation/index.ts | 9 + .../installation/installation_service.ts | 36 ++++ .../public/services/installation/types.ts | 18 ++ .../ai_infra/product_doc_base/public/types.ts | 12 +- .../product_doc_base/server/plugin.ts | 28 +++- .../product_doc_base/server/routes/index.ts | 23 +++ .../server/routes/installation.ts | 102 ++++++++++++ .../product_doc_install_service.ts | 9 +- .../package_installer/package_installer.ts | 7 + .../server/services/search/perform_search.ts | 10 +- .../kibana.jsonc | 2 +- .../public/constants.ts | 3 + .../hooks/use_get_product_doc_status.ts | 32 ++++ .../public/hooks/use_install_product_doc.ts | 57 +++++++ .../public/hooks/use_uninstall_product_doc.ts | 57 +++++++ .../public/plugin.ts | 12 +- .../settings_tab/product_doc_entry.tsx | 156 ++++++++++++++++++ .../components/settings_tab/settings_tab.tsx | 3 + 25 files changed, 642 insertions(+), 71 deletions(-) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx diff --git a/x-pack/packages/ai-infra/product-doc-common/src/indices.ts b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts index 3c19bf0d198a..b48cacf79fd2 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/indices.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts @@ -11,5 +11,5 @@ export const productDocIndexPrefix = '.kibana-ai-product-doc'; export const productDocIndexPattern = `${productDocIndexPrefix}-*`; export const getProductDocIndexName = (productName: ProductName): string => { - return `${productDocIndexPrefix}-${productName}`; + return `${productDocIndexPrefix}-${productName.toLowerCase()}`; }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts index 04a650078b02..cbdc659225b2 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { encode } from 'gpt-tokenizer'; import type { OutputAPI } from '@kbn/inference-plugin/common/output'; import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server'; import type { RetrieveDocumentationAPI } from './types'; @@ -18,18 +19,28 @@ export const retrieveDocumentation = outputAPI: OutputAPI; searchDocAPI: ProductDocSearchAPI; }): RetrieveDocumentationAPI => - async ({ searchTerm, connectorId, functionCalling }) => { - const searchResults = await searchDocAPI({ query: searchTerm, max: 3 }); + async ({ searchTerm, connectorId, products, functionCalling, max = 3 }) => { + const searchResults = await searchDocAPI({ query: searchTerm, products, max }); + + console.log('*** retrieveDocumentation => found ' + searchResults.results.length); const processedDocuments = await Promise.all( searchResults.results.map(async (document) => { - const { chunks } = await extractRelevantChunks({ - searchTerm, - documentContent: document.content, - outputAPI, - connectorId, - functionCalling, - }); + const tokenCount = countTokens(document.content); + + let chunks: string[]; + if (tokenCount > 250) { + const extractResponse = await extractRelevantChunks({ + searchTerm, + documentContent: document.content, + outputAPI, + connectorId, + functionCalling, + }); + chunks = extractResponse.chunks; + } else { + chunks = [document.content]; + } return { title: document.title, @@ -39,7 +50,13 @@ export const retrieveDocumentation = }) ); + console.log(`retrieved documents: ${processedDocuments.map((doc) => doc.title)}`); + return { documents: processedDocuments.filter((doc) => doc.chunks.length > 0), }; }; + +const countTokens = (text: string): number => { + return encode(text).length; +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts index 8283b5f6ceec..115c14b706ca 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -6,17 +6,13 @@ */ import type { KibanaRequest } from '@kbn/core/server'; -import type { InferenceServerStart } from '@kbn/inference-plugin/server'; -import type { OutputAPI } from '@kbn/inference-plugin/common/output'; -import type { ChatCompleteAPI } from '@kbn/inference-plugin/common/chat_complete'; import type { FunctionCallingMode } from '@kbn/inference-plugin/common/chat_complete'; -import type { - ProductDocBaseStartContract, - ProductDocSearchAPI, -} from '@kbn/product-doc-base-plugin/server'; +import type { ProductName } from '@kbn/product-doc-common'; export interface RetrieveDocumentationParams { request: KibanaRequest; + max?: number; + products?: ProductName[]; connectorId: string; searchTerm: string; functionCalling?: FunctionCallingMode; diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts new file mode 100644 index 000000000000..d578a8fb9222 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; +import type { ProductInstallState, InstallationStatus } from '../install_status'; + +export const INSTALLATION_STATUS_API_PATH = '/internal/product_doc_base/status'; +export const PERFORM_INSTALL_API_PATH = '/internal/product_doc_base/install'; +export const UNINSTALL_API_PATH = '/internal/product_doc_base/uninstall'; + +export interface InstallationStatusResponse { + overall: InstallationStatus; + perProducts: Record; +} + +export interface PerformInstallResponse { + installed: boolean; +} + +export interface UninstallResponse { + success: boolean; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts index 5a7da36adb89..e48b6f8866c3 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts @@ -18,3 +18,8 @@ export interface ProductDocInstallStatus { lastInstallationFailureReason: string | undefined; indexName?: string; } + +export interface ProductInstallState { + status: InstallationStatus; + version?: string; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts index 0a674f4704d6..d04402aaadb9 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts @@ -8,19 +8,19 @@ import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; import { KnowledgeBaseRegistryPlugin } from './plugin'; import type { - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies, + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies, PublicPluginConfig, } from './types'; -export type { KnowledgeBaseRegistrySetupContract, KnowledgeBaseRegistryStartContract }; +export type { ProductDocBasePluginSetup, ProductDocBasePluginStart }; export const plugin: PluginInitializer< - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies > = (pluginInitializerContext: PluginInitializerContext) => new KnowledgeBaseRegistryPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx index 7897b7bcc2ff..c426cb9dfdbb 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx +++ b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx @@ -9,19 +9,20 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kb import type { Logger } from '@kbn/logging'; import type { PublicPluginConfig, - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies, + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies, } from './types'; +import { InstallationService } from './services/installation'; export class KnowledgeBaseRegistryPlugin implements Plugin< - KnowledgeBaseRegistrySetupContract, - KnowledgeBaseRegistryStartContract, - KnowledgeBaseRegistrySetupDependencies, - KnowledgeBaseRegistryStartDependencies + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies > { logger: Logger; @@ -30,19 +31,21 @@ export class KnowledgeBaseRegistryPlugin this.logger = context.logger.get(); } setup( - coreSetup: CoreSetup< - KnowledgeBaseRegistryStartDependencies, - KnowledgeBaseRegistryStartContract - >, - pluginsSetup: KnowledgeBaseRegistrySetupDependencies - ): KnowledgeBaseRegistrySetupContract { + coreSetup: CoreSetup, + pluginsSetup: PluginSetupDependencies + ): ProductDocBasePluginSetup { return {}; } - start( - coreStart: CoreStart, - pluginsStart: KnowledgeBaseRegistryStartDependencies - ): KnowledgeBaseRegistryStartContract { - return {}; + start(coreStart: CoreStart, pluginsStart: PluginStartDependencies): ProductDocBasePluginStart { + const installationService = new InstallationService({ http: coreStart.http }); + + return { + installation: { + getStatus: () => installationService.getInstallationStatus(), + install: () => installationService.install(), + uninstall: () => installationService.uninstall(), + }, + }; } } diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts new file mode 100644 index 000000000000..2eee8613d77d --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { InstallationService } from './installation_service'; +export type { InstallationAPI } from './types'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts new file mode 100644 index 000000000000..5c90a9bb0dca --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { HttpSetup } from '@kbn/core-http-browser'; +import { + INSTALLATION_STATUS_API_PATH, + PERFORM_INSTALL_API_PATH, + UNINSTALL_API_PATH, + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../../common/http_api/installation'; + +export class InstallationService { + private readonly http: HttpSetup; + + constructor({ http }: { http: HttpSetup }) { + this.http = http; + } + + async getInstallationStatus(): Promise { + return await this.http.get(INSTALLATION_STATUS_API_PATH); + } + + async install(): Promise { + return await this.http.post(PERFORM_INSTALL_API_PATH); + } + + async uninstall(): Promise { + return await this.http.post(UNINSTALL_API_PATH); + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts new file mode 100644 index 000000000000..5c01c84b2462 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../../common/http_api/installation'; + +export interface InstallationAPI { + getStatus(): Promise; + install(): Promise; + uninstall(): Promise; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/types.ts b/x-pack/plugins/ai_infra/product_doc_base/public/types.ts index eb99333d37e8..1d06b0e08fa2 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/types.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/public/types.ts @@ -5,14 +5,18 @@ * 2.0. */ +import type { InstallationAPI } from './services/installation'; + /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface PublicPluginConfig {} -export interface KnowledgeBaseRegistrySetupDependencies {} +export interface PluginSetupDependencies {} -export interface KnowledgeBaseRegistryStartDependencies {} +export interface PluginStartDependencies {} -export interface KnowledgeBaseRegistrySetupContract {} +export interface ProductDocBasePluginSetup {} -export interface KnowledgeBaseRegistryStartContract {} +export interface ProductDocBasePluginStart { + installation: InstallationAPI; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index bb64231c1ed7..8d323e70c7a6 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -23,6 +23,7 @@ import { PackageInstaller } from './services/package_installer'; import { InferenceEndpointManager } from './services/inference_endpoint'; import { ProductDocInstallClient } from './services/doc_install_status'; import { SearchService } from './services/search'; +import { registerRoutes } from './routes'; export class KnowledgeBaseRegistryPlugin implements @@ -34,6 +35,8 @@ export class KnowledgeBaseRegistryPlugin > { logger: Logger; + private installClient?: ProductDocInstallClient; + private packageInstaller?: PackageInstaller; constructor(private readonly context: PluginInitializerContext) { this.logger = context.logger.get(); @@ -44,6 +47,23 @@ export class KnowledgeBaseRegistryPlugin ): ProductDocBaseSetupContract { coreSetup.savedObjects.registerType(knowledgeBaseProductDocInstallSavedObjectType); + const router = coreSetup.http.createRouter(); + registerRoutes({ + router, + getInstallClient: () => { + if (!this.installClient) { + throw new Error('getInstallClient called before #start'); + } + return this.installClient; + }, + getInstaller: () => { + if (!this.packageInstaller) { + throw new Error('getInstaller called before #start'); + } + return this.packageInstaller; + }, + }); + return {}; } @@ -55,13 +75,14 @@ export class KnowledgeBaseRegistryPlugin core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName]) ); const productDocClient = new ProductDocInstallClient({ soClient }); + this.installClient = productDocClient; const endpointManager = new InferenceEndpointManager({ esClient: core.elasticsearch.client.asInternalUser, logger: this.logger.get('endpoint-manager'), }); - const packageInstaller = new PackageInstaller({ + this.packageInstaller = new PackageInstaller({ esClient: core.elasticsearch.client.asInternalUser, productDocClient, endpointManager, @@ -77,7 +98,7 @@ export class KnowledgeBaseRegistryPlugin }); // TODO: see if we should be using taskManager for that. - packageInstaller.ensureUpToDate({}).catch((err) => { + this.packageInstaller.ensureUpToDate({}).catch((err) => { this.logger.error(`Error checking if product documentation is up to date: ${err.message}`); }); @@ -85,8 +106,7 @@ export class KnowledgeBaseRegistryPlugin delay(10) .then(async () => { // this.logger.info('*** test installing packages'); - // await packageInstaller.installAll({}); - + // await this.packageInstaller.installAll({}); // const results = await searchService.search({ // query: 'How to create a space in Kibana?', // products: ['kibana'], diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts new file mode 100644 index 000000000000..254328ab46f6 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter } from '@kbn/core/server'; +import { registerInstallationRoutes } from './installation'; +import type { ProductDocInstallClient } from '../services/doc_install_status'; +import type { PackageInstaller } from '../services/package_installer'; + +export const registerRoutes = ({ + router, + getInstallClient, + getInstaller, +}: { + router: IRouter; + getInstallClient: () => ProductDocInstallClient; + getInstaller: () => PackageInstaller; +}) => { + registerInstallationRoutes({ getInstaller, getInstallClient, router }); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts new file mode 100644 index 000000000000..d8638e8f014c --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter } from '@kbn/core/server'; +import { + INSTALLATION_STATUS_API_PATH, + PERFORM_INSTALL_API_PATH, + UNINSTALL_API_PATH, + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../common/http_api/installation'; +import { InstallationStatus } from '../../common/install_status'; +import type { ProductDocInstallClient } from '../services/doc_install_status'; +import type { PackageInstaller } from '../services/package_installer'; + +export const registerInstallationRoutes = ({ + router, + getInstallClient, + getInstaller, +}: { + router: IRouter; + getInstallClient: () => ProductDocInstallClient; + getInstaller: () => PackageInstaller; +}) => { + router.get( + { path: INSTALLATION_STATUS_API_PATH, validate: false, options: { access: 'internal' } }, + async (ctx, req, res) => { + // TODO: use installer instead + const installClient = getInstallClient(); + const installStatus = await installClient.getInstallationStatus(); + const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status)); + + return res.ok({ + body: { + perProducts: installStatus, + overall: overallStatus, + }, + }); + } + ); + + router.post( + { + path: PERFORM_INSTALL_API_PATH, + validate: false, + options: { + access: 'internal', + timeout: { idleSocket: 20 * 60 * 1000 }, // 20 minutes, install can take time. + }, + }, + async (ctx, req, res) => { + const installer = getInstaller(); + + // TODO: use ensureUpToDate with parameter to install non-installed + await installer.installAll({}); + + return res.ok({ + body: { + installed: true, + }, + }); + } + ); + + router.post( + { + path: UNINSTALL_API_PATH, + validate: false, + options: { + access: 'internal', + }, + }, + async (ctx, req, res) => { + const installer = getInstaller(); + + // TODO: use ensureUpToDate with parameter to install non-installed + await installer.uninstallAll(); + + return res.ok({ + body: { + success: true, + }, + }); + } + ); +}; + +const getOverallStatus = (statuses: InstallationStatus[]): InstallationStatus => { + for (const status of statusOrder) { + if (statuses.includes(status)) { + return status; + } + } + return 'installed'; +}; + +const statusOrder: InstallationStatus[] = ['error', 'installing', 'uninstalled', 'installed']; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts index d28902bf4ab5..fa2db1e3bf4e 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts @@ -8,16 +8,11 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; import { ProductName, DocumentationProduct } from '@kbn/product-doc-common'; -import { InstallationStatus } from '../../../common/install_status'; +import type { ProductInstallState } from '../../../common/install_status'; import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; import type { KnowledgeBaseProductDocInstallAttributes as TypeAttributes } from '../../saved_objects'; // import { soToModel } from './model_conversion'; -interface ProductInstallState { - status: InstallationStatus; - version?: string; -} - export class ProductDocInstallClient { private soClient: SavedObjectsClientContract; @@ -37,7 +32,7 @@ export class ProductDocInstallClient { } */ - async getInstallationStatus() { + async getInstallationStatus(): Promise> { const response = await this.soClient.find({ type: typeName, perPage: 100, diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts index ef5ca27cad57..8570e4209114 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -208,6 +208,13 @@ export class PackageInstaller { await this.productDocClient.setUninstalled(productName); } + + async uninstallAll() { + const allProducts = Object.values(DocumentationProduct); + for (const productName of allProducts) { + await this.uninstallPackage({ productName }); + } + } } const selectVersion = (currentVersion: string, availableVersions: string[]): string => { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts index 40c70ef785fb..468554649468 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts @@ -27,11 +27,11 @@ export const performSearch = async ({ query: { bool: { filter: [ - { - bool: { - must: [{ term: { version: '8.15' } }], - }, - }, + // { + // bool: { + // must: [{ term: { version: '8.15' } }], + // }, + // }, ], should: [ { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc index ddf00c84c0ac..f7b830183995 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc @@ -8,7 +8,7 @@ "browser": true, "configPath": ["xpack", "observabilityAiAssistantManagement"], "requiredPlugins": ["management", "observabilityAIAssistant", "observabilityShared"], - "optionalPlugins": ["actions", "home", "serverless", "enterpriseSearch"], + "optionalPlugins": ["actions", "home", "serverless", "enterpriseSearch", "productDocBase"], "requiredBundles": ["kibanaReact", "logsDataAccess"] } } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts index a680da5ed3f9..3bfe3dff3f9f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts @@ -9,6 +9,9 @@ export const REACT_QUERY_KEYS = { GET_GENAI_CONNECTORS: 'get_genai_connectors', GET_KB_ENTRIES: 'get_kb_entries', GET_KB_USER_INSTRUCTIONS: 'get_kb_user_instructions', + GET_PRODUCT_DOC_STATUS: 'get_product_doc_status', + INSTALL_PRODUCT_DOC: 'install_product_doc', + UNINSTALL_PRODUCT_DOC: 'uninstall_product_doc', CREATE_KB_ENTRIES: 'create_kb_entry', IMPORT_KB_ENTRIES: 'import_kb_entry', }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts new file mode 100644 index 000000000000..ef95d51f78d4 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from '@tanstack/react-query'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +export function useGetProductDocStatus() { + const { productDocBase } = useKibana().services; + + const { isLoading, isError, isSuccess, isRefetching, data, refetch } = useQuery({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + queryFn: async () => { + return productDocBase!.installation.getStatus(); + }, + keepPreviousData: false, + refetchOnWindowFocus: false, + }); + + return { + status: data, + refetch, + isLoading, + isRefetching, + isSuccess, + isError, + }; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts new file mode 100644 index 000000000000..a34399882d7b --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; +import { i18n } from '@kbn/i18n'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import type { PerformInstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +type ServerError = IHttpFetchError; + +export function useInstallProductDoc() { + const { + productDocBase, + notifications: { toasts }, + } = useKibana().services; + const queryClient = useQueryClient(); + + return useMutation( + [REACT_QUERY_KEYS.INSTALL_PRODUCT_DOC], + () => { + return productDocBase!.installation.install(); + }, + { + onSuccess: (_data) => { + toasts.addSuccess( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.installProductDoc.successNotification', + { + defaultMessage: 'The Elastic documentation was successfully installed', + } + ) + ); + + queryClient.invalidateQueries({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + refetchType: 'all', + }); + }, + onError: (error) => { + toasts.addError(new Error(error.body?.message ?? error.message), { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.installProductDoc.errorNotification', + { + defaultMessage: 'Something went wrong while installing the Elastic documentation', + } + ), + }); + }, + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts new file mode 100644 index 000000000000..8c4774a5e8f6 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; +import { i18n } from '@kbn/i18n'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import type { UninstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +type ServerError = IHttpFetchError; + +export function useUninstallProductDoc() { + const { + productDocBase, + notifications: { toasts }, + } = useKibana().services; + const queryClient = useQueryClient(); + + return useMutation( + [REACT_QUERY_KEYS.UNINSTALL_PRODUCT_DOC], + () => { + return productDocBase!.installation.uninstall(); + }, + { + onSuccess: (_data) => { + toasts.addSuccess( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.uninstallProductDoc.successNotification', + { + defaultMessage: 'The Elastic documentation was successfully uninstalled', + } + ) + ); + + queryClient.invalidateQueries({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + refetchType: 'all', + }); + }, + onError: (error) => { + toasts.addError(new Error(error.body?.message ?? error.message), { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.uninstallProductDoc.errorNotification', + { + defaultMessage: 'Something went wrong while uninstalling the Elastic documentation', + } + ), + }); + }, + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts index 53da619c7ad1..348922339174 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts @@ -6,11 +6,12 @@ */ import { i18n } from '@kbn/i18n'; -import { CoreSetup, Plugin } from '@kbn/core/public'; -import { ManagementSetup } from '@kbn/management-plugin/public'; -import { HomePublicPluginSetup } from '@kbn/home-plugin/public'; -import { ServerlessPluginStart } from '@kbn/serverless/public'; -import { EnterpriseSearchPublicStart } from '@kbn/enterprise-search-plugin/public'; +import type { CoreSetup, Plugin } from '@kbn/core/public'; +import type { ManagementSetup } from '@kbn/management-plugin/public'; +import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; +import type { ServerlessPluginStart } from '@kbn/serverless/public'; +import type { EnterpriseSearchPublicStart } from '@kbn/enterprise-search-plugin/public'; +import type { ProductDocBasePluginStart } from '@kbn/product-doc-base-plugin/public'; import type { ObservabilityAIAssistantPublicSetup, @@ -33,6 +34,7 @@ export interface StartDependencies { observabilityAIAssistant: ObservabilityAIAssistantPublicStart; serverless?: ServerlessPluginStart; enterpriseSearch?: EnterpriseSearchPublicStart; + productDocBase?: ProductDocBasePluginStart; } export class AiAssistantManagementObservabilityPlugin diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx new file mode 100644 index 000000000000..1c63ddabf923 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useState, useCallback, useMemo } from 'react'; +import { + EuiButton, + EuiDescribedFormGroup, + EuiFormRow, + EuiText, + EuiFlexGroup, + EuiFlexItem, + EuiHealth, + EuiLoadingSpinner, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { useKibana } from '../../../hooks/use_kibana'; +import { useGetProductDocStatus } from '../../../hooks/use_get_product_doc_status'; +import { useInstallProductDoc } from '../../../hooks/use_install_product_doc'; +import { useUninstallProductDoc } from '../../../hooks/use_uninstall_product_doc'; + +export function ProductDocEntry() { + const { overlays } = useKibana().services; + + const [isInstalled, setInstalled] = useState(true); + const [isInstalling, setInstalling] = useState(false); + + const { mutateAsync: installProductDoc } = useInstallProductDoc(); + const { mutateAsync: uninstallProductDoc } = useUninstallProductDoc(); + const { status, isLoading: isStatusLoading } = useGetProductDocStatus(); + + useEffect(() => { + if (status) { + setInstalled(status.overall === 'installed'); + } + }, [status]); + + const onClickInstall = useCallback(() => { + setInstalling(true); + installProductDoc().then(() => { + setInstalling(false); + setInstalled(true); + }); + }, [installProductDoc]); + + const onClickUninstall = useCallback(() => { + overlays + .openConfirm( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocUninstallConfirmText', + { + defaultMessage: `Are you sure you want to uninstall the Elastic documentation?`, + } + ), + { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocUninstallConfirmTitle', + { + defaultMessage: `Uninstalling Elastic documentation`, + } + ), + } + ) + .then((confirmed) => { + if (confirmed) { + uninstallProductDoc().then(() => { + setInstalling(false); + setInstalled(false); + }); + } + }); + }, [overlays, uninstallProductDoc]); + + const content = useMemo(() => { + if (isStatusLoading) { + return <>; + } + if (isInstalling) { + return ( + + + + + + + ); + } + if (isInstalled) { + return ( + + + + Installed + + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.uninstallProductDocButtonLabel', + { defaultMessage: 'Uninstall' } + )} + + + + ); + } + return ( + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.installProductDocButtonLabel', + { defaultMessage: 'Install' } + )} + + + + ); + }, [isInstalled, isInstalling, isStatusLoading, onClickInstall, onClickUninstall]); + + return ( + + {i18n.translate('xpack.observabilityAiAssistantManagement.settingsPage.productDocLabel', { + defaultMessage: 'Product documentation', + })} + + } + description={ +

+ {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocDescription', + { + defaultMessage: `Install Elastic documentation to improve the assistant's efficiency.`, + } + )} +

+ } + > + {content} + + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx index 4ec17f34610e..7cc3ec5bcdd0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx @@ -10,10 +10,12 @@ import { EuiButton, EuiDescribedFormGroup, EuiFormRow, EuiPanel } from '@elastic import { i18n } from '@kbn/i18n'; import { useKibana } from '../../../hooks/use_kibana'; import { UISettings } from './ui_settings'; +import { ProductDocEntry } from './product_doc_entry'; export function SettingsTab() { const { application: { navigateToApp }, + productDocBase, } = useKibana().services; const handleNavigateToConnectors = () => { @@ -104,6 +106,7 @@ export function SettingsTab() { + {productDocBase ? : undefined} ); From 75aaaf62aa37ffd4d0baf6ef32afda0f39fa6489 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 10:01:54 +0200 Subject: [PATCH 31/97] add task availability check, improve summarizer --- .../ai_infra/llm_tasks/server/plugin.ts | 6 +- .../extract_relevant_chunks.ts | 38 +++---- .../retrieve_documentation.ts | 100 +++++++++++------- .../tasks/retrieve_documentation/types.ts | 7 +- .../ai_infra/llm_tasks/server/types.ts | 1 + .../ai_infra/llm_tasks/server/utils/tokens.ts | 21 ++++ .../product_doc_base/server/plugin.ts | 26 ++--- .../ai_infra/product_doc_base/server/types.ts | 1 + .../kibana.jsonc | 3 +- .../server/functions/documentation.ts | 71 +++++++++++++ .../server/functions/index.ts | 2 + .../server/types.ts | 2 + 12 files changed, 195 insertions(+), 83 deletions(-) create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts create mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts index 489d71a66889..7cf3b11557d2 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts @@ -27,7 +27,7 @@ export class LlmTasksPlugin { logger: Logger; - constructor(context: PluginInitializerContext) { + constructor(private readonly context: PluginInitializerContext) { this.logger = context.logger.get(); } setup( @@ -40,10 +40,14 @@ export class LlmTasksPlugin start(core: CoreStart, startDependencies: PluginStartDependencies): LlmTasksPluginStart { const { inference, productDocBase } = startDependencies; return { + retrieveDocumentationAvailable: async () => { + return await startDependencies.productDocBase.isInstalled(); + }, retrieveDocumentation: (options) => { return retrieveDocumentation({ outputAPI: inference.getClient({ request: options.request }).output, searchDocAPI: productDocBase.search, + logger: this.context.logger.get('tasks.retrieve-documentation'), })(options); }, }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts index 58f0ec1e57d2..f32f4aafdaf3 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts @@ -11,29 +11,26 @@ import type { FunctionCallingMode } from '@kbn/inference-plugin/common/chat_comp import type { OutputAPI } from '@kbn/inference-plugin/common/output'; import { withoutOutputUpdateEvents } from '@kbn/inference-plugin/common/output/without_output_update_events'; -const extractRelevantChunksSchema = { +const summarizeDocumentSchema = { type: 'object', properties: { useful: { type: 'boolean', - description: `Whether the provided document has any useful information related to the user's query`, + description: `Whether the provided document has any useful information related to the user's query.`, }, - chunks: { - type: 'array', - items: { - type: 'string', - }, - description: `The chunks of text of the document that are relevant to the user's query. Can be empty`, + summary: { + type: 'string', + description: `The condensed version of the document that can be used to answer the question. Can be empty.`, }, }, required: ['useful'], } as const satisfies ToolSchema; -interface ExtractRelevantChunksResponse { - chunks: string[]; +interface SummarizeDocumentResponse { + summary: string; } -export const extractRelevantChunks = async ({ +export const summarizeDocument = async ({ searchTerm, documentContent, connectorId, @@ -45,21 +42,20 @@ export const extractRelevantChunks = async ({ outputAPI: OutputAPI; connectorId: string; functionCalling?: FunctionCallingMode; -}): Promise => { +}): Promise => { const result = await lastValueFrom( outputAPI('extract_relevant_chunks', { connectorId, functionCalling, - system: `You are an Elastic assistant in charge of helping answering the user question + system: `You are an Elastic assistant in charge of helping answering the user question. - Given a search query and a document, your current task will be to extract - the parts of the document that could be useful in answering the question. - - - If multiple parts are useful, return them all. - - If you think nothing in the document could help answering the question, return an empty list. + Given a question and a document, please provide a condensed version of the document + that can be used to answer the question. + - All useful information should be included in the condensed version. + - If you think the document isn't relevant at all to answer the question, return an empty text. `, input: ` - ## Search query + ## User question ${searchTerm} @@ -67,11 +63,11 @@ export const extractRelevantChunks = async ({ ${documentContent} `, - schema: extractRelevantChunksSchema, + schema: summarizeDocumentSchema, }).pipe(withoutOutputUpdateEvents()) ); return { - chunks: result.output.chunks ?? [], + summary: result.output.summary ?? '', }; }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts index cbdc659225b2..879dcf1373f6 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -5,58 +5,76 @@ * 2.0. */ -import { encode } from 'gpt-tokenizer'; +import type { Logger } from '@kbn/logging'; import type { OutputAPI } from '@kbn/inference-plugin/common/output'; import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server'; +import { truncate, count as countTokens } from '../../utils/tokens'; import type { RetrieveDocumentationAPI } from './types'; -import { extractRelevantChunks } from './extract_relevant_chunks'; +import { summarizeDocument } from './extract_relevant_chunks'; + +// if content length greater, then we'll trigger the summary task +const MIN_TOKENS_TO_SUMMARIZE = 500; +// maximum token length of generated summaries - will truncate if greater +const MAX_SUMMARY_TOKEN_LENGTH = 1000; export const retrieveDocumentation = ({ outputAPI, searchDocAPI, + logger: log, }: { outputAPI: OutputAPI; searchDocAPI: ProductDocSearchAPI; + logger: Logger; }): RetrieveDocumentationAPI => async ({ searchTerm, connectorId, products, functionCalling, max = 3 }) => { - const searchResults = await searchDocAPI({ query: searchTerm, products, max }); - - console.log('*** retrieveDocumentation => found ' + searchResults.results.length); - - const processedDocuments = await Promise.all( - searchResults.results.map(async (document) => { - const tokenCount = countTokens(document.content); - - let chunks: string[]; - if (tokenCount > 250) { - const extractResponse = await extractRelevantChunks({ - searchTerm, - documentContent: document.content, - outputAPI, - connectorId, - functionCalling, - }); - chunks = extractResponse.chunks; - } else { - chunks = [document.content]; - } - - return { - title: document.title, - url: document.url, - chunks, - }; - }) - ); - - console.log(`retrieved documents: ${processedDocuments.map((doc) => doc.title)}`); - - return { - documents: processedDocuments.filter((doc) => doc.chunks.length > 0), - }; - }; + try { + const { results } = await searchDocAPI({ query: searchTerm, products, max }); + + log.debug(`searching with term=[${searchTerm}] returned ${results.length} documents`); + + const processedDocuments = await Promise.all( + results.map(async (document) => { + const tokenCount = countTokens(document.content); + const summarize = tokenCount >= MIN_TOKENS_TO_SUMMARIZE; + log.debug( + `processing doc [${document.url}] - tokens : [${tokenCount}] - summarize: [${summarize}]` + ); -const countTokens = (text: string): number => { - return encode(text).length; -}; + let content: string; + if (summarize) { + const extractResponse = await summarizeDocument({ + searchTerm, + documentContent: document.content, + outputAPI, + connectorId, + functionCalling, + }); + content = truncate(extractResponse.summary, MAX_SUMMARY_TOKEN_LENGTH); + } else { + content = document.content; + } + + log.debug(`done processing document [${document.url}]`); + return { + title: document.title, + url: document.url, + content, + }; + }) + ); + + log.debug(() => { + const docsAsJson = JSON.stringify(processedDocuments); + return `searching with term=[${searchTerm}] - results: ${docsAsJson}`; + }); + + return { + success: true, + documents: processedDocuments.filter((doc) => doc.content.length > 0), + }; + } catch (e) { + log.error(`Error retrieving documentation: ${e.message}. Returning empty results.`); + return { success: false, documents: [] }; + } + }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts index 115c14b706ca..2d0e5fb46f71 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -18,14 +18,15 @@ export interface RetrieveDocumentationParams { functionCalling?: FunctionCallingMode; } -export interface DocumentRelevantChunks { +export interface RetrievedDocument { title: string; url: string; - chunks: string[]; + content: string; } export interface RetrieveDocumentationResult { - documents: DocumentRelevantChunks[]; + success: boolean; + documents: RetrievedDocument[]; } export type RetrieveDocumentationAPI = ( diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts index df7872d6cdce..cef03ee31b29 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts @@ -21,5 +21,6 @@ export interface PluginStartDependencies { export interface LlmTasksPluginSetup {} export interface LlmTasksPluginStart { + retrieveDocumentationAvailable: () => Promise; retrieveDocumentation: RetrieveDocumentationAPI; } diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts new file mode 100644 index 000000000000..cb469144255b --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { encode, decode } from 'gpt-tokenizer'; + +export const count = (text: string): number => { + return encode(text).length; +}; + +export const truncate = (text: string, maxTokens: number): string => { + const encoded = encode(text); + if (encoded.length > maxTokens) { + const truncated = encoded.slice(0, maxTokens); + return decode(truncated); + } + return text; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index 8d323e70c7a6..efa0e22c4e1b 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -102,24 +102,18 @@ export class KnowledgeBaseRegistryPlugin this.logger.error(`Error checking if product documentation is up to date: ${err.message}`); }); - // TODO: remove - delay(10) - .then(async () => { - // this.logger.info('*** test installing packages'); - // await this.packageInstaller.installAll({}); - // const results = await searchService.search({ - // query: 'How to create a space in Kibana?', - // products: ['kibana'], - // ); - // console.log(JSON.stringify(results.results[0])); - }) - .catch((e) => { - this.logger.error('*** ERROR', e); - }); return { + isInstalled: async () => { + // TODO: should also check license, probably + + // TODO: something less naive + const installStatus = await productDocClient.getInstallationStatus(); + const installed = Object.values(installStatus).some( + (status) => status.status === 'installed' + ); + return installed; + }, search: searchService.search.bind(searchService), }; } } - -const delay = (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds * 1000)); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts index 6a56752d2f88..80a7cdb76603 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts @@ -17,4 +17,5 @@ export interface ProductDocBaseSetupContract {} export interface ProductDocBaseStartContract { search: SearchApi; + isInstalled: () => Promise; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc index 1414912d3916..d4ecd65da29b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc @@ -25,7 +25,8 @@ "alerting", "features", "inference", - "logsDataAccess" + "logsDataAccess", + "llmTasks" ], "requiredBundles": ["kibanaReact", "esqlDataGrid"], "optionalPlugins": ["cloud"], diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts new file mode 100644 index 000000000000..e79f8e88f04a --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FunctionVisibility } from '@kbn/observability-ai-assistant-plugin/common'; +import type { FunctionRegistrationParameters } from '.'; + +export const RETRIEVE_DOCUMENTATION_NAME = 'retrieve_elastic_doc'; + +export async function registerDocumentationFunction({ + functions, + resources, + pluginsStart: { llmTasks }, +}: FunctionRegistrationParameters) { + const isProductDocAvailable = (await llmTasks.retrieveDocumentationAvailable()) ?? false; + + functions.registerFunction( + { + name: RETRIEVE_DOCUMENTATION_NAME, + visibility: isProductDocAvailable + ? FunctionVisibility.AssistantOnly + : FunctionVisibility.Internal, + description: `Use this function to retrieve documentation about Elastic products. + You can retrieve documentation about the Elastic stack, such as Kibana and Elasticsearch, + or for Elastic solutions, such as Elastic Security, Elastic Observability or Elastic Enterprise Search + `, + parameters: { + type: 'object', + properties: { + query: { + description: 'The query to use to retrieve documentation', + type: 'string', + }, + product: { + description: `Filter the product to retrieve documentation for + Possible options are: + - "kibana": Kibana product + - "elasticsearch": Elasticsearch product + - "observability": Elastic Observability solution + - "security": Elastic Security solution + If not specified, will search against all products + `, + type: 'string' as const, + enum: ['kibana', 'elasticsearch', 'observability', 'security'], + }, + }, + required: ['query'], + } as const, + }, + async ({ arguments: { query, product }, connectorId, useSimulatedFunctionCalling }) => { + const response = await llmTasks!.retrieveDocumentation({ + searchTerm: query, + products: product ? [product] : undefined, + max: 3, + connectorId, + request: resources.request, + functionCalling: useSimulatedFunctionCalling ? 'simulated' : 'native', + }); + + return { + content: { + documents: response.documents, + }, + }; + }, + ['all'] + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts index 7554164a55a6..ba876ad9457b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts @@ -12,6 +12,7 @@ import { registerLensFunction } from './lens'; import { registerVisualizeESQLFunction } from './visualize_esql'; import { ObservabilityAIAssistantAppPluginStartDependencies } from '../types'; import { registerChangesFunction } from './changes'; +import { registerDocumentationFunction } from './documentation'; export type FunctionRegistrationParameters = Omit< Parameters[0], @@ -24,4 +25,5 @@ export const registerFunctions = async (registrationParameters: FunctionRegistra registerVisualizeESQLFunction(registrationParameters); registerAlertsFunction(registrationParameters); registerChangesFunction(registrationParameters); + await registerDocumentationFunction(registrationParameters); }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts index fc39e0b7fb24..a1196be6a829 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts @@ -37,6 +37,7 @@ import type { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plu import type { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import type { InferenceServerStart, InferenceServerSetup } from '@kbn/inference-plugin/server'; import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/server'; +import type { LlmTasksPluginStart } from '@kbn/llm-tasks-plugin/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ObservabilityAIAssistantAppServerStart {} @@ -57,6 +58,7 @@ export interface ObservabilityAIAssistantAppPluginStartDependencies { serverless?: ServerlessPluginStart; inference: InferenceServerStart; logsDataAccess: LogsDataAccessPluginStart; + llmTasks: LlmTasksPluginStart; } export interface ObservabilityAIAssistantAppPluginSetupDependencies { From cc727d27bae0e25c4b14652ebb160d39a14078ba Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 14 Oct 2024 08:22:24 +0000 Subject: [PATCH 32/97] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 1 + docs/developer/plugin-list.asciidoc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c06127a1532c..e1077d3944ee 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -569,6 +569,7 @@ src/plugins/links @elastic/kibana-presentation packages/kbn-lint-packages-cli @elastic/kibana-operations packages/kbn-lint-ts-projects-cli @elastic/kibana-operations x-pack/plugins/lists @elastic/security-detection-engine +x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra examples/locator_examples @elastic/appex-sharedux examples/locator_explorer @elastic/appex-sharedux packages/kbn-logging @elastic/kibana-core diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 71713d716198..1051e80883aa 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana. using the CURL scripts in the scripts folder. +|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks] +|This plugin contains various LLM tasks + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess] |Exposes services to access logs data. From 0e6e0d65c7f9da6f0bf1a26cdfb17e03ca7d780b Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 14 Oct 2024 08:23:11 +0000 Subject: [PATCH 33/97] [CI] Auto-commit changed files from 'node scripts/notice' --- x-pack/plugins/ai_infra/llm_tasks/tsconfig.json | 4 ++-- x-pack/plugins/ai_infra/product_doc_base/tsconfig.json | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json index 4f44448828fd..fbcfedb5972a 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json +++ b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json @@ -19,7 +19,7 @@ "@kbn/logging", "@kbn/config-schema", "@kbn/product-doc-common", - "@kbn/core-saved-objects-server", - "@kbn/utils", + "@kbn/inference-plugin", + "@kbn/product-doc-base-plugin", ] } diff --git a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json index 4f44448828fd..93c9ca2ac860 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json +++ b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json @@ -21,5 +21,6 @@ "@kbn/product-doc-common", "@kbn/core-saved-objects-server", "@kbn/utils", + "@kbn/core-http-browser", ] } From 1f11493aa4c54cb328b7e7f53e0eb28d5208f426 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 14 Oct 2024 08:23:25 +0000 Subject: [PATCH 34/97] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- .../observability_ai_assistant_app/tsconfig.json | 1 + .../observability_ai_assistant_management/tsconfig.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json index af04a677f5e9..2673a20c725d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json @@ -69,6 +69,7 @@ "@kbn/cloud-plugin", "@kbn/logs-data-access-plugin", "@kbn/ai-assistant-common", + "@kbn/llm-tasks-plugin", ], "exclude": [ "target/**/*" diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json index d8a03acbae61..45ecd988294e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json @@ -21,7 +21,8 @@ "@kbn/observability-shared-plugin", "@kbn/config-schema", "@kbn/core-ui-settings-common", - "@kbn/logs-data-access-plugin" + "@kbn/logs-data-access-plugin", + "@kbn/product-doc-base-plugin" ], "exclude": ["target/**/*"] } From e4d9f9b9c1100388c4b44b784d9723653f6ce57c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 10:34:07 +0200 Subject: [PATCH 35/97] working on config page --- .../settings_tab/product_doc_entry.tsx | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx index 1c63ddabf923..6eae22809810 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx @@ -24,7 +24,10 @@ import { useInstallProductDoc } from '../../../hooks/use_install_product_doc'; import { useUninstallProductDoc } from '../../../hooks/use_uninstall_product_doc'; export function ProductDocEntry() { - const { overlays } = useKibana().services; + const { + overlays, + notifications: { toasts }, + } = useKibana().services; const [isInstalled, setInstalled] = useState(true); const [isInstalling, setInstalling] = useState(false); @@ -40,12 +43,20 @@ export function ProductDocEntry() { }, [status]); const onClickInstall = useCallback(() => { + toasts.addSuccess( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocInstallToastText', + { + defaultMessage: 'Installing Elastic documentation, this can take a few minutes.', + } + ) + ); setInstalling(true); installProductDoc().then(() => { setInstalling(false); setInstalled(true); }); - }, [installProductDoc]); + }, [installProductDoc, toasts]); const onClickUninstall = useCallback(() => { overlays @@ -135,16 +146,22 @@ export function ProductDocEntry() { title={

{i18n.translate('xpack.observabilityAiAssistantManagement.settingsPage.productDocLabel', { - defaultMessage: 'Product documentation', + defaultMessage: 'Elastic documentation', })}

} description={

+ + {i18n.translate('xpack.observabilityAiAssistantManagement.settingsPage.techPreview', { + defaultMessage: '[technical preview] ', + })} + {i18n.translate( 'xpack.observabilityAiAssistantManagement.settingsPage.productDocDescription', { - defaultMessage: `Install Elastic documentation to improve the assistant's efficiency.`, + defaultMessage: + "Install Elastic documentation to improve the assistant's efficiency.", } )}

From 2deea8b8e4a002234c77797e18747fa102d83897 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 10:39:48 +0200 Subject: [PATCH 36/97] revert old mappings --- .../current_fields.json | 7 ----- .../current_mappings.json | 26 ------------------- .../check_registered_types.test.ts | 1 - .../group3/type_registrations.test.ts | 1 + .../product_doc_base/common/consts.ts | 2 +- 5 files changed, 2 insertions(+), 35 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 315919e3a532..3bd084fc677a 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -727,13 +727,6 @@ "use_space_awareness_migration_status" ], "inventory-view": [], - "knowledge_base_entry": [ - "installed_by", - "installed_by.type", - "name", - "source", - "source.type" - ], "kql-telemetry": [], "legacy-url-alias": [ "disabled", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 8e90f3e4b91c..9eb4375f082d 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2403,32 +2403,6 @@ "dynamic": false, "properties": {} }, - "knowledge_base_entry": { - "dynamic": false, - "properties": { - "installed_by": { - "dynamic": false, - "properties": { - "type": { - "type": "keyword" - } - }, - "type": "object" - }, - "name": { - "type": "keyword" - }, - "source": { - "dynamic": false, - "properties": { - "type": { - "type": "keyword" - } - }, - "type": "object" - } - } - }, "kql-telemetry": { "dynamic": false, "properties": {} diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index d52522978dff..d6cfcae558b2 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -126,7 +126,6 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", "ingest_manager_settings": "e794576a05d19dd5306a1e23cbb82c09bffabd65", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", - "knowledge_base_entry": "5a9b5821eb2c22ed66e6f8298c45d519ac13de5c", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", "legacy-url-alias": "9b8cca3fbb2da46fd12823d3cd38fdf1c9f24bc8", "lens": "5cfa2c52b979b4f8df56dd13c477e152183468b9", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index ce65c320cb54..b47e7451baf5 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -116,6 +116,7 @@ const previouslyRegisteredTypes = [ 'osquery-usage-metric', 'osquery-manager-usage-metric', 'policy-settings-protection-updates-note', + 'product-doc-install-status', 'query', 'rules-settings', 'sample-data-telemetry', diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts index bc83730d6389..850e624fd057 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts @@ -5,6 +5,6 @@ * 2.0. */ -export const productDocInstallStatusSavedObjectTypeName = 'kb_product_doc_install_status'; +export const productDocInstallStatusSavedObjectTypeName = 'product-doc-install-status'; export const internalElserInferenceId = 'kibana-internal-elser2'; From 55bf175c184d01ff06032661eaf2e3fab404736f Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 10:40:58 +0200 Subject: [PATCH 37/97] update limits --- packages/kbn-optimizer/limits.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 369284743247..bc8ad4b71f0f 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -99,6 +99,7 @@ pageLoadAssetSize: licensing: 29004 links: 44490 lists: 22900 + llmTasks: 22500 logsDataAccess: 16759 logsExplorer: 60000 logsShared: 281060 From 084c78cc495da026bc9b078060b43fd123676c59 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 10:56:25 +0200 Subject: [PATCH 38/97] rename things --- x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts | 4 ++-- .../product_doc_base/server/saved_objects/index.ts | 4 ++-- .../server/saved_objects/product_doc_install.ts | 7 ++++--- .../server/services/doc_install_status/model_conversion.ts | 4 ++-- .../doc_install_status/product_doc_install_service.ts | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index efa0e22c4e1b..bdbd4f4e3a83 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -18,7 +18,7 @@ import type { ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies, } from './types'; -import { knowledgeBaseProductDocInstallSavedObjectType } from './saved_objects'; +import { productDocInstallStatusSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; import { InferenceEndpointManager } from './services/inference_endpoint'; import { ProductDocInstallClient } from './services/doc_install_status'; @@ -45,7 +45,7 @@ export class KnowledgeBaseRegistryPlugin coreSetup: CoreSetup, pluginsSetup: ProductDocBaseSetupDependencies ): ProductDocBaseSetupContract { - coreSetup.savedObjects.registerType(knowledgeBaseProductDocInstallSavedObjectType); + coreSetup.savedObjects.registerType(productDocInstallStatusSavedObjectType); const router = coreSetup.http.createRouter(); registerRoutes({ diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts index 3715eab90a00..f87c6d37eb66 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts @@ -6,6 +6,6 @@ */ export { - knowledgeBaseProductDocInstallSavedObjectType, - type KnowledgeBaseProductDocInstallAttributes, + productDocInstallStatusSavedObjectType, + type ProductDocInstallStatusAttributes, } from './product_doc_install'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts index 9c4fff582be7..464d404d1fcc 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts @@ -6,15 +6,16 @@ */ import type { SavedObjectsType } from '@kbn/core/server'; +import type { ProductName } from '@kbn/product-doc-common'; import { productDocInstallStatusSavedObjectTypeName } from '../../common/consts'; import type { InstallationStatus } from '../../common/install_status'; /** - * Interface describing the raw attributes of the KB Entry SO type. + * Interface describing the raw attributes of the product doc install SO type. * Contains more fields than the mappings, which only list * indexed fields. */ -export interface KnowledgeBaseProductDocInstallAttributes { +export interface ProductDocInstallStatusAttributes { product_name: ProductName; product_version: string; installation_status: InstallationStatus; @@ -23,7 +24,7 @@ export interface KnowledgeBaseProductDocInstallAttributes { index_name?: string; } -export const knowledgeBaseProductDocInstallSavedObjectType: SavedObjectsType = +export const productDocInstallStatusSavedObjectType: SavedObjectsType = { name: productDocInstallStatusSavedObjectTypeName, hidden: true, diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts index f917cdb7d883..86bda77ac9fa 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts @@ -7,10 +7,10 @@ import type { SavedObject } from '@kbn/core/server'; import type { ProductDocInstallStatus } from '../../../common/saved_objects'; -import type { KnowledgeBaseProductDocInstallAttributes } from '../../saved_objects'; +import type { ProductDocInstallStatusAttributes } from '../../saved_objects'; export const soToModel = ( - so: SavedObject + so: SavedObject ): ProductDocInstallStatus => { return { id: so.id, diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts index fa2db1e3bf4e..25ff6913b8b9 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts @@ -10,7 +10,7 @@ import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; import { ProductName, DocumentationProduct } from '@kbn/product-doc-common'; import type { ProductInstallState } from '../../../common/install_status'; import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; -import type { KnowledgeBaseProductDocInstallAttributes as TypeAttributes } from '../../saved_objects'; +import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects'; // import { soToModel } from './model_conversion'; export class ProductDocInstallClient { From 33fd53897b548d59a1c1d7fbaeccf47bb2e3dffe Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 13:01:34 +0200 Subject: [PATCH 39/97] fix imports --- .../server/services/doc_install_status/model_conversion.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts index 86bda77ac9fa..cf77bb9222a1 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts @@ -6,7 +6,7 @@ */ import type { SavedObject } from '@kbn/core/server'; -import type { ProductDocInstallStatus } from '../../../common/saved_objects'; +import type { ProductDocInstallStatus } from '../../../common/install_status'; import type { ProductDocInstallStatusAttributes } from '../../saved_objects'; export const soToModel = ( From 23bb7fd9b585a97fa653a6a09fbf2d4e3e730f75 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 13:20:14 +0200 Subject: [PATCH 40/97] fix jest file --- x-pack/plugins/ai_infra/llm_tasks/jest.config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js index aaf90e7fafaf..26ef1f865cde 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js +++ b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js @@ -11,7 +11,6 @@ module.exports = { roots: [ '/x-pack/plugins/ai_infra/llm_tasks/public', '/x-pack/plugins/ai_infra/llm_tasks/server', - '/x-pack/plugins/ai_infra/llm_tasks/common', ], setupFiles: [], collectCoverage: true, From 05bb3b918668aa5e2d984ff40ceece6886836963 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 13:27:43 +0200 Subject: [PATCH 41/97] remove type that never existed ever --- .../saved_objects/migrations/group3/type_registrations.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index b47e7451baf5..f745edd7a4e1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -93,7 +93,6 @@ const previouslyRegisteredTypes = [ 'ingest-package-policies', 'ingest_manager_settings', 'inventory-view', - 'knowledge_base_entry', 'kql-telemetry', 'legacy-url-alias', 'lens', From c1c01a9e51e71cbc36526c5b94a1e2354953e805 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 14 Oct 2024 21:47:27 +0200 Subject: [PATCH 42/97] silence i18n check's output --- .buildkite/scripts/steps/checks/i18n.sh | 2 +- src/dev/i18n_tools/bin/run_i18n_check.ts | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.buildkite/scripts/steps/checks/i18n.sh b/.buildkite/scripts/steps/checks/i18n.sh index 090512e391d7..45d2474a1085 100755 --- a/.buildkite/scripts/steps/checks/i18n.sh +++ b/.buildkite/scripts/steps/checks/i18n.sh @@ -5,4 +5,4 @@ set -euo pipefail source .buildkite/scripts/common/util.sh echo --- Check i18n -node scripts/i18n_check +node scripts/i18n_check --silent diff --git a/src/dev/i18n_tools/bin/run_i18n_check.ts b/src/dev/i18n_tools/bin/run_i18n_check.ts index 9d7265f84520..a4be2678a0af 100644 --- a/src/dev/i18n_tools/bin/run_i18n_check.ts +++ b/src/dev/i18n_tools/bin/run_i18n_check.ts @@ -9,7 +9,6 @@ import { Listr } from 'listr2'; import { run } from '@kbn/dev-cli-runner'; -import { ToolingLog } from '@kbn/tooling-log'; import { getTimeReporter } from '@kbn/ci-stats-reporter'; import { isFailError } from '@kbn/dev-cli-errors'; import { I18nCheckTaskContext, MessageDescriptor } from '../types'; @@ -24,13 +23,7 @@ import { import { TaskReporter } from '../utils/task_reporter'; import { flagFailError, isDefined, undefinedOrBoolean } from '../utils/verify_bin_flags'; -const toolingLog = new ToolingLog({ - level: 'info', - writeTo: process.stdout, -}); - const runStartTime = Date.now(); -const reportTime = getTimeReporter(toolingLog, 'scripts/i18n_check'); const skipOnNoTranslations = ({ config }: I18nCheckTaskContext) => !config?.translations.length && 'No translations found.'; @@ -50,9 +43,12 @@ run( namespace: namespace, fix = false, path, + silent, }, log, }) => { + const reportTime = getTimeReporter(log, 'scripts/i18n_check'); + if ( fix && (isDefined(ignoreIncompatible) || @@ -131,13 +127,13 @@ run( { concurrent: false, exitOnError: true, - renderer: process.env.CI ? 'verbose' : ('default' as any), + renderer: (silent && 'silent') || (process.env.CI ? 'verbose' : ('default' as any)), } ); try { const messages: Map = new Map(); - const taskReporter = new TaskReporter({ toolingLog }); + const taskReporter = new TaskReporter({ toolingLog: log }); await list.run({ messages, taskReporter }); reportTime(runStartTime, 'total', { From 25d6132f72fab26c226372cf46ca091f9666881c Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 14 Oct 2024 20:23:04 +0000 Subject: [PATCH 43/97] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../current_fields.json | 7 +++++++ .../current_mappings.json | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index fa55bd4800c8..f22b67410b99 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -854,6 +854,13 @@ "policy-settings-protection-updates-note": [ "note" ], + "product-doc-install-status": [ + "index_name", + "installation_status", + "last_installation_date", + "product_name", + "product_version" + ], "query": [ "description", "title", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 9f94d36af50f..3f307837f61d 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2837,6 +2837,26 @@ } } }, + "product-doc-install-status": { + "dynamic": false, + "properties": { + "index_name": { + "type": "keyword" + }, + "installation_status": { + "type": "keyword" + }, + "last_installation_date": { + "type": "integer" + }, + "product_name": { + "type": "keyword" + }, + "product_version": { + "type": "keyword" + } + } + }, "query": { "dynamic": false, "properties": { From f67405e36853518652ad41c4ed4cd5cf913c7c00 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 14 Oct 2024 20:41:02 +0000 Subject: [PATCH 44/97] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx b/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx index 53297e3d0663..aa591d263029 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx +++ b/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx @@ -30,19 +30,13 @@ export class KnowledgeBaseRegistryPlugin this.logger = context.logger.get(); } setup( - coreSetup: CoreSetup< - PluginStartDependencies, - LlmTasksPluginStart - >, + coreSetup: CoreSetup, pluginsSetup: PluginSetupDependencies ): LlmTasksPluginSetup { return {}; } - start( - coreStart: CoreStart, - pluginsStart: PluginStartDependencies - ): LlmTasksPluginStart { + start(coreStart: CoreStart, pluginsStart: PluginStartDependencies): LlmTasksPluginStart { return {}; } } From 0b13919e756d2cd92da10481f9ca22c41eb3a1dc Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:26:52 +0000 Subject: [PATCH 45/97] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 53198f9746cf..cc92e6c75409 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -145,6 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () => "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75", "osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4", "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352", + "product-doc-install-status": "bf0f1cb29850b2b5c45351d93fa7abfea3a39907", "query": "501bece68f26fe561286a488eabb1a8ab12f1137", "risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb", "rules-settings": "892a2918ebaeba809a612b8d97cec0b07c800b5f", From 81c4ddf432ef4e1eb5a99114d1be72153ecceaec Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 15 Oct 2024 12:07:51 +0200 Subject: [PATCH 46/97] set logging to quiet in i18n.sh, still log the error in case something happens in i18n_check --- .buildkite/scripts/steps/checks/i18n.sh | 2 +- src/dev/i18n_tools/bin/run_i18n_check.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/checks/i18n.sh b/.buildkite/scripts/steps/checks/i18n.sh index 45d2474a1085..f23ed2d63759 100755 --- a/.buildkite/scripts/steps/checks/i18n.sh +++ b/.buildkite/scripts/steps/checks/i18n.sh @@ -5,4 +5,4 @@ set -euo pipefail source .buildkite/scripts/common/util.sh echo --- Check i18n -node scripts/i18n_check --silent +node scripts/i18n_check --quiet diff --git a/src/dev/i18n_tools/bin/run_i18n_check.ts b/src/dev/i18n_tools/bin/run_i18n_check.ts index a4be2678a0af..ff00148ab301 100644 --- a/src/dev/i18n_tools/bin/run_i18n_check.ts +++ b/src/dev/i18n_tools/bin/run_i18n_check.ts @@ -44,6 +44,7 @@ run( fix = false, path, silent, + quiet, }, log, }) => { @@ -127,7 +128,9 @@ run( { concurrent: false, exitOnError: true, - renderer: (silent && 'silent') || (process.env.CI ? 'verbose' : ('default' as any)), + forceTTY: false, + renderer: + ((silent || quiet) && 'silent') || (process.env.CI ? 'verbose' : ('default' as any)), } ); @@ -146,6 +149,7 @@ run( reportTime(runStartTime, 'error', { success: false, }); + log.error(error); } else { log.error('Unhandled exception!'); log.error(error); From 2fcfdee1cba3f1fd63eaceba3e2bde311dd1d0e8 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 14 Oct 2024 15:36:53 +0200 Subject: [PATCH 47/97] self-review, first batch --- .../src/artifact/product_name.ts | 46 ++++++++--------- .../src/tasks/extract_documentation.ts | 4 +- .../ai-infra/product-doc-common/README.md | 2 +- .../product-doc-common/src/artifact.test.ts | 49 +++++++++++++++++++ .../product-doc-common/src/artifact.ts | 2 +- .../src/artifact_content.test.ts | 22 +++++++++ .../product-doc-common/src/documents.ts | 18 +++++-- .../plugins/ai_infra/llm_tasks/jest.config.js | 1 - .../plugins/ai_infra/llm_tasks/kibana.jsonc | 2 +- .../ai_infra/llm_tasks/public/index.ts | 26 ---------- .../ai_infra/llm_tasks/public/plugin.tsx | 42 ---------------- .../ai_infra/llm_tasks/public/types.ts | 18 ------- .../ai_infra/llm_tasks/server/plugin.ts | 6 +-- .../retrieve_documentation.ts | 2 +- ...levant_chunks.ts => summarize_document.ts} | 0 .../server/services/search/perform_search.ts | 8 +-- 16 files changed, 117 insertions(+), 131 deletions(-) create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts create mode 100644 x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts delete mode 100644 x-pack/plugins/ai_infra/llm_tasks/public/index.ts delete mode 100644 x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx delete mode 100644 x-pack/plugins/ai_infra/llm_tasks/public/types.ts rename x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/{extract_relevant_chunks.ts => summarize_document.ts} (100%) diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts index 8164b1213054..e4ca33849a52 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts @@ -7,32 +7,32 @@ import type { ProductName } from '@kbn/product-doc-common'; -export const getSourceProductName = (productName: ProductName) => { - switch (productName) { - case 'elasticsearch': - return 'Elasticsearch'; - case 'observability': - return 'Observability'; - case 'security': - return 'Security'; - case 'kibana': - return 'Kibana'; - default: - throw new Error(`Unknown product name: ${productName}`); +const productNameToSourceNamesMap: Record = { + kibana: ['Kibana'], + elasticsearch: ['Elasticsearch'], + security: ['Security'], + observability: ['Observability'], +}; + +const sourceNameToProductName = Object.entries(productNameToSourceNamesMap).reduce< + Record +>((map, [productName, sourceNames]) => { + sourceNames.forEach((sourceName) => { + map[sourceName] = productName as ProductName; + }); + return map; +}, {}); + +export const getSourceNamesFromProductName = (productName: ProductName): string[] => { + if (!productNameToSourceNamesMap[productName]) { + throw new Error(`Unknown product name: ${productName}`); } + return productNameToSourceNamesMap[productName]; }; export const getProductNameFromSource = (source: string): ProductName => { - switch (source) { - case 'Elasticsearch': - return 'elasticsearch'; - case 'Observability': - return 'observability'; - case 'Security': - return 'security'; - case 'Kibana': - return 'kibana'; - default: - throw new Error(`Unknown source product name: ${source}`); + if (!sourceNameToProductName[source]) { + throw new Error(`Unknown source name: ${source}`); } + return sourceNameToProductName[source]; }; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts index d5b321ccfafb..6aa8bb49b0cf 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts @@ -9,7 +9,7 @@ import type { Client } from '@elastic/elasticsearch'; import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import type { ToolingLog } from '@kbn/tooling-log'; import type { ProductName } from '@kbn/product-doc-common'; -import { getSourceProductName, getProductNameFromSource } from '../artifact/product_name'; +import { getSourceNamesFromProductName, getProductNameFromSource } from '../artifact/product_name'; /** the list of fields to import from the source cluster */ const fields = [ @@ -78,7 +78,7 @@ export const extractDocumentation = async ({ query: { bool: { must: [ - { term: { product_name: getSourceProductName(productName) } }, + { terms: { product_name: getSourceNamesFromProductName(productName) } }, { term: { version: stackVersion } }, { exists: { field: 'ai_fields.ai_summary' } }, ], diff --git a/x-pack/packages/ai-infra/product-doc-common/README.md b/x-pack/packages/ai-infra/product-doc-common/README.md index 6a309c5de351..ff20c0e0fd0e 100644 --- a/x-pack/packages/ai-infra/product-doc-common/README.md +++ b/x-pack/packages/ai-infra/product-doc-common/README.md @@ -1,3 +1,3 @@ # @kbn/product-doc-common -Empty package generated by @kbn/generate +Common types and utilities for the product documentation feature. diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts new file mode 100644 index 000000000000..fb8af932ffb4 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getArtifactName, parseArtifactName } from './artifact'; + +describe('getArtifactName', () => { + it('builds the name based on the provided product name and version', () => { + expect( + getArtifactName({ + productName: 'kibana', + productVersion: '8.16', + }) + ).toEqual('kb-product-doc-kibana-8.16.zip'); + }); + + it('excludes the extension when excludeExtension is true', () => { + expect( + getArtifactName({ + productName: 'elasticsearch', + productVersion: '8.17', + excludeExtension: true, + }) + ).toEqual('kb-product-doc-elasticsearch-8.17'); + }); +}); + +describe('parseArtifactName', () => { + it('parses an artifact name with extension', () => { + expect(parseArtifactName('kb-product-doc-kibana-8.16.zip')).toEqual({ + productName: 'kibana', + productVersion: '8.16', + }); + }); + + it('parses an artifact name without extension', () => { + expect(parseArtifactName('kb-product-doc-security-8.17')).toEqual({ + productName: 'security', + productVersion: '8.17', + }); + }); + + it('returns undefined if the provided string does not match the artifact name pattern', () => { + expect(parseArtifactName('some-wrong-name')).toEqual(undefined); + }); +}); diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts index d252a04424b6..4c69ade2ca5e 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts @@ -16,7 +16,7 @@ export const getArtifactName = ({ productVersion, excludeExtension = false, }: { - productName: string; + productName: ProductName; productVersion: string; excludeExtension?: boolean; }): string => { diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts new file mode 100644 index 000000000000..7f5e87cb3a23 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArtifactContentFilePath } from './artifact_content'; + +describe('isArtifactContentFilePath', () => { + it('returns true for filenames matching the pattern', () => { + expect(isArtifactContentFilePath('content-0.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content-007.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content-9042.ndjson')).toEqual(true); + }); + + it('returns false for filenames not matching the pattern', () => { + expect(isArtifactContentFilePath('content-0')).toEqual(false); + expect(isArtifactContentFilePath('content.ndjson')).toEqual(false); + expect(isArtifactContentFilePath('content-9042.json')).toEqual(false); + }); +}); diff --git a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts index fcb1ac323190..ed14139378f1 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts @@ -7,17 +7,25 @@ import type { ProductName } from './product'; -// TODO: proper structure (e.g. semantic field +// don't need to define the other props +interface SemanticTextField { + text: string; +} + +interface SemanticTextArrayField { + text: string[]; +} + export interface ProductDocumentationAttributes { content_title: string; - content_body: string; + content_body: SemanticTextField; product_name: ProductName; root_type: string; slug: string; url: string; version: string; - ai_subtitle: string; - ai_summary: string; - ai_questions_answered: string[]; + ai_subtitle: SemanticTextField; + ai_summary: SemanticTextField; + ai_questions_answered: SemanticTextArrayField[]; ai_tags: string[]; } diff --git a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js index 26ef1f865cde..65fd78fd7a88 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js +++ b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js @@ -9,7 +9,6 @@ module.exports = { preset: '@kbn/test', rootDir: '../../../..', roots: [ - '/x-pack/plugins/ai_infra/llm_tasks/public', '/x-pack/plugins/ai_infra/llm_tasks/server', ], setupFiles: [], diff --git a/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc index a6b45924c613..1ef211d01210 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc +++ b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc @@ -5,7 +5,7 @@ "plugin": { "id": "llmTasks", "server": true, - "browser": true, + "browser": false, "configPath": ["xpack", "llmTasks"], "requiredPlugins": ["inference", "productDocBase"], "requiredBundles": [], diff --git a/x-pack/plugins/ai_infra/llm_tasks/public/index.ts b/x-pack/plugins/ai_infra/llm_tasks/public/index.ts deleted file mode 100644 index 229b2bd3c01b..000000000000 --- a/x-pack/plugins/ai_infra/llm_tasks/public/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; -import { KnowledgeBaseRegistryPlugin } from './plugin'; -import type { - LlmTasksPluginSetup, - LlmTasksPluginStart, - PluginSetupDependencies, - PluginStartDependencies, - PublicPluginConfig, -} from './types'; - -export type { LlmTasksPluginSetup, LlmTasksPluginStart }; - -export const plugin: PluginInitializer< - LlmTasksPluginSetup, - LlmTasksPluginStart, - PluginSetupDependencies, - PluginStartDependencies -> = (pluginInitializerContext: PluginInitializerContext) => - new KnowledgeBaseRegistryPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx b/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx deleted file mode 100644 index aa591d263029..000000000000 --- a/x-pack/plugins/ai_infra/llm_tasks/public/plugin.tsx +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; -import type { Logger } from '@kbn/logging'; -import type { - PublicPluginConfig, - LlmTasksPluginSetup, - LlmTasksPluginStart, - PluginSetupDependencies, - PluginStartDependencies, -} from './types'; - -export class KnowledgeBaseRegistryPlugin - implements - Plugin< - LlmTasksPluginSetup, - LlmTasksPluginStart, - PluginSetupDependencies, - PluginStartDependencies - > -{ - logger: Logger; - - constructor(context: PluginInitializerContext) { - this.logger = context.logger.get(); - } - setup( - coreSetup: CoreSetup, - pluginsSetup: PluginSetupDependencies - ): LlmTasksPluginSetup { - return {}; - } - - start(coreStart: CoreStart, pluginsStart: PluginStartDependencies): LlmTasksPluginStart { - return {}; - } -} diff --git a/x-pack/plugins/ai_infra/llm_tasks/public/types.ts b/x-pack/plugins/ai_infra/llm_tasks/public/types.ts deleted file mode 100644 index feeb691ba509..000000000000 --- a/x-pack/plugins/ai_infra/llm_tasks/public/types.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/* eslint-disable @typescript-eslint/no-empty-interface*/ - -export interface PublicPluginConfig {} - -export interface PluginSetupDependencies {} - -export interface PluginStartDependencies {} - -export interface LlmTasksPluginSetup {} - -export interface LlmTasksPluginStart {} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts index 7cf3b11557d2..4e34a48452f0 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts @@ -25,9 +25,9 @@ export class LlmTasksPlugin PluginStartDependencies > { - logger: Logger; + private logger: Logger; - constructor(private readonly context: PluginInitializerContext) { + constructor(context: PluginInitializerContext) { this.logger = context.logger.get(); } setup( @@ -47,7 +47,7 @@ export class LlmTasksPlugin return retrieveDocumentation({ outputAPI: inference.getClient({ request: options.request }).output, searchDocAPI: productDocBase.search, - logger: this.context.logger.get('tasks.retrieve-documentation'), + logger: this.logger.get('tasks.retrieve-documentation'), })(options); }, }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts index 879dcf1373f6..d3a210707a7b 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -10,7 +10,7 @@ import type { OutputAPI } from '@kbn/inference-plugin/common/output'; import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server'; import { truncate, count as countTokens } from '../../utils/tokens'; import type { RetrieveDocumentationAPI } from './types'; -import { summarizeDocument } from './extract_relevant_chunks'; +import { summarizeDocument } from './summarize_document'; // if content length greater, then we'll trigger the summary task const MIN_TOKENS_TO_SUMMARIZE = 500; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts similarity index 100% rename from x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/extract_relevant_chunks.ts rename to x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts index 468554649468..a437e182a0d2 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts @@ -26,13 +26,7 @@ export const performSearch = async ({ size, query: { bool: { - filter: [ - // { - // bool: { - // must: [{ term: { version: '8.15' } }], - // }, - // }, - ], + filter: [], should: [ { multi_match: { From 0059e66ddbc22e551ba24403de7d41b287f31e39 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 15 Oct 2024 14:51:05 +0200 Subject: [PATCH 48/97] adding tests --- .../product-doc-common/src/manifest.ts | 4 +- .../tasks/retrieve_documentation/types.ts | 15 +- .../ai_infra/llm_tasks/server/types.ts | 10 ++ .../ai_infra/product_doc_base/README.md | 5 +- .../common/http_api/installation.ts | 4 +- .../ai_infra/product_doc_base/public/index.ts | 4 +- .../product_doc_base/public/plugin.tsx | 2 +- .../installation/installation_service.test.ts | 67 ++++++++ .../installation/installation_service.ts | 8 +- .../ai_infra/product_doc_base/server/index.ts | 4 +- .../product_doc_base/server/plugin.ts | 4 +- .../server/routes/installation.ts | 10 +- .../utils/get_model_install_status.ts | 1 - .../steps/create_index.test.ts | 84 ++++++++++ .../steps/fetch_artifact_versions.test.ts | 152 ++++++++++++++++++ .../steps/fetch_artifact_versions.ts | 8 +- .../steps/validate_artifact_archive.test.ts | 73 +++++++++ .../utils/archive_accessors.test.ts | 78 +++++++++ .../utils/archive_accessors.ts | 4 +- .../package_installer/utils/semver.test.ts | 29 ++++ .../package_installer/utils/semver.ts | 9 +- 21 files changed, 538 insertions(+), 37 deletions(-) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts diff --git a/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts index 6649791d0c5f..6c246cf58fd5 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts @@ -5,8 +5,10 @@ * 2.0. */ +import type { ProductName } from './product'; + export interface ArtifactManifest { formatVersion: string; - productName: string; + productName: ProductName; productVersion: string; } diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts index 2d0e5fb46f71..caeedd72ca30 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -10,11 +10,22 @@ import type { FunctionCallingMode } from '@kbn/inference-plugin/common/chat_comp import type { ProductName } from '@kbn/product-doc-common'; export interface RetrieveDocumentationParams { - request: KibanaRequest; + /** + * The search term to perform semantic text with. + * E.g. "What is Kibana Lens?" + */ + searchTerm: string; + /** + * Maximum number of documents to return + * Defaults to 3. + */ max?: number; + /** + * Optional list of products to restrict the search to + */ products?: ProductName[]; + request: KibanaRequest; connectorId: string; - searchTerm: string; functionCalling?: FunctionCallingMode; } diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts index cef03ee31b29..bed92bccc0fc 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts @@ -21,6 +21,16 @@ export interface PluginStartDependencies { export interface LlmTasksPluginSetup {} export interface LlmTasksPluginStart { + /** + * Checks if all prerequisites to use the `retrieveDocumentation` task + * are respected. Can be used to check if the task can be registered + * as LLM tool for example. + */ retrieveDocumentationAvailable: () => Promise; + /** + * Perform the `retrieveDocumentation` task. + * + * @see RetrieveDocumentationAPI + */ retrieveDocumentation: RetrieveDocumentationAPI; } diff --git a/x-pack/plugins/ai_infra/product_doc_base/README.md b/x-pack/plugins/ai_infra/product_doc_base/README.md index db52c35a8f8f..0ff6c34dd278 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/README.md +++ b/x-pack/plugins/ai_infra/product_doc_base/README.md @@ -1,4 +1,3 @@ -# Knowledge base registry plugin - -This plugin contains the registry for the knowledge base. +# Product documentation base plugin +This plugin contains the product documentation base service. diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts index d578a8fb9222..0237bd2c3b48 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts @@ -9,8 +9,8 @@ import type { ProductName } from '@kbn/product-doc-common'; import type { ProductInstallState, InstallationStatus } from '../install_status'; export const INSTALLATION_STATUS_API_PATH = '/internal/product_doc_base/status'; -export const PERFORM_INSTALL_API_PATH = '/internal/product_doc_base/install'; -export const UNINSTALL_API_PATH = '/internal/product_doc_base/uninstall'; +export const INSTALL_ALL_API_PATH = '/internal/product_doc_base/install'; +export const UNINSTALL_ALL_API_PATH = '/internal/product_doc_base/uninstall'; export interface InstallationStatusResponse { overall: InstallationStatus; diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts index d04402aaadb9..b5ccbf029a73 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts @@ -6,7 +6,7 @@ */ import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; -import { KnowledgeBaseRegistryPlugin } from './plugin'; +import { ProductDocBasePlugin } from './plugin'; import type { ProductDocBasePluginSetup, ProductDocBasePluginStart, @@ -23,4 +23,4 @@ export const plugin: PluginInitializer< PluginSetupDependencies, PluginStartDependencies > = (pluginInitializerContext: PluginInitializerContext) => - new KnowledgeBaseRegistryPlugin(pluginInitializerContext); + new ProductDocBasePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx index c426cb9dfdbb..6f2c989b6e45 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx +++ b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx @@ -16,7 +16,7 @@ import type { } from './types'; import { InstallationService } from './services/installation'; -export class KnowledgeBaseRegistryPlugin +export class ProductDocBasePlugin implements Plugin< ProductDocBasePluginSetup, diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts new file mode 100644 index 000000000000..52b1fae22393 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServiceMock } from '@kbn/core/public/mocks'; +import { InstallationService } from './installation_service'; +import { + INSTALLATION_STATUS_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, +} from '../../../common/http_api/installation'; + +describe('InstallationService', () => { + let http: ReturnType; + let service: InstallationService; + + beforeEach(() => { + http = httpServiceMock.createSetupContract(); + service = new InstallationService({ http }); + }); + + describe('#getInstallationStatus', () => { + it('calls the endpoint with the right parameters', async () => { + await service.getInstallationStatus(); + expect(http.get).toHaveBeenCalledTimes(1); + expect(http.get).toHaveBeenCalledWith(INSTALLATION_STATUS_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { stubbed: true }; + http.get.mockResolvedValue(expected); + + const response = await service.getInstallationStatus(); + expect(response).toEqual(expected); + }); + }); + describe('#install', () => { + it('calls the endpoint with the right parameters', async () => { + await service.install(); + expect(http.post).toHaveBeenCalledTimes(1); + expect(http.post).toHaveBeenCalledWith(INSTALL_ALL_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { stubbed: true }; + http.post.mockResolvedValue(expected); + + const response = await service.install(); + expect(response).toEqual(expected); + }); + }); + describe('#uninstall', () => { + it('calls the endpoint with the right parameters', async () => { + await service.uninstall(); + expect(http.post).toHaveBeenCalledTimes(1); + expect(http.post).toHaveBeenCalledWith(UNINSTALL_ALL_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { stubbed: true }; + http.post.mockResolvedValue(expected); + + const response = await service.uninstall(); + expect(response).toEqual(expected); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts index 5c90a9bb0dca..17601db60589 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts @@ -8,8 +8,8 @@ import type { HttpSetup } from '@kbn/core-http-browser'; import { INSTALLATION_STATUS_API_PATH, - PERFORM_INSTALL_API_PATH, - UNINSTALL_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, InstallationStatusResponse, PerformInstallResponse, UninstallResponse, @@ -27,10 +27,10 @@ export class InstallationService { } async install(): Promise { - return await this.http.post(PERFORM_INSTALL_API_PATH); + return await this.http.post(INSTALL_ALL_API_PATH); } async uninstall(): Promise { - return await this.http.post(UNINSTALL_API_PATH); + return await this.http.post(UNINSTALL_ALL_API_PATH); } } diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts index 9d08b297573c..805a0f2ea8c4 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts @@ -13,7 +13,7 @@ import type { ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies, } from './types'; -import { KnowledgeBaseRegistryPlugin } from './plugin'; +import { ProductDocBasePlugin } from './plugin'; export { config } from './config'; @@ -26,4 +26,4 @@ export const plugin: PluginInitializer< ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies > = async (pluginInitializerContext: PluginInitializerContext) => - new KnowledgeBaseRegistryPlugin(pluginInitializerContext); + new ProductDocBasePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index bdbd4f4e3a83..46c46480da8f 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -25,7 +25,7 @@ import { ProductDocInstallClient } from './services/doc_install_status'; import { SearchService } from './services/search'; import { registerRoutes } from './routes'; -export class KnowledgeBaseRegistryPlugin +export class ProductDocBasePlugin implements Plugin< ProductDocBaseSetupContract, @@ -34,7 +34,7 @@ export class KnowledgeBaseRegistryPlugin ProductDocBaseStartDependencies > { - logger: Logger; + private logger: Logger; private installClient?: ProductDocInstallClient; private packageInstaller?: PackageInstaller; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index d8638e8f014c..a555cdd92a0e 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -8,8 +8,8 @@ import type { IRouter } from '@kbn/core/server'; import { INSTALLATION_STATUS_API_PATH, - PERFORM_INSTALL_API_PATH, - UNINSTALL_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, InstallationStatusResponse, PerformInstallResponse, UninstallResponse, @@ -30,7 +30,6 @@ export const registerInstallationRoutes = ({ router.get( { path: INSTALLATION_STATUS_API_PATH, validate: false, options: { access: 'internal' } }, async (ctx, req, res) => { - // TODO: use installer instead const installClient = getInstallClient(); const installStatus = await installClient.getInstallationStatus(); const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status)); @@ -46,7 +45,7 @@ export const registerInstallationRoutes = ({ router.post( { - path: PERFORM_INSTALL_API_PATH, + path: INSTALL_ALL_API_PATH, validate: false, options: { access: 'internal', @@ -69,7 +68,7 @@ export const registerInstallationRoutes = ({ router.post( { - path: UNINSTALL_API_PATH, + path: UNINSTALL_ALL_API_PATH, validate: false, options: { access: 'internal', @@ -78,7 +77,6 @@ export const registerInstallationRoutes = ({ async (ctx, req, res) => { const installer = getInstaller(); - // TODO: use ensureUpToDate with parameter to install non-installed await installer.uninstallAll(); return res.ok({ diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts index 49da426a1925..be6caa34d0ad 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts @@ -12,7 +12,6 @@ export const getModelInstallStatus = async ({ inferenceId, taskType = 'sparse_embedding', client, - log, }: { inferenceId: string; taskType?: InferenceTaskType; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts new file mode 100644 index 000000000000..fca8b5283c30 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { createIndex } from './create_index'; +import { internalElserInferenceId } from '../../../../common/consts'; + +describe('createIndex', () => { + let log: MockedLogger; + let esClient: ElasticsearchClient; + + beforeEach(() => { + log = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('calls esClient.indices.create with the right parameters', async () => { + const mappings: MappingTypeMapping = { + properties: {}, + }; + const indexName = '.some-index'; + + await createIndex({ + indexName, + mappings, + log, + esClient, + }); + + expect(esClient.indices.create).toHaveBeenCalledTimes(1); + expect(esClient.indices.create).toHaveBeenCalledWith({ + index: indexName, + mappings, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }, + }); + }); + + it('rewrites the inference_id attribute of semantic_text fields in the mapping', async () => { + const mappings: MappingTypeMapping = { + properties: { + semantic: { + type: 'semantic_text', + inference_id: '.elser', + }, + bool: { + type: 'boolean', + }, + }, + }; + + await createIndex({ + indexName: '.some-index', + mappings, + log, + esClient, + }); + + expect(esClient.indices.create).toHaveBeenCalledWith( + expect.objectContaining({ + mappings: { + properties: { + semantic: { + type: 'semantic_text', + inference_id: internalElserInferenceId, + }, + bool: { + type: 'boolean', + }, + }, + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts new file mode 100644 index 000000000000..e320a2488c98 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fetch, { Response } from 'node-fetch'; +import { fetchArtifactVersions } from './fetch_artifact_versions'; +import { getArtifactName, DocumentationProduct, ProductName } from '@kbn/product-doc-common'; + +jest.mock('node-fetch'); +const fetchMock = fetch as jest.MockedFn; + +const createResponse = ({ + artifactNames, + truncated = false, +}: { + artifactNames: string[]; + truncated?: boolean; +}) => { + return ` + + kibana-ai-assistant-kb-artifacts + + + ${truncated} + ${artifactNames.map( + (artifactName) => ` + + ${artifactName} + 1728486063097626 + 1 + 2024-10-09T15:01:03.137Z + "e0584955969eccf2a16b8829f768cb1f" + 36781438 + ` + )} + + `; +}; + +const artifactRepositoryUrl = 'https://lost.com'; + +const expectVersions = ( + versions: Partial> +): Record => { + const response = {} as Record; + Object.values(DocumentationProduct).forEach((productName) => { + response[productName] = []; + }); + return { + ...response, + ...versions, + }; +}; + +describe('fetchArtifactVersions', () => { + beforeEach(() => { + fetchMock.mockReset(); + }); + + const mockResponse = (responseText: string) => { + const response = { + text: () => Promise.resolve(responseText), + }; + fetchMock.mockResolvedValue(response as Response); + }; + + it('calls fetch with the right parameters', async () => { + mockResponse(createResponse({ artifactNames: [] })); + + await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith(`${artifactRepositoryUrl}?max-keys=1000`); + }); + + it('returns the list of versions from the repository', async () => { + const artifactNames = [ + getArtifactName({ productName: 'kibana', productVersion: '8.16' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }), + ]; + mockResponse(createResponse({ artifactNames })); + + const versions = await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(versions).toEqual( + expectVersions({ + kibana: ['8.16'], + elasticsearch: ['8.16'], + }) + ); + }); + + it('retrieve all versions for each product', async () => { + const artifactNames = [ + getArtifactName({ productName: 'kibana', productVersion: '8.15' }), + getArtifactName({ productName: 'kibana', productVersion: '8.16' }), + getArtifactName({ productName: 'kibana', productVersion: '8.17' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '9.0' }), + ]; + mockResponse(createResponse({ artifactNames })); + + const versions = await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(versions).toEqual( + expectVersions({ + kibana: ['8.15', '8.16', '8.17'], + elasticsearch: ['8.16', '9.0'], + }) + ); + }); + + it('throws an error if the response is truncated', async () => { + mockResponse(createResponse({ artifactNames: [], truncated: true })); + + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects + .toThrowErrorMatchingInlineSnapshot(` + "Unhandled error. (Error: bucket content is truncated, cannot retrieve all versions + at /Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts:29:15 + at Parser. (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:308:18) + at Parser.emit (node:events:519:28) + at SAXParser.onclosetag (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:266:26) + at emit (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:625:35) + at emitNode (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:630:5) + at closeTag (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:890:7) + at SAXParser.write (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:1437:13) + at Parser.Object..exports.Parser.Parser.parseString (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:327:31) + at Parser.parseString (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:5:59) + at Object..exports.parseString (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:373:19) + at /Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts:22:16 + at new Promise () + at fetchArtifactVersions (/Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts:21:10) + at processTicksAndRejections (node:internal/process/task_queues:95:5) + at Object. (/Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts:119:5))" + `); + }); + + it('throws an error if the response is not valid xml', async () => { + mockResponse('some plain text'); + + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects + .toThrowErrorMatchingInlineSnapshot(` + "Non-whitespace before first tag. + Line: 0 + Column: 1 + Char: s" + `); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts index c3e32ca39e21..69c6db2d5d8a 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts @@ -36,7 +36,7 @@ export const fetchArtifactVersions = async ({ record[product] = []; }); - result.ListBucketResult.Contents.forEach((contentEntry) => { + result.ListBucketResult.Contents?.forEach((contentEntry) => { const artifactName = contentEntry.Key[0]; const parsed = parseArtifactName(artifactName); if (parsed) { @@ -52,8 +52,8 @@ export const fetchArtifactVersions = async ({ interface ListBucketResponse { ListBucketResult: { - Name: string[]; - IsTruncated: string[]; - Contents: Array<{ Key: string[] }>; + Name?: string[]; + IsTruncated?: string[]; + Contents?: Array<{ Key: string[] }>; }; } diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts new file mode 100644 index 000000000000..607277aaf346 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ZipArchive } from '../utils/zip_archive'; +import { validateArtifactArchive } from './validate_artifact_archive'; + +const createMockArchive = (entryPaths: string[]): ZipArchive => { + return { + hasEntry: (entryPath) => entryPaths.includes(entryPath), + getEntryPaths: () => entryPaths, + getEntryContent: () => { + throw new Error('non implemented'); + }, + close: () => undefined, + }; +}; + +describe('validateArtifactArchive', () => { + it('validates that the archive contains all the mandatory files', () => { + const archive = createMockArchive([ + 'manifest.json', + 'mappings.json', + 'content/content-1.ndjson', + ]); + + const validation = validateArtifactArchive(archive); + + expect(validation).toEqual({ valid: true }); + }); + + it('does not validate if the archive does not contain a manifest', () => { + const archive = createMockArchive(['something.txt']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "Manifest file not found", + "valid": false, + } + `); + }); + + it('does not validate if the archive does not contain mappings', () => { + const archive = createMockArchive(['manifest.json']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "Mapping file not found", + "valid": false, + } + `); + }); + + it('does not validate if the archive does not contain content files', () => { + const archive = createMockArchive(['manifest.json', 'mappings.json']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "No content files were found", + "valid": false, + } + `); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts new file mode 100644 index 000000000000..9d42be652d74 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ArtifactManifest } from '@kbn/product-doc-common'; +import type { ZipArchive } from './zip_archive'; +import { loadManifestFile, loadMappingFile } from './archive_accessors'; + +const createMockArchive = (entries: Record): ZipArchive => { + return { + hasEntry: (entryPath) => Object.keys(entries).includes(entryPath), + getEntryPaths: () => Object.keys(entries), + getEntryContent: async (entryPath) => Buffer.from(entries[entryPath]), + close: () => undefined, + }; +}; + +describe('loadManifestFile', () => { + it('parses the manifest from the archive', async () => { + const manifest: ArtifactManifest = { + formatVersion: '1.0.0', + productName: 'kibana', + productVersion: '8.16', + }; + const archive = createMockArchive({ 'manifest.json': JSON.stringify(manifest) }); + + const parsedManifest = await loadManifestFile(archive); + + expect(parsedManifest).toEqual(manifest); + }); + + it('throws if the archive does not contain the manifest', async () => { + const archive = createMockArchive({}); + + await expect(loadManifestFile(archive)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Could not load archive file: \\"manifest.json\\" not found in archive"` + ); + }); + + it('throws if the manifest cannot be parsed', async () => { + const archive = createMockArchive({ 'manifest.json': '{}}}{' }); + + await expect(loadManifestFile(archive)).rejects.toThrowError(); + }); +}); + +describe('loadMappingFile', () => { + it('parses the manifest from the archive', async () => { + const mappings: MappingTypeMapping = { + properties: { + foo: { type: 'text' }, + }, + }; + const archive = createMockArchive({ 'mappings.json': JSON.stringify(mappings) }); + + const parsedMappings = await loadMappingFile(archive); + + expect(parsedMappings).toEqual(mappings); + }); + + it('throws if the archive does not contain the manifest', async () => { + const archive = createMockArchive({}); + + await expect(loadMappingFile(archive)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Could not load archive file: \\"mappings.json\\" not found in archive"` + ); + }); + + it('throws if the manifest cannot be parsed', async () => { + const archive = createMockArchive({ 'mappings.json': '{}}}{' }); + + await expect(loadMappingFile(archive)).rejects.toThrowError(); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts index d5e7b87181b3..a4ec4f4418f3 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts @@ -13,9 +13,7 @@ const manifestEntryPath = 'manifest.json'; const mappingsEntryPath = 'mappings.json'; export const loadManifestFile = async (archive: ZipArchive): Promise => { - const manifest = await parseEntryContent(manifestEntryPath, archive); - // TODO: schema validation - return manifest; + return await parseEntryContent(manifestEntryPath, archive); }; export const loadMappingFile = async (archive: ZipArchive): Promise => { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts new file mode 100644 index 000000000000..9bc20f2eecdb --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { latestVersion, majorMinor } from './semver'; + +describe('majorMinor', () => { + it('returns the version in a {major.minor} format', () => { + expect(majorMinor('9.17.5')).toEqual('9.17'); + }); + it('ignores qualifiers', () => { + expect(majorMinor('10.42.9000-snap')).toEqual('10.42'); + }); + it('accepts {major.minor} format as input', () => { + expect(majorMinor('8.16')).toEqual('8.16'); + }); +}); + +describe('latestVersion', () => { + it('returns the highest version from the list', () => { + expect(latestVersion(['7.16.3', '8.1.4', '6.14.2'])).toEqual('8.1.4'); + }); + it('accepts versions in a {major.minor} format', () => { + expect(latestVersion(['9.16', '9.3'])).toEqual('9.16'); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts index 6b405f2ceb4e..b4e38215af90 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts @@ -9,16 +9,17 @@ import Semver from 'semver'; export const latestVersion = (versions: string[]): string => { let latest: string = versions[0]; - for (let i = 1; 1 < versions.length; i++) { - if (Semver.gt(versions[i], latest)) { - latest = versions[i]; + for (let i = 1; i < versions.length; i++) { + const current = versions[i]; + if (Semver.gt(Semver.coerce(current)!, Semver.coerce(latest)!)) { + latest = current; } } return latest; }; export const majorMinor = (version: string): string => { - const parsed = Semver.parse(version); + const parsed = Semver.coerce(version); if (!parsed) { throw new Error(`Not a valid semver version: [${version}]`); } From 74485177051e7d65aa48e55d3be6b4ba1197129b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 16 Oct 2024 09:23:53 +0200 Subject: [PATCH 49/97] more tests and fixes --- .../product-doc-common/src/documents.ts | 2 +- .../utils/test_data/test_archive_1.zip | Bin 0 -> 800 bytes .../utils/zip_archive.test.ts | 43 +++++++++++++++ .../package_installer/utils/zip_archive.ts | 23 ++++---- .../services/search/search_service.test.ts | 51 ++++++++++++++++++ .../get_indices_for_product_names.test.ts | 22 ++++++++ .../services/search/utils/map_result.test.ts | 46 ++++++++++++++++ 7 files changed, 175 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts diff --git a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts index ed14139378f1..8dbdbfb5f206 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts @@ -26,6 +26,6 @@ export interface ProductDocumentationAttributes { version: string; ai_subtitle: SemanticTextField; ai_summary: SemanticTextField; - ai_questions_answered: SemanticTextArrayField[]; + ai_questions_answered: SemanticTextArrayField; ai_tags: string[]; } diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip new file mode 100644 index 0000000000000000000000000000000000000000..fce195d2c4db26c8590ec97f45dc9dfc3fcf8346 GIT binary patch literal 800 zcmWIWW@h1H00EW;|H!p|Div%%HVAVu$S{6KKJgobc3FlUtVrDpZQZmq# zR2(K5B20>^@{a^M6%N>ezROE3E=f(%2YYRIHedQ4AV%{VssZbSK3(Sk8Uez>2m>JI z#3S4UGOQTWO)zx<-i%Cg%(y~b0_;X$*fK0>1Tm50fE5x47>-1khZ*+B=6S))gT@5V zJWx#FF%L5akjv#`n9z0fnhJgZ-fMKX{g6ueX eU>HITgM~aK)!;Ii6_m^vSb=aFP-_V&3K#%85|UK_ literal 0 HcmV?d00001 diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts new file mode 100644 index 000000000000..71cd5891c5e5 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; +import { openZipArchive, ZipArchive } from './zip_archive'; + +const ZIP_PATH = Path.resolve(__dirname, './test_data/test_archive_1.zip'); + +describe('ZipArchive', () => { + let archive: ZipArchive; + + beforeAll(async () => { + archive = await openZipArchive(ZIP_PATH); + }); + + afterAll(() => { + archive?.close(); + }); + + test('#getEntryPaths returns the path of all entries', () => { + expect(archive.getEntryPaths().sort()).toEqual([ + 'nested/', + 'nested/nested_1.txt', + 'text_1.txt', + 'text_2.txt', + 'text_3.txt', + ]); + }); + + test('#hasEntry returns true if the entry exists, false otherwise', () => { + expect(archive.hasEntry('nested/nested_1.txt')).toBe(true); + expect(archive.hasEntry('not_an_entry')).toBe(false); + }); + + test('#getEntryContent returns the content of the entry', async () => { + const buffer = await archive.getEntryContent('text_1.txt'); + expect(buffer.toString('utf-8')).toEqual('text_1'); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts index fee6d886e7b1..dbc4ec1b3e41 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { buffer } from 'stream/consumers'; import yauzl from 'yauzl'; export interface ZipArchive { @@ -23,7 +22,7 @@ export const openZipArchive = async (archivePath: string): Promise = return reject(err ?? 'No zip file'); } - zipFile!.on('entry', function (entry) { + zipFile!.on('entry', (entry) => { entries.push(entry); zipFile.readEntry(); }); @@ -72,18 +71,20 @@ class ZipArchiveImpl implements ZipArchive { const getZipEntryContent = async (zipFile: yauzl.ZipFile, entry: yauzl.Entry): Promise => { return new Promise((resolve, reject) => { - zipFile.openReadStream(entry, function (err, readStream) { + zipFile.openReadStream(entry, (err, readStream) => { if (err) { return reject(err); } else { - buffer(readStream!).then( - (result) => { - resolve(result); - }, - (error) => { - reject(error); - } - ); + const chunks: Buffer[] = []; + readStream!.on('data', (chunk: Buffer) => { + chunks.push(chunk); + }); + readStream!.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + readStream!.on('error', () => { + reject(); + }); } }); }); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts new file mode 100644 index 000000000000..c8053ca981e7 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { SearchService } from './search_service'; +import { getIndicesForProductNames } from './utils'; + +import { performSearch } from './perform_search'; +jest.mock('./perform_search'); +const performSearchMock = performSearch as jest.MockedFn; + +describe('SearchService', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let service: SearchService; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + service = new SearchService({ logger, esClient }); + + performSearchMock.mockResolvedValue([]); + }); + + afterEach(() => { + performSearchMock.mockReset(); + }); + + describe('#search', () => { + it('calls `performSearch` with the right parameters', async () => { + await service.search({ + query: 'What is Kibana?', + products: ['kibana'], + max: 42, + }); + + expect(performSearchMock).toHaveBeenCalledTimes(1); + expect(performSearchMock).toHaveBeenCalledWith({ + searchQuery: 'What is Kibana?', + size: 42, + index: getIndicesForProductNames(['kibana']), + client: esClient, + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts new file mode 100644 index 000000000000..0293d086d4f1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { productDocIndexPattern, getProductDocIndexName } from '@kbn/product-doc-common'; +import { getIndicesForProductNames } from './get_indices_for_product_names'; + +describe('getIndicesForProductNames', () => { + it('returns the index pattern when product names are not specified', () => { + expect(getIndicesForProductNames(undefined)).toEqual(productDocIndexPattern); + expect(getIndicesForProductNames([])).toEqual(productDocIndexPattern); + }); + it('returns individual index names when product names are specified', () => { + expect(getIndicesForProductNames(['kibana', 'elasticsearch'])).toEqual([ + getProductDocIndexName('kibana'), + getProductDocIndexName('elasticsearch'), + ]); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts new file mode 100644 index 000000000000..34fe40355013 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; +import { mapResult } from './map_result'; + +const createHit = ( + attrs: ProductDocumentationAttributes +): SearchHit => { + return { + _index: '.foo', + _source: attrs, + }; +}; + +describe('mapResult', () => { + it('returns the expected shape', () => { + const input = createHit({ + content_title: 'content_title', + content_body: { text: 'content_body' }, + product_name: 'kibana', + root_type: 'documentation', + slug: 'foo.html', + url: 'http://lost.com/foo.html', + version: '8.16', + ai_subtitle: { text: 'ai_subtitle' }, + ai_summary: { text: 'ai_summary' }, + ai_questions_answered: { text: ['question A'] }, + ai_tags: ['foo', 'bar', 'test'], + }); + + const output = mapResult(input); + + expect(output).toEqual({ + content: 'content_body', + productName: 'kibana', + title: 'content_title', + url: 'http://lost.com/foo.html', + }); + }); +}); From 77caf771b139a10cb49a0dc43b2fa5282f027312 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 16 Oct 2024 11:20:25 +0200 Subject: [PATCH 50/97] more tests and fixes bis --- .../llm_tasks/server/utils/tokens.test.ts | 27 +++++ .../product_doc_base/common/install_status.ts | 3 + .../server/routes/installation.ts | 1 - .../model_conversion.test.ts | 44 +++++++ .../steps/populate_index.test.ts | 109 ++++++++++++++++++ .../server/functions/documentation.ts | 12 +- .../public/hooks/use_install_product_doc.ts | 6 +- .../public/hooks/use_uninstall_product_doc.ts | 6 +- 8 files changed, 197 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts new file mode 100644 index 000000000000..dce97eaea9b7 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { count, truncate } from './tokens'; + +describe('count', () => { + it('returns the token count of a given text', () => { + expect(count('some short sentence')).toBeGreaterThan(1); + }); +}); + +describe('truncate', () => { + it('truncates text that exceed the specified maximum token count', () => { + const text = 'some sentence that is likely longer than 5 tokens.'; + const output = truncate(text, 5); + expect(output.length).toBeLessThan(text.length); + }); + it('keeps text with a smaller amount of tokens unchanged', () => { + const text = 'some sentence that is likely less than 100 tokens.'; + const output = truncate(text, 100); + expect(output.length).toEqual(text.length); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts index e48b6f8866c3..81102d43c1ff 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts @@ -9,6 +9,9 @@ import type { ProductName } from '@kbn/product-doc-common'; export type InstallationStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; +/** + * DTO representation of the product doc install status SO + */ export interface ProductDocInstallStatus { id: string; productName: ProductName; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index a555cdd92a0e..97827a523dc1 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -55,7 +55,6 @@ export const registerInstallationRoutes = ({ async (ctx, req, res) => { const installer = getInstaller(); - // TODO: use ensureUpToDate with parameter to install non-installed await installer.installAll({}); return res.ok({ diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts new file mode 100644 index 000000000000..6460d8452dc2 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject } from '@kbn/core/server'; +import type { ProductDocInstallStatusAttributes } from '../../saved_objects'; +import { soToModel } from './model_conversion'; + +const createObj = ( + attrs: ProductDocInstallStatusAttributes +): SavedObject => { + return { + id: 'some-id', + type: 'product-doc-install-status', + attributes: attrs, + references: [], + }; +}; + +describe('soToModel', () => { + it('converts the SO to the expected shape', () => { + const input = createObj({ + product_name: 'kibana', + product_version: '8.16', + installation_status: 'installed', + last_installation_date: 9000, + index_name: '.kibana', + }); + + const output = soToModel(input); + + expect(output).toEqual({ + id: 'some-id', + productName: 'kibana', + productVersion: '8.16', + indexName: '.kibana', + installationStatus: 'installed', + lastInstallationDate: expect.any(Date), + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts new file mode 100644 index 000000000000..2f301f9928e9 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { times } from 'lodash'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { internalElserInferenceId } from '../../../../common/consts'; +import type { ZipArchive } from '../utils/zip_archive'; +import { populateIndex } from './populate_index'; + +const createMockArchive = (entries: Record): ZipArchive => { + return { + hasEntry: (entryPath) => Object.keys(entries).includes(entryPath), + getEntryPaths: () => Object.keys(entries), + getEntryContent: async (entryPath) => Buffer.from(entries[entryPath]), + close: () => undefined, + }; +}; + +const createContentFile = (count: number, offset: number = 0): string => { + return times(count) + .map((i) => JSON.stringify({ idx: offset + i })) + .join('\n'); +}; + +describe('populateIndex', () => { + let log: MockedLogger; + let esClient: ReturnType; + + beforeEach(() => { + log = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('calls `esClient.bulk` once per content file', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': createContentFile(2), + 'content/content-1.ndjson': createContentFile(2), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(2); + }); + + it('calls `esClient.bulk` with the right payload', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': createContentFile(2), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(1); + expect(esClient.bulk).toHaveBeenCalledWith({ + refresh: false, + operations: [ + { index: { _index: '.foo' } }, + { idx: 0 }, + { index: { _index: '.foo' } }, + { idx: 1 }, + ], + }); + }); + + it('rewrites the inference_id of semantic fields', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': JSON.stringify({ + semantic: { text: 'foo', inference: { inference_id: '.some-inference' } }, + }), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(1); + expect(esClient.bulk).toHaveBeenCalledWith({ + refresh: false, + operations: [ + { index: { _index: '.foo' } }, + { + semantic: { + inference: { + inference_id: internalElserInferenceId, + }, + text: 'foo', + }, + }, + ], + }); + }); +}); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts index e79f8e88f04a..62a068c0e6ea 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { DocumentationProduct } from '@kbn/product-doc-common'; import { FunctionVisibility } from '@kbn/observability-ai-assistant-plugin/common'; import type { FunctionRegistrationParameters } from '.'; @@ -31,11 +32,14 @@ export async function registerDocumentationFunction({ type: 'object', properties: { query: { - description: 'The query to use to retrieve documentation', - type: 'string', + description: `The query to use to retrieve documentation + Examples: + - "How to enable TLS for Elasticsearch?" + - "What is Kibana Lens?"`, + type: 'string' as const, }, product: { - description: `Filter the product to retrieve documentation for + description: `If specified, will filter the products to retrieve documentation for Possible options are: - "kibana": Kibana product - "elasticsearch": Elasticsearch product @@ -44,7 +48,7 @@ export async function registerDocumentationFunction({ If not specified, will search against all products `, type: 'string' as const, - enum: ['kibana', 'elasticsearch', 'observability', 'security'], + enum: Object.values(DocumentationProduct), }, }, required: ['query'], diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts index a34399882d7b..cb32efa7e390 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; -import { i18n } from '@kbn/i18n'; import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; +import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; import type { PerformInstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; import { REACT_QUERY_KEYS } from '../constants'; import { useKibana } from './use_kibana'; @@ -27,7 +27,7 @@ export function useInstallProductDoc() { return productDocBase!.installation.install(); }, { - onSuccess: (_data) => { + onSuccess: () => { toasts.addSuccess( i18n.translate( 'xpack.observabilityAiAssistantManagement.kb.installProductDoc.successNotification', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts index 8c4774a5e8f6..4aa3b5423faa 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; -import { i18n } from '@kbn/i18n'; import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; +import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; import type { UninstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; import { REACT_QUERY_KEYS } from '../constants'; import { useKibana } from './use_kibana'; @@ -27,7 +27,7 @@ export function useUninstallProductDoc() { return productDocBase!.installation.uninstall(); }, { - onSuccess: (_data) => { + onSuccess: () => { toasts.addSuccess( i18n.translate( 'xpack.observabilityAiAssistantManagement.kb.uninstallProductDoc.successNotification', From 6a15d09eea4319acd6ffe70989335ccab468e886 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 16 Oct 2024 15:09:57 +0200 Subject: [PATCH 51/97] almost done --- .../retrieve_documentation.test.ts | 50 +++++++++++++++ .../retrieve_documentation.ts | 4 +- .../summarize_document.ts | 12 ++-- .../product_doc_base/server/plugin.test.ts | 61 +++++++++++++++++++ .../product_doc_base/server/plugin.ts | 2 +- .../endpoint_manager.test.ts | 58 ++++++++++++++++++ .../package_installer/package_installer.ts | 9 +-- 7 files changed, 180 insertions(+), 16 deletions(-) create mode 100644 x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts new file mode 100644 index 000000000000..b551093aac43 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServerMock } from '@kbn/core/server/mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { retrieveDocumentation } from './retrieve_documentation'; + +describe('retrieveDocumentation', () => { + let logger: MockedLogger; + let outputAPI: jest.Mock; + let searchDocAPI: jest.Mock; + let retrieve: ReturnType; + + beforeEach(() => { + logger = loggerMock.create(); + outputAPI = jest.fn(); + searchDocAPI = jest.fn(); + retrieve = retrieveDocumentation({ logger, searchDocAPI, outputAPI }); + }); + + it('calls the search API with the right parameters', async () => { + searchDocAPI.mockResolvedValue({ results: [] }); + const request = httpServerMock.createKibanaRequest(); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + products: ['kibana'], + request, + max: 5, + connectorId: '.my-connector', + functionCalling: 'simulated', + }); + + expect(result).toEqual({ + success: true, + documents: [], + }); + + expect(searchDocAPI).toHaveBeenCalledTimes(1); + expect(searchDocAPI).toHaveBeenCalledWith({ + query: 'What is Kibana?', + products: ['kibana'], + max: 5, + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts index d3a210707a7b..f411733087b9 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -12,8 +12,8 @@ import { truncate, count as countTokens } from '../../utils/tokens'; import type { RetrieveDocumentationAPI } from './types'; import { summarizeDocument } from './summarize_document'; -// if content length greater, then we'll trigger the summary task -const MIN_TOKENS_TO_SUMMARIZE = 500; +// if document content length greater, then we'll trigger the summary task +const MIN_TOKENS_TO_SUMMARIZE = 750; // maximum token length of generated summaries - will truncate if greater const MAX_SUMMARY_TOKEN_LENGTH = 1000; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts index f32f4aafdaf3..d5ade61151fe 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts @@ -44,16 +44,18 @@ export const summarizeDocument = async ({ functionCalling?: FunctionCallingMode; }): Promise => { const result = await lastValueFrom( - outputAPI('extract_relevant_chunks', { + outputAPI('summarize_document', { connectorId, functionCalling, - system: `You are an Elastic assistant in charge of helping answering the user question. + system: `You are an helpful Elastic assistant, and your current task is to help answering the user question. Given a question and a document, please provide a condensed version of the document that can be used to answer the question. - - All useful information should be included in the condensed version. - - If you think the document isn't relevant at all to answer the question, return an empty text. - `, + - Please try to limit the length of the output to approximately 4000 characters, or 800 words. + - Try to include all relevant information that could be used to answer the question in the condensed version. If this + can't be done without exceeding the 4000chars/800words requirement, please only include the information that you think + are the most relevant and the most helpful to answer the question. + - If you think the document isn't relevant at all to answer the question, just return an empty text`, input: ` ## User question diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts new file mode 100644 index 000000000000..b2beccdded66 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { coreMock } from '@kbn/core/server/mocks'; +import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; +import { ProductDocBasePlugin } from './plugin'; + +jest.mock('./services/package_installer'); +jest.mock('./services/search'); +jest.mock('./services/doc_install_status'); +jest.mock('./routes'); +import { registerRoutes } from './routes'; +import { PackageInstaller } from './services/package_installer'; + +const PackageInstallMock = PackageInstaller as jest.Mock; + +describe('ProductDocBasePlugin', () => { + let initContext: ReturnType; + let plugin: ProductDocBasePlugin; + + beforeEach(() => { + initContext = coreMock.createPluginInitializerContext(); + plugin = new ProductDocBasePlugin(initContext); + + PackageInstallMock.mockReturnValue({ ensureUpToDate: jest.fn().mockResolvedValue({}) }); + }); + + describe('#setup', () => { + it('register the routes', () => { + plugin.setup(coreMock.createSetup(), {}); + + expect(registerRoutes).toHaveBeenCalledTimes(1); + }); + it('register the product-doc SO type', () => { + const coreSetup = coreMock.createSetup(); + plugin.setup(coreSetup, {}); + + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledTimes(1); + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledWith( + expect.objectContaining({ + name: productDocInstallStatusSavedObjectTypeName, + }) + ); + }); + }); + + describe('#start', () => { + it('returns a contract with the expected shape', () => { + plugin.setup(coreMock.createSetup(), {}); + const startContract = plugin.start(coreMock.createStart(), {}); + expect(startContract).toEqual({ + isInstalled: expect.any(Function), + search: expect.any(Function), + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index 46c46480da8f..be9b2f5fbd47 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -97,7 +97,7 @@ export class ProductDocBasePlugin logger: this.logger.get('search-service'), }); - // TODO: see if we should be using taskManager for that. + // should we use taskManager for this? this.packageInstaller.ensureUpToDate({}).catch((err) => { this.logger.error(`Error checking if product documentation is up to date: ${err.message}`); }); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts new file mode 100644 index 000000000000..e5dabaaa9b7f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { InferenceEndpointManager } from './endpoint_manager'; + +jest.mock('./utils'); +import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils'; +const installElserMock = installElser as jest.MockedFn; +const getModelInstallStatusMock = getModelInstallStatus as jest.MockedFn< + typeof getModelInstallStatus +>; +const waitUntilModelDeployedMock = waitUntilModelDeployed as jest.MockedFn< + typeof waitUntilModelDeployed +>; + +describe('InferenceEndpointManager', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let endpointManager: InferenceEndpointManager; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + + endpointManager = new InferenceEndpointManager({ esClient, logger }); + }); + + afterEach(() => { + installElserMock.mockReset(); + getModelInstallStatusMock.mockReset(); + waitUntilModelDeployedMock.mockReset(); + }); + + describe('#ensureInternalElserInstalled', () => { + it('installs ELSER if not already installed', async () => { + getModelInstallStatusMock.mockResolvedValue({ installed: true }); + + await endpointManager.ensureInternalElserInstalled(); + + expect(installElserMock).not.toHaveBeenCalled(); + expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1); + }); + it('does not install ELSER if already present', async () => { + getModelInstallStatusMock.mockResolvedValue({ installed: false }); + + await endpointManager.ensureInternalElserInstalled(); + + expect(installElserMock).toHaveBeenCalledTimes(1); + expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts index 8570e4209114..b60c3b572d03 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -15,13 +15,7 @@ import { } from '@kbn/product-doc-common'; import type { ProductDocInstallClient } from '../doc_install_status'; import type { InferenceEndpointManager } from '../inference_endpoint'; -import { - downloadToDisk, - openZipArchive, - loadManifestFile, - loadMappingFile, - type ZipArchive, -} from './utils'; +import { downloadToDisk, openZipArchive, loadMappingFile, type ZipArchive } from './utils'; import { majorMinor, latestVersion } from './utils/semver'; import { validateArtifactArchive, @@ -160,7 +154,6 @@ export class PackageInstaller { validateArtifactArchive(zipArchive); - const manifest = await loadManifestFile(zipArchive); const mappings = await loadMappingFile(zipArchive); const indexName = getProductDocIndexName(productName); From d3624d02a1e2842613bc66b459690969936d8893 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 17 Oct 2024 09:02:56 +0200 Subject: [PATCH 52/97] last batch of unit tests --- .../product_doc_install_service.test.ts | 65 +++++ .../product_doc_install_service.ts | 13 - .../doc_install_status/service.mock.ts | 24 ++ .../inference_endpoint/service.mock.ts | 20 ++ .../package_installer.test.mocks.ts | 36 +++ .../package_installer.test.ts | 255 ++++++++++++++++++ .../package_installer/package_installer.ts | 6 +- 7 files changed, 403 insertions(+), 16 deletions(-) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts new file mode 100644 index 000000000000..81249038a129 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from '@kbn/core/server'; +import { DocumentationProduct } from '@kbn/product-doc-common'; +import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { ProductDocInstallClient } from './product_doc_install_service'; + +const createObj = (attrs: TypeAttributes): SavedObjectsFindResult => { + return { + id: attrs.product_name, + type: 'type', + references: [], + attributes: attrs, + score: 42, + }; +}; + +describe('ProductDocInstallClient', () => { + let soClient: ReturnType; + let service: ProductDocInstallClient; + + beforeEach(() => { + soClient = savedObjectsClientMock.create(); + service = new ProductDocInstallClient({ soClient }); + }); + + describe('getInstallationStatus', () => { + it('returns the installation status based on existing entries', async () => { + soClient.find.mockResolvedValue({ + saved_objects: [ + createObj({ + product_name: 'kibana', + product_version: '8.15', + installation_status: 'installed', + }), + createObj({ + product_name: 'elasticsearch', + product_version: '8.15', + installation_status: 'installing', + }), + ], + total: 2, + per_page: 100, + page: 1, + }); + + const installStatus = await service.getInstallationStatus(); + + expect(Object.keys(installStatus).sort()).toEqual(Object.keys(DocumentationProduct).sort()); + expect(installStatus.kibana).toEqual({ + status: 'installed', + version: '8.15', + }); + expect(installStatus.security).toEqual({ + status: 'uninstalled', + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts index 25ff6913b8b9..24625ebc5158 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts @@ -11,7 +11,6 @@ import { ProductName, DocumentationProduct } from '@kbn/product-doc-common'; import type { ProductInstallState } from '../../../common/install_status'; import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects'; -// import { soToModel } from './model_conversion'; export class ProductDocInstallClient { private soClient: SavedObjectsClientContract; @@ -20,18 +19,6 @@ export class ProductDocInstallClient { this.soClient = soClient; } - /* - async getForProduct(productName: string): Promise { - const objectId = getObjectIdFromProductName(productName); - try { - const object = await this.soClient.get(typeName, objectId); - return soToModel(object); - } catch (e) { - // TODO - } - } - */ - async getInstallationStatus(): Promise> { const response = await this.soClient.find({ type: typeName, diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts new file mode 100644 index 000000000000..c2a0adbac9f2 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductDocInstallClient } from './product_doc_install_service'; + +export type InstallClientMock = jest.Mocked; + +const createInstallClientMock = (): InstallClientMock => { + return { + getInstallationStatus: jest.fn(), + setInstallationStarted: jest.fn(), + setInstallationSuccessful: jest.fn(), + setInstallationFailed: jest.fn(), + setUninstalled: jest.fn(), + } as unknown as InstallClientMock; +}; + +export const installClientMock = { + create: createInstallClientMock, +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts new file mode 100644 index 000000000000..e9715c4ad2ac --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceEndpointManager } from './endpoint_manager'; + +export type InferenceEndpointManagerMock = jest.Mocked; + +const createMock = (): InferenceEndpointManagerMock => { + return { + ensureInternalElserInstalled: jest.fn(), + } as unknown as InferenceEndpointManagerMock; +}; + +export const inferenceManagerMock = { + create: createMock, +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts new file mode 100644 index 000000000000..3b7b7c234800 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const validateArtifactArchiveMock = jest.fn(); +export const fetchArtifactVersionsMock = jest.fn(); +export const createIndexMock = jest.fn(); +export const populateIndexMock = jest.fn(); + +jest.doMock('./steps', () => { + const actual = jest.requireActual('./steps'); + return { + ...actual, + validateArtifactArchive: validateArtifactArchiveMock, + fetchArtifactVersions: fetchArtifactVersionsMock, + createIndex: createIndexMock, + populateIndex: populateIndexMock, + }; +}); + +export const downloadToDiskMock = jest.fn(); +export const openZipArchiveMock = jest.fn(); +export const loadMappingFileMock = jest.fn(); + +jest.doMock('./utils', () => { + const actual = jest.requireActual('./utils'); + return { + ...actual, + downloadToDisk: downloadToDiskMock, + openZipArchive: openZipArchiveMock, + loadMappingFile: loadMappingFileMock, + }; +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts new file mode 100644 index 000000000000..e68bd0e9c505 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + downloadToDiskMock, + createIndexMock, + populateIndexMock, + loadMappingFileMock, + openZipArchiveMock, + validateArtifactArchiveMock, + fetchArtifactVersionsMock, +} from './package_installer.test.mocks'; + +import { + getArtifactName, + getProductDocIndexName, + DocumentationProduct, + ProductName, +} from '@kbn/product-doc-common'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { installClientMock } from '../doc_install_status/service.mock'; +import { inferenceManagerMock } from '../inference_endpoint/service.mock'; +import type { ProductInstallState } from '../../../common/install_status'; +import { PackageInstaller } from './package_installer'; + +const artifactsFolder = '/lost'; +const artifactRepositoryUrl = 'https://repository.com'; +const kibanaVersion = '8.16.3'; + +const callOrder = (fn: { mock: { invocationCallOrder: number[] } }): number => { + return fn.mock.invocationCallOrder[0]; +}; + +describe('PackageInstaller', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let productDocClient: ReturnType; + let endpointManager: ReturnType; + + let packageInstaller: PackageInstaller; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + productDocClient = installClientMock.create(); + endpointManager = inferenceManagerMock.create(); + packageInstaller = new PackageInstaller({ + artifactsFolder, + logger, + esClient, + productDocClient, + endpointManager, + artifactRepositoryUrl, + kibanaVersion, + }); + }); + + afterEach(() => { + downloadToDiskMock.mockReset(); + createIndexMock.mockReset(); + populateIndexMock.mockReset(); + loadMappingFileMock.mockReset(); + openZipArchiveMock.mockReset(); + validateArtifactArchiveMock.mockReset(); + fetchArtifactVersionsMock.mockReset(); + }); + + describe('installPackage', () => { + it('calls the steps with the right parameters', async () => { + const zipArchive = { + close: jest.fn(), + }; + openZipArchiveMock.mockResolvedValue(zipArchive); + + const mappings = Symbol('mappings'); + loadMappingFileMock.mockResolvedValue(mappings); + + await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }); + + const artifactName = getArtifactName({ + productName: 'kibana', + productVersion: '8.16', + }); + const indexName = getProductDocIndexName('kibana'); + expect(endpointManager.ensureInternalElserInstalled).toHaveBeenCalledTimes(1); + + expect(downloadToDiskMock).toHaveBeenCalledTimes(1); + expect(downloadToDiskMock).toHaveBeenCalledWith( + `${artifactRepositoryUrl}/${artifactName}`, + `${artifactsFolder}/${artifactName}` + ); + + expect(openZipArchiveMock).toHaveBeenCalledTimes(1); + expect(openZipArchiveMock).toHaveBeenCalledWith(`${artifactsFolder}/${artifactName}`); + + expect(loadMappingFileMock).toHaveBeenCalledTimes(1); + expect(loadMappingFileMock).toHaveBeenCalledWith(zipArchive); + + expect(createIndexMock).toHaveBeenCalledTimes(1); + expect(createIndexMock).toHaveBeenCalledWith({ + indexName, + mappings, + esClient, + log: logger, + }); + + expect(populateIndexMock).toHaveBeenCalledTimes(1); + expect(populateIndexMock).toHaveBeenCalledWith({ + indexName, + archive: zipArchive, + esClient, + log: logger, + }); + + expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledTimes(1); + expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledWith('kibana', indexName); + + expect(zipArchive.close).toHaveBeenCalledTimes(1); + + expect(productDocClient.setInstallationFailed).not.toHaveBeenCalled(); + }); + + it('executes the steps in the right order', async () => { + await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }); + + expect(callOrder(endpointManager.ensureInternalElserInstalled)).toBeLessThan( + callOrder(downloadToDiskMock) + ); + expect(callOrder(downloadToDiskMock)).toBeLessThan(callOrder(openZipArchiveMock)); + expect(callOrder(openZipArchiveMock)).toBeLessThan(callOrder(loadMappingFileMock)); + expect(callOrder(loadMappingFileMock)).toBeLessThan(callOrder(createIndexMock)); + expect(callOrder(createIndexMock)).toBeLessThan(callOrder(populateIndexMock)); + expect(callOrder(populateIndexMock)).toBeLessThan( + callOrder(productDocClient.setInstallationSuccessful) + ); + }); + + it('closes the archive and calls setInstallationFailed if the installation fails', async () => { + const zipArchive = { + close: jest.fn(), + }; + openZipArchiveMock.mockResolvedValue(zipArchive); + + populateIndexMock.mockImplementation(async () => { + throw new Error('something bad'); + }); + + await expect( + packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }) + ).rejects.toThrowError(); + + expect(productDocClient.setInstallationSuccessful).not.toHaveBeenCalled(); + + expect(zipArchive.close).toHaveBeenCalledTimes(1); + + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Error during documentation installation') + ); + + expect(productDocClient.setInstallationFailed).toHaveBeenCalledTimes(1); + expect(productDocClient.setInstallationFailed).toHaveBeenCalledWith( + 'kibana', + 'something bad' + ); + }); + }); + + describe('installALl', () => { + it('installs all the packages to their latest version', async () => { + jest.spyOn(packageInstaller, 'installPackage'); + + fetchArtifactVersionsMock.mockResolvedValue({ + kibana: ['8.15', '8.16'], + elasticsearch: ['8.15'], + }); + + await packageInstaller.installAll({}); + + expect(packageInstaller.installPackage).toHaveBeenCalledTimes(2); + + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'kibana', + productVersion: '8.16', + }); + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'elasticsearch', + productVersion: '8.15', + }); + }); + }); + + describe('ensureUpToDate', () => { + it('updates the installed packages to the latest version', async () => { + fetchArtifactVersionsMock.mockResolvedValue({ + kibana: ['8.15', '8.16'], + security: ['8.15', '8.16'], + elasticsearch: ['8.15'], + }); + + productDocClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'installed', version: '8.15' }, + security: { status: 'installed', version: '8.16' }, + elasticsearch: { status: 'uninstalled' }, + } as Record); + + jest.spyOn(packageInstaller, 'installPackage'); + + await packageInstaller.ensureUpToDate({}); + + expect(packageInstaller.installPackage).toHaveBeenCalledTimes(1); + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'kibana', + productVersion: '8.16', + }); + }); + }); + + describe('uninstallPackage', () => { + it('performs the uninstall steps', async () => { + await packageInstaller.uninstallPackage({ productName: 'kibana' }); + + expect(esClient.indices.delete).toHaveBeenCalledTimes(1); + expect(esClient.indices.delete).toHaveBeenCalledWith( + { + index: getProductDocIndexName('kibana'), + }, + expect.objectContaining({ ignore: [404] }) + ); + + expect(productDocClient.setUninstalled).toHaveBeenCalledTimes(1); + expect(productDocClient.setUninstalled).toHaveBeenCalledWith('kibana'); + }); + }); + + describe('uninstallAll', () => { + it('calls uninstall for all packages', async () => { + jest.spyOn(packageInstaller, 'uninstallPackage'); + + await packageInstaller.uninstallAll(); + + expect(packageInstaller.uninstallPackage).toHaveBeenCalledTimes( + Object.keys(DocumentationProduct).length + ); + Object.values(DocumentationProduct).forEach((productName) => { + expect(packageInstaller.uninstallPackage).toHaveBeenCalledWith({ productName }); + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts index b60c3b572d03..463b698238f3 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -132,6 +132,8 @@ export class PackageInstaller { `Starting installing documentation for product [${productName}] and version [${productVersion}]` ); + productVersion = majorMinor(productVersion); + await this.uninstallPackage({ productName }); let zipArchive: ZipArchive | undefined; @@ -184,9 +186,7 @@ export class PackageInstaller { await this.productDocClient.setInstallationFailed(productName, e.message); throw e; } finally { - if (zipArchive) { - zipArchive.close(); - } + zipArchive?.close(); } } From b0ac940a4de9ef04bc0cc6550f1d4f447a33780d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 17 Oct 2024 07:20:13 +0000 Subject: [PATCH 53/97] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- docs/developer/plugin-list.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 1051e80883aa..7031b89e3b61 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -772,7 +772,7 @@ Elastic. |{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase] -|This plugin contains the registry for the knowledge base. +|This plugin contains the product documentation base service. |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling] From 57978296c4ea9eed97c2a456eba3debbcf800290 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 17 Oct 2024 07:20:41 +0000 Subject: [PATCH 54/97] [CI] Auto-commit changed files from 'node scripts/notice' --- x-pack/plugins/ai_infra/llm_tasks/tsconfig.json | 1 + x-pack/plugins/ai_infra/product_doc_base/tsconfig.json | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json index fbcfedb5972a..d156e75fae01 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json +++ b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json @@ -21,5 +21,6 @@ "@kbn/product-doc-common", "@kbn/inference-plugin", "@kbn/product-doc-base-plugin", + "@kbn/logging-mocks", ] } diff --git a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json index 93c9ca2ac860..79d40e48af72 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json +++ b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json @@ -22,5 +22,6 @@ "@kbn/core-saved-objects-server", "@kbn/utils", "@kbn/core-http-browser", + "@kbn/logging-mocks", ] } From 904416e770029a5db3b91f3a612dce88193cefc7 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 17 Oct 2024 07:21:10 +0000 Subject: [PATCH 55/97] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- .../observability_ai_assistant_app/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json index 2673a20c725d..c175bfc1b5a1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json @@ -70,6 +70,7 @@ "@kbn/logs-data-access-plugin", "@kbn/ai-assistant-common", "@kbn/llm-tasks-plugin", + "@kbn/product-doc-common", ], "exclude": [ "target/**/*" From 2c51c37927f06259f1c71ebf87beadc9ad72af48 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 17 Oct 2024 09:55:12 +0200 Subject: [PATCH 56/97] remove bundle from limits --- packages/kbn-optimizer/limits.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index bc8ad4b71f0f..369284743247 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -99,7 +99,6 @@ pageLoadAssetSize: licensing: 29004 links: 44490 lists: 22900 - llmTasks: 22500 logsDataAccess: 16759 logsExplorer: 60000 logsShared: 281060 From 3c79e40f2aa6701868b2b91018e067f6a528709c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 17 Oct 2024 11:38:42 +0200 Subject: [PATCH 57/97] fix failures --- .../src/artifact/manifest.ts | 4 +-- .../src/tasks/create_artifact.ts | 4 +-- .../src/artifact_content.test.ts | 13 ++++---- .../steps/fetch_artifact_versions.test.ts | 31 +++---------------- 4 files changed, 15 insertions(+), 37 deletions(-) diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts index a6916f6fb9af..a8aa927c5ef1 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts @@ -5,13 +5,13 @@ * 2.0. */ -import type { ArtifactManifest } from '@kbn/product-doc-common'; +import type { ArtifactManifest, ProductName } from '@kbn/product-doc-common'; export const getArtifactManifest = ({ productName, stackVersion, }: { - productName: string; + productName: ProductName; stackVersion: string; }): ArtifactManifest => { return { diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts index e289b894a639..056887a41a4d 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts @@ -8,7 +8,7 @@ import Path from 'path'; import AdmZip from 'adm-zip'; import type { ToolingLog } from '@kbn/tooling-log'; -import { getArtifactName } from '@kbn/product-doc-common'; +import { getArtifactName, type ProductName } from '@kbn/product-doc-common'; import { getArtifactMappings } from '../artifact/mappings'; import { getArtifactManifest } from '../artifact/manifest'; @@ -21,7 +21,7 @@ export const createArtifact = async ({ }: { buildFolder: string; targetFolder: string; - productName: string; + productName: ProductName; stackVersion: string; log: ToolingLog; }) => { diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts index 7f5e87cb3a23..3f97aaf94f88 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts @@ -9,14 +9,15 @@ import { isArtifactContentFilePath } from './artifact_content'; describe('isArtifactContentFilePath', () => { it('returns true for filenames matching the pattern', () => { - expect(isArtifactContentFilePath('content-0.ndjson')).toEqual(true); - expect(isArtifactContentFilePath('content-007.ndjson')).toEqual(true); - expect(isArtifactContentFilePath('content-9042.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content/content-0.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content/content-007.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content/content-9042.ndjson')).toEqual(true); }); it('returns false for filenames not matching the pattern', () => { - expect(isArtifactContentFilePath('content-0')).toEqual(false); - expect(isArtifactContentFilePath('content.ndjson')).toEqual(false); - expect(isArtifactContentFilePath('content-9042.json')).toEqual(false); + expect(isArtifactContentFilePath('content-0.ndjson')).toEqual(false); + expect(isArtifactContentFilePath('content/content-0')).toEqual(false); + expect(isArtifactContentFilePath('content/content.ndjson')).toEqual(false); + expect(isArtifactContentFilePath('content/content-9042.json')).toEqual(false); }); }); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts index e320a2488c98..805008ccab69 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts @@ -116,37 +116,14 @@ describe('fetchArtifactVersions', () => { it('throws an error if the response is truncated', async () => { mockResponse(createResponse({ artifactNames: [], truncated: true })); - await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects - .toThrowErrorMatchingInlineSnapshot(` - "Unhandled error. (Error: bucket content is truncated, cannot retrieve all versions - at /Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts:29:15 - at Parser. (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:308:18) - at Parser.emit (node:events:519:28) - at SAXParser.onclosetag (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:266:26) - at emit (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:625:35) - at emitNode (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:630:5) - at closeTag (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:890:7) - at SAXParser.write (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/sax/lib/sax.js:1437:13) - at Parser.Object..exports.Parser.Parser.parseString (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:327:31) - at Parser.parseString (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:5:59) - at Object..exports.parseString (/Users/pgayvallet/DEV/workspaces/elastic/kibana/node_modules/xml2js/lib/parser.js:373:19) - at /Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts:22:16 - at new Promise () - at fetchArtifactVersions (/Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts:21:10) - at processTicksAndRejections (node:internal/process/task_queues:95:5) - at Object. (/Users/pgayvallet/DEV/workspaces/elastic/kibana/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts:119:5))" - `); + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects.toThrowError( + /bucket content is truncated/ + ); }); it('throws an error if the response is not valid xml', async () => { mockResponse('some plain text'); - await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects - .toThrowErrorMatchingInlineSnapshot(` - "Non-whitespace before first tag. - Line: 0 - Column: 1 - Char: s" - `); + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects.toThrowError(); }); }); From 83709a8434de71568132a33c27e890b4ca018d36 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 17 Oct 2024 11:41:55 +0200 Subject: [PATCH 58/97] lint --- x-pack/plugins/ai_infra/llm_tasks/jest.config.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js index 65fd78fd7a88..2a6206d4304b 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js +++ b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js @@ -8,9 +8,7 @@ module.exports = { preset: '@kbn/test', rootDir: '../../../..', - roots: [ - '/x-pack/plugins/ai_infra/llm_tasks/server', - ], + roots: ['/x-pack/plugins/ai_infra/llm_tasks/server'], setupFiles: [], collectCoverage: true, collectCoverageFrom: [ From 0382e2bb16517a0f66b7c5f3dc7b915b3cd1457c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 17 Oct 2024 12:32:04 +0200 Subject: [PATCH 59/97] add endpoint access control --- .../product_doc_base/server/routes/installation.ts | 10 ++++++++-- .../observability_ai_assistant/server/plugin.ts | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index 97827a523dc1..bd1c10bc7227 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -28,7 +28,11 @@ export const registerInstallationRoutes = ({ getInstaller: () => PackageInstaller; }) => { router.get( - { path: INSTALLATION_STATUS_API_PATH, validate: false, options: { access: 'internal' } }, + { + path: INSTALLATION_STATUS_API_PATH, + validate: false, + options: { access: 'internal', tags: ['access:manage_llm_product_doc'] }, + }, async (ctx, req, res) => { const installClient = getInstallClient(); const installStatus = await installClient.getInstallationStatus(); @@ -49,7 +53,8 @@ export const registerInstallationRoutes = ({ validate: false, options: { access: 'internal', - timeout: { idleSocket: 20 * 60 * 1000 }, // 20 minutes, install can take time. + tags: ['access:manage_llm_product_doc'], + timeout: { idleSocket: 20 * 60 * 1000 }, // install can take time. }, }, async (ctx, req, res) => { @@ -71,6 +76,7 @@ export const registerInstallationRoutes = ({ validate: false, options: { access: 'internal', + tags: ['access:manage_llm_product_doc'], }, }, async (ctx, req, res) => { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts index 50687920478a..91f861d6e215 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts @@ -78,7 +78,7 @@ export class ObservabilityAIAssistantPlugin privileges: { all: { app: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'kibana'], - api: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'ai_assistant'], + api: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'ai_assistant', 'manage_llm_product_doc'], catalogue: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID], savedObject: { all: [ From 63a0e6b9907d7aff9d93fcf8b7ef0ad0fb233e5a Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 18 Oct 2024 08:54:30 +0200 Subject: [PATCH 60/97] add basic document processing to improve content --- .../src/build_artifacts.ts | 5 +- .../src/tasks/index.ts | 1 + .../src/tasks/process_documents.ts | 59 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts index 230a2a38cc69..551f58bc6830 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts @@ -19,6 +19,7 @@ import { createArtifact, cleanupFolders, deleteIndex, + processDocuments, } from './tasks'; import type { TaskConfig } from './types'; @@ -106,7 +107,7 @@ const buildArtifact = async ({ const targetIndex = getTargetIndexName({ productName, stackVersion }); - const documents = await extractDocumentation({ + let documents = await extractDocumentation({ client: sourceClient, index: 'search-docs-1', log, @@ -114,6 +115,8 @@ const buildArtifact = async ({ stackVersion, }); + documents = await processDocuments({ documents, log }); + await createTargetIndex({ client: embeddingClient, indexName: targetIndex, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts index bba81d70f7bb..ec94e4c135c1 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts @@ -14,3 +14,4 @@ export { checkConnectivity } from './check_connectivity'; export { createArtifact } from './create_artifact'; export { cleanupFolders } from './cleanup_folders'; export { deleteIndex } from './delete_index'; +export { processDocuments } from './process_documents'; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts new file mode 100644 index 000000000000..69141ca167ab --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { uniqBy } from 'lodash'; +import { encode } from 'gpt-tokenizer'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { ExtractedDocument } from './extract_documentation'; + +export const processDocuments = async ({ + documents, + log, +}: { + documents: ExtractedDocument[]; + log: ToolingLog; +}): Promise => { + log.info('Starting processing documents.'); + const initialCount = documents.length; + documents = removeDuplicates(documents); + const noDupCount = documents.length; + log.info(`Removed ${initialCount - noDupCount} duplicates`); + documents.forEach(processDocument); + documents = filterEmptyDocs(documents); + log.info(`Removed ${noDupCount - documents.length} empty documents`); + log.info('Done processing documents.'); + return documents; +}; + +const removeDuplicates = (documents: ExtractedDocument[]): ExtractedDocument[] => { + return uniqBy(documents, (doc) => doc.slug); +}; + +/** + * Filter "this content has moved" or "deleted pages" type of documents, just based on token count. + */ +const filterEmptyDocs = (documents: ExtractedDocument[]): ExtractedDocument[] => { + return documents.filter((doc) => { + const tokenCount = encode(doc.content_body).length; + if (tokenCount < 100) { + return false; + } + return true; + }); +}; + +const processDocument = (document: ExtractedDocument) => { + document.content_body = document.content_body + // remove those "edit" button text that got embedded into titles. + .replaceAll(/([a-zA-Z])edit\n/g, (match) => { + return `${match[0]}\n`; + }) + // limit to 2 consecutive carriage return + .replaceAll(/\n\n+/g, '\n\n'); + + return document; +}; From 9e96d0815657977c3a35380072c102703f92aac1 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 18 Oct 2024 08:59:49 +0200 Subject: [PATCH 61/97] bump summary threshold to 1k --- .../tasks/retrieve_documentation/retrieve_documentation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts index f411733087b9..63b44352887b 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -13,7 +13,7 @@ import type { RetrieveDocumentationAPI } from './types'; import { summarizeDocument } from './summarize_document'; // if document content length greater, then we'll trigger the summary task -const MIN_TOKENS_TO_SUMMARIZE = 750; +const MIN_TOKENS_TO_SUMMARIZE = 1000; // maximum token length of generated summaries - will truncate if greater const MAX_SUMMARY_TOKEN_LENGTH = 1000; From 9807a21fa13a48ec98f02708df4ab973f0969767 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 18 Oct 2024 09:38:51 +0200 Subject: [PATCH 62/97] improve summary prompt --- .../server/tasks/retrieve_documentation/summarize_document.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts index d5ade61151fe..11febdb82a95 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts @@ -51,9 +51,9 @@ export const summarizeDocument = async ({ Given a question and a document, please provide a condensed version of the document that can be used to answer the question. - - Please try to limit the length of the output to approximately 4000 characters, or 800 words. + - Limit the length of the output to 800 words. - Try to include all relevant information that could be used to answer the question in the condensed version. If this - can't be done without exceeding the 4000chars/800words requirement, please only include the information that you think + can't be done without exceeding the 800 words limit requirement, then only include the information that you think are the most relevant and the most helpful to answer the question. - If you think the document isn't relevant at all to answer the question, just return an empty text`, input: ` From 4f2398494e90a03d512662929770f1c5885c2a15 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 18 Oct 2024 10:05:55 +0200 Subject: [PATCH 63/97] change comment --- x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index be9b2f5fbd47..6ae1a08016c2 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -104,9 +104,7 @@ export class ProductDocBasePlugin return { isInstalled: async () => { - // TODO: should also check license, probably - - // TODO: something less naive + // can probably be improved. But is a boolean good enough then const installStatus = await productDocClient.getInstallationStatus(); const installed = Object.values(installStatus).some( (status) => status.status === 'installed' From ef794c3e2b15678e4403073c2587447ea9a4ec6c Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 23 Oct 2024 08:35:11 +0200 Subject: [PATCH 64/97] remove empty filter --- .../product_doc_base/server/services/search/perform_search.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts index a437e182a0d2..2c971501e015 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts @@ -26,7 +26,6 @@ export const performSearch = async ({ size, query: { bool: { - filter: [], should: [ { multi_match: { From dec690316e3ac7e165c565a8d183231b34cb9ce4 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 23 Oct 2024 08:40:49 +0200 Subject: [PATCH 65/97] address review comments on i18n script changes --- .buildkite/scripts/steps/checks/i18n.sh | 2 +- src/dev/i18n_tools/bin/run_i18n_check.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/steps/checks/i18n.sh b/.buildkite/scripts/steps/checks/i18n.sh index f23ed2d63759..45d2474a1085 100755 --- a/.buildkite/scripts/steps/checks/i18n.sh +++ b/.buildkite/scripts/steps/checks/i18n.sh @@ -5,4 +5,4 @@ set -euo pipefail source .buildkite/scripts/common/util.sh echo --- Check i18n -node scripts/i18n_check --quiet +node scripts/i18n_check --silent diff --git a/src/dev/i18n_tools/bin/run_i18n_check.ts b/src/dev/i18n_tools/bin/run_i18n_check.ts index ff00148ab301..ce3e87105108 100644 --- a/src/dev/i18n_tools/bin/run_i18n_check.ts +++ b/src/dev/i18n_tools/bin/run_i18n_check.ts @@ -87,7 +87,7 @@ run( const kibanaRootPaths = ['./src', './packages', './x-pack']; const rootPaths = Array().concat(path || kibanaRootPaths); - const list = new Listr( + const list = new Listr( [ { title: 'Checking .i18nrc.json files', @@ -129,8 +129,7 @@ run( concurrent: false, exitOnError: true, forceTTY: false, - renderer: - ((silent || quiet) && 'silent') || (process.env.CI ? 'verbose' : ('default' as any)), + renderer: silent ? 'silent' : process.env.CI ? 'verbose' : 'default', } ); From c4824f69735ddb20583a870a47a566bd62c90277 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 30 Oct 2024 10:22:21 +0100 Subject: [PATCH 66/97] add license check for doc installation --- .../ai_infra/product_doc_base/kibana.jsonc | 2 +- .../product_doc_base/server/plugin.ts | 34 +++++++++---------- .../product_doc_base/server/routes/index.ts | 11 +++--- .../server/routes/installation.ts | 27 +++++++++------ .../services/package_installer/index.ts | 1 + .../package_installer/utils/check_license.ts | 13 +++++++ .../services/package_installer/utils/index.ts | 1 + .../ai_infra/product_doc_base/server/types.ts | 13 ++++++- .../settings_tab/product_doc_entry.tsx | 29 +++++++--------- 9 files changed, 77 insertions(+), 54 deletions(-) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/check_license.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc index 4e5ebdd3c068..7053a73517e0 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc +++ b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc @@ -7,7 +7,7 @@ "server": true, "browser": true, "configPath": ["xpack", "productDocBase"], - "requiredPlugins": [], + "requiredPlugins": ["licensing"], "requiredBundles": [], "optionalPlugins": [], "extraPublicDirs": [] diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index 6ae1a08016c2..3ff052a40e6c 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -12,11 +12,12 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kb import { SavedObjectsClient } from '@kbn/core/server'; import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; import type { ProductDocBaseConfig } from './config'; -import type { +import { ProductDocBaseSetupContract, ProductDocBaseStartContract, ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies, + InternalRouteServices, } from './types'; import { productDocInstallStatusSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; @@ -35,8 +36,7 @@ export class ProductDocBasePlugin > { private logger: Logger; - private installClient?: ProductDocInstallClient; - private packageInstaller?: PackageInstaller; + private routeServices?: InternalRouteServices; constructor(private readonly context: PluginInitializerContext) { this.logger = context.logger.get(); @@ -50,17 +50,11 @@ export class ProductDocBasePlugin const router = coreSetup.http.createRouter(); registerRoutes({ router, - getInstallClient: () => { - if (!this.installClient) { - throw new Error('getInstallClient called before #start'); + getServices: () => { + if (!this.routeServices) { + throw new Error('getServices called before #start'); } - return this.installClient; - }, - getInstaller: () => { - if (!this.packageInstaller) { - throw new Error('getInstaller called before #start'); - } - return this.packageInstaller; + return this.routeServices; }, }); @@ -69,20 +63,20 @@ export class ProductDocBasePlugin start( core: CoreStart, - pluginsStart: ProductDocBaseStartDependencies + { licensing }: ProductDocBaseStartDependencies ): ProductDocBaseStartContract { const soClient = new SavedObjectsClient( core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName]) ); const productDocClient = new ProductDocInstallClient({ soClient }); - this.installClient = productDocClient; + const installClient = productDocClient; const endpointManager = new InferenceEndpointManager({ esClient: core.elasticsearch.client.asInternalUser, logger: this.logger.get('endpoint-manager'), }); - this.packageInstaller = new PackageInstaller({ + const packageInstaller = new PackageInstaller({ esClient: core.elasticsearch.client.asInternalUser, productDocClient, endpointManager, @@ -98,10 +92,16 @@ export class ProductDocBasePlugin }); // should we use taskManager for this? - this.packageInstaller.ensureUpToDate({}).catch((err) => { + packageInstaller.ensureUpToDate({}).catch((err) => { this.logger.error(`Error checking if product documentation is up to date: ${err.message}`); }); + this.routeServices = { + packageInstaller, + installClient, + licensing, + }; + return { isInstalled: async () => { // can probably be improved. But is a boolean good enough then diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts index 254328ab46f6..2c9278217b3f 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts @@ -7,17 +7,14 @@ import type { IRouter } from '@kbn/core/server'; import { registerInstallationRoutes } from './installation'; -import type { ProductDocInstallClient } from '../services/doc_install_status'; -import type { PackageInstaller } from '../services/package_installer'; +import type { InternalRouteServices } from '../types'; export const registerRoutes = ({ router, - getInstallClient, - getInstaller, + getServices, }: { router: IRouter; - getInstallClient: () => ProductDocInstallClient; - getInstaller: () => PackageInstaller; + getServices: () => InternalRouteServices; }) => { - registerInstallationRoutes({ getInstaller, getInstallClient, router }); + registerInstallationRoutes({ getServices, router }); }; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index bd1c10bc7227..247fad4beef4 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -15,17 +15,15 @@ import { UninstallResponse, } from '../../common/http_api/installation'; import { InstallationStatus } from '../../common/install_status'; -import type { ProductDocInstallClient } from '../services/doc_install_status'; -import type { PackageInstaller } from '../services/package_installer'; +import type { InternalRouteServices } from '../types'; +import { checkLicense } from '../services/package_installer'; export const registerInstallationRoutes = ({ router, - getInstallClient, - getInstaller, + getServices, }: { router: IRouter; - getInstallClient: () => ProductDocInstallClient; - getInstaller: () => PackageInstaller; + getServices: () => InternalRouteServices; }) => { router.get( { @@ -34,7 +32,7 @@ export const registerInstallationRoutes = ({ options: { access: 'internal', tags: ['access:manage_llm_product_doc'] }, }, async (ctx, req, res) => { - const installClient = getInstallClient(); + const { installClient } = getServices(); const installStatus = await installClient.getInstallationStatus(); const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status)); @@ -58,9 +56,16 @@ export const registerInstallationRoutes = ({ }, }, async (ctx, req, res) => { - const installer = getInstaller(); + const { packageInstaller, licensing } = getServices(); - await installer.installAll({}); + const license = await licensing.getLicense(); + if (!checkLicense(license)) { + return res.badRequest({ + body: 'Elastic documentation requires an enterprise license', + }); + } + + await packageInstaller.installAll({}); return res.ok({ body: { @@ -80,9 +85,9 @@ export const registerInstallationRoutes = ({ }, }, async (ctx, req, res) => { - const installer = getInstaller(); + const { packageInstaller } = getServices(); - await installer.uninstallAll(); + await packageInstaller.uninstallAll(); return res.ok({ body: { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts index a9edb7c38fda..1b0a9595b96d 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts @@ -6,3 +6,4 @@ */ export { PackageInstaller } from './package_installer'; +export { checkLicense } from './utils'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/check_license.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/check_license.ts new file mode 100644 index 000000000000..d4af5b7ebdb2 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/check_license.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ILicense } from '@kbn/licensing-plugin/server'; + +export const checkLicense = (license: ILicense): boolean => { + const result = license.check('elastic documentation', 'enterprise'); + return result.state === 'valid'; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts index a612a8c6e9f4..a980b86df623 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts @@ -8,3 +8,4 @@ export { downloadToDisk } from './download'; export { openZipArchive, type ZipArchive } from './zip_archive'; export { loadManifestFile, loadMappingFile } from './archive_accessors'; +export { checkLicense } from './check_license'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts index 80a7cdb76603..0debfa98a6a2 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts @@ -5,13 +5,18 @@ * 2.0. */ +import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; import type { SearchApi } from './services/search'; +import { ProductDocInstallClient } from './services/doc_install_status'; +import { PackageInstaller } from './services/package_installer'; /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ProductDocBaseSetupDependencies {} -export interface ProductDocBaseStartDependencies {} +export interface ProductDocBaseStartDependencies { + licensing: LicensingPluginStart; +} export interface ProductDocBaseSetupContract {} @@ -19,3 +24,9 @@ export interface ProductDocBaseStartContract { search: SearchApi; isInstalled: () => Promise; } + +export interface InternalRouteServices { + installClient: ProductDocInstallClient; + packageInstaller: PackageInstaller; + licensing: LicensingPluginStart; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx index 6eae22809810..db5ba257201b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx @@ -24,10 +24,7 @@ import { useInstallProductDoc } from '../../../hooks/use_install_product_doc'; import { useUninstallProductDoc } from '../../../hooks/use_uninstall_product_doc'; export function ProductDocEntry() { - const { - overlays, - notifications: { toasts }, - } = useKibana().services; + const { overlays } = useKibana().services; const [isInstalled, setInstalled] = useState(true); const [isInstalling, setInstalling] = useState(false); @@ -43,20 +40,18 @@ export function ProductDocEntry() { }, [status]); const onClickInstall = useCallback(() => { - toasts.addSuccess( - i18n.translate( - 'xpack.observabilityAiAssistantManagement.settingsPage.productDocInstallToastText', - { - defaultMessage: 'Installing Elastic documentation, this can take a few minutes.', - } - ) - ); setInstalling(true); - installProductDoc().then(() => { - setInstalling(false); - setInstalled(true); - }); - }, [installProductDoc, toasts]); + installProductDoc().then( + () => { + setInstalling(false); + setInstalled(true); + }, + () => { + setInstalling(false); + setInstalled(false); + } + ); + }, [installProductDoc]); const onClickUninstall = useCallback(() => { overlays From 30ae93570fc8cb3ccafb3cb190c77c80cd3d898f Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 30 Oct 2024 10:25:15 +0100 Subject: [PATCH 67/97] add timeout on endpoint creation --- .../inference_endpoint/utils/install_elser.ts | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts index 8b75cd5b0c41..0e92d765a3d1 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts @@ -16,17 +16,20 @@ export const installElser = async ({ client: ElasticsearchClient; log: Logger; }) => { - await client.inference.put({ - task_type: 'sparse_embedding', - inference_id: inferenceId, - inference_config: { - service: 'elasticsearch', - service_settings: { - num_allocations: 1, - num_threads: 1, - model_id: '.elser_model_2', + await client.inference.put( + { + task_type: 'sparse_embedding', + inference_id: inferenceId, + inference_config: { + service: 'elasticsearch', + service_settings: { + num_allocations: 1, + num_threads: 1, + model_id: '.elser_model_2', + }, + task_settings: {}, }, - task_settings: {}, }, - }); + { requestTimeout: 5 * 60 * 1000 } + ); }; From 5ebcef49308ef8a3bb624cbef9f545e4a57a5cba Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 30 Oct 2024 10:27:17 +0100 Subject: [PATCH 68/97] fix tool registration due to API change --- .../server/functions/documentation.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts index 62a068c0e6ea..dc384a0d3120 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts @@ -69,7 +69,6 @@ export async function registerDocumentationFunction({ documents: response.documents, }, }; - }, - ['all'] + } ); } From 64648b03ff65214ada36a02d537425f1e8b80f0c Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 30 Oct 2024 09:38:11 +0000 Subject: [PATCH 69/97] [CI] Auto-commit changed files from 'node scripts/notice' --- x-pack/plugins/ai_infra/product_doc_base/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json index 79d40e48af72..9d6fa2ff9f36 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json +++ b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json @@ -23,5 +23,6 @@ "@kbn/utils", "@kbn/core-http-browser", "@kbn/logging-mocks", + "@kbn/licensing-plugin", ] } From 6306a52eba8f62d1cc4fc4feecb44dfbd2077f8c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 30 Oct 2024 12:10:39 +0100 Subject: [PATCH 70/97] fix plugin tests --- .../plugins/ai_infra/product_doc_base/server/plugin.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts index b2beccdded66..da3d478d6e06 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts @@ -6,6 +6,7 @@ */ import { coreMock } from '@kbn/core/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; import { ProductDocBasePlugin } from './plugin'; @@ -51,7 +52,9 @@ describe('ProductDocBasePlugin', () => { describe('#start', () => { it('returns a contract with the expected shape', () => { plugin.setup(coreMock.createSetup(), {}); - const startContract = plugin.start(coreMock.createStart(), {}); + const startContract = plugin.start(coreMock.createStart(), { + licensing: licensingMock.createStart(), + }); expect(startContract).toEqual({ isInstalled: expect.any(Function), search: expect.any(Function), From 3c13779d5762207b2fcc3ceffe02711cd693d4f6 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:02:49 +0000 Subject: [PATCH 71/97] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 105deccf167d..48643074f180 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -394,7 +394,6 @@ packages/kbn-lens-embeddable-utils @elastic/obs-ux-infra_services-team @elastic/ packages/kbn-lens-formula-docs @elastic/kibana-visualizations packages/kbn-lint-packages-cli @elastic/kibana-operations packages/kbn-lint-ts-projects-cli @elastic/kibana-operations -x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra packages/kbn-logging @elastic/kibana-core packages/kbn-logging-mocks @elastic/kibana-core packages/kbn-managed-content-badge @elastic/kibana-visualizations @@ -433,8 +432,6 @@ packages/kbn-picomatcher @elastic/kibana-operations packages/kbn-plugin-check @elastic/appex-sharedux packages/kbn-plugin-generator @elastic/kibana-operations packages/kbn-plugin-helpers @elastic/kibana-operations -x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra -x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra packages/kbn-profiling-utils @elastic/obs-ux-infra_services-team packages/kbn-react-field @elastic/kibana-data-discovery packages/kbn-react-hooks @elastic/obs-ux-logs-team @@ -769,6 +766,7 @@ x-pack/examples/triggers_actions_ui_example @elastic/response-ops x-pack/examples/ui_actions_enhanced_examples @elastic/appex-sharedux x-pack/packages/ai-infra/inference-common @elastic/appex-ai-infra x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra +x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra x-pack/packages/index-management/index_management_shared_types @elastic/kibana-management x-pack/packages/kbn-ai-assistant @elastic/search-kibana x-pack/packages/kbn-ai-assistant-common @elastic/search-kibana @@ -855,6 +853,8 @@ x-pack/packages/security/role_management_model @elastic/kibana-security x-pack/packages/security/ui_components @elastic/kibana-security x-pack/performance @elastic/appex-qa x-pack/plugins/actions @elastic/response-ops +x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra +x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra x-pack/plugins/aiops @elastic/ml-ui x-pack/plugins/alerting @elastic/response-ops x-pack/plugins/banners @elastic/appex-sharedux From c85e9cc0cc48e823813f3d1fbcea96d49dad5eda Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 7 Nov 2024 10:04:32 +0100 Subject: [PATCH 72/97] fix types for new inference package --- .../retrieve_documentation.ts | 2 +- .../summarize_document.ts | 24 +++++++------------ .../tasks/retrieve_documentation/types.ts | 2 +- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts index 63b44352887b..b0e66f5fb4c3 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -6,7 +6,7 @@ */ import type { Logger } from '@kbn/logging'; -import type { OutputAPI } from '@kbn/inference-plugin/common/output'; +import type { OutputAPI } from '@kbn/inference-common'; import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server'; import { truncate, count as countTokens } from '../../utils/tokens'; import type { RetrieveDocumentationAPI } from './types'; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts index 11febdb82a95..2c7b879bd6dc 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts @@ -5,11 +5,7 @@ * 2.0. */ -import { lastValueFrom } from 'rxjs'; -import type { ToolSchema } from '@kbn/inference-plugin/common'; -import type { FunctionCallingMode } from '@kbn/inference-plugin/common/chat_complete'; -import type { OutputAPI } from '@kbn/inference-plugin/common/output'; -import { withoutOutputUpdateEvents } from '@kbn/inference-plugin/common/output/without_output_update_events'; +import type { ToolSchema, FunctionCallingMode, OutputAPI } from '@kbn/inference-common'; const summarizeDocumentSchema = { type: 'object', @@ -43,11 +39,11 @@ export const summarizeDocument = async ({ connectorId: string; functionCalling?: FunctionCallingMode; }): Promise => { - const result = await lastValueFrom( - outputAPI('summarize_document', { - connectorId, - functionCalling, - system: `You are an helpful Elastic assistant, and your current task is to help answering the user question. + const result = await outputAPI({ + id: 'summarize_document', + connectorId, + functionCalling, + system: `You are an helpful Elastic assistant, and your current task is to help answering the user question. Given a question and a document, please provide a condensed version of the document that can be used to answer the question. @@ -56,7 +52,7 @@ export const summarizeDocument = async ({ can't be done without exceeding the 800 words limit requirement, then only include the information that you think are the most relevant and the most helpful to answer the question. - If you think the document isn't relevant at all to answer the question, just return an empty text`, - input: ` + input: ` ## User question ${searchTerm} @@ -65,10 +61,8 @@ export const summarizeDocument = async ({ ${documentContent} `, - schema: summarizeDocumentSchema, - }).pipe(withoutOutputUpdateEvents()) - ); - + schema: summarizeDocumentSchema, + }); return { summary: result.output.summary ?? '', }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts index caeedd72ca30..ae4e8eb36468 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -6,7 +6,7 @@ */ import type { KibanaRequest } from '@kbn/core/server'; -import type { FunctionCallingMode } from '@kbn/inference-plugin/common/chat_complete'; +import type { FunctionCallingMode } from '@kbn/inference-common'; import type { ProductName } from '@kbn/product-doc-common'; export interface RetrieveDocumentationParams { From 0f0a4947be287ed375bebd7d71918777a1664d3b Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:16:14 +0000 Subject: [PATCH 73/97] [CI] Auto-commit changed files from 'node scripts/notice' --- x-pack/plugins/ai_infra/llm_tasks/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json index d156e75fae01..03b87827d941 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json +++ b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json @@ -22,5 +22,6 @@ "@kbn/inference-plugin", "@kbn/product-doc-base-plugin", "@kbn/logging-mocks", + "@kbn/inference-common", ] } From d7dc1bc8f34461443c366b94a1718e2659646bdc Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 7 Nov 2024 12:58:44 +0100 Subject: [PATCH 74/97] add token / reduction parameters --- .../retrieve_documentation.ts | 46 +++++++++++-------- .../tasks/retrieve_documentation/types.ts | 31 ++++++++++++- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts index b0e66f5fb4c3..96f966e48360 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -12,10 +12,8 @@ import { truncate, count as countTokens } from '../../utils/tokens'; import type { RetrieveDocumentationAPI } from './types'; import { summarizeDocument } from './summarize_document'; -// if document content length greater, then we'll trigger the summary task -const MIN_TOKENS_TO_SUMMARIZE = 1000; -// maximum token length of generated summaries - will truncate if greater -const MAX_SUMMARY_TOKEN_LENGTH = 1000; +const MAX_DOCUMENTS_DEFAULT = 3; +const MAX_TOKENS_DEFAULT = 1000; export const retrieveDocumentation = ({ @@ -27,7 +25,15 @@ export const retrieveDocumentation = searchDocAPI: ProductDocSearchAPI; logger: Logger; }): RetrieveDocumentationAPI => - async ({ searchTerm, connectorId, products, functionCalling, max = 3 }) => { + async ({ + searchTerm, + connectorId, + products, + functionCalling, + max = MAX_DOCUMENTS_DEFAULT, + maxDocumentTokens = MAX_TOKENS_DEFAULT, + tokenReductionStrategy = 'summarize', + }) => { try { const { results } = await searchDocAPI({ query: searchTerm, products, max }); @@ -36,23 +42,25 @@ export const retrieveDocumentation = const processedDocuments = await Promise.all( results.map(async (document) => { const tokenCount = countTokens(document.content); - const summarize = tokenCount >= MIN_TOKENS_TO_SUMMARIZE; + const docHasTooManyTokens = tokenCount >= maxDocumentTokens; log.debug( - `processing doc [${document.url}] - tokens : [${tokenCount}] - summarize: [${summarize}]` + `processing doc [${document.url}] - tokens : [${tokenCount}] - tooManyTokens: [${docHasTooManyTokens}]` ); - let content: string; - if (summarize) { - const extractResponse = await summarizeDocument({ - searchTerm, - documentContent: document.content, - outputAPI, - connectorId, - functionCalling, - }); - content = truncate(extractResponse.summary, MAX_SUMMARY_TOKEN_LENGTH); - } else { - content = document.content; + let content = document.content; + if (docHasTooManyTokens) { + if (tokenReductionStrategy === 'summarize') { + const extractResponse = await summarizeDocument({ + searchTerm, + documentContent: document.content, + outputAPI, + connectorId, + functionCalling, + }); + content = truncate(extractResponse.summary, maxDocumentTokens); + } else { + content = truncate(document.content, maxDocumentTokens); + } } log.debug(`done processing document [${document.url}]`); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts index ae4e8eb36468..1e0637fcd344 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -9,6 +9,9 @@ import type { KibanaRequest } from '@kbn/core/server'; import type { FunctionCallingMode } from '@kbn/inference-common'; import type { ProductName } from '@kbn/product-doc-common'; +/** + * Parameters for {@link RetrieveDocumentationAPI} + */ export interface RetrieveDocumentationParams { /** * The search term to perform semantic text with. @@ -16,15 +19,39 @@ export interface RetrieveDocumentationParams { */ searchTerm: string; /** - * Maximum number of documents to return + * Maximum number of documents to return. * Defaults to 3. */ max?: number; /** - * Optional list of products to restrict the search to + * Optional list of products to restrict the search to. */ products?: ProductName[]; + /** + * The maximum number of tokens to return *per document*. + * Documents exceeding this limit will go through token reduction. + * + * Defaults to `1000`. + */ + maxDocumentTokens?: number; + /** + * The token reduction strategy to apply for documents exceeding max token count. + * - truncate: Will keep the N first tokens + * - summarize: Will call the LLM asking to generate a contextualized summary of the document + * + * Overall, `summarize` is way more efficient, but significantly slower, given that an additional + * LLM call will be performed. + * + * Defaults to `summarize` + */ + tokenReductionStrategy?: 'truncate' | 'summarize'; + /** + * The request that initiated the task. + */ request: KibanaRequest; + /** + * Id of the LLM connector to use for the task. + */ connectorId: string; functionCalling?: FunctionCallingMode; } From 17d089c2066c62672be548062d1eafcaacf6d219 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 11 Nov 2024 09:21:42 +0100 Subject: [PATCH 75/97] Address review nits --- .../product-doc-common/src/artifact.test.ts | 15 +++++++ .../product-doc-common/src/artifact.ts | 2 +- x-pack/plugins/ai_infra/llm_tasks/README.md | 43 ++++++++++++++++++- .../summarize_document.ts | 2 +- .../ai_infra/llm_tasks/server/types.ts | 6 +++ .../product_doc_base/common/consts.ts | 4 ++ .../server/routes/installation.ts | 21 +++++++-- .../saved_objects/product_doc_install.ts | 2 +- .../settings_tab/product_doc_entry.tsx | 5 ++- 9 files changed, 92 insertions(+), 8 deletions(-) diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts index fb8af932ffb4..2b6362dbf4aa 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts @@ -26,6 +26,17 @@ describe('getArtifactName', () => { }) ).toEqual('kb-product-doc-elasticsearch-8.17'); }); + + it('generates a lowercase name', () => { + expect( + getArtifactName({ + // @ts-expect-error testing + productName: 'ElasticSearch', + productVersion: '8.17', + excludeExtension: true, + }) + ).toEqual('kb-product-doc-elasticsearch-8.17'); + }); }); describe('parseArtifactName', () => { @@ -46,4 +57,8 @@ describe('parseArtifactName', () => { it('returns undefined if the provided string does not match the artifact name pattern', () => { expect(parseArtifactName('some-wrong-name')).toEqual(undefined); }); + + it('returns undefined if the provided string is not strictly lowercase', () => { + expect(parseArtifactName('kb-product-doc-Security-8.17')).toEqual(undefined); + }); }); diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts index 4c69ade2ca5e..1a6745abd733 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts @@ -8,7 +8,7 @@ import { type ProductName, DocumentationProduct } from './product'; // kb-product-doc-elasticsearch-8.15.zip -const artifactNameRegexp = /^kb-product-doc-([a-zA-Z]+)-([0-9]+\.[0-9]+)(\.zip)?$/; +const artifactNameRegexp = /^kb-product-doc-([a-z]+)-([0-9]+\.[0-9]+)(\.zip)?$/; const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); export const getArtifactName = ({ diff --git a/x-pack/plugins/ai_infra/llm_tasks/README.md b/x-pack/plugins/ai_infra/llm_tasks/README.md index a7a37a4e0d62..e019d456cd65 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/README.md +++ b/x-pack/plugins/ai_infra/llm_tasks/README.md @@ -1,4 +1,45 @@ # LLM Tasks plugin -This plugin contains various LLM tasks +This plugin contains various LLM tasks. +## Retrieve documentation + +This task allows to retrieve documents from our Elastic product documentation. + +The task depends on the `product-doc-base` plugin, as this dependency is used +to install and manage the product documentation. + +### Checking if the task is available + +A `retrieveDocumentationAvailable` API is exposed from the start contract, that +should be used to assert that the `retrieve_doc` task can be used in the current +context. + +That API receive the inbound request as parameter. + +Example: +```ts +if (await llmTasksStart.retrieveDocumentationAvailable({ request })) { + // task is available +} else { + // task is not available +} +``` + +### Executing the task + +The task is executed as an API of the plugin's start contract, and can be invoked +as any other lifecycle API would. + +Example: +```ts +const result = await llmTasksStart.retrieveDocumentation({ + searchTerm: "How to create a space in Kibana?", + request, + connectorId: 'my-connector-id', +}); + +const { success, documents } = result; +``` + +The exhaustive list of options for the task is available on the `RetrieveDocumentationParams` type's TS doc. diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts index 2c7b879bd6dc..47bbdc4b3a6f 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts @@ -43,7 +43,7 @@ export const summarizeDocument = async ({ id: 'summarize_document', connectorId, functionCalling, - system: `You are an helpful Elastic assistant, and your current task is to help answering the user question. + system: `You are an helpful Elastic assistant, and your current task is to help answer the user's question. Given a question and a document, please provide a condensed version of the document that can be used to answer the question. diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts index bed92bccc0fc..d550e4398b50 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts @@ -18,8 +18,14 @@ export interface PluginStartDependencies { productDocBase: ProductDocBaseStartContract; } +/** + * Describes public llmTasks plugin contract returned at the `setup` stage. + */ export interface LlmTasksPluginSetup {} +/** + * Describes public llmTasks plugin contract returned at the `start` stage. + */ export interface LlmTasksPluginStart { /** * Checks if all prerequisites to use the `retrieveDocumentation` task diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts index 850e624fd057..1622df5ed865 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts @@ -7,4 +7,8 @@ export const productDocInstallStatusSavedObjectTypeName = 'product-doc-install-status'; +/** + * The id of the inference endpoint we're creating for our product doc indices. + * Could be replaced with the default elser 2 endpoint once the default endpoint feature is available. + */ export const internalElserInferenceId = 'kibana-internal-elser2'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index 247fad4beef4..4d7218eacac9 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -29,7 +29,14 @@ export const registerInstallationRoutes = ({ { path: INSTALLATION_STATUS_API_PATH, validate: false, - options: { access: 'internal', tags: ['access:manage_llm_product_doc'] }, + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, + }, }, async (ctx, req, res) => { const { installClient } = getServices(); @@ -51,7 +58,11 @@ export const registerInstallationRoutes = ({ validate: false, options: { access: 'internal', - tags: ['access:manage_llm_product_doc'], + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, timeout: { idleSocket: 20 * 60 * 1000 }, // install can take time. }, }, @@ -81,7 +92,11 @@ export const registerInstallationRoutes = ({ validate: false, options: { access: 'internal', - tags: ['access:manage_llm_product_doc'], + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, }, }, async (ctx, req, res) => { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts index 464d404d1fcc..47cf7eb50cdd 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts @@ -35,7 +35,7 @@ export const productDocInstallStatusSavedObjectType: SavedObjectsType - Installed + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.installProductDocInstalledLabel', + { defaultMessage: 'Installed' } + )} From 2dc68c2bbc2967922542dc519d69cf93828842ac Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 11 Nov 2024 09:44:13 +0100 Subject: [PATCH 76/97] more review nits --- .../services/package_installer/package_installer.ts | 11 ++++++----- .../package_installer/steps/populate_index.ts | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts index 463b698238f3..7739219c15dc 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -66,11 +66,12 @@ export class PackageInstaller { * Will not upgrade products that are not already installed */ async ensureUpToDate({}: {}) { - const repositoryVersions = await fetchArtifactVersions({ - artifactRepositoryUrl: this.artifactRepositoryUrl, - }); - - const installStatuses = await this.productDocClient.getInstallationStatus(); + const [repositoryVersions, installStatuses] = await Promise.all([ + fetchArtifactVersions({ + artifactRepositoryUrl: this.artifactRepositoryUrl, + }), + this.productDocClient.getInstallationStatus(), + ]); const toUpdate: Array<{ productName: ProductName; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts index 234b886b6f52..017757ca90b9 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts @@ -68,7 +68,7 @@ const indexContentFile = async ({ }); if (response.errors) { - const error = response.items.find((item) => item.index!.error)!.index!.error!; + const error = response.items.find((item) => item.index?.error)?.index?.error ?? 'unknown error'; throw new Error(`Error indexing documents: ${JSON.stringify(error)}`); } }; From e31a91340c454be66ce1c034b3a2fbd9b70572e3 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 11 Nov 2024 09:54:25 +0100 Subject: [PATCH 77/97] Add instructions for the retrieve doc function --- .../server/functions/documentation.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts index dc384a0d3120..6f1ed379f633 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts @@ -18,6 +18,14 @@ export async function registerDocumentationFunction({ }: FunctionRegistrationParameters) { const isProductDocAvailable = (await llmTasks.retrieveDocumentationAvailable()) ?? false; + functions.registerInstruction(({ availableFunctionNames }) => { + return availableFunctionNames.includes(RETRIEVE_DOCUMENTATION_NAME) + ? `When asked questions about the Elastic stack, You should use the ${RETRIEVE_DOCUMENTATION_NAME} function before answering, + to retrieve documentation related to the question. Consider that the documentation returned by the function + is always more up to date and accurate than any own internal knowledge you might have.` + : undefined; + }); + functions.registerFunction( { name: RETRIEVE_DOCUMENTATION_NAME, From 57ed9d4e20a16d766ce1fc912f562be09a210910 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 11 Nov 2024 09:56:45 +0100 Subject: [PATCH 78/97] nit --- .../server/functions/documentation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts index 6f1ed379f633..00072e0c79c4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts @@ -20,7 +20,7 @@ export async function registerDocumentationFunction({ functions.registerInstruction(({ availableFunctionNames }) => { return availableFunctionNames.includes(RETRIEVE_DOCUMENTATION_NAME) - ? `When asked questions about the Elastic stack, You should use the ${RETRIEVE_DOCUMENTATION_NAME} function before answering, + ? `When asked questions about the Elastic stack or products, You should use the ${RETRIEVE_DOCUMENTATION_NAME} function before answering, to retrieve documentation related to the question. Consider that the documentation returned by the function is always more up to date and accurate than any own internal knowledge you might have.` : undefined; From 3a087188eadf50a8134a2c7491c60ffe5df67945 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 11 Nov 2024 09:06:31 +0000 Subject: [PATCH 79/97] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- docs/developer/plugin-list.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index cc9d9bf40577..6915662b1eec 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -691,7 +691,7 @@ using the CURL scripts in the scripts folder. |{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks] -|This plugin contains various LLM tasks +|This plugin contains various LLM tasks. |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess] From 4feb090386351ad7c5c19806166c144728c69fa2 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 11 Nov 2024 12:53:46 +0100 Subject: [PATCH 80/97] update mappings --- packages/kbn-check-mappings-update-cli/current_mappings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 2be841c53db8..d36ded6f03ed 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2847,7 +2847,7 @@ "type": "keyword" }, "last_installation_date": { - "type": "integer" + "type": "date" }, "product_name": { "type": "keyword" From 7bb232256339070e03923b3dab31cfb05b31a713 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 11 Nov 2024 12:13:51 +0000 Subject: [PATCH 81/97] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 25652a79c917..73c59a31059e 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -145,7 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () => "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75", "osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4", "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352", - "product-doc-install-status": "bf0f1cb29850b2b5c45351d93fa7abfea3a39907", + "product-doc-install-status": "ca6e96840228e4cc2f11bae24a0797f4f7238c8c", "query": "501bece68f26fe561286a488eabb1a8ab12f1137", "risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb", "rules-settings": "ba57ef1881b3dcbf48fbfb28902d8f74442190b2", From 7af9f4e0958905f72da21f0857220eff12e218e6 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 11 Nov 2024 16:09:36 +0100 Subject: [PATCH 82/97] add more unit tests for retrieveDocumentation --- .../retrieve_documentation.test.ts | 134 +++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts index b551093aac43..5722b73ca039 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts @@ -7,24 +7,51 @@ import { httpServerMock } from '@kbn/core/server/mocks'; import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import type { DocSearchResult } from '@kbn/product-doc-base-plugin/server/services/search'; + import { retrieveDocumentation } from './retrieve_documentation'; +import { truncate, count as countTokens } from '../../utils/tokens'; +jest.mock('../../utils/tokens'); +const truncateMock = truncate as jest.MockedFn; +const countTokensMock = countTokens as jest.MockedFn; + +import { summarizeDocument } from './summarize_document'; +jest.mock('./summarize_document'); +const summarizeDocumentMock = summarizeDocument as jest.MockedFn; describe('retrieveDocumentation', () => { let logger: MockedLogger; + let request: ReturnType; let outputAPI: jest.Mock; let searchDocAPI: jest.Mock; let retrieve: ReturnType; + const createResult = (parts: Partial = {}): DocSearchResult => { + return { + title: 'title', + content: 'content', + url: 'url', + productName: 'kibana', + ...parts, + }; + }; + beforeEach(() => { logger = loggerMock.create(); + request = httpServerMock.createKibanaRequest(); outputAPI = jest.fn(); searchDocAPI = jest.fn(); retrieve = retrieveDocumentation({ logger, searchDocAPI, outputAPI }); }); + afterEach(() => { + summarizeDocumentMock.mockReset(); + truncateMock.mockReset(); + countTokensMock.mockReset(); + }); + it('calls the search API with the right parameters', async () => { searchDocAPI.mockResolvedValue({ results: [] }); - const request = httpServerMock.createKibanaRequest(); const result = await retrieve({ searchTerm: 'What is Kibana?', @@ -47,4 +74,109 @@ describe('retrieveDocumentation', () => { max: 5, }); }); + + it('reduces the document length using the truncate strategy', async () => { + searchDocAPI.mockResolvedValue({ + results: [ + createResult({ content: 'content-1' }), + createResult({ content: 'content-2' }), + createResult({ content: 'content-3' }), + ], + }); + + countTokensMock.mockImplementation((text) => { + if (text === 'content-2') { + return 150; + } else { + return 50; + } + }); + truncateMock.mockReturnValue('truncated'); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'truncate', + }); + + expect(result.documents.length).toEqual(3); + expect(result.documents[0].content).toEqual('content-1'); + expect(result.documents[1].content).toEqual('truncated'); + expect(result.documents[2].content).toEqual('content-3'); + + expect(truncateMock).toHaveBeenCalledTimes(1); + expect(truncateMock).toHaveBeenCalledWith('content-2', 100); + }); + + it('reduces the document length using the summarize strategy', async () => { + searchDocAPI.mockResolvedValue({ + results: [ + createResult({ content: 'content-1' }), + createResult({ content: 'content-2' }), + createResult({ content: 'content-3' }), + ], + }); + + countTokensMock.mockImplementation((text) => { + if (text === 'content-2') { + return 50; + } else { + return 150; + } + }); + truncateMock.mockImplementation((text) => text); + + summarizeDocumentMock.mockImplementation(({ documentContent }) => { + return Promise.resolve({ summary: `${documentContent}-summarized` }); + }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'summarize', + }); + + expect(result.documents.length).toEqual(3); + expect(result.documents[0].content).toEqual('content-1-summarized'); + expect(result.documents[1].content).toEqual('content-2'); + expect(result.documents[2].content).toEqual('content-3-summarized'); + + expect(truncateMock).toHaveBeenCalledTimes(2); + expect(truncateMock).toHaveBeenCalledWith('content-1-summarized', 100); + expect(truncateMock).toHaveBeenCalledWith('content-3-summarized', 100); + }); + + it('logs an error and return an empty list of docs in case of error', async () => { + searchDocAPI.mockResolvedValue({ + results: [createResult({ content: 'content-1' })], + }); + countTokensMock.mockImplementation(() => { + return 150; + }); + summarizeDocumentMock.mockImplementation(() => { + throw new Error('woups'); + }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'summarize', + }); + + expect(result).toEqual({ + success: false, + documents: [], + }); + + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Error retrieving documentation') + ); + }); }); From c00c642c8cf017d3c8e45c88204d40cf0e35e345 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 12 Nov 2024 15:17:48 +0100 Subject: [PATCH 83/97] use task manager for installation workflows --- .../ai_infra/product_doc_base/kibana.jsonc | 2 +- .../product_doc_base/server/plugin.test.ts | 35 ++++++++-- .../product_doc_base/server/plugin.ts | 45 +++++++----- .../product_doc_base/server/routes/index.ts | 4 +- .../server/routes/installation.ts | 15 ++-- .../server/tasks/ensure_up_to_date.ts | 70 +++++++++++++++++++ .../product_doc_base/server/tasks/index.ts | 29 ++++++++ .../server/tasks/install_all.ts | 70 +++++++++++++++++++ .../server/tasks/uninstall_all.ts | 70 +++++++++++++++++++ .../product_doc_base/server/tasks/utils.ts | 50 +++++++++++++ .../ai_infra/product_doc_base/server/types.ts | 18 +++-- 11 files changed, 371 insertions(+), 37 deletions(-) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc index 7053a73517e0..268b4a70c992 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc +++ b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc @@ -7,7 +7,7 @@ "server": true, "browser": true, "configPath": ["xpack", "productDocBase"], - "requiredPlugins": ["licensing"], + "requiredPlugins": ["licensing", "taskManager"], "requiredBundles": [], "optionalPlugins": [], "extraPublicDirs": [] diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts index da3d478d6e06..d1ce6ad72b97 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts @@ -7,38 +7,51 @@ import { coreMock } from '@kbn/core/server/mocks'; import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; import { ProductDocBasePlugin } from './plugin'; +import { ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies } from './types'; jest.mock('./services/package_installer'); jest.mock('./services/search'); jest.mock('./services/doc_install_status'); jest.mock('./routes'); +jest.mock('./tasks'); import { registerRoutes } from './routes'; import { PackageInstaller } from './services/package_installer'; +import { registerTaskDefinitions, scheduleEnsureUpToDateTask } from './tasks'; const PackageInstallMock = PackageInstaller as jest.Mock; describe('ProductDocBasePlugin', () => { let initContext: ReturnType; let plugin: ProductDocBasePlugin; + let pluginSetupDeps: ProductDocBaseSetupDependencies; + let pluginStartDeps: ProductDocBaseStartDependencies; beforeEach(() => { initContext = coreMock.createPluginInitializerContext(); plugin = new ProductDocBasePlugin(initContext); + pluginSetupDeps = { + taskManager: taskManagerMock.createSetup(), + }; + pluginStartDeps = { + licensing: licensingMock.createStart(), + taskManager: taskManagerMock.createStart(), + }; PackageInstallMock.mockReturnValue({ ensureUpToDate: jest.fn().mockResolvedValue({}) }); }); describe('#setup', () => { it('register the routes', () => { - plugin.setup(coreMock.createSetup(), {}); + plugin.setup(coreMock.createSetup(), pluginSetupDeps); expect(registerRoutes).toHaveBeenCalledTimes(1); }); it('register the product-doc SO type', () => { const coreSetup = coreMock.createSetup(); - plugin.setup(coreSetup, {}); + plugin.setup(coreSetup, pluginSetupDeps); expect(coreSetup.savedObjects.registerType).toHaveBeenCalledTimes(1); expect(coreSetup.savedObjects.registerType).toHaveBeenCalledWith( @@ -47,18 +60,28 @@ describe('ProductDocBasePlugin', () => { }) ); }); + it('register the task definitions', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + + expect(registerTaskDefinitions).toHaveBeenCalledTimes(1); + }); }); describe('#start', () => { it('returns a contract with the expected shape', () => { - plugin.setup(coreMock.createSetup(), {}); - const startContract = plugin.start(coreMock.createStart(), { - licensing: licensingMock.createStart(), - }); + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + const startContract = plugin.start(coreMock.createStart(), pluginStartDeps); expect(startContract).toEqual({ isInstalled: expect.any(Function), search: expect.any(Function), }); }); + + it('schedules the update task', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + plugin.start(coreMock.createStart(), pluginStartDeps); + + expect(scheduleEnsureUpToDateTask).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index 3ff052a40e6c..8ee0f8e47bb2 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -17,7 +17,7 @@ import { ProductDocBaseStartContract, ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies, - InternalRouteServices, + InternalServices, } from './types'; import { productDocInstallStatusSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; @@ -25,6 +25,7 @@ import { InferenceEndpointManager } from './services/inference_endpoint'; import { ProductDocInstallClient } from './services/doc_install_status'; import { SearchService } from './services/search'; import { registerRoutes } from './routes'; +import { registerTaskDefinitions, scheduleEnsureUpToDateTask } from './tasks'; export class ProductDocBasePlugin implements @@ -36,26 +37,33 @@ export class ProductDocBasePlugin > { private logger: Logger; - private routeServices?: InternalRouteServices; + private internalServices?: InternalServices; constructor(private readonly context: PluginInitializerContext) { this.logger = context.logger.get(); } setup( coreSetup: CoreSetup, - pluginsSetup: ProductDocBaseSetupDependencies + { taskManager }: ProductDocBaseSetupDependencies ): ProductDocBaseSetupContract { + const getServices = () => { + if (!this.internalServices) { + throw new Error('getServices called before #start'); + } + return this.internalServices; + }; + coreSetup.savedObjects.registerType(productDocInstallStatusSavedObjectType); + registerTaskDefinitions({ + taskManager, + getServices, + }); + const router = coreSetup.http.createRouter(); registerRoutes({ router, - getServices: () => { - if (!this.routeServices) { - throw new Error('getServices called before #start'); - } - return this.routeServices; - }, + getServices, }); return {}; @@ -63,13 +71,12 @@ export class ProductDocBasePlugin start( core: CoreStart, - { licensing }: ProductDocBaseStartDependencies + { licensing, taskManager }: ProductDocBaseStartDependencies ): ProductDocBaseStartContract { const soClient = new SavedObjectsClient( core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName]) ); const productDocClient = new ProductDocInstallClient({ soClient }); - const installClient = productDocClient; const endpointManager = new InferenceEndpointManager({ esClient: core.elasticsearch.client.asInternalUser, @@ -91,17 +98,19 @@ export class ProductDocBasePlugin logger: this.logger.get('search-service'), }); - // should we use taskManager for this? - packageInstaller.ensureUpToDate({}).catch((err) => { - this.logger.error(`Error checking if product documentation is up to date: ${err.message}`); - }); - - this.routeServices = { + this.internalServices = { + logger: this.logger, packageInstaller, - installClient, + installClient: productDocClient, licensing, + taskManager, }; + const tasksLogger = this.logger.get('tasks'); + scheduleEnsureUpToDateTask({ taskManager, logger: tasksLogger }).catch((err) => { + tasksLogger.error(`Error checking if product documentation is up to date: ${err.message}`); + }); + return { isInstalled: async () => { // can probably be improved. But is a boolean good enough then diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts index 2c9278217b3f..66660c199d81 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts @@ -7,14 +7,14 @@ import type { IRouter } from '@kbn/core/server'; import { registerInstallationRoutes } from './installation'; -import type { InternalRouteServices } from '../types'; +import type { InternalServices } from '../types'; export const registerRoutes = ({ router, getServices, }: { router: IRouter; - getServices: () => InternalRouteServices; + getServices: () => InternalServices; }) => { registerInstallationRoutes({ getServices, router }); }; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index 4d7218eacac9..18d33da3d4b7 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -15,15 +15,16 @@ import { UninstallResponse, } from '../../common/http_api/installation'; import { InstallationStatus } from '../../common/install_status'; -import type { InternalRouteServices } from '../types'; +import type { InternalServices } from '../types'; import { checkLicense } from '../services/package_installer'; +import { scheduleInstallAllTask, scheduleUninstallAllTask, waitUntilTaskCompleted } from '../tasks'; export const registerInstallationRoutes = ({ router, getServices, }: { router: IRouter; - getServices: () => InternalRouteServices; + getServices: () => InternalServices; }) => { router.get( { @@ -67,7 +68,7 @@ export const registerInstallationRoutes = ({ }, }, async (ctx, req, res) => { - const { packageInstaller, licensing } = getServices(); + const { licensing, taskManager, logger } = getServices(); const license = await licensing.getLicense(); if (!checkLicense(license)) { @@ -76,7 +77,8 @@ export const registerInstallationRoutes = ({ }); } - await packageInstaller.installAll({}); + const taskId = await scheduleInstallAllTask({ taskManager, logger }); + await waitUntilTaskCompleted({ taskId, taskManager, timeout: 10 * 60 }); return res.ok({ body: { @@ -100,9 +102,10 @@ export const registerInstallationRoutes = ({ }, }, async (ctx, req, res) => { - const { packageInstaller } = getServices(); + const { taskManager, logger } = getServices(); - await packageInstaller.uninstallAll(); + const taskId = await scheduleUninstallAllTask({ taskManager, logger }); + await waitUntilTaskCompleted({ taskId, taskManager, timeout: 10 * 60 * 1000 }); return res.ok({ body: { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts new file mode 100644 index 000000000000..d971561914ff --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const ENSURE_DOC_UP_TO_DATE_TASK_TYPE = 'ProductDocBase:EnsureUpToDate'; +export const ENSURE_DOC_UP_TO_DATE_TASK_ID = 'ProductDocBase:EnsureUpToDate'; + +export const registerEnsureUpToDateTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [ENSURE_DOC_UP_TO_DATE_TASK_TYPE]: { + title: 'Ensure product documentation up to date task', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.ensureUpToDate({}); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleEnsureUpToDateTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: ENSURE_DOC_UP_TO_DATE_TASK_ID, + taskType: ENSURE_DOC_UP_TO_DATE_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(ENSURE_DOC_UP_TO_DATE_TASK_ID); + + logger.info(`Task ${ENSURE_DOC_UP_TO_DATE_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return ENSURE_DOC_UP_TO_DATE_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts new file mode 100644 index 000000000000..865044a6e8a8 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { registerEnsureUpToDateTaskDefinition } from './ensure_up_to_date'; +import { registerInstallAllTaskDefinition } from './install_all'; +import { registerUninstallAllTaskDefinition } from './uninstall_all'; + +export const registerTaskDefinitions = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + registerEnsureUpToDateTaskDefinition({ getServices, taskManager }); + registerInstallAllTaskDefinition({ getServices, taskManager }); + registerUninstallAllTaskDefinition({ getServices, taskManager }); +}; + +export { scheduleEnsureUpToDateTask } from './ensure_up_to_date'; +export { scheduleInstallAllTask } from './install_all'; +export { scheduleUninstallAllTask } from './uninstall_all'; +export { waitUntilTaskCompleted } from './utils'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts new file mode 100644 index 000000000000..0d2cc48fb06b --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const INSTALL_ALL_TASK_TYPE = 'ProductDocBase:InstallAll'; +export const INSTALL_ALL_TASK_ID = 'ProductDocBase:InstallAll'; + +export const registerInstallAllTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [INSTALL_ALL_TASK_TYPE]: { + title: 'Install all product documentation artifacts', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.installAll({}); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleInstallAllTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: INSTALL_ALL_TASK_ID, + taskType: INSTALL_ALL_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(INSTALL_ALL_TASK_ID); + + logger.info(`Task ${INSTALL_ALL_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return INSTALL_ALL_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts new file mode 100644 index 000000000000..6a88fec205dd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const UNINSTALL_ALL_TASK_TYPE = 'ProductDocBase:UninstallAll'; +export const UNINSTALL_ALL_TASK_ID = 'ProductDocBase:UninstallAll'; + +export const registerUninstallAllTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [UNINSTALL_ALL_TASK_TYPE]: { + title: 'Uninstall all product documentation artifacts', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.uninstallAll(); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleUninstallAllTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: UNINSTALL_ALL_TASK_ID, + taskType: UNINSTALL_ALL_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(UNINSTALL_ALL_TASK_ID); + + logger.info(`Task ${UNINSTALL_ALL_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return UNINSTALL_ALL_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts new file mode 100644 index 000000000000..7a2c5800b0b0 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; + +export const isTaskCurrentlyRunningError = (err: Error): boolean => { + return err.message?.includes('currently running'); +}; + +export const waitUntilTaskCompleted = async ({ + taskManager, + taskId, + timeout = 120_000, + interval = 5_000, +}: { + taskManager: TaskManagerStartContract; + taskId: string; + timeout?: number; + interval?: number; +}): Promise => { + const start = Date.now(); + const max = start + timeout; + let now = start; + while (now < max) { + try { + const taskInstance = await taskManager.get(taskId); + const { status } = taskInstance; + if (status === 'idle' || status === 'claiming' || status === 'running') { + await sleep(interval); + now = Date.now(); + } else { + return false; + } + } catch (e) { + if (SavedObjectsErrorHelpers.isNotFoundError(e)) { + // not found means the task was completed and the entry removed + return true; + } + } + } + + throw new Error(`Timeout waiting for task ${taskId} to complete.`); +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts index 0debfa98a6a2..ad24de5ff303 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts @@ -5,17 +5,25 @@ * 2.0. */ +import type { Logger } from '@kbn/logging'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; import type { SearchApi } from './services/search'; -import { ProductDocInstallClient } from './services/doc_install_status'; -import { PackageInstaller } from './services/package_installer'; +import type { ProductDocInstallClient } from './services/doc_install_status'; +import type { PackageInstaller } from './services/package_installer'; /* eslint-disable @typescript-eslint/no-empty-interface*/ -export interface ProductDocBaseSetupDependencies {} +export interface ProductDocBaseSetupDependencies { + taskManager: TaskManagerSetupContract; +} export interface ProductDocBaseStartDependencies { licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; } export interface ProductDocBaseSetupContract {} @@ -25,8 +33,10 @@ export interface ProductDocBaseStartContract { isInstalled: () => Promise; } -export interface InternalRouteServices { +export interface InternalServices { + logger: Logger; installClient: ProductDocInstallClient; packageInstaller: PackageInstaller; licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; } From 5fdd08dd5bccd52b199c0a3f5021ebabb0bf57fe Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:38:06 +0000 Subject: [PATCH 84/97] [CI] Auto-commit changed files from 'node scripts/notice' --- x-pack/plugins/ai_infra/product_doc_base/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json index 9d6fa2ff9f36..9a2d1969556b 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json +++ b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json @@ -24,5 +24,6 @@ "@kbn/core-http-browser", "@kbn/logging-mocks", "@kbn/licensing-plugin", + "@kbn/task-manager-plugin", ] } From 0764c0ef1c15c6c754d216895c401d84500054c8 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 13 Nov 2024 09:35:52 +0100 Subject: [PATCH 85/97] create global manager for documentation installation --- .../ai_infra/llm_tasks/server/plugin.ts | 3 +- .../product_doc_base/server/plugin.ts | 28 +-- .../server/routes/installation.ts | 38 +--- .../utils => doc_manager}/check_license.ts | 0 .../services/doc_manager/doc_manager.ts | 162 ++++++++++++++++++ .../server/services/doc_manager/index.ts | 15 ++ .../server/services/doc_manager/types.ts | 79 +++++++++ .../services/package_installer/utils/index.ts | 1 - .../product_doc_base/server/tasks/index.ts | 8 +- .../product_doc_base/server/tasks/utils.ts | 18 ++ .../ai_infra/product_doc_base/server/types.ts | 4 +- 11 files changed, 309 insertions(+), 47 deletions(-) rename x-pack/plugins/ai_infra/product_doc_base/server/services/{package_installer/utils => doc_manager}/check_license.ts (100%) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts index 4e34a48452f0..d10c495ece15 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts @@ -41,7 +41,8 @@ export class LlmTasksPlugin const { inference, productDocBase } = startDependencies; return { retrieveDocumentationAvailable: async () => { - return await startDependencies.productDocBase.isInstalled(); + const docBaseStatus = await startDependencies.productDocBase.management.getStatus(); + return docBaseStatus.status === 'installed'; }, retrieveDocumentation: (options) => { return retrieveDocumentation({ diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index 8ee0f8e47bb2..bd3decfbb6d5 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -23,9 +23,10 @@ import { productDocInstallStatusSavedObjectType } from './saved_objects'; import { PackageInstaller } from './services/package_installer'; import { InferenceEndpointManager } from './services/inference_endpoint'; import { ProductDocInstallClient } from './services/doc_install_status'; +import { DocumentationManager } from './services/doc_manager'; import { SearchService } from './services/search'; import { registerRoutes } from './routes'; -import { registerTaskDefinitions, scheduleEnsureUpToDateTask } from './tasks'; +import { registerTaskDefinitions } from './tasks'; export class ProductDocBasePlugin implements @@ -98,27 +99,32 @@ export class ProductDocBasePlugin logger: this.logger.get('search-service'), }); + const documentationManager = new DocumentationManager({ + logger: this.logger.get('doc-manager'), + docInstallClient: productDocClient, + licensing, + taskManager, + }); + this.internalServices = { logger: this.logger, packageInstaller, installClient: productDocClient, + documentationManager, licensing, taskManager, }; - const tasksLogger = this.logger.get('tasks'); - scheduleEnsureUpToDateTask({ taskManager, logger: tasksLogger }).catch((err) => { - tasksLogger.error(`Error checking if product documentation is up to date: ${err.message}`); + documentationManager.update().catch((err) => { + this.logger.error(`Error scheduling product documentation update task: ${err.message}`); }); return { - isInstalled: async () => { - // can probably be improved. But is a boolean good enough then - const installStatus = await productDocClient.getInstallationStatus(); - const installed = Object.values(installStatus).some( - (status) => status.status === 'installed' - ); - return installed; + management: { + install: documentationManager.install.bind(documentationManager), + update: documentationManager.update.bind(documentationManager), + uninstall: documentationManager.uninstall.bind(documentationManager), + getStatus: documentationManager.getStatus.bind(documentationManager), }, search: searchService.search.bind(searchService), }; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index 18d33da3d4b7..b90353d69294 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -14,10 +14,7 @@ import { PerformInstallResponse, UninstallResponse, } from '../../common/http_api/installation'; -import { InstallationStatus } from '../../common/install_status'; import type { InternalServices } from '../types'; -import { checkLicense } from '../services/package_installer'; -import { scheduleInstallAllTask, scheduleUninstallAllTask, waitUntilTaskCompleted } from '../tasks'; export const registerInstallationRoutes = ({ router, @@ -40,9 +37,9 @@ export const registerInstallationRoutes = ({ }, }, async (ctx, req, res) => { - const { installClient } = getServices(); + const { installClient, documentationManager } = getServices(); const installStatus = await installClient.getInstallationStatus(); - const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status)); + const { status: overallStatus } = await documentationManager.getStatus(); return res.ok({ body: { @@ -68,17 +65,12 @@ export const registerInstallationRoutes = ({ }, }, async (ctx, req, res) => { - const { licensing, taskManager, logger } = getServices(); + const { documentationManager } = getServices(); - const license = await licensing.getLicense(); - if (!checkLicense(license)) { - return res.badRequest({ - body: 'Elastic documentation requires an enterprise license', - }); - } - - const taskId = await scheduleInstallAllTask({ taskManager, logger }); - await waitUntilTaskCompleted({ taskId, taskManager, timeout: 10 * 60 }); + await documentationManager.install({ + force: false, + wait: true, + }); return res.ok({ body: { @@ -102,10 +94,9 @@ export const registerInstallationRoutes = ({ }, }, async (ctx, req, res) => { - const { taskManager, logger } = getServices(); + const { documentationManager } = getServices(); - const taskId = await scheduleUninstallAllTask({ taskManager, logger }); - await waitUntilTaskCompleted({ taskId, taskManager, timeout: 10 * 60 * 1000 }); + await documentationManager.uninstall({ wait: true }); return res.ok({ body: { @@ -115,14 +106,3 @@ export const registerInstallationRoutes = ({ } ); }; - -const getOverallStatus = (statuses: InstallationStatus[]): InstallationStatus => { - for (const status of statusOrder) { - if (statuses.includes(status)) { - return status; - } - } - return 'installed'; -}; - -const statusOrder: InstallationStatus[] = ['error', 'installing', 'uninstalled', 'installed']; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/check_license.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts similarity index 100% rename from x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/check_license.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts new file mode 100644 index 000000000000..6d17b09b9e91 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import { type TaskManagerStartContract, TaskStatus } from '@kbn/task-manager-plugin/server'; +import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { InstallationStatus } from '../../../common/install_status'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import { + INSTALL_ALL_TASK_ID, + scheduleInstallAllTask, + scheduleUninstallAllTask, + scheduleEnsureUpToDateTask, + getTaskStatus, + waitUntilTaskCompleted, +} from '../../tasks'; +import { checkLicense } from './check_license'; +import type { + DocumentationManagerAPI, + DocGetStatusResponse, + DocInstallOptions, + DocUninstallOptions, + DocUpdateOptions, +} from './types'; + +const TEN_MIN_IN_MS = 10 * 60 * 1000; + +/** + * High-level installation service, handling product documentation + * installation as unary operations, abstracting away the fact + * that documentation is composed of multiple entities. + */ +export class DocumentationManager implements DocumentationManagerAPI { + private logger: Logger; + private taskManager: TaskManagerStartContract; + private licensing: LicensingPluginStart; + private docInstallClient: ProductDocInstallClient; + + constructor({ + logger, + taskManager, + licensing, + docInstallClient, + }: { + logger: Logger; + taskManager: TaskManagerStartContract; + licensing: LicensingPluginStart; + docInstallClient: ProductDocInstallClient; + }) { + this.logger = logger; + this.taskManager = taskManager; + this.licensing = licensing; + this.docInstallClient = docInstallClient; + } + + async install(options: DocInstallOptions = {}): Promise { + const { force = false, wait = false } = options; + + const { status } = await this.getStatus(); + if (!force && status === 'installed') { + return; + } + + const license = await this.licensing.getLicense(); + if (!checkLicense(license)) { + throw new Error('Elastic documentation requires an enterprise license'); + } + + const taskId = await scheduleInstallAllTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async update(options: DocUpdateOptions = {}): Promise { + const { wait = false } = options; + + const taskId = await scheduleEnsureUpToDateTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async uninstall(options: DocUninstallOptions = {}): Promise { + const { wait = false } = options; + + const taskId = await scheduleUninstallAllTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async getStatus(): Promise { + const taskStatus = await getTaskStatus({ + taskManager: this.taskManager, + taskId: INSTALL_ALL_TASK_ID, + }); + if (taskStatus !== 'not_scheduled') { + const status = convertTaskStatus(taskStatus); + if (status !== 'unknown') { + return { status }; + } + } + + const installStatus = await this.docInstallClient.getInstallationStatus(); + const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status)); + return { status: overallStatus }; + } +} + +const convertTaskStatus = (taskStatus: TaskStatus): InstallationStatus | 'unknown' => { + switch (taskStatus) { + case TaskStatus.Idle: + case TaskStatus.Claiming: + case TaskStatus.Running: + return 'installing'; + case TaskStatus.Failed: + return 'error'; + case TaskStatus.Unrecognized: + case TaskStatus.DeadLetter: + case TaskStatus.ShouldDelete: + return 'unknown'; + } +}; + +const getOverallStatus = (statuses: InstallationStatus[]): InstallationStatus => { + const statusOrder: InstallationStatus[] = ['error', 'installing', 'uninstalled', 'installed']; + for (const status of statusOrder) { + if (statuses.includes(status)) { + return status; + } + } + return 'installed'; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts new file mode 100644 index 000000000000..588b5e2f5cc6 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { DocumentationManager } from './doc_manager'; +export type { + DocumentationManagerAPI, + DocUninstallOptions, + DocInstallOptions, + DocUpdateOptions, + DocGetStatusResponse, +} from './types'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts new file mode 100644 index 000000000000..852c7305c3ac --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InstallationStatus } from '../../../common/install_status'; + +/** + * APIs to manage the product documentation. + */ +export interface DocumentationManagerAPI { + /** + * Install the product documentation. + * By default, will only try to install if not already present. + * Can use the `force` option to forcefully reinstall. + */ + install(options?: DocInstallOptions): Promise; + /** + * Update the product documentation to the latest version. + * No-op if the product documentation is not currently installed. + */ + update(options?: DocUpdateOptions): Promise; + /** + * Uninstall the product documentation. + * No-op if the product documentation is not currently installed. + */ + uninstall(options?: DocUninstallOptions): Promise; + /** + * Returns the overall installation status of the documentation. + */ + getStatus(): Promise; +} + +/** + * Return type for {@link DocumentationManagerAPI.getStatus} + */ +export interface DocGetStatusResponse { + status: InstallationStatus; +} + +/** + * Options for {@link DocumentationManagerAPI.install} + */ +export interface DocInstallOptions { + /** + * If true, will reinstall the documentation even if already present. + * Defaults to `false` + */ + force?: boolean; + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} + +/** + * Options for {@link DocumentationManagerAPI.uninstall} + */ +export interface DocUninstallOptions { + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} + +/** + * Options for {@link DocumentationManagerAPI.update} + */ +export interface DocUpdateOptions { + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts index a980b86df623..a612a8c6e9f4 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts @@ -8,4 +8,3 @@ export { downloadToDisk } from './download'; export { openZipArchive, type ZipArchive } from './zip_archive'; export { loadManifestFile, loadMappingFile } from './archive_accessors'; -export { checkLicense } from './check_license'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts index 865044a6e8a8..0b5833055fd8 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts @@ -23,7 +23,7 @@ export const registerTaskDefinitions = ({ registerUninstallAllTaskDefinition({ getServices, taskManager }); }; -export { scheduleEnsureUpToDateTask } from './ensure_up_to_date'; -export { scheduleInstallAllTask } from './install_all'; -export { scheduleUninstallAllTask } from './uninstall_all'; -export { waitUntilTaskCompleted } from './utils'; +export { scheduleEnsureUpToDateTask, ENSURE_DOC_UP_TO_DATE_TASK_ID } from './ensure_up_to_date'; +export { scheduleInstallAllTask, INSTALL_ALL_TASK_ID } from './install_all'; +export { scheduleUninstallAllTask, UNINSTALL_ALL_TASK_ID } from './uninstall_all'; +export { waitUntilTaskCompleted, getTaskStatus } from './utils'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts index 7a2c5800b0b0..13d9e26dc167 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts @@ -8,6 +8,24 @@ import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; +export const getTaskStatus = async ({ + taskManager, + taskId, +}: { + taskManager: TaskManagerStartContract; + taskId: string; +}) => { + try { + const taskInstance = await taskManager.get(taskId); + return taskInstance.status; + } catch (e) { + if (isTaskCurrentlyRunningError(e)) { + return 'not_scheduled' as const; + } + throw e; + } +}; + export const isTaskCurrentlyRunningError = (err: Error): boolean => { return err.message?.includes('currently running'); }; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts index ad24de5ff303..f00943b69670 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts @@ -14,6 +14,7 @@ import type { import type { SearchApi } from './services/search'; import type { ProductDocInstallClient } from './services/doc_install_status'; import type { PackageInstaller } from './services/package_installer'; +import type { DocumentationManager, DocumentationManagerAPI } from './services/doc_manager'; /* eslint-disable @typescript-eslint/no-empty-interface*/ @@ -30,13 +31,14 @@ export interface ProductDocBaseSetupContract {} export interface ProductDocBaseStartContract { search: SearchApi; - isInstalled: () => Promise; + management: DocumentationManagerAPI; } export interface InternalServices { logger: Logger; installClient: ProductDocInstallClient; packageInstaller: PackageInstaller; + documentationManager: DocumentationManager; licensing: LicensingPluginStart; taskManager: TaskManagerStartContract; } From be74940db5a59f0535b9c395c2bf7bf31beca0fe Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 13 Nov 2024 09:49:47 +0100 Subject: [PATCH 86/97] fix getTaskStatus --- .../plugins/ai_infra/product_doc_base/server/tasks/utils.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts index 13d9e26dc167..7b8d20234c24 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts @@ -19,8 +19,9 @@ export const getTaskStatus = async ({ const taskInstance = await taskManager.get(taskId); return taskInstance.status; } catch (e) { - if (isTaskCurrentlyRunningError(e)) { - return 'not_scheduled' as const; + // not found means the task was completed and the entry removed + if (SavedObjectsErrorHelpers.isNotFoundError(e)) { + return 'not_scheduled'; } throw e; } From 5f7dbdaf80cb46cc2025b54e9b0160c29a9491e7 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 13 Nov 2024 10:26:04 +0100 Subject: [PATCH 87/97] fixes due to merge --- .../inference_endpoint/utils/wait_until_model_deployed.ts | 1 - .../product_doc_base/server/services/package_installer/index.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts index 53f022174fef..83775ed80f5a 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts @@ -25,7 +25,6 @@ export const waitUntilModelDeployed = async ({ model_id: modelId, }); const deploymentStats = statsRes.trained_model_stats[0]?.deployment_stats; - // @ts-expect-error deploymentStats.nodes not defined as array even if it is. if (!deploymentStats || deploymentStats.nodes.length === 0) { log.debug(`ML model [${modelId}] was not deployed - attempt ${i + 1} of ${maxRetries}`); await sleep(delay); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts index 1b0a9595b96d..a9edb7c38fda 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts @@ -6,4 +6,3 @@ */ export { PackageInstaller } from './package_installer'; -export { checkLicense } from './utils'; From 27abf468bf205d9666fb83be1abc7a419e43ae27 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 13 Nov 2024 15:45:25 +0100 Subject: [PATCH 88/97] add audit logging for product doc --- docs/user/security/audit-logging.asciidoc | 9 ++++ .../product_doc_base/server/plugin.test.ts | 13 ++++- .../product_doc_base/server/plugin.ts | 1 + .../server/routes/installation.ts | 6 ++- .../services/doc_manager/doc_manager.ts | 47 +++++++++++++++++-- .../server/services/doc_manager/types.ts | 19 ++++++++ 6 files changed, 89 insertions(+), 6 deletions(-) diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc index 1ac40bcc7764..7e6a7eee1c19 100644 --- a/docs/user/security/audit-logging.asciidoc +++ b/docs/user/security/audit-logging.asciidoc @@ -148,6 +148,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Creating trained model. | `failure` | Failed to create trained model. +.1+| `product_documentation_create` +| `unknown` | User requested to install the product documentation. + 3+a| ====== Type: change @@ -334,6 +337,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Updating trained model deployment. | `failure` | Failed to update trained model deployment. +.1+| `product_documentation_update` +| `unknown` | User requested to update the product documentation. + 3+a| ====== Type: deletion @@ -425,6 +431,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Deleting trained model. | `failure` | Failed to delete trained model. +.1+| `product_documentation_delete` +| `unknown` | User requested to delete the product documentation. + 3+a| ====== Type: access diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts index d1ce6ad72b97..bd5d6a720dd7 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts @@ -43,6 +43,10 @@ describe('ProductDocBasePlugin', () => { PackageInstallMock.mockReturnValue({ ensureUpToDate: jest.fn().mockResolvedValue({}) }); }); + afterEach(() => { + (scheduleEnsureUpToDateTask as jest.Mock).mockReset(); + }); + describe('#setup', () => { it('register the routes', () => { plugin.setup(coreMock.createSetup(), pluginSetupDeps); @@ -63,7 +67,7 @@ describe('ProductDocBasePlugin', () => { it('register the task definitions', () => { plugin.setup(coreMock.createSetup(), pluginSetupDeps); - expect(registerTaskDefinitions).toHaveBeenCalledTimes(1); + expect(registerTaskDefinitions).toHaveBeenCalledTimes(3); }); }); @@ -72,7 +76,12 @@ describe('ProductDocBasePlugin', () => { plugin.setup(coreMock.createSetup(), pluginSetupDeps); const startContract = plugin.start(coreMock.createStart(), pluginStartDeps); expect(startContract).toEqual({ - isInstalled: expect.any(Function), + management: { + getStatus: expect.any(Function), + install: expect.any(Function), + uninstall: expect.any(Function), + update: expect.any(Function), + }, search: expect.any(Function), }); }); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts index bd3decfbb6d5..c8ed100cabb1 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -104,6 +104,7 @@ export class ProductDocBasePlugin docInstallClient: productDocClient, licensing, taskManager, + auditService: core.security.audit, }); this.internalServices = { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index b90353d69294..08f4ea71c37f 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -68,6 +68,7 @@ export const registerInstallationRoutes = ({ const { documentationManager } = getServices(); await documentationManager.install({ + request: req, force: false, wait: true, }); @@ -96,7 +97,10 @@ export const registerInstallationRoutes = ({ async (ctx, req, res) => { const { documentationManager } = getServices(); - await documentationManager.uninstall({ wait: true }); + await documentationManager.uninstall({ + request: req, + wait: true, + }); return res.ok({ body: { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts index 6d17b09b9e91..53ce55ae3be0 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts @@ -6,6 +6,7 @@ */ import type { Logger } from '@kbn/logging'; +import type { CoreAuditService } from '@kbn/core/server'; import { type TaskManagerStartContract, TaskStatus } from '@kbn/task-manager-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; import type { InstallationStatus } from '../../../common/install_status'; @@ -39,26 +40,30 @@ export class DocumentationManager implements DocumentationManagerAPI { private taskManager: TaskManagerStartContract; private licensing: LicensingPluginStart; private docInstallClient: ProductDocInstallClient; + private auditService: CoreAuditService; constructor({ logger, taskManager, licensing, docInstallClient, + auditService, }: { logger: Logger; taskManager: TaskManagerStartContract; licensing: LicensingPluginStart; docInstallClient: ProductDocInstallClient; + auditService: CoreAuditService; }) { this.logger = logger; this.taskManager = taskManager; this.licensing = licensing; this.docInstallClient = docInstallClient; + this.auditService = auditService; } async install(options: DocInstallOptions = {}): Promise { - const { force = false, wait = false } = options; + const { request, force = false, wait = false } = options; const { status } = await this.getStatus(); if (!force && status === 'installed') { @@ -75,6 +80,18 @@ export class DocumentationManager implements DocumentationManagerAPI { logger: this.logger, }); + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting installation of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_create', + category: ['database'], + type: ['creation'], + outcome: 'unknown', + }, + }); + } + if (wait) { await waitUntilTaskCompleted({ taskManager: this.taskManager, @@ -85,13 +102,25 @@ export class DocumentationManager implements DocumentationManagerAPI { } async update(options: DocUpdateOptions = {}): Promise { - const { wait = false } = options; + const { request, wait = false } = options; const taskId = await scheduleEnsureUpToDateTask({ taskManager: this.taskManager, logger: this.logger, }); + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting update of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_update', + category: ['database'], + type: ['change'], + outcome: 'unknown', + }, + }); + } + if (wait) { await waitUntilTaskCompleted({ taskManager: this.taskManager, @@ -102,13 +131,25 @@ export class DocumentationManager implements DocumentationManagerAPI { } async uninstall(options: DocUninstallOptions = {}): Promise { - const { wait = false } = options; + const { request, wait = false } = options; const taskId = await scheduleUninstallAllTask({ taskManager: this.taskManager, logger: this.logger, }); + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting deletion of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_delete', + category: ['database'], + type: ['deletion'], + outcome: 'unknown', + }, + }); + } + if (wait) { await waitUntilTaskCompleted({ taskManager: this.taskManager, diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts index 852c7305c3ac..5a954a5ffb0f 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { KibanaRequest } from '@kbn/core/server'; import type { InstallationStatus } from '../../../common/install_status'; /** @@ -44,6 +45,12 @@ export interface DocGetStatusResponse { * Options for {@link DocumentationManagerAPI.install} */ export interface DocInstallOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; /** * If true, will reinstall the documentation even if already present. * Defaults to `false` @@ -60,6 +67,12 @@ export interface DocInstallOptions { * Options for {@link DocumentationManagerAPI.uninstall} */ export interface DocUninstallOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; /** * If true, the returned promise will wait until the update task has completed before resolving. * Defaults to `false` @@ -71,6 +84,12 @@ export interface DocUninstallOptions { * Options for {@link DocumentationManagerAPI.update} */ export interface DocUpdateOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; /** * If true, the returned promise will wait until the update task has completed before resolving. * Defaults to `false` From 34ba91c42175e38b276974cff5f8d6a5ab943333 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 14 Nov 2024 09:57:17 +0100 Subject: [PATCH 89/97] add tests for doc manager --- .../services/doc_manager/doc_manager.test.ts | 247 ++++++++++++++++++ .../services/doc_manager/doc_manager.ts | 1 + 2 files changed, 248 insertions(+) create mode 100644 x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts new file mode 100644 index 000000000000..0be913ee6dd7 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import { securityServiceMock, httpServerMock } from '@kbn/core/server/mocks'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import { DocumentationManager } from './doc_manager'; + +jest.mock('../../tasks'); +import { + scheduleInstallAllTask, + scheduleUninstallAllTask, + scheduleEnsureUpToDateTask, + getTaskStatus, + waitUntilTaskCompleted, +} from '../../tasks'; + +const scheduleInstallAllTaskMock = scheduleInstallAllTask as jest.MockedFn< + typeof scheduleInstallAllTask +>; +const scheduleUninstallAllTaskMock = scheduleUninstallAllTask as jest.MockedFn< + typeof scheduleUninstallAllTask +>; +const scheduleEnsureUpToDateTaskMock = scheduleEnsureUpToDateTask as jest.MockedFn< + typeof scheduleEnsureUpToDateTask +>; +const waitUntilTaskCompletedMock = waitUntilTaskCompleted as jest.MockedFn< + typeof waitUntilTaskCompleted +>; +const getTaskStatusMock = getTaskStatus as jest.MockedFn; + +describe('DocumentationManager', () => { + let logger: MockedLogger; + let taskManager: ReturnType; + let licensing: ReturnType; + let auditService: ReturnType['audit']; + let docInstallClient: jest.Mocked; + + let docManager: DocumentationManager; + + beforeEach(() => { + logger = loggerMock.create(); + taskManager = taskManagerMock.createStart(); + licensing = licensingMock.createStart(); + auditService = securityServiceMock.createStart().audit; + + docInstallClient = { + getInstallationStatus: jest.fn(), + } as unknown as jest.Mocked; + + docManager = new DocumentationManager({ + logger, + taskManager, + licensing, + auditService, + docInstallClient, + }); + }); + + afterEach(() => { + scheduleInstallAllTaskMock.mockReset(); + scheduleUninstallAllTaskMock.mockReset(); + scheduleEnsureUpToDateTaskMock.mockReset(); + waitUntilTaskCompletedMock.mockReset(); + getTaskStatusMock.mockReset(); + }); + + describe('#install', () => { + beforeEach(() => { + licensing.getLicense.mockResolvedValue( + licensingMock.createLicense({ license: { type: 'enterprise' } }) + ); + + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleInstallAllTask`', async () => { + await docManager.install({}); + + expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleInstallAllTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.install({ wait: true }); + + expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('does not call scheduleInstallAllTask if already installed and not force', async () => { + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'installed' }, + } as Awaited>); + + await docManager.install({ wait: true }); + + expect(scheduleInstallAllTaskMock).not.toHaveBeenCalled(); + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.install({ force: false, wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_create', + category: ['database'], + type: ['creation'], + outcome: 'unknown', + }, + }); + }); + + it('throws an error if license level is not sufficient', async () => { + licensing.getLicense.mockResolvedValue( + licensingMock.createLicense({ license: { type: 'basic' } }) + ); + + await expect( + docManager.install({ force: false, wait: false }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Elastic documentation requires an enterprise license"` + ); + }); + }); + + describe('#update', () => { + beforeEach(() => { + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleEnsureUpToDateTask`', async () => { + await docManager.update({}); + + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.update({ wait: true }); + + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.update({ wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_update', + category: ['database'], + type: ['change'], + outcome: 'unknown', + }, + }); + }); + }); + + describe('#uninstall', () => { + beforeEach(() => { + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleUninstallAllTask`', async () => { + await docManager.uninstall({}); + + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.uninstall({ wait: true }); + + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.uninstall({ wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_delete', + category: ['database'], + type: ['deletion'], + outcome: 'unknown', + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts index 53ce55ae3be0..40dc53e19cee 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts @@ -188,6 +188,7 @@ const convertTaskStatus = (taskStatus: TaskStatus): InstallationStatus | 'unknow case TaskStatus.Unrecognized: case TaskStatus.DeadLetter: case TaskStatus.ShouldDelete: + default: return 'unknown'; } }; From 407430da59a069eab9fbd2fe96443dcf53e6bd56 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 14 Nov 2024 12:36:44 +0100 Subject: [PATCH 90/97] fix response ops ftr test --- .../test_suites/task_manager/check_registered_task_types.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index c8056c2ee205..f6355d0ae5fa 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -49,6 +49,9 @@ export default function ({ getService }: FtrProviderContext) { 'Fleet-Usage-Logger', 'Fleet-Usage-Sender', 'ML:saved-objects-sync', + 'ProductDocBase:EnsureUpToDate', + 'ProductDocBase:InstallAll', + 'ProductDocBase:UninstallAll', 'SLO:ORPHAN_SUMMARIES-CLEANUP-TASK', 'Synthetics:Clean-Up-Package-Policies', 'UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects', From f2cd9e6ea441d70bc2af9146f341484469ef5e44 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 14 Nov 2024 14:42:53 +0100 Subject: [PATCH 91/97] remove semantic text for ai_subtitle --- .../product-doc-artifact-builder/README.md | 48 ++++++++++++++++++- .../src/artifact/mappings.ts | 5 +- .../src/tasks/create_index.ts | 5 +- .../product-doc-common/src/documents.ts | 2 +- .../server/services/search/perform_search.ts | 8 +--- 5 files changed, 51 insertions(+), 17 deletions(-) diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md b/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md index eb64d53b5b8f..49949def3e5e 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md @@ -1,3 +1,49 @@ # @kbn/product-doc-artifact-builder -Script to build the knowledge base artifacts +Script to build the knowledge base artifacts. + +## How to run + +``` +node scripts/build_product_doc_artifacts.js --stack-version {version} --product-name {product} +``` + +### parameters + +#### `stack-version`: + +the stack version to generate the artifacts for. + +#### `product-name`: + +(multi-value) the list of products to generate artifacts for. + +possible values: +- "kibana" +- "elasticsearch" +- "observability" +- "security" + +#### `target-folder`: + +The folder to generate the artifacts in. + +Defaults to `{REPO_ROOT}/build-kb-artifacts`. + +#### `build-folder`: + +The folder to use for temporary files. + +Defaults to `{REPO_ROOT}/build/temp-kb-artifacts` + +#### Cluster infos + +- params for the source cluster: +`sourceClusterUrl` / env.KIBANA_SOURCE_CLUSTER_URL +`sourceClusterUsername` / env.KIBANA_SOURCE_CLUSTER_USERNAME +`sourceClusterPassword` / env.KIBANA_SOURCE_CLUSTER_PASSWORD + +- params for the embedding cluster: +`embeddingClusterUrl` / env.KIBANA_EMBEDDING_CLUSTER_URL +`embeddingClusterUsername` / env.KIBANA_EMBEDDING_CLUSTER_USERNAME +`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD \ No newline at end of file diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts index ae84ae60616a..979845ec3184 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts @@ -21,10 +21,7 @@ export const getArtifactMappings = (inferenceEndpoint: string): MappingTypeMappi slug: { type: 'keyword' }, url: { type: 'keyword' }, version: { type: 'version' }, - ai_subtitle: { - type: 'semantic_text', - inference_id: inferenceEndpoint, - }, + ai_subtitle: { type: 'text' }, ai_summary: { type: 'semantic_text', inference_id: inferenceEndpoint, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts index e4f24725883a..d26ffc980f3a 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts @@ -21,10 +21,7 @@ const mappings: MappingTypeMapping = { slug: { type: 'keyword' }, url: { type: 'keyword' }, version: { type: 'version' }, - ai_subtitle: { - type: 'semantic_text', - inference_id: 'kibana-elser2', - }, + ai_subtitle: { type: 'text' }, ai_summary: { type: 'semantic_text', inference_id: 'kibana-elser2', diff --git a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts index 8dbdbfb5f206..ef81b3d6411c 100644 --- a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts @@ -24,7 +24,7 @@ export interface ProductDocumentationAttributes { slug: string; url: string; version: string; - ai_subtitle: SemanticTextField; + ai_subtitle: string; ai_summary: SemanticTextField; ai_questions_answered: SemanticTextArrayField; ai_tags: string[]; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts index 2c971501e015..03c3b72f86f9 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts @@ -35,7 +35,7 @@ export const performSearch = async ({ fields: [ 'content_title', 'content_body.text', - 'ai_subtitle.text', + 'ai_subtitle', 'ai_summary.text', 'ai_questions_answered.text', 'ai_tags', @@ -63,12 +63,6 @@ export const performSearch = async ({ query: searchQuery, }, }, - { - semantic: { - field: 'ai_subtitle', - query: searchQuery, - }, - }, { semantic: { field: 'ai_summary', From 3858b635fcb9d7dccfaa4676775fa967ef83d3b6 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 14 Nov 2024 14:54:37 +0100 Subject: [PATCH 92/97] tweak summarize prompt --- .../tasks/retrieve_documentation/summarize_document.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts index 47bbdc4b3a6f..815cbc94d08f 100644 --- a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts @@ -45,12 +45,10 @@ export const summarizeDocument = async ({ functionCalling, system: `You are an helpful Elastic assistant, and your current task is to help answer the user's question. - Given a question and a document, please provide a condensed version of the document - that can be used to answer the question. - - Limit the length of the output to 800 words. - - Try to include all relevant information that could be used to answer the question in the condensed version. If this - can't be done without exceeding the 800 words limit requirement, then only include the information that you think - are the most relevant and the most helpful to answer the question. + Given a question and a document, please provide a condensed version of the document that can be used to answer the question. + - Limit the length of the output to 500 words. + - Try to include all relevant information that could be used to answer the question. If this + can't be done within the 500 words limit, then only include the most relevant information related to the question. - If you think the document isn't relevant at all to answer the question, just return an empty text`, input: ` ## User question From 1ddad2f6cd6b23333a6d2070e50d705ec79ab2fd Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:15:07 +0000 Subject: [PATCH 93/97] [CI] Auto-commit changed files from 'make api-docs' --- oas_docs/output/kibana.serverless.yaml | 12 +++--------- oas_docs/output/kibana.yaml | 12 +++--------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 32d38c356914..9f0c38baded7 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -16693,14 +16693,10 @@ paths: type: object properties: active: - description: >- - When false, the enrollment API key is revoked and - cannot be used for enrolling Elastic Agents. + description: When false, the enrollment API key is revoked and cannot be used for enrolling Elastic Agents. type: boolean api_key: - description: >- - The enrollment API key (token) used for enrolling - Elastic Agents. + description: The enrollment API key (token) used for enrolling Elastic Agents. type: string api_key_id: description: The ID of the API key in the Security API. @@ -16713,9 +16709,7 @@ paths: description: The name of the enrollment API key. type: string policy_id: - description: >- - The ID of the agent policy the Elastic Agent will be - enrolled in. + description: The ID of the agent policy the Elastic Agent will be enrolled in. type: string required: - id diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 8a3d4d3634b8..50fd92fdc8a9 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -19477,14 +19477,10 @@ paths: type: object properties: active: - description: >- - When false, the enrollment API key is revoked and - cannot be used for enrolling Elastic Agents. + description: When false, the enrollment API key is revoked and cannot be used for enrolling Elastic Agents. type: boolean api_key: - description: >- - The enrollment API key (token) used for enrolling - Elastic Agents. + description: The enrollment API key (token) used for enrolling Elastic Agents. type: string api_key_id: description: The ID of the API key in the Security API. @@ -19497,9 +19493,7 @@ paths: description: The name of the enrollment API key. type: string policy_id: - description: >- - The ID of the agent policy the Elastic Agent will be - enrolled in. + description: The ID of the agent policy the Elastic Agent will be enrolled in. type: string required: - id From 7063ce4377b09f5e9d27152981eb0534416d0d5f Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 15 Nov 2024 10:13:00 +0100 Subject: [PATCH 94/97] fix type in test --- .../server/services/search/utils/map_result.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts index 34fe40355013..56e8ce4875cc 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts @@ -28,7 +28,7 @@ describe('mapResult', () => { slug: 'foo.html', url: 'http://lost.com/foo.html', version: '8.16', - ai_subtitle: { text: 'ai_subtitle' }, + ai_subtitle: 'ai_subtitle', ai_summary: { text: 'ai_summary' }, ai_questions_answered: { text: ['question A'] }, ai_tags: ['foo', 'bar', 'test'], From a34e114038cb3ff9595857ade5cf1664371dbc78 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 18 Nov 2024 08:17:13 +0100 Subject: [PATCH 95/97] update doc --- docs/user/security/audit-logging.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc index 7e6a7eee1c19..ef12f4303c1b 100644 --- a/docs/user/security/audit-logging.asciidoc +++ b/docs/user/security/audit-logging.asciidoc @@ -149,7 +149,7 @@ Refer to the corresponding {es} logs for potential write errors. | `failure` | Failed to create trained model. .1+| `product_documentation_create` -| `unknown` | User requested to install the product documentation. +| `unknown` | User requested to install the product documentation for use in AI Assistants. 3+a| ====== Type: change @@ -338,7 +338,7 @@ Refer to the corresponding {es} logs for potential write errors. | `failure` | Failed to update trained model deployment. .1+| `product_documentation_update` -| `unknown` | User requested to update the product documentation. +| `unknown` | User requested to update the product documentation for use in AI Assistants. 3+a| ====== Type: deletion @@ -432,7 +432,7 @@ Refer to the corresponding {es} logs for potential write errors. | `failure` | Failed to delete trained model. .1+| `product_documentation_delete` -| `unknown` | User requested to delete the product documentation. +| `unknown` | User requested to delete the product documentation for use in AI Assistants. 3+a| ====== Type: access From b9145effcdc675f3e97fc25339e1332f5b3a4c97 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 18 Nov 2024 09:12:40 +0100 Subject: [PATCH 96/97] check status after installation --- .../installation/installation_service.test.ts | 14 +++++++++++++- .../services/installation/installation_service.ts | 6 +++++- .../product_doc_base/server/routes/installation.ts | 5 ++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts index 52b1fae22393..294aeb99e0fd 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts @@ -37,18 +37,30 @@ describe('InstallationService', () => { }); }); describe('#install', () => { + beforeEach(() => { + http.post.mockResolvedValue({ installed: true }); + }); + it('calls the endpoint with the right parameters', async () => { await service.install(); expect(http.post).toHaveBeenCalledTimes(1); expect(http.post).toHaveBeenCalledWith(INSTALL_ALL_API_PATH); }); it('returns the value from the server', async () => { - const expected = { stubbed: true }; + const expected = { installed: true }; http.post.mockResolvedValue(expected); const response = await service.install(); expect(response).toEqual(expected); }); + it('throws when the server returns installed: false', async () => { + const expected = { installed: false }; + http.post.mockResolvedValue(expected); + + await expect(service.install()).rejects.toThrowErrorMatchingInlineSnapshot( + `"Installation did not complete successfully"` + ); + }); }); describe('#uninstall', () => { it('calls the endpoint with the right parameters', async () => { diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts index 17601db60589..ff347f52cb53 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts @@ -27,7 +27,11 @@ export class InstallationService { } async install(): Promise { - return await this.http.post(INSTALL_ALL_API_PATH); + const response = await this.http.post(INSTALL_ALL_API_PATH); + if (!response.installed) { + throw new Error('Installation did not complete successfully'); + } + return response; } async uninstall(): Promise { diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts index 08f4ea71c37f..dbede9f7d94d 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -73,9 +73,12 @@ export const registerInstallationRoutes = ({ wait: true, }); + // check status after installation in case of failure + const { status } = await documentationManager.getStatus(); + return res.ok({ body: { - installed: true, + installed: status === 'installed', }, }); } From 040956d7205478f7c0db994b13e6421994490748 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 18 Nov 2024 09:16:11 +0100 Subject: [PATCH 97/97] remove unused return type --- .../plugins/ai_infra/product_doc_base/server/tasks/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts index 7b8d20234c24..e32ea02a11b0 100644 --- a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts @@ -41,7 +41,7 @@ export const waitUntilTaskCompleted = async ({ taskId: string; timeout?: number; interval?: number; -}): Promise => { +}): Promise => { const start = Date.now(); const max = start + timeout; let now = start; @@ -53,12 +53,12 @@ export const waitUntilTaskCompleted = async ({ await sleep(interval); now = Date.now(); } else { - return false; + return; } } catch (e) { if (SavedObjectsErrorHelpers.isNotFoundError(e)) { // not found means the task was completed and the entry removed - return true; + return; } } }