diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0894fbff896ad..d7e2cf7fda612 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -768,6 +768,7 @@ x-pack/examples/triggers_actions_ui_example @elastic/response-ops x-pack/examples/ui_actions_enhanced_examples @elastic/appex-sharedux x-pack/packages/ai-infra/inference-common @elastic/appex-ai-infra x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra +x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared @elastic/kibana-management x-pack/packages/index-management/index_management_shared_types @elastic/kibana-management x-pack/packages/kbn-ai-assistant @elastic/search-kibana @@ -857,6 +858,8 @@ x-pack/packages/security/role_management_model @elastic/kibana-security x-pack/packages/security/ui_components @elastic/kibana-security x-pack/performance @elastic/appex-qa x-pack/plugins/actions @elastic/response-ops +x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra +x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra x-pack/plugins/aiops @elastic/ml-ui x-pack/plugins/alerting @elastic/response-ops x-pack/plugins/banners @elastic/appex-sharedux diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 71ab26400f496..ea31863576115 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana. using the CURL scripts in the scripts folder. +|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks] +|This plugin contains various LLM tasks. + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess] |Exposes services to access logs data. @@ -767,6 +771,10 @@ Elastic. |This plugin helps users learn how to use the Painless scripting language. +|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase] +|This plugin contains the product documentation base service. + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling] |Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views. diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc index 1ac40bcc7764a..ef12f4303c1b4 100644 --- a/docs/user/security/audit-logging.asciidoc +++ b/docs/user/security/audit-logging.asciidoc @@ -148,6 +148,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Creating trained model. | `failure` | Failed to create trained model. +.1+| `product_documentation_create` +| `unknown` | User requested to install the product documentation for use in AI Assistants. + 3+a| ====== Type: change @@ -334,6 +337,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Updating trained model deployment. | `failure` | Failed to update trained model deployment. +.1+| `product_documentation_update` +| `unknown` | User requested to update the product documentation for use in AI Assistants. + 3+a| ====== Type: deletion @@ -425,6 +431,9 @@ Refer to the corresponding {es} logs for potential write errors. | `success` | Deleting trained model. | `failure` | Failed to delete trained model. +.1+| `product_documentation_delete` +| `unknown` | User requested to delete the product documentation for use in AI Assistants. + 3+a| ====== Type: access diff --git a/package.json b/package.json index ebbf26f25002c..b67d4b90fdf95 100644 --- a/package.json +++ b/package.json @@ -617,6 +617,7 @@ "@kbn/licensing-plugin": "link:x-pack/plugins/licensing", "@kbn/links-plugin": "link:src/plugins/links", "@kbn/lists-plugin": "link:x-pack/plugins/lists", + "@kbn/llm-tasks-plugin": "link:x-pack/plugins/ai_infra/llm_tasks", "@kbn/locator-examples-plugin": "link:examples/locator_examples", "@kbn/locator-explorer-plugin": "link:examples/locator_explorer", "@kbn/logging": "link:packages/kbn-logging", @@ -721,6 +722,8 @@ "@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel", "@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing", "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util", + "@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base", + "@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common", "@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access", "@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling", "@kbn/profiling-utils": "link:packages/kbn-profiling-utils", diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index a5642cee10958..020b9a97753b4 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -855,6 +855,13 @@ "policy-settings-protection-updates-note": [ "note" ], + "product-doc-install-status": [ + "index_name", + "installation_status", + "last_installation_date", + "product_name", + "product_version" + ], "query": [ "description", "title", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 61f680509c133..2409b7578da84 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2841,6 +2841,26 @@ } } }, + "product-doc-install-status": { + "dynamic": false, + "properties": { + "index_name": { + "type": "keyword" + }, + "installation_status": { + "type": "keyword" + }, + "last_installation_date": { + "type": "date" + }, + "product_name": { + "type": "keyword" + }, + "product_version": { + "type": "keyword" + } + } + }, "query": { "dynamic": false, "properties": { diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 58424700d9bf6..32a7bc827907e 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -124,6 +124,7 @@ pageLoadAssetSize: painlessLab: 179748 presentationPanel: 55463 presentationUtil: 58834 + productDocBase: 22500 profiling: 36694 remoteClusters: 51327 reporting: 58600 diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 28a1e8e1eb538..0f186fba94b54 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -145,6 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () => "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75", "osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4", "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352", + "product-doc-install-status": "ca6e96840228e4cc2f11bae24a0797f4f7238c8c", "query": "501bece68f26fe561286a488eabb1a8ab12f1137", "risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb", "rules-settings": "ba57ef1881b3dcbf48fbfb28902d8f74442190b2", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index ba06073e454a9..3ceba522d08cb 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -115,6 +115,7 @@ const previouslyRegisteredTypes = [ 'osquery-usage-metric', 'osquery-manager-usage-metric', 'policy-settings-protection-updates-note', + 'product-doc-install-status', 'query', 'rules-settings', 'sample-data-telemetry', diff --git a/tsconfig.base.json b/tsconfig.base.json index 26fe060916a92..3e1d80208f5b4 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1148,6 +1148,8 @@ "@kbn/lint-ts-projects-cli/*": ["packages/kbn-lint-ts-projects-cli/*"], "@kbn/lists-plugin": ["x-pack/plugins/lists"], "@kbn/lists-plugin/*": ["x-pack/plugins/lists/*"], + "@kbn/llm-tasks-plugin": ["x-pack/plugins/ai_infra/llm_tasks"], + "@kbn/llm-tasks-plugin/*": ["x-pack/plugins/ai_infra/llm_tasks/*"], "@kbn/locator-examples-plugin": ["examples/locator_examples"], "@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"], "@kbn/locator-explorer-plugin": ["examples/locator_explorer"], @@ -1390,6 +1392,10 @@ "@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"], "@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"], "@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"], + "@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"], + "@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"], + "@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"], + "@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"], "@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"], "@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/observability_solution/profiling_data_access/*"], "@kbn/profiling-plugin": ["x-pack/plugins/observability_solution/profiling"], diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md b/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md index eb64d53b5b8f7..49949def3e5e7 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md @@ -1,3 +1,49 @@ # @kbn/product-doc-artifact-builder -Script to build the knowledge base artifacts +Script to build the knowledge base artifacts. + +## How to run + +``` +node scripts/build_product_doc_artifacts.js --stack-version {version} --product-name {product} +``` + +### parameters + +#### `stack-version`: + +the stack version to generate the artifacts for. + +#### `product-name`: + +(multi-value) the list of products to generate artifacts for. + +possible values: +- "kibana" +- "elasticsearch" +- "observability" +- "security" + +#### `target-folder`: + +The folder to generate the artifacts in. + +Defaults to `{REPO_ROOT}/build-kb-artifacts`. + +#### `build-folder`: + +The folder to use for temporary files. + +Defaults to `{REPO_ROOT}/build/temp-kb-artifacts` + +#### Cluster infos + +- params for the source cluster: +`sourceClusterUrl` / env.KIBANA_SOURCE_CLUSTER_URL +`sourceClusterUsername` / env.KIBANA_SOURCE_CLUSTER_USERNAME +`sourceClusterPassword` / env.KIBANA_SOURCE_CLUSTER_PASSWORD + +- params for the embedding cluster: +`embeddingClusterUrl` / env.KIBANA_EMBEDDING_CLUSTER_URL +`embeddingClusterUsername` / env.KIBANA_EMBEDDING_CLUSTER_USERNAME +`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD \ No newline at end of file diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts index cbebcdc22981b..a8aa927c5ef1f 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts @@ -5,17 +5,13 @@ * 2.0. */ -export interface ArtifactManifest { - formatVersion: string; - productName: string; - productVersion: string; -} +import type { ArtifactManifest, ProductName } from '@kbn/product-doc-common'; export const getArtifactManifest = ({ productName, stackVersion, }: { - productName: string; + productName: ProductName; stackVersion: string; }): ArtifactManifest => { return { diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts index ae84ae60616a3..979845ec31844 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts @@ -21,10 +21,7 @@ export const getArtifactMappings = (inferenceEndpoint: string): MappingTypeMappi slug: { type: 'keyword' }, url: { type: 'keyword' }, version: { type: 'version' }, - ai_subtitle: { - type: 'semantic_text', - inference_id: inferenceEndpoint, - }, + ai_subtitle: { type: 'text' }, ai_summary: { type: 'semantic_text', inference_id: inferenceEndpoint, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts index cfcc141323f4f..e4ca33849a527 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts @@ -5,7 +5,34 @@ * 2.0. */ -/** - * The allowed product names, as found in the source's cluster - */ -export const sourceProductNames = ['Kibana', 'Elasticsearch', 'Security', 'Observability']; +import type { ProductName } from '@kbn/product-doc-common'; + +const productNameToSourceNamesMap: Record = { + kibana: ['Kibana'], + elasticsearch: ['Elasticsearch'], + security: ['Security'], + observability: ['Observability'], +}; + +const sourceNameToProductName = Object.entries(productNameToSourceNamesMap).reduce< + Record +>((map, [productName, sourceNames]) => { + sourceNames.forEach((sourceName) => { + map[sourceName] = productName as ProductName; + }); + return map; +}, {}); + +export const getSourceNamesFromProductName = (productName: ProductName): string[] => { + if (!productNameToSourceNamesMap[productName]) { + throw new Error(`Unknown product name: ${productName}`); + } + return productNameToSourceNamesMap[productName]; +}; + +export const getProductNameFromSource = (source: string): ProductName => { + if (!sourceNameToProductName[source]) { + throw new Error(`Unknown source name: ${source}`); + } + return sourceNameToProductName[source]; +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts index bbde3310f8e3a..551f58bc68308 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts @@ -8,6 +8,7 @@ import Path from 'path'; import { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; +import type { ProductName } from '@kbn/product-doc-common'; import { // checkConnectivity, createTargetIndex, @@ -18,6 +19,7 @@ import { createArtifact, cleanupFolders, deleteIndex, + processDocuments, } from './tasks'; import type { TaskConfig } from './types'; @@ -93,7 +95,7 @@ const buildArtifact = async ({ sourceClient, log, }: { - productName: string; + productName: ProductName; stackVersion: string; buildFolder: string; targetFolder: string; @@ -105,7 +107,7 @@ const buildArtifact = async ({ const targetIndex = getTargetIndexName({ productName, stackVersion }); - const documents = await extractDocumentation({ + let documents = await extractDocumentation({ client: sourceClient, index: 'search-docs-1', log, @@ -113,6 +115,8 @@ const buildArtifact = async ({ stackVersion, }); + documents = await processDocuments({ documents, log }); + await createTargetIndex({ client: embeddingClient, indexName: targetIndex, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts index 49af1d158db83..e8d0d9486e331 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts @@ -6,19 +6,19 @@ */ import Path from 'path'; -import { REPO_ROOT } from '@kbn/repo-info'; import yargs from 'yargs'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { DocumentationProduct } from '@kbn/product-doc-common'; import type { TaskConfig } from './types'; import { buildArtifacts } from './build_artifacts'; -import { sourceProductNames } from './artifact/product_name'; function options(y: yargs.Argv) { return y .option('productName', { describe: 'name of products to generate documentation for', array: true, - choices: sourceProductNames, - default: ['Kibana'], + choices: Object.values(DocumentationProduct), + default: [DocumentationProduct.kibana], }) .option('stackVersion', { describe: 'The stack version to generate documentation for', diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts index 343099876585a..056887a41a4d2 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts @@ -8,9 +8,9 @@ import Path from 'path'; import AdmZip from 'adm-zip'; import type { ToolingLog } from '@kbn/tooling-log'; +import { getArtifactName, type ProductName } from '@kbn/product-doc-common'; import { getArtifactMappings } from '../artifact/mappings'; import { getArtifactManifest } from '../artifact/manifest'; -import { getArtifactName } from '../artifact/artifact_name'; export const createArtifact = async ({ productName, @@ -21,7 +21,7 @@ export const createArtifact = async ({ }: { buildFolder: string; targetFolder: string; - productName: string; + productName: ProductName; stackVersion: string; log: ToolingLog; }) => { diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts index 8b0e7323c2886..73cf8f0109228 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts @@ -10,7 +10,7 @@ import Fs from 'fs/promises'; import type { Client } from '@elastic/elasticsearch'; import type { ToolingLog } from '@kbn/tooling-log'; -const fileSizeLimit = 250_000; +const fileSizeLimit = 500_000; export const createChunkFiles = async ({ index, diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts index e4f24725883ab..d26ffc980f3ab 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts @@ -21,10 +21,7 @@ const mappings: MappingTypeMapping = { slug: { type: 'keyword' }, url: { type: 'keyword' }, version: { type: 'version' }, - ai_subtitle: { - type: 'semantic_text', - inference_id: 'kibana-elser2', - }, + ai_subtitle: { type: 'text' }, ai_summary: { type: 'semantic_text', inference_id: 'kibana-elser2', diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts index f1dd051394bbd..6aa8bb49b0cfd 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts @@ -8,6 +8,8 @@ import type { Client } from '@elastic/elasticsearch'; import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import type { ToolingLog } from '@kbn/tooling-log'; +import type { ProductName } from '@kbn/product-doc-common'; +import { getSourceNamesFromProductName, getProductNameFromSource } from '../artifact/product_name'; /** the list of fields to import from the source cluster */ const fields = [ @@ -27,7 +29,7 @@ const fields = [ export interface ExtractedDocument { content_title: string; content_body: string; - product_name: string; + product_name: ProductName; root_type: string; slug: string; url: string; @@ -43,7 +45,7 @@ const convertHit = (hit: SearchHit): ExtractedDocument => { return { content_title: source.content_title, content_body: source.content_body, - product_name: source.product_name, + product_name: getProductNameFromSource(source.product_name), root_type: 'documentation', slug: source.slug, url: source.url, @@ -65,7 +67,7 @@ export const extractDocumentation = async ({ client: Client; index: string; stackVersion: string; - productName: string; + productName: ProductName; log: ToolingLog; }) => { log.info(`Starting to extract documents from source cluster`); @@ -76,7 +78,7 @@ export const extractDocumentation = async ({ query: { bool: { must: [ - { term: { product_name: productName } }, + { terms: { product_name: getSourceNamesFromProductName(productName) } }, { term: { version: stackVersion } }, { exists: { field: 'ai_fields.ai_summary' } }, ], diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts index 0c63431362329..ec94e4c135c17 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts @@ -10,8 +10,8 @@ export { indexDocuments } from './index_documents'; export { createTargetIndex } from './create_index'; export { installElser } from './install_elser'; export { createChunkFiles } from './create_chunk_files'; -export { performSemanticSearch } from './perform_semantic_search'; export { checkConnectivity } from './check_connectivity'; export { createArtifact } from './create_artifact'; export { cleanupFolders } from './cleanup_folders'; export { deleteIndex } from './delete_index'; +export { processDocuments } from './process_documents'; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts new file mode 100644 index 0000000000000..69141ca167ab4 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/process_documents.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { uniqBy } from 'lodash'; +import { encode } from 'gpt-tokenizer'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { ExtractedDocument } from './extract_documentation'; + +export const processDocuments = async ({ + documents, + log, +}: { + documents: ExtractedDocument[]; + log: ToolingLog; +}): Promise => { + log.info('Starting processing documents.'); + const initialCount = documents.length; + documents = removeDuplicates(documents); + const noDupCount = documents.length; + log.info(`Removed ${initialCount - noDupCount} duplicates`); + documents.forEach(processDocument); + documents = filterEmptyDocs(documents); + log.info(`Removed ${noDupCount - documents.length} empty documents`); + log.info('Done processing documents.'); + return documents; +}; + +const removeDuplicates = (documents: ExtractedDocument[]): ExtractedDocument[] => { + return uniqBy(documents, (doc) => doc.slug); +}; + +/** + * Filter "this content has moved" or "deleted pages" type of documents, just based on token count. + */ +const filterEmptyDocs = (documents: ExtractedDocument[]): ExtractedDocument[] => { + return documents.filter((doc) => { + const tokenCount = encode(doc.content_body).length; + if (tokenCount < 100) { + return false; + } + return true; + }); +}; + +const processDocument = (document: ExtractedDocument) => { + document.content_body = document.content_body + // remove those "edit" button text that got embedded into titles. + .replaceAll(/([a-zA-Z])edit\n/g, (match) => { + return `${match[0]}\n`; + }) + // limit to 2 consecutive carriage return + .replaceAll(/\n\n+/g, '\n\n'); + + return document; +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts index d2acfb5774500..1eb4a4348d218 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts @@ -5,8 +5,10 @@ * 2.0. */ +import type { ProductName } from '@kbn/product-doc-common'; + export interface TaskConfig { - productNames: string[]; + productNames: ProductName[]; stackVersion: string; buildFolder: string; targetFolder: string; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json b/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json index 508d4c715d0a7..68ff27852c4d1 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/tsconfig.json @@ -16,5 +16,6 @@ "kbn_references": [ "@kbn/tooling-log", "@kbn/repo-info", + "@kbn/product-doc-common", ] } diff --git a/x-pack/packages/ai-infra/product-doc-common/README.md b/x-pack/packages/ai-infra/product-doc-common/README.md new file mode 100644 index 0000000000000..ff20c0e0fd0e7 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/README.md @@ -0,0 +1,3 @@ +# @kbn/product-doc-common + +Common types and utilities for the product documentation feature. diff --git a/x-pack/packages/ai-infra/product-doc-common/index.ts b/x-pack/packages/ai-infra/product-doc-common/index.ts new file mode 100644 index 0000000000000..1a96737138991 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getArtifactName, parseArtifactName } from './src/artifact'; +export { type ArtifactManifest } from './src/manifest'; +export { DocumentationProduct, type ProductName } from './src/product'; +export { isArtifactContentFilePath } from './src/artifact_content'; +export { + productDocIndexPrefix, + productDocIndexPattern, + getProductDocIndexName, +} from './src/indices'; +export type { ProductDocumentationAttributes } from './src/documents'; diff --git a/x-pack/packages/ai-infra/product-doc-common/jest.config.js b/x-pack/packages/ai-infra/product-doc-common/jest.config.js new file mode 100644 index 0000000000000..e6cae43806c8d --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../../..', + roots: ['/x-pack/packages/ai-infra/product-doc-common'], +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc b/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc new file mode 100644 index 0000000000000..16336c1fc8e27 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/product-doc-common", + "owner": "@elastic/appex-ai-infra" +} diff --git a/x-pack/packages/ai-infra/product-doc-common/package.json b/x-pack/packages/ai-infra/product-doc-common/package.json new file mode 100644 index 0000000000000..839d411a2efb9 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/product-doc-common", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} \ No newline at end of file diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts new file mode 100644 index 0000000000000..2b6362dbf4aad --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getArtifactName, parseArtifactName } from './artifact'; + +describe('getArtifactName', () => { + it('builds the name based on the provided product name and version', () => { + expect( + getArtifactName({ + productName: 'kibana', + productVersion: '8.16', + }) + ).toEqual('kb-product-doc-kibana-8.16.zip'); + }); + + it('excludes the extension when excludeExtension is true', () => { + expect( + getArtifactName({ + productName: 'elasticsearch', + productVersion: '8.17', + excludeExtension: true, + }) + ).toEqual('kb-product-doc-elasticsearch-8.17'); + }); + + it('generates a lowercase name', () => { + expect( + getArtifactName({ + // @ts-expect-error testing + productName: 'ElasticSearch', + productVersion: '8.17', + excludeExtension: true, + }) + ).toEqual('kb-product-doc-elasticsearch-8.17'); + }); +}); + +describe('parseArtifactName', () => { + it('parses an artifact name with extension', () => { + expect(parseArtifactName('kb-product-doc-kibana-8.16.zip')).toEqual({ + productName: 'kibana', + productVersion: '8.16', + }); + }); + + it('parses an artifact name without extension', () => { + expect(parseArtifactName('kb-product-doc-security-8.17')).toEqual({ + productName: 'security', + productVersion: '8.17', + }); + }); + + it('returns undefined if the provided string does not match the artifact name pattern', () => { + expect(parseArtifactName('some-wrong-name')).toEqual(undefined); + }); + + it('returns undefined if the provided string is not strictly lowercase', () => { + expect(parseArtifactName('kb-product-doc-Security-8.17')).toEqual(undefined); + }); +}); diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts new file mode 100644 index 0000000000000..1a6745abd733d --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type ProductName, DocumentationProduct } from './product'; + +// kb-product-doc-elasticsearch-8.15.zip +const artifactNameRegexp = /^kb-product-doc-([a-z]+)-([0-9]+\.[0-9]+)(\.zip)?$/; +const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); + +export const getArtifactName = ({ + productName, + productVersion, + excludeExtension = false, +}: { + productName: ProductName; + productVersion: string; + excludeExtension?: boolean; +}): string => { + const ext = excludeExtension ? '' : '.zip'; + return `kb-product-doc-${productName}-${productVersion}${ext}`.toLowerCase(); +}; + +export const parseArtifactName = (artifactName: string) => { + const match = artifactNameRegexp.exec(artifactName); + if (match) { + const productName = match[1].toLowerCase() as ProductName; + const productVersion = match[2].toLowerCase(); + if (allowedProductNames.includes(productName)) { + return { + productName, + productVersion, + }; + } + } +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts new file mode 100644 index 0000000000000..3f97aaf94f880 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.test.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArtifactContentFilePath } from './artifact_content'; + +describe('isArtifactContentFilePath', () => { + it('returns true for filenames matching the pattern', () => { + expect(isArtifactContentFilePath('content/content-0.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content/content-007.ndjson')).toEqual(true); + expect(isArtifactContentFilePath('content/content-9042.ndjson')).toEqual(true); + }); + + it('returns false for filenames not matching the pattern', () => { + expect(isArtifactContentFilePath('content-0.ndjson')).toEqual(false); + expect(isArtifactContentFilePath('content/content-0')).toEqual(false); + expect(isArtifactContentFilePath('content/content.ndjson')).toEqual(false); + expect(isArtifactContentFilePath('content/content-9042.json')).toEqual(false); + }); +}); diff --git a/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts new file mode 100644 index 0000000000000..757e6664bb588 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/artifact_content.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/; + +export const isArtifactContentFilePath = (path: string): boolean => { + return contentFileRegexp.test(path); +}; diff --git a/x-pack/packages/ai-infra/product-doc-common/src/documents.ts b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts new file mode 100644 index 0000000000000..ef81b3d6411cc --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/documents.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from './product'; + +// don't need to define the other props +interface SemanticTextField { + text: string; +} + +interface SemanticTextArrayField { + text: string[]; +} + +export interface ProductDocumentationAttributes { + content_title: string; + content_body: SemanticTextField; + product_name: ProductName; + root_type: string; + slug: string; + url: string; + version: string; + ai_subtitle: string; + ai_summary: SemanticTextField; + ai_questions_answered: SemanticTextArrayField; + ai_tags: string[]; +} diff --git a/x-pack/packages/ai-infra/product-doc-common/src/indices.ts b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts new file mode 100644 index 0000000000000..b48cacf79fd23 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/indices.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from './product'; + +export const productDocIndexPrefix = '.kibana-ai-product-doc'; +export const productDocIndexPattern = `${productDocIndexPrefix}-*`; + +export const getProductDocIndexName = (productName: ProductName): string => { + return `${productDocIndexPrefix}-${productName.toLowerCase()}`; +}; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts similarity index 59% rename from x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts rename to x-pack/packages/ai-infra/product-doc-common/src/manifest.ts index 678b17088c7b4..6c246cf58fd5f 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/artifact_name.ts +++ b/x-pack/packages/ai-infra/product-doc-common/src/manifest.ts @@ -5,12 +5,10 @@ * 2.0. */ -export const getArtifactName = ({ - productName, - productVersion, -}: { - productName: string; +import type { ProductName } from './product'; + +export interface ArtifactManifest { + formatVersion: string; + productName: ProductName; productVersion: string; -}): string => { - return `kibana-kb-${productName}-${productVersion}.zip`.toLowerCase(); -}; +} diff --git a/x-pack/packages/ai-infra/product-doc-common/src/product.ts b/x-pack/packages/ai-infra/product-doc-common/src/product.ts new file mode 100644 index 0000000000000..417033f5083ec --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/src/product.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum DocumentationProduct { + kibana = 'kibana', + elasticsearch = 'elasticsearch', + observability = 'observability', + security = 'security', +} + +export type ProductName = keyof typeof DocumentationProduct; diff --git a/x-pack/packages/ai-infra/product-doc-common/tsconfig.json b/x-pack/packages/ai-infra/product-doc-common/tsconfig.json new file mode 100644 index 0000000000000..0d78dace105e1 --- /dev/null +++ b/x-pack/packages/ai-infra/product-doc-common/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [] +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/README.md b/x-pack/plugins/ai_infra/llm_tasks/README.md new file mode 100644 index 0000000000000..e019d456cd65a --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/README.md @@ -0,0 +1,45 @@ +# LLM Tasks plugin + +This plugin contains various LLM tasks. + +## Retrieve documentation + +This task allows to retrieve documents from our Elastic product documentation. + +The task depends on the `product-doc-base` plugin, as this dependency is used +to install and manage the product documentation. + +### Checking if the task is available + +A `retrieveDocumentationAvailable` API is exposed from the start contract, that +should be used to assert that the `retrieve_doc` task can be used in the current +context. + +That API receive the inbound request as parameter. + +Example: +```ts +if (await llmTasksStart.retrieveDocumentationAvailable({ request })) { + // task is available +} else { + // task is not available +} +``` + +### Executing the task + +The task is executed as an API of the plugin's start contract, and can be invoked +as any other lifecycle API would. + +Example: +```ts +const result = await llmTasksStart.retrieveDocumentation({ + searchTerm: "How to create a space in Kibana?", + request, + connectorId: 'my-connector-id', +}); + +const { success, documents } = result; +``` + +The exhaustive list of options for the task is available on the `RetrieveDocumentationParams` type's TS doc. diff --git a/x-pack/plugins/ai_infra/llm_tasks/jest.config.js b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js new file mode 100644 index 0000000000000..2a6206d4304b9 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/jest.config.js @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: ['/x-pack/plugins/ai_infra/llm_tasks/server'], + setupFiles: [], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/ai_infra/llm_tasks/{public,server,common}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc new file mode 100644 index 0000000000000..1ef211d01210e --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc @@ -0,0 +1,15 @@ +{ + "type": "plugin", + "id": "@kbn/llm-tasks-plugin", + "owner": "@elastic/appex-ai-infra", + "plugin": { + "id": "llmTasks", + "server": true, + "browser": false, + "configPath": ["xpack", "llmTasks"], + "requiredPlugins": ["inference", "productDocBase"], + "requiredBundles": [], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/config.ts b/x-pack/plugins/ai_infra/llm_tasks/server/config.ts new file mode 100644 index 0000000000000..c509af8bda64b --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from '@kbn/core/server'; + +const configSchema = schema.object({}); + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: {}, +}; + +export type LlmTasksConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/index.ts new file mode 100644 index 0000000000000..1b18426dc2c34 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server'; +import type { LlmTasksConfig } from './config'; +import type { + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { LlmTasksPlugin } from './plugin'; + +export { config } from './config'; + +export type { LlmTasksPluginSetup, LlmTasksPluginStart }; + +export const plugin: PluginInitializer< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new LlmTasksPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts new file mode 100644 index 0000000000000..d10c495ece159 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import type { LlmTasksConfig } from './config'; +import type { + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { retrieveDocumentation } from './tasks'; + +export class LlmTasksPlugin + implements + Plugin< + LlmTasksPluginSetup, + LlmTasksPluginStart, + PluginSetupDependencies, + PluginStartDependencies + > +{ + private logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + setupDependencies: PluginSetupDependencies + ): LlmTasksPluginSetup { + return {}; + } + + start(core: CoreStart, startDependencies: PluginStartDependencies): LlmTasksPluginStart { + const { inference, productDocBase } = startDependencies; + return { + retrieveDocumentationAvailable: async () => { + const docBaseStatus = await startDependencies.productDocBase.management.getStatus(); + return docBaseStatus.status === 'installed'; + }, + retrieveDocumentation: (options) => { + return retrieveDocumentation({ + outputAPI: inference.getClient({ request: options.request }).output, + searchDocAPI: productDocBase.search, + logger: this.logger.get('tasks.retrieve-documentation'), + })(options); + }, + }; + } +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts new file mode 100644 index 0000000000000..41d3911823449 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { retrieveDocumentation } from './retrieve_documentation'; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts new file mode 100644 index 0000000000000..22bf0745bd77f --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { retrieveDocumentation } from './retrieve_documentation'; +export type { + RetrieveDocumentationAPI, + RetrieveDocumentationResult, + RetrieveDocumentationParams, +} from './types'; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts new file mode 100644 index 0000000000000..5722b73ca039c --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.test.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServerMock } from '@kbn/core/server/mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import type { DocSearchResult } from '@kbn/product-doc-base-plugin/server/services/search'; + +import { retrieveDocumentation } from './retrieve_documentation'; +import { truncate, count as countTokens } from '../../utils/tokens'; +jest.mock('../../utils/tokens'); +const truncateMock = truncate as jest.MockedFn; +const countTokensMock = countTokens as jest.MockedFn; + +import { summarizeDocument } from './summarize_document'; +jest.mock('./summarize_document'); +const summarizeDocumentMock = summarizeDocument as jest.MockedFn; + +describe('retrieveDocumentation', () => { + let logger: MockedLogger; + let request: ReturnType; + let outputAPI: jest.Mock; + let searchDocAPI: jest.Mock; + let retrieve: ReturnType; + + const createResult = (parts: Partial = {}): DocSearchResult => { + return { + title: 'title', + content: 'content', + url: 'url', + productName: 'kibana', + ...parts, + }; + }; + + beforeEach(() => { + logger = loggerMock.create(); + request = httpServerMock.createKibanaRequest(); + outputAPI = jest.fn(); + searchDocAPI = jest.fn(); + retrieve = retrieveDocumentation({ logger, searchDocAPI, outputAPI }); + }); + + afterEach(() => { + summarizeDocumentMock.mockReset(); + truncateMock.mockReset(); + countTokensMock.mockReset(); + }); + + it('calls the search API with the right parameters', async () => { + searchDocAPI.mockResolvedValue({ results: [] }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + products: ['kibana'], + request, + max: 5, + connectorId: '.my-connector', + functionCalling: 'simulated', + }); + + expect(result).toEqual({ + success: true, + documents: [], + }); + + expect(searchDocAPI).toHaveBeenCalledTimes(1); + expect(searchDocAPI).toHaveBeenCalledWith({ + query: 'What is Kibana?', + products: ['kibana'], + max: 5, + }); + }); + + it('reduces the document length using the truncate strategy', async () => { + searchDocAPI.mockResolvedValue({ + results: [ + createResult({ content: 'content-1' }), + createResult({ content: 'content-2' }), + createResult({ content: 'content-3' }), + ], + }); + + countTokensMock.mockImplementation((text) => { + if (text === 'content-2') { + return 150; + } else { + return 50; + } + }); + truncateMock.mockReturnValue('truncated'); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'truncate', + }); + + expect(result.documents.length).toEqual(3); + expect(result.documents[0].content).toEqual('content-1'); + expect(result.documents[1].content).toEqual('truncated'); + expect(result.documents[2].content).toEqual('content-3'); + + expect(truncateMock).toHaveBeenCalledTimes(1); + expect(truncateMock).toHaveBeenCalledWith('content-2', 100); + }); + + it('reduces the document length using the summarize strategy', async () => { + searchDocAPI.mockResolvedValue({ + results: [ + createResult({ content: 'content-1' }), + createResult({ content: 'content-2' }), + createResult({ content: 'content-3' }), + ], + }); + + countTokensMock.mockImplementation((text) => { + if (text === 'content-2') { + return 50; + } else { + return 150; + } + }); + truncateMock.mockImplementation((text) => text); + + summarizeDocumentMock.mockImplementation(({ documentContent }) => { + return Promise.resolve({ summary: `${documentContent}-summarized` }); + }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'summarize', + }); + + expect(result.documents.length).toEqual(3); + expect(result.documents[0].content).toEqual('content-1-summarized'); + expect(result.documents[1].content).toEqual('content-2'); + expect(result.documents[2].content).toEqual('content-3-summarized'); + + expect(truncateMock).toHaveBeenCalledTimes(2); + expect(truncateMock).toHaveBeenCalledWith('content-1-summarized', 100); + expect(truncateMock).toHaveBeenCalledWith('content-3-summarized', 100); + }); + + it('logs an error and return an empty list of docs in case of error', async () => { + searchDocAPI.mockResolvedValue({ + results: [createResult({ content: 'content-1' })], + }); + countTokensMock.mockImplementation(() => { + return 150; + }); + summarizeDocumentMock.mockImplementation(() => { + throw new Error('woups'); + }); + + const result = await retrieve({ + searchTerm: 'What is Kibana?', + request, + connectorId: '.my-connector', + maxDocumentTokens: 100, + tokenReductionStrategy: 'summarize', + }); + + expect(result).toEqual({ + success: false, + documents: [], + }); + + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Error retrieving documentation') + ); + }); +}); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts new file mode 100644 index 0000000000000..96f966e483601 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/retrieve_documentation.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { OutputAPI } from '@kbn/inference-common'; +import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server'; +import { truncate, count as countTokens } from '../../utils/tokens'; +import type { RetrieveDocumentationAPI } from './types'; +import { summarizeDocument } from './summarize_document'; + +const MAX_DOCUMENTS_DEFAULT = 3; +const MAX_TOKENS_DEFAULT = 1000; + +export const retrieveDocumentation = + ({ + outputAPI, + searchDocAPI, + logger: log, + }: { + outputAPI: OutputAPI; + searchDocAPI: ProductDocSearchAPI; + logger: Logger; + }): RetrieveDocumentationAPI => + async ({ + searchTerm, + connectorId, + products, + functionCalling, + max = MAX_DOCUMENTS_DEFAULT, + maxDocumentTokens = MAX_TOKENS_DEFAULT, + tokenReductionStrategy = 'summarize', + }) => { + try { + const { results } = await searchDocAPI({ query: searchTerm, products, max }); + + log.debug(`searching with term=[${searchTerm}] returned ${results.length} documents`); + + const processedDocuments = await Promise.all( + results.map(async (document) => { + const tokenCount = countTokens(document.content); + const docHasTooManyTokens = tokenCount >= maxDocumentTokens; + log.debug( + `processing doc [${document.url}] - tokens : [${tokenCount}] - tooManyTokens: [${docHasTooManyTokens}]` + ); + + let content = document.content; + if (docHasTooManyTokens) { + if (tokenReductionStrategy === 'summarize') { + const extractResponse = await summarizeDocument({ + searchTerm, + documentContent: document.content, + outputAPI, + connectorId, + functionCalling, + }); + content = truncate(extractResponse.summary, maxDocumentTokens); + } else { + content = truncate(document.content, maxDocumentTokens); + } + } + + log.debug(`done processing document [${document.url}]`); + return { + title: document.title, + url: document.url, + content, + }; + }) + ); + + log.debug(() => { + const docsAsJson = JSON.stringify(processedDocuments); + return `searching with term=[${searchTerm}] - results: ${docsAsJson}`; + }); + + return { + success: true, + documents: processedDocuments.filter((doc) => doc.content.length > 0), + }; + } catch (e) { + log.error(`Error retrieving documentation: ${e.message}. Returning empty results.`); + return { success: false, documents: [] }; + } + }; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts new file mode 100644 index 0000000000000..815cbc94d08f8 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/summarize_document.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolSchema, FunctionCallingMode, OutputAPI } from '@kbn/inference-common'; + +const summarizeDocumentSchema = { + type: 'object', + properties: { + useful: { + type: 'boolean', + description: `Whether the provided document has any useful information related to the user's query.`, + }, + summary: { + type: 'string', + description: `The condensed version of the document that can be used to answer the question. Can be empty.`, + }, + }, + required: ['useful'], +} as const satisfies ToolSchema; + +interface SummarizeDocumentResponse { + summary: string; +} + +export const summarizeDocument = async ({ + searchTerm, + documentContent, + connectorId, + outputAPI, + functionCalling, +}: { + searchTerm: string; + documentContent: string; + outputAPI: OutputAPI; + connectorId: string; + functionCalling?: FunctionCallingMode; +}): Promise => { + const result = await outputAPI({ + id: 'summarize_document', + connectorId, + functionCalling, + system: `You are an helpful Elastic assistant, and your current task is to help answer the user's question. + + Given a question and a document, please provide a condensed version of the document that can be used to answer the question. + - Limit the length of the output to 500 words. + - Try to include all relevant information that could be used to answer the question. If this + can't be done within the 500 words limit, then only include the most relevant information related to the question. + - If you think the document isn't relevant at all to answer the question, just return an empty text`, + input: ` + ## User question + + ${searchTerm} + + ## Document + + ${documentContent} + `, + schema: summarizeDocumentSchema, + }); + return { + summary: result.output.summary ?? '', + }; +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts new file mode 100644 index 0000000000000..1e0637fcd344c --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/tasks/retrieve_documentation/types.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRequest } from '@kbn/core/server'; +import type { FunctionCallingMode } from '@kbn/inference-common'; +import type { ProductName } from '@kbn/product-doc-common'; + +/** + * Parameters for {@link RetrieveDocumentationAPI} + */ +export interface RetrieveDocumentationParams { + /** + * The search term to perform semantic text with. + * E.g. "What is Kibana Lens?" + */ + searchTerm: string; + /** + * Maximum number of documents to return. + * Defaults to 3. + */ + max?: number; + /** + * Optional list of products to restrict the search to. + */ + products?: ProductName[]; + /** + * The maximum number of tokens to return *per document*. + * Documents exceeding this limit will go through token reduction. + * + * Defaults to `1000`. + */ + maxDocumentTokens?: number; + /** + * The token reduction strategy to apply for documents exceeding max token count. + * - truncate: Will keep the N first tokens + * - summarize: Will call the LLM asking to generate a contextualized summary of the document + * + * Overall, `summarize` is way more efficient, but significantly slower, given that an additional + * LLM call will be performed. + * + * Defaults to `summarize` + */ + tokenReductionStrategy?: 'truncate' | 'summarize'; + /** + * The request that initiated the task. + */ + request: KibanaRequest; + /** + * Id of the LLM connector to use for the task. + */ + connectorId: string; + functionCalling?: FunctionCallingMode; +} + +export interface RetrievedDocument { + title: string; + url: string; + content: string; +} + +export interface RetrieveDocumentationResult { + success: boolean; + documents: RetrievedDocument[]; +} + +export type RetrieveDocumentationAPI = ( + options: RetrieveDocumentationParams +) => Promise; diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/types.ts b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts new file mode 100644 index 0000000000000..d550e4398b509 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceServerStart } from '@kbn/inference-plugin/server'; +import type { ProductDocBaseStartContract } from '@kbn/product-doc-base-plugin/server'; +import type { RetrieveDocumentationAPI } from './tasks/retrieve_documentation'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface PluginSetupDependencies {} + +export interface PluginStartDependencies { + inference: InferenceServerStart; + productDocBase: ProductDocBaseStartContract; +} + +/** + * Describes public llmTasks plugin contract returned at the `setup` stage. + */ +export interface LlmTasksPluginSetup {} + +/** + * Describes public llmTasks plugin contract returned at the `start` stage. + */ +export interface LlmTasksPluginStart { + /** + * Checks if all prerequisites to use the `retrieveDocumentation` task + * are respected. Can be used to check if the task can be registered + * as LLM tool for example. + */ + retrieveDocumentationAvailable: () => Promise; + /** + * Perform the `retrieveDocumentation` task. + * + * @see RetrieveDocumentationAPI + */ + retrieveDocumentation: RetrieveDocumentationAPI; +} diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts new file mode 100644 index 0000000000000..dce97eaea9b75 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { count, truncate } from './tokens'; + +describe('count', () => { + it('returns the token count of a given text', () => { + expect(count('some short sentence')).toBeGreaterThan(1); + }); +}); + +describe('truncate', () => { + it('truncates text that exceed the specified maximum token count', () => { + const text = 'some sentence that is likely longer than 5 tokens.'; + const output = truncate(text, 5); + expect(output.length).toBeLessThan(text.length); + }); + it('keeps text with a smaller amount of tokens unchanged', () => { + const text = 'some sentence that is likely less than 100 tokens.'; + const output = truncate(text, 100); + expect(output.length).toEqual(text.length); + }); +}); diff --git a/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts new file mode 100644 index 0000000000000..cb469144255b7 --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { encode, decode } from 'gpt-tokenizer'; + +export const count = (text: string): number => { + return encode(text).length; +}; + +export const truncate = (text: string, maxTokens: number): string => { + const encoded = encode(text); + if (encoded.length > maxTokens) { + const truncated = encoded.slice(0, maxTokens); + return decode(truncated); + } + return text; +}; diff --git a/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json new file mode 100644 index 0000000000000..03b87827d941a --- /dev/null +++ b/x-pack/plugins/ai_infra/llm_tasks/tsconfig.json @@ -0,0 +1,27 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + "scripts/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [ + "@kbn/core", + "@kbn/logging", + "@kbn/config-schema", + "@kbn/product-doc-common", + "@kbn/inference-plugin", + "@kbn/product-doc-base-plugin", + "@kbn/logging-mocks", + "@kbn/inference-common", + ] +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/README.md b/x-pack/plugins/ai_infra/product_doc_base/README.md new file mode 100644 index 0000000000000..0ff6c34dd2785 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/README.md @@ -0,0 +1,3 @@ +# Product documentation base plugin + +This plugin contains the product documentation base service. diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts new file mode 100644 index 0000000000000..1622df5ed865c --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/common/consts.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const productDocInstallStatusSavedObjectTypeName = 'product-doc-install-status'; + +/** + * The id of the inference endpoint we're creating for our product doc indices. + * Could be replaced with the default elser 2 endpoint once the default endpoint feature is available. + */ +export const internalElserInferenceId = 'kibana-internal-elser2'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts new file mode 100644 index 0000000000000..0237bd2c3b488 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/common/http_api/installation.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; +import type { ProductInstallState, InstallationStatus } from '../install_status'; + +export const INSTALLATION_STATUS_API_PATH = '/internal/product_doc_base/status'; +export const INSTALL_ALL_API_PATH = '/internal/product_doc_base/install'; +export const UNINSTALL_ALL_API_PATH = '/internal/product_doc_base/uninstall'; + +export interface InstallationStatusResponse { + overall: InstallationStatus; + perProducts: Record; +} + +export interface PerformInstallResponse { + installed: boolean; +} + +export interface UninstallResponse { + success: boolean; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts new file mode 100644 index 0000000000000..81102d43c1ff3 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/common/install_status.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; + +export type InstallationStatus = 'installed' | 'uninstalled' | 'installing' | 'error'; + +/** + * DTO representation of the product doc install status SO + */ +export interface ProductDocInstallStatus { + id: string; + productName: ProductName; + productVersion: string; + installationStatus: InstallationStatus; + lastInstallationDate: Date | undefined; + lastInstallationFailureReason: string | undefined; + indexName?: string; +} + +export interface ProductInstallState { + status: InstallationStatus; + version?: string; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/jest.config.js b/x-pack/plugins/ai_infra/product_doc_base/jest.config.js new file mode 100644 index 0000000000000..fc06be251a6f7 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/jest.config.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: [ + '/x-pack/plugins/ai_infra/product_doc_base/public', + '/x-pack/plugins/ai_infra/product_doc_base/server', + '/x-pack/plugins/ai_infra/product_doc_base/common', + ], + setupFiles: [], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/ai_infra/product_doc_base/{public,server,common}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc new file mode 100644 index 0000000000000..268b4a70c9921 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc @@ -0,0 +1,15 @@ +{ + "type": "plugin", + "id": "@kbn/product-doc-base-plugin", + "owner": "@elastic/appex-ai-infra", + "plugin": { + "id": "productDocBase", + "server": true, + "browser": true, + "configPath": ["xpack", "productDocBase"], + "requiredPlugins": ["licensing", "taskManager"], + "requiredBundles": [], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts new file mode 100644 index 0000000000000..b5ccbf029a73e --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { ProductDocBasePlugin } from './plugin'; +import type { + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies, + PublicPluginConfig, +} from './types'; + +export type { ProductDocBasePluginSetup, ProductDocBasePluginStart }; + +export const plugin: PluginInitializer< + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies +> = (pluginInitializerContext: PluginInitializerContext) => + new ProductDocBasePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx new file mode 100644 index 0000000000000..6f2c989b6e45d --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; +import type { Logger } from '@kbn/logging'; +import type { + PublicPluginConfig, + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies, +} from './types'; +import { InstallationService } from './services/installation'; + +export class ProductDocBasePlugin + implements + Plugin< + ProductDocBasePluginSetup, + ProductDocBasePluginStart, + PluginSetupDependencies, + PluginStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + pluginsSetup: PluginSetupDependencies + ): ProductDocBasePluginSetup { + return {}; + } + + start(coreStart: CoreStart, pluginsStart: PluginStartDependencies): ProductDocBasePluginStart { + const installationService = new InstallationService({ http: coreStart.http }); + + return { + installation: { + getStatus: () => installationService.getInstallationStatus(), + install: () => installationService.install(), + uninstall: () => installationService.uninstall(), + }, + }; + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts new file mode 100644 index 0000000000000..2eee8613d77dc --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { InstallationService } from './installation_service'; +export type { InstallationAPI } from './types'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts new file mode 100644 index 0000000000000..294aeb99e0fd8 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServiceMock } from '@kbn/core/public/mocks'; +import { InstallationService } from './installation_service'; +import { + INSTALLATION_STATUS_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, +} from '../../../common/http_api/installation'; + +describe('InstallationService', () => { + let http: ReturnType; + let service: InstallationService; + + beforeEach(() => { + http = httpServiceMock.createSetupContract(); + service = new InstallationService({ http }); + }); + + describe('#getInstallationStatus', () => { + it('calls the endpoint with the right parameters', async () => { + await service.getInstallationStatus(); + expect(http.get).toHaveBeenCalledTimes(1); + expect(http.get).toHaveBeenCalledWith(INSTALLATION_STATUS_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { stubbed: true }; + http.get.mockResolvedValue(expected); + + const response = await service.getInstallationStatus(); + expect(response).toEqual(expected); + }); + }); + describe('#install', () => { + beforeEach(() => { + http.post.mockResolvedValue({ installed: true }); + }); + + it('calls the endpoint with the right parameters', async () => { + await service.install(); + expect(http.post).toHaveBeenCalledTimes(1); + expect(http.post).toHaveBeenCalledWith(INSTALL_ALL_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { installed: true }; + http.post.mockResolvedValue(expected); + + const response = await service.install(); + expect(response).toEqual(expected); + }); + it('throws when the server returns installed: false', async () => { + const expected = { installed: false }; + http.post.mockResolvedValue(expected); + + await expect(service.install()).rejects.toThrowErrorMatchingInlineSnapshot( + `"Installation did not complete successfully"` + ); + }); + }); + describe('#uninstall', () => { + it('calls the endpoint with the right parameters', async () => { + await service.uninstall(); + expect(http.post).toHaveBeenCalledTimes(1); + expect(http.post).toHaveBeenCalledWith(UNINSTALL_ALL_API_PATH); + }); + it('returns the value from the server', async () => { + const expected = { stubbed: true }; + http.post.mockResolvedValue(expected); + + const response = await service.uninstall(); + expect(response).toEqual(expected); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts new file mode 100644 index 0000000000000..ff347f52cb531 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/installation_service.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { HttpSetup } from '@kbn/core-http-browser'; +import { + INSTALLATION_STATUS_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../../common/http_api/installation'; + +export class InstallationService { + private readonly http: HttpSetup; + + constructor({ http }: { http: HttpSetup }) { + this.http = http; + } + + async getInstallationStatus(): Promise { + return await this.http.get(INSTALLATION_STATUS_API_PATH); + } + + async install(): Promise { + const response = await this.http.post(INSTALL_ALL_API_PATH); + if (!response.installed) { + throw new Error('Installation did not complete successfully'); + } + return response; + } + + async uninstall(): Promise { + return await this.http.post(UNINSTALL_ALL_API_PATH); + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts new file mode 100644 index 0000000000000..5c01c84b24625 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/services/installation/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../../common/http_api/installation'; + +export interface InstallationAPI { + getStatus(): Promise; + install(): Promise; + uninstall(): Promise; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/public/types.ts b/x-pack/plugins/ai_infra/product_doc_base/public/types.ts new file mode 100644 index 0000000000000..1d06b0e08fa23 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/public/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InstallationAPI } from './services/installation'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface PublicPluginConfig {} + +export interface PluginSetupDependencies {} + +export interface PluginStartDependencies {} + +export interface ProductDocBasePluginSetup {} + +export interface ProductDocBasePluginStart { + installation: InstallationAPI; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/config.ts b/x-pack/plugins/ai_infra/product_doc_base/server/config.ts new file mode 100644 index 0000000000000..bd0892d582701 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/config.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from '@kbn/core/server'; + +const configSchema = schema.object({ + artifactRepositoryUrl: schema.string({ + defaultValue: 'https://kibana-knowledge-base-artifacts.elastic.co', + }), +}); + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: {}, +}; + +export type ProductDocBaseConfig = TypeOf; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts new file mode 100644 index 0000000000000..805a0f2ea8c41 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server'; +import type { ProductDocBaseConfig } from './config'; +import type { + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies, +} from './types'; +import { ProductDocBasePlugin } from './plugin'; + +export { config } from './config'; + +export type { ProductDocBaseSetupContract, ProductDocBaseStartContract }; +export type { SearchApi as ProductDocSearchAPI } from './services/search/types'; + +export const plugin: PluginInitializer< + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new ProductDocBasePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts new file mode 100644 index 0000000000000..bd5d6a720dd71 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { coreMock } from '@kbn/core/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; +import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; +import { ProductDocBasePlugin } from './plugin'; +import { ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies } from './types'; + +jest.mock('./services/package_installer'); +jest.mock('./services/search'); +jest.mock('./services/doc_install_status'); +jest.mock('./routes'); +jest.mock('./tasks'); +import { registerRoutes } from './routes'; +import { PackageInstaller } from './services/package_installer'; +import { registerTaskDefinitions, scheduleEnsureUpToDateTask } from './tasks'; + +const PackageInstallMock = PackageInstaller as jest.Mock; + +describe('ProductDocBasePlugin', () => { + let initContext: ReturnType; + let plugin: ProductDocBasePlugin; + let pluginSetupDeps: ProductDocBaseSetupDependencies; + let pluginStartDeps: ProductDocBaseStartDependencies; + + beforeEach(() => { + initContext = coreMock.createPluginInitializerContext(); + plugin = new ProductDocBasePlugin(initContext); + pluginSetupDeps = { + taskManager: taskManagerMock.createSetup(), + }; + pluginStartDeps = { + licensing: licensingMock.createStart(), + taskManager: taskManagerMock.createStart(), + }; + + PackageInstallMock.mockReturnValue({ ensureUpToDate: jest.fn().mockResolvedValue({}) }); + }); + + afterEach(() => { + (scheduleEnsureUpToDateTask as jest.Mock).mockReset(); + }); + + describe('#setup', () => { + it('register the routes', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + + expect(registerRoutes).toHaveBeenCalledTimes(1); + }); + it('register the product-doc SO type', () => { + const coreSetup = coreMock.createSetup(); + plugin.setup(coreSetup, pluginSetupDeps); + + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledTimes(1); + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledWith( + expect.objectContaining({ + name: productDocInstallStatusSavedObjectTypeName, + }) + ); + }); + it('register the task definitions', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + + expect(registerTaskDefinitions).toHaveBeenCalledTimes(3); + }); + }); + + describe('#start', () => { + it('returns a contract with the expected shape', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + const startContract = plugin.start(coreMock.createStart(), pluginStartDeps); + expect(startContract).toEqual({ + management: { + getStatus: expect.any(Function), + install: expect.any(Function), + uninstall: expect.any(Function), + update: expect.any(Function), + }, + search: expect.any(Function), + }); + }); + + it('schedules the update task', () => { + plugin.setup(coreMock.createSetup(), pluginSetupDeps); + plugin.start(coreMock.createStart(), pluginStartDeps); + + expect(scheduleEnsureUpToDateTask).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts new file mode 100644 index 0000000000000..c8ed100cabb16 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; +import type { Logger } from '@kbn/logging'; +import { getDataPath } from '@kbn/utils'; +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import { SavedObjectsClient } from '@kbn/core/server'; +import { productDocInstallStatusSavedObjectTypeName } from '../common/consts'; +import type { ProductDocBaseConfig } from './config'; +import { + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies, + InternalServices, +} from './types'; +import { productDocInstallStatusSavedObjectType } from './saved_objects'; +import { PackageInstaller } from './services/package_installer'; +import { InferenceEndpointManager } from './services/inference_endpoint'; +import { ProductDocInstallClient } from './services/doc_install_status'; +import { DocumentationManager } from './services/doc_manager'; +import { SearchService } from './services/search'; +import { registerRoutes } from './routes'; +import { registerTaskDefinitions } from './tasks'; + +export class ProductDocBasePlugin + implements + Plugin< + ProductDocBaseSetupContract, + ProductDocBaseStartContract, + ProductDocBaseSetupDependencies, + ProductDocBaseStartDependencies + > +{ + private logger: Logger; + private internalServices?: InternalServices; + + constructor(private readonly context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + { taskManager }: ProductDocBaseSetupDependencies + ): ProductDocBaseSetupContract { + const getServices = () => { + if (!this.internalServices) { + throw new Error('getServices called before #start'); + } + return this.internalServices; + }; + + coreSetup.savedObjects.registerType(productDocInstallStatusSavedObjectType); + + registerTaskDefinitions({ + taskManager, + getServices, + }); + + const router = coreSetup.http.createRouter(); + registerRoutes({ + router, + getServices, + }); + + return {}; + } + + start( + core: CoreStart, + { licensing, taskManager }: ProductDocBaseStartDependencies + ): ProductDocBaseStartContract { + const soClient = new SavedObjectsClient( + core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName]) + ); + const productDocClient = new ProductDocInstallClient({ soClient }); + + const endpointManager = new InferenceEndpointManager({ + esClient: core.elasticsearch.client.asInternalUser, + logger: this.logger.get('endpoint-manager'), + }); + + const packageInstaller = new PackageInstaller({ + esClient: core.elasticsearch.client.asInternalUser, + productDocClient, + endpointManager, + kibanaVersion: this.context.env.packageInfo.version, + artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'), + artifactRepositoryUrl: this.context.config.get().artifactRepositoryUrl, + logger: this.logger.get('package-installer'), + }); + + const searchService = new SearchService({ + esClient: core.elasticsearch.client.asInternalUser, + logger: this.logger.get('search-service'), + }); + + const documentationManager = new DocumentationManager({ + logger: this.logger.get('doc-manager'), + docInstallClient: productDocClient, + licensing, + taskManager, + auditService: core.security.audit, + }); + + this.internalServices = { + logger: this.logger, + packageInstaller, + installClient: productDocClient, + documentationManager, + licensing, + taskManager, + }; + + documentationManager.update().catch((err) => { + this.logger.error(`Error scheduling product documentation update task: ${err.message}`); + }); + + return { + management: { + install: documentationManager.install.bind(documentationManager), + update: documentationManager.update.bind(documentationManager), + uninstall: documentationManager.uninstall.bind(documentationManager), + getStatus: documentationManager.getStatus.bind(documentationManager), + }, + search: searchService.search.bind(searchService), + }; + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts new file mode 100644 index 0000000000000..66660c199d819 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter } from '@kbn/core/server'; +import { registerInstallationRoutes } from './installation'; +import type { InternalServices } from '../types'; + +export const registerRoutes = ({ + router, + getServices, +}: { + router: IRouter; + getServices: () => InternalServices; +}) => { + registerInstallationRoutes({ getServices, router }); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts new file mode 100644 index 0000000000000..dbede9f7d94d3 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/routes/installation.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter } from '@kbn/core/server'; +import { + INSTALLATION_STATUS_API_PATH, + INSTALL_ALL_API_PATH, + UNINSTALL_ALL_API_PATH, + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '../../common/http_api/installation'; +import type { InternalServices } from '../types'; + +export const registerInstallationRoutes = ({ + router, + getServices, +}: { + router: IRouter; + getServices: () => InternalServices; +}) => { + router.get( + { + path: INSTALLATION_STATUS_API_PATH, + validate: false, + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, + }, + }, + async (ctx, req, res) => { + const { installClient, documentationManager } = getServices(); + const installStatus = await installClient.getInstallationStatus(); + const { status: overallStatus } = await documentationManager.getStatus(); + + return res.ok({ + body: { + perProducts: installStatus, + overall: overallStatus, + }, + }); + } + ); + + router.post( + { + path: INSTALL_ALL_API_PATH, + validate: false, + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, + timeout: { idleSocket: 20 * 60 * 1000 }, // install can take time. + }, + }, + async (ctx, req, res) => { + const { documentationManager } = getServices(); + + await documentationManager.install({ + request: req, + force: false, + wait: true, + }); + + // check status after installation in case of failure + const { status } = await documentationManager.getStatus(); + + return res.ok({ + body: { + installed: status === 'installed', + }, + }); + } + ); + + router.post( + { + path: UNINSTALL_ALL_API_PATH, + validate: false, + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['manage_llm_product_doc'], + }, + }, + }, + }, + async (ctx, req, res) => { + const { documentationManager } = getServices(); + + await documentationManager.uninstall({ + request: req, + wait: true, + }); + + return res.ok({ + body: { + success: true, + }, + }); + } + ); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts new file mode 100644 index 0000000000000..f87c6d37eb66f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { + productDocInstallStatusSavedObjectType, + type ProductDocInstallStatusAttributes, +} from './product_doc_install'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts new file mode 100644 index 0000000000000..47cf7eb50cdd1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/saved_objects/product_doc_install.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsType } from '@kbn/core/server'; +import type { ProductName } from '@kbn/product-doc-common'; +import { productDocInstallStatusSavedObjectTypeName } from '../../common/consts'; +import type { InstallationStatus } from '../../common/install_status'; + +/** + * Interface describing the raw attributes of the product doc install SO type. + * Contains more fields than the mappings, which only list + * indexed fields. + */ +export interface ProductDocInstallStatusAttributes { + product_name: ProductName; + product_version: string; + installation_status: InstallationStatus; + last_installation_date?: number; + last_installation_failure_reason?: string; + index_name?: string; +} + +export const productDocInstallStatusSavedObjectType: SavedObjectsType = + { + name: productDocInstallStatusSavedObjectTypeName, + hidden: true, + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: { + product_name: { type: 'keyword' }, + product_version: { type: 'keyword' }, + installation_status: { type: 'keyword' }, + last_installation_date: { type: 'date' }, + index_name: { type: 'keyword' }, + }, + }, + management: { + importableAndExportable: false, + }, + modelVersions: {}, + }; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts new file mode 100644 index 0000000000000..d55cb303b1908 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ProductDocInstallClient } from './product_doc_install_service'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts new file mode 100644 index 0000000000000..6460d8452dc2b --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject } from '@kbn/core/server'; +import type { ProductDocInstallStatusAttributes } from '../../saved_objects'; +import { soToModel } from './model_conversion'; + +const createObj = ( + attrs: ProductDocInstallStatusAttributes +): SavedObject => { + return { + id: 'some-id', + type: 'product-doc-install-status', + attributes: attrs, + references: [], + }; +}; + +describe('soToModel', () => { + it('converts the SO to the expected shape', () => { + const input = createObj({ + product_name: 'kibana', + product_version: '8.16', + installation_status: 'installed', + last_installation_date: 9000, + index_name: '.kibana', + }); + + const output = soToModel(input); + + expect(output).toEqual({ + id: 'some-id', + productName: 'kibana', + productVersion: '8.16', + indexName: '.kibana', + installationStatus: 'installed', + lastInstallationDate: expect.any(Date), + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts new file mode 100644 index 0000000000000..cf77bb9222a15 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/model_conversion.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject } from '@kbn/core/server'; +import type { ProductDocInstallStatus } from '../../../common/install_status'; +import type { ProductDocInstallStatusAttributes } from '../../saved_objects'; + +export const soToModel = ( + so: SavedObject +): ProductDocInstallStatus => { + return { + id: so.id, + productName: so.attributes.product_name, + productVersion: so.attributes.product_version, + installationStatus: so.attributes.installation_status, + indexName: so.attributes.index_name, + lastInstallationDate: so.attributes.last_installation_date + ? new Date(so.attributes.last_installation_date) + : undefined, + lastInstallationFailureReason: so.attributes.last_installation_failure_reason, + }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts new file mode 100644 index 0000000000000..81249038a1294 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from '@kbn/core/server'; +import { DocumentationProduct } from '@kbn/product-doc-common'; +import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { ProductDocInstallClient } from './product_doc_install_service'; + +const createObj = (attrs: TypeAttributes): SavedObjectsFindResult => { + return { + id: attrs.product_name, + type: 'type', + references: [], + attributes: attrs, + score: 42, + }; +}; + +describe('ProductDocInstallClient', () => { + let soClient: ReturnType; + let service: ProductDocInstallClient; + + beforeEach(() => { + soClient = savedObjectsClientMock.create(); + service = new ProductDocInstallClient({ soClient }); + }); + + describe('getInstallationStatus', () => { + it('returns the installation status based on existing entries', async () => { + soClient.find.mockResolvedValue({ + saved_objects: [ + createObj({ + product_name: 'kibana', + product_version: '8.15', + installation_status: 'installed', + }), + createObj({ + product_name: 'elasticsearch', + product_version: '8.15', + installation_status: 'installing', + }), + ], + total: 2, + per_page: 100, + page: 1, + }); + + const installStatus = await service.getInstallationStatus(); + + expect(Object.keys(installStatus).sort()).toEqual(Object.keys(DocumentationProduct).sort()); + expect(installStatus.kibana).toEqual({ + status: 'installed', + version: '8.15', + }); + expect(installStatus.security).toEqual({ + status: 'uninstalled', + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts new file mode 100644 index 0000000000000..24625ebc51586 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/product_doc_install_service.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; +import { ProductName, DocumentationProduct } from '@kbn/product-doc-common'; +import type { ProductInstallState } from '../../../common/install_status'; +import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts'; +import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects'; + +export class ProductDocInstallClient { + private soClient: SavedObjectsClientContract; + + constructor({ soClient }: { soClient: SavedObjectsClientContract }) { + this.soClient = soClient; + } + + async getInstallationStatus(): Promise> { + const response = await this.soClient.find({ + type: typeName, + perPage: 100, + }); + + const installStatus = Object.values(DocumentationProduct).reduce((memo, product) => { + memo[product] = { status: 'uninstalled' }; + return memo; + }, {} as Record); + + response.saved_objects.forEach(({ attributes }) => { + installStatus[attributes.product_name as ProductName] = { + status: attributes.installation_status, + version: attributes.product_version, + }; + }); + + return installStatus; + } + + async setInstallationStarted(fields: { productName: ProductName; productVersion: string }) { + const { productName, productVersion } = fields; + const objectId = getObjectIdFromProductName(productName); + const attributes = { + product_name: productName, + product_version: productVersion, + installation_status: 'installing' as const, + last_installation_failure_reason: '', + }; + await this.soClient.update(typeName, objectId, attributes, { + upsert: attributes, + }); + } + + async setInstallationSuccessful(productName: ProductName, indexName: string) { + const objectId = getObjectIdFromProductName(productName); + await this.soClient.update(typeName, objectId, { + installation_status: 'installed', + index_name: indexName, + }); + } + + async setInstallationFailed(productName: ProductName, failureReason: string) { + const objectId = getObjectIdFromProductName(productName); + await this.soClient.update(typeName, objectId, { + installation_status: 'error', + last_installation_failure_reason: failureReason, + }); + } + + async setUninstalled(productName: ProductName) { + const objectId = getObjectIdFromProductName(productName); + try { + await this.soClient.update(typeName, objectId, { + installation_status: 'uninstalled', + last_installation_failure_reason: '', + }); + } catch (e) { + if (!SavedObjectsErrorHelpers.isNotFoundError(e)) { + throw e; + } + } + } +} + +const getObjectIdFromProductName = (productName: ProductName) => + `kb-product-doc-${productName}-status`.toLowerCase(); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts new file mode 100644 index 0000000000000..c2a0adbac9f29 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_install_status/service.mock.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductDocInstallClient } from './product_doc_install_service'; + +export type InstallClientMock = jest.Mocked; + +const createInstallClientMock = (): InstallClientMock => { + return { + getInstallationStatus: jest.fn(), + setInstallationStarted: jest.fn(), + setInstallationSuccessful: jest.fn(), + setInstallationFailed: jest.fn(), + setUninstalled: jest.fn(), + } as unknown as InstallClientMock; +}; + +export const installClientMock = { + create: createInstallClientMock, +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts new file mode 100644 index 0000000000000..d4af5b7ebdb22 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/check_license.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ILicense } from '@kbn/licensing-plugin/server'; + +export const checkLicense = (license: ILicense): boolean => { + const result = license.check('elastic documentation', 'enterprise'); + return result.state === 'valid'; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts new file mode 100644 index 0000000000000..0be913ee6dd71 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import { securityServiceMock, httpServerMock } from '@kbn/core/server/mocks'; +import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import { DocumentationManager } from './doc_manager'; + +jest.mock('../../tasks'); +import { + scheduleInstallAllTask, + scheduleUninstallAllTask, + scheduleEnsureUpToDateTask, + getTaskStatus, + waitUntilTaskCompleted, +} from '../../tasks'; + +const scheduleInstallAllTaskMock = scheduleInstallAllTask as jest.MockedFn< + typeof scheduleInstallAllTask +>; +const scheduleUninstallAllTaskMock = scheduleUninstallAllTask as jest.MockedFn< + typeof scheduleUninstallAllTask +>; +const scheduleEnsureUpToDateTaskMock = scheduleEnsureUpToDateTask as jest.MockedFn< + typeof scheduleEnsureUpToDateTask +>; +const waitUntilTaskCompletedMock = waitUntilTaskCompleted as jest.MockedFn< + typeof waitUntilTaskCompleted +>; +const getTaskStatusMock = getTaskStatus as jest.MockedFn; + +describe('DocumentationManager', () => { + let logger: MockedLogger; + let taskManager: ReturnType; + let licensing: ReturnType; + let auditService: ReturnType['audit']; + let docInstallClient: jest.Mocked; + + let docManager: DocumentationManager; + + beforeEach(() => { + logger = loggerMock.create(); + taskManager = taskManagerMock.createStart(); + licensing = licensingMock.createStart(); + auditService = securityServiceMock.createStart().audit; + + docInstallClient = { + getInstallationStatus: jest.fn(), + } as unknown as jest.Mocked; + + docManager = new DocumentationManager({ + logger, + taskManager, + licensing, + auditService, + docInstallClient, + }); + }); + + afterEach(() => { + scheduleInstallAllTaskMock.mockReset(); + scheduleUninstallAllTaskMock.mockReset(); + scheduleEnsureUpToDateTaskMock.mockReset(); + waitUntilTaskCompletedMock.mockReset(); + getTaskStatusMock.mockReset(); + }); + + describe('#install', () => { + beforeEach(() => { + licensing.getLicense.mockResolvedValue( + licensingMock.createLicense({ license: { type: 'enterprise' } }) + ); + + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleInstallAllTask`', async () => { + await docManager.install({}); + + expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleInstallAllTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.install({ wait: true }); + + expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('does not call scheduleInstallAllTask if already installed and not force', async () => { + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'installed' }, + } as Awaited>); + + await docManager.install({ wait: true }); + + expect(scheduleInstallAllTaskMock).not.toHaveBeenCalled(); + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.install({ force: false, wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_create', + category: ['database'], + type: ['creation'], + outcome: 'unknown', + }, + }); + }); + + it('throws an error if license level is not sufficient', async () => { + licensing.getLicense.mockResolvedValue( + licensingMock.createLicense({ license: { type: 'basic' } }) + ); + + await expect( + docManager.install({ force: false, wait: false }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Elastic documentation requires an enterprise license"` + ); + }); + }); + + describe('#update', () => { + beforeEach(() => { + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleEnsureUpToDateTask`', async () => { + await docManager.update({}); + + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.update({ wait: true }); + + expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.update({ wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_update', + category: ['database'], + type: ['change'], + outcome: 'unknown', + }, + }); + }); + }); + + describe('#uninstall', () => { + beforeEach(() => { + getTaskStatusMock.mockResolvedValue('not_scheduled'); + + docInstallClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'uninstalled' }, + } as Awaited>); + }); + + it('calls `scheduleUninstallAllTask`', async () => { + await docManager.uninstall({}); + + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledWith({ + taskManager, + logger, + }); + + expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled(); + }); + + it('calls waitUntilTaskCompleted if wait=true', async () => { + await docManager.uninstall({ wait: true }); + + expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1); + expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1); + }); + + it('records an audit log when request is provided', async () => { + const request = httpServerMock.createKibanaRequest(); + + const auditLog = auditService.withoutRequest; + auditService.asScoped = jest.fn(() => auditLog); + + await docManager.uninstall({ wait: false, request }); + + expect(auditLog.log).toHaveBeenCalledTimes(1); + expect(auditLog.log).toHaveBeenCalledWith({ + message: expect.any(String), + event: { + action: 'product_documentation_delete', + category: ['database'], + type: ['deletion'], + outcome: 'unknown', + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts new file mode 100644 index 0000000000000..40dc53e19ceea --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/doc_manager.ts @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { CoreAuditService } from '@kbn/core/server'; +import { type TaskManagerStartContract, TaskStatus } from '@kbn/task-manager-plugin/server'; +import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { InstallationStatus } from '../../../common/install_status'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import { + INSTALL_ALL_TASK_ID, + scheduleInstallAllTask, + scheduleUninstallAllTask, + scheduleEnsureUpToDateTask, + getTaskStatus, + waitUntilTaskCompleted, +} from '../../tasks'; +import { checkLicense } from './check_license'; +import type { + DocumentationManagerAPI, + DocGetStatusResponse, + DocInstallOptions, + DocUninstallOptions, + DocUpdateOptions, +} from './types'; + +const TEN_MIN_IN_MS = 10 * 60 * 1000; + +/** + * High-level installation service, handling product documentation + * installation as unary operations, abstracting away the fact + * that documentation is composed of multiple entities. + */ +export class DocumentationManager implements DocumentationManagerAPI { + private logger: Logger; + private taskManager: TaskManagerStartContract; + private licensing: LicensingPluginStart; + private docInstallClient: ProductDocInstallClient; + private auditService: CoreAuditService; + + constructor({ + logger, + taskManager, + licensing, + docInstallClient, + auditService, + }: { + logger: Logger; + taskManager: TaskManagerStartContract; + licensing: LicensingPluginStart; + docInstallClient: ProductDocInstallClient; + auditService: CoreAuditService; + }) { + this.logger = logger; + this.taskManager = taskManager; + this.licensing = licensing; + this.docInstallClient = docInstallClient; + this.auditService = auditService; + } + + async install(options: DocInstallOptions = {}): Promise { + const { request, force = false, wait = false } = options; + + const { status } = await this.getStatus(); + if (!force && status === 'installed') { + return; + } + + const license = await this.licensing.getLicense(); + if (!checkLicense(license)) { + throw new Error('Elastic documentation requires an enterprise license'); + } + + const taskId = await scheduleInstallAllTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting installation of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_create', + category: ['database'], + type: ['creation'], + outcome: 'unknown', + }, + }); + } + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async update(options: DocUpdateOptions = {}): Promise { + const { request, wait = false } = options; + + const taskId = await scheduleEnsureUpToDateTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting update of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_update', + category: ['database'], + type: ['change'], + outcome: 'unknown', + }, + }); + } + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async uninstall(options: DocUninstallOptions = {}): Promise { + const { request, wait = false } = options; + + const taskId = await scheduleUninstallAllTask({ + taskManager: this.taskManager, + logger: this.logger, + }); + + if (request) { + this.auditService.asScoped(request).log({ + message: `User is requesting deletion of product documentation for AI Assistants. Task ID=[${taskId}]`, + event: { + action: 'product_documentation_delete', + category: ['database'], + type: ['deletion'], + outcome: 'unknown', + }, + }); + } + + if (wait) { + await waitUntilTaskCompleted({ + taskManager: this.taskManager, + taskId, + timeout: TEN_MIN_IN_MS, + }); + } + } + + async getStatus(): Promise { + const taskStatus = await getTaskStatus({ + taskManager: this.taskManager, + taskId: INSTALL_ALL_TASK_ID, + }); + if (taskStatus !== 'not_scheduled') { + const status = convertTaskStatus(taskStatus); + if (status !== 'unknown') { + return { status }; + } + } + + const installStatus = await this.docInstallClient.getInstallationStatus(); + const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status)); + return { status: overallStatus }; + } +} + +const convertTaskStatus = (taskStatus: TaskStatus): InstallationStatus | 'unknown' => { + switch (taskStatus) { + case TaskStatus.Idle: + case TaskStatus.Claiming: + case TaskStatus.Running: + return 'installing'; + case TaskStatus.Failed: + return 'error'; + case TaskStatus.Unrecognized: + case TaskStatus.DeadLetter: + case TaskStatus.ShouldDelete: + default: + return 'unknown'; + } +}; + +const getOverallStatus = (statuses: InstallationStatus[]): InstallationStatus => { + const statusOrder: InstallationStatus[] = ['error', 'installing', 'uninstalled', 'installed']; + for (const status of statusOrder) { + if (statuses.includes(status)) { + return status; + } + } + return 'installed'; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts new file mode 100644 index 0000000000000..588b5e2f5cc65 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { DocumentationManager } from './doc_manager'; +export type { + DocumentationManagerAPI, + DocUninstallOptions, + DocInstallOptions, + DocUpdateOptions, + DocGetStatusResponse, +} from './types'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts new file mode 100644 index 0000000000000..5a954a5ffb0fd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/doc_manager/types.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRequest } from '@kbn/core/server'; +import type { InstallationStatus } from '../../../common/install_status'; + +/** + * APIs to manage the product documentation. + */ +export interface DocumentationManagerAPI { + /** + * Install the product documentation. + * By default, will only try to install if not already present. + * Can use the `force` option to forcefully reinstall. + */ + install(options?: DocInstallOptions): Promise; + /** + * Update the product documentation to the latest version. + * No-op if the product documentation is not currently installed. + */ + update(options?: DocUpdateOptions): Promise; + /** + * Uninstall the product documentation. + * No-op if the product documentation is not currently installed. + */ + uninstall(options?: DocUninstallOptions): Promise; + /** + * Returns the overall installation status of the documentation. + */ + getStatus(): Promise; +} + +/** + * Return type for {@link DocumentationManagerAPI.getStatus} + */ +export interface DocGetStatusResponse { + status: InstallationStatus; +} + +/** + * Options for {@link DocumentationManagerAPI.install} + */ +export interface DocInstallOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; + /** + * If true, will reinstall the documentation even if already present. + * Defaults to `false` + */ + force?: boolean; + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} + +/** + * Options for {@link DocumentationManagerAPI.uninstall} + */ +export interface DocUninstallOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} + +/** + * Options for {@link DocumentationManagerAPI.update} + */ +export interface DocUpdateOptions { + /** + * When the operation was requested by a user, the request that initiated it. + * + * If not provided, the call will be considered as being done on behalf of system. + */ + request?: KibanaRequest; + /** + * If true, the returned promise will wait until the update task has completed before resolving. + * Defaults to `false` + */ + wait?: boolean; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts new file mode 100644 index 0000000000000..e5dabaaa9b7f7 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { InferenceEndpointManager } from './endpoint_manager'; + +jest.mock('./utils'); +import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils'; +const installElserMock = installElser as jest.MockedFn; +const getModelInstallStatusMock = getModelInstallStatus as jest.MockedFn< + typeof getModelInstallStatus +>; +const waitUntilModelDeployedMock = waitUntilModelDeployed as jest.MockedFn< + typeof waitUntilModelDeployed +>; + +describe('InferenceEndpointManager', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let endpointManager: InferenceEndpointManager; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + + endpointManager = new InferenceEndpointManager({ esClient, logger }); + }); + + afterEach(() => { + installElserMock.mockReset(); + getModelInstallStatusMock.mockReset(); + waitUntilModelDeployedMock.mockReset(); + }); + + describe('#ensureInternalElserInstalled', () => { + it('installs ELSER if not already installed', async () => { + getModelInstallStatusMock.mockResolvedValue({ installed: true }); + + await endpointManager.ensureInternalElserInstalled(); + + expect(installElserMock).not.toHaveBeenCalled(); + expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1); + }); + it('does not install ELSER if already present', async () => { + getModelInstallStatusMock.mockResolvedValue({ installed: false }); + + await endpointManager.ensureInternalElserInstalled(); + + expect(installElserMock).toHaveBeenCalledTimes(1); + expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts new file mode 100644 index 0000000000000..4f7467501d61d --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/endpoint_manager.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { internalElserInferenceId } from '../../../common/consts'; +import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils'; + +export class InferenceEndpointManager { + private readonly log: Logger; + private readonly esClient: ElasticsearchClient; + + constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) { + this.log = logger; + this.esClient = esClient; + } + + async ensureInternalElserInstalled() { + const { installed } = await getModelInstallStatus({ + inferenceId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + if (!installed) { + await installElser({ + inferenceId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + } + + await waitUntilModelDeployed({ + modelId: internalElserInferenceId, + client: this.esClient, + log: this.log, + }); + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts new file mode 100644 index 0000000000000..e4098ff58fe51 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { InferenceEndpointManager } from './endpoint_manager'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts new file mode 100644 index 0000000000000..e9715c4ad2acd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/service.mock.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceEndpointManager } from './endpoint_manager'; + +export type InferenceEndpointManagerMock = jest.Mocked; + +const createMock = (): InferenceEndpointManagerMock => { + return { + ensureInternalElserInstalled: jest.fn(), + } as unknown as InferenceEndpointManagerMock; +}; + +export const inferenceManagerMock = { + create: createMock, +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts new file mode 100644 index 0000000000000..be6caa34d0ad1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/get_model_install_status.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceTaskType } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const getModelInstallStatus = async ({ + inferenceId, + taskType = 'sparse_embedding', + client, +}: { + inferenceId: string; + taskType?: InferenceTaskType; + client: ElasticsearchClient; + log: Logger; +}) => { + const getInferenceRes = await client.inference.get( + { + task_type: taskType, + inference_id: inferenceId, + }, + { ignore: [404] } + ); + + const installed = (getInferenceRes.endpoints ?? []).some( + (endpoint) => endpoint.inference_id === inferenceId + ); + + return { installed }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts new file mode 100644 index 0000000000000..089997557f301 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { waitUntilModelDeployed } from './wait_until_model_deployed'; +export { getModelInstallStatus } from './get_model_install_status'; +export { installElser } from './install_elser'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts new file mode 100644 index 0000000000000..0e92d765a3d17 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/install_elser.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const installElser = async ({ + inferenceId, + client, + log, +}: { + inferenceId: string; + client: ElasticsearchClient; + log: Logger; +}) => { + await client.inference.put( + { + task_type: 'sparse_embedding', + inference_id: inferenceId, + inference_config: { + service: 'elasticsearch', + service_settings: { + num_allocations: 1, + num_threads: 1, + model_id: '.elser_model_2', + }, + task_settings: {}, + }, + }, + { requestTimeout: 5 * 60 * 1000 } + ); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts new file mode 100644 index 0000000000000..83775ed80f5a0 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/inference_endpoint/utils/wait_until_model_deployed.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +export const waitUntilModelDeployed = async ({ + modelId, + client, + log, + maxRetries = 20, + delay = 2000, +}: { + modelId: string; + client: ElasticsearchClient; + log: Logger; + maxRetries?: number; + delay?: number; +}) => { + for (let i = 0; i < maxRetries; i++) { + const statsRes = await client.ml.getTrainedModelsStats({ + model_id: modelId, + }); + const deploymentStats = statsRes.trained_model_stats[0]?.deployment_stats; + if (!deploymentStats || deploymentStats.nodes.length === 0) { + log.debug(`ML model [${modelId}] was not deployed - attempt ${i + 1} of ${maxRetries}`); + await sleep(delay); + continue; + } + return; + } + + throw new Error(`Timeout waiting for ML model ${modelId} to be deployed`); +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts new file mode 100644 index 0000000000000..a9edb7c38fdaa --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { PackageInstaller } from './package_installer'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts new file mode 100644 index 0000000000000..3b7b7c234800f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.mocks.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const validateArtifactArchiveMock = jest.fn(); +export const fetchArtifactVersionsMock = jest.fn(); +export const createIndexMock = jest.fn(); +export const populateIndexMock = jest.fn(); + +jest.doMock('./steps', () => { + const actual = jest.requireActual('./steps'); + return { + ...actual, + validateArtifactArchive: validateArtifactArchiveMock, + fetchArtifactVersions: fetchArtifactVersionsMock, + createIndex: createIndexMock, + populateIndex: populateIndexMock, + }; +}); + +export const downloadToDiskMock = jest.fn(); +export const openZipArchiveMock = jest.fn(); +export const loadMappingFileMock = jest.fn(); + +jest.doMock('./utils', () => { + const actual = jest.requireActual('./utils'); + return { + ...actual, + downloadToDisk: downloadToDiskMock, + openZipArchive: openZipArchiveMock, + loadMappingFile: loadMappingFileMock, + }; +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts new file mode 100644 index 0000000000000..e68bd0e9c5058 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.test.ts @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + downloadToDiskMock, + createIndexMock, + populateIndexMock, + loadMappingFileMock, + openZipArchiveMock, + validateArtifactArchiveMock, + fetchArtifactVersionsMock, +} from './package_installer.test.mocks'; + +import { + getArtifactName, + getProductDocIndexName, + DocumentationProduct, + ProductName, +} from '@kbn/product-doc-common'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { installClientMock } from '../doc_install_status/service.mock'; +import { inferenceManagerMock } from '../inference_endpoint/service.mock'; +import type { ProductInstallState } from '../../../common/install_status'; +import { PackageInstaller } from './package_installer'; + +const artifactsFolder = '/lost'; +const artifactRepositoryUrl = 'https://repository.com'; +const kibanaVersion = '8.16.3'; + +const callOrder = (fn: { mock: { invocationCallOrder: number[] } }): number => { + return fn.mock.invocationCallOrder[0]; +}; + +describe('PackageInstaller', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let productDocClient: ReturnType; + let endpointManager: ReturnType; + + let packageInstaller: PackageInstaller; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + productDocClient = installClientMock.create(); + endpointManager = inferenceManagerMock.create(); + packageInstaller = new PackageInstaller({ + artifactsFolder, + logger, + esClient, + productDocClient, + endpointManager, + artifactRepositoryUrl, + kibanaVersion, + }); + }); + + afterEach(() => { + downloadToDiskMock.mockReset(); + createIndexMock.mockReset(); + populateIndexMock.mockReset(); + loadMappingFileMock.mockReset(); + openZipArchiveMock.mockReset(); + validateArtifactArchiveMock.mockReset(); + fetchArtifactVersionsMock.mockReset(); + }); + + describe('installPackage', () => { + it('calls the steps with the right parameters', async () => { + const zipArchive = { + close: jest.fn(), + }; + openZipArchiveMock.mockResolvedValue(zipArchive); + + const mappings = Symbol('mappings'); + loadMappingFileMock.mockResolvedValue(mappings); + + await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }); + + const artifactName = getArtifactName({ + productName: 'kibana', + productVersion: '8.16', + }); + const indexName = getProductDocIndexName('kibana'); + expect(endpointManager.ensureInternalElserInstalled).toHaveBeenCalledTimes(1); + + expect(downloadToDiskMock).toHaveBeenCalledTimes(1); + expect(downloadToDiskMock).toHaveBeenCalledWith( + `${artifactRepositoryUrl}/${artifactName}`, + `${artifactsFolder}/${artifactName}` + ); + + expect(openZipArchiveMock).toHaveBeenCalledTimes(1); + expect(openZipArchiveMock).toHaveBeenCalledWith(`${artifactsFolder}/${artifactName}`); + + expect(loadMappingFileMock).toHaveBeenCalledTimes(1); + expect(loadMappingFileMock).toHaveBeenCalledWith(zipArchive); + + expect(createIndexMock).toHaveBeenCalledTimes(1); + expect(createIndexMock).toHaveBeenCalledWith({ + indexName, + mappings, + esClient, + log: logger, + }); + + expect(populateIndexMock).toHaveBeenCalledTimes(1); + expect(populateIndexMock).toHaveBeenCalledWith({ + indexName, + archive: zipArchive, + esClient, + log: logger, + }); + + expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledTimes(1); + expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledWith('kibana', indexName); + + expect(zipArchive.close).toHaveBeenCalledTimes(1); + + expect(productDocClient.setInstallationFailed).not.toHaveBeenCalled(); + }); + + it('executes the steps in the right order', async () => { + await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }); + + expect(callOrder(endpointManager.ensureInternalElserInstalled)).toBeLessThan( + callOrder(downloadToDiskMock) + ); + expect(callOrder(downloadToDiskMock)).toBeLessThan(callOrder(openZipArchiveMock)); + expect(callOrder(openZipArchiveMock)).toBeLessThan(callOrder(loadMappingFileMock)); + expect(callOrder(loadMappingFileMock)).toBeLessThan(callOrder(createIndexMock)); + expect(callOrder(createIndexMock)).toBeLessThan(callOrder(populateIndexMock)); + expect(callOrder(populateIndexMock)).toBeLessThan( + callOrder(productDocClient.setInstallationSuccessful) + ); + }); + + it('closes the archive and calls setInstallationFailed if the installation fails', async () => { + const zipArchive = { + close: jest.fn(), + }; + openZipArchiveMock.mockResolvedValue(zipArchive); + + populateIndexMock.mockImplementation(async () => { + throw new Error('something bad'); + }); + + await expect( + packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' }) + ).rejects.toThrowError(); + + expect(productDocClient.setInstallationSuccessful).not.toHaveBeenCalled(); + + expect(zipArchive.close).toHaveBeenCalledTimes(1); + + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('Error during documentation installation') + ); + + expect(productDocClient.setInstallationFailed).toHaveBeenCalledTimes(1); + expect(productDocClient.setInstallationFailed).toHaveBeenCalledWith( + 'kibana', + 'something bad' + ); + }); + }); + + describe('installALl', () => { + it('installs all the packages to their latest version', async () => { + jest.spyOn(packageInstaller, 'installPackage'); + + fetchArtifactVersionsMock.mockResolvedValue({ + kibana: ['8.15', '8.16'], + elasticsearch: ['8.15'], + }); + + await packageInstaller.installAll({}); + + expect(packageInstaller.installPackage).toHaveBeenCalledTimes(2); + + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'kibana', + productVersion: '8.16', + }); + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'elasticsearch', + productVersion: '8.15', + }); + }); + }); + + describe('ensureUpToDate', () => { + it('updates the installed packages to the latest version', async () => { + fetchArtifactVersionsMock.mockResolvedValue({ + kibana: ['8.15', '8.16'], + security: ['8.15', '8.16'], + elasticsearch: ['8.15'], + }); + + productDocClient.getInstallationStatus.mockResolvedValue({ + kibana: { status: 'installed', version: '8.15' }, + security: { status: 'installed', version: '8.16' }, + elasticsearch: { status: 'uninstalled' }, + } as Record); + + jest.spyOn(packageInstaller, 'installPackage'); + + await packageInstaller.ensureUpToDate({}); + + expect(packageInstaller.installPackage).toHaveBeenCalledTimes(1); + expect(packageInstaller.installPackage).toHaveBeenCalledWith({ + productName: 'kibana', + productVersion: '8.16', + }); + }); + }); + + describe('uninstallPackage', () => { + it('performs the uninstall steps', async () => { + await packageInstaller.uninstallPackage({ productName: 'kibana' }); + + expect(esClient.indices.delete).toHaveBeenCalledTimes(1); + expect(esClient.indices.delete).toHaveBeenCalledWith( + { + index: getProductDocIndexName('kibana'), + }, + expect.objectContaining({ ignore: [404] }) + ); + + expect(productDocClient.setUninstalled).toHaveBeenCalledTimes(1); + expect(productDocClient.setUninstalled).toHaveBeenCalledWith('kibana'); + }); + }); + + describe('uninstallAll', () => { + it('calls uninstall for all packages', async () => { + jest.spyOn(packageInstaller, 'uninstallPackage'); + + await packageInstaller.uninstallAll(); + + expect(packageInstaller.uninstallPackage).toHaveBeenCalledTimes( + Object.keys(DocumentationProduct).length + ); + Object.values(DocumentationProduct).forEach((productName) => { + expect(packageInstaller.uninstallPackage).toHaveBeenCalledWith({ productName }); + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts new file mode 100644 index 0000000000000..7739219c15dc6 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/package_installer.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { + getArtifactName, + getProductDocIndexName, + DocumentationProduct, + type ProductName, +} from '@kbn/product-doc-common'; +import type { ProductDocInstallClient } from '../doc_install_status'; +import type { InferenceEndpointManager } from '../inference_endpoint'; +import { downloadToDisk, openZipArchive, loadMappingFile, type ZipArchive } from './utils'; +import { majorMinor, latestVersion } from './utils/semver'; +import { + validateArtifactArchive, + fetchArtifactVersions, + createIndex, + populateIndex, +} from './steps'; + +interface PackageInstallerOpts { + artifactsFolder: string; + logger: Logger; + esClient: ElasticsearchClient; + productDocClient: ProductDocInstallClient; + endpointManager: InferenceEndpointManager; + artifactRepositoryUrl: string; + kibanaVersion: string; +} + +export class PackageInstaller { + private readonly log: Logger; + private readonly artifactsFolder: string; + private readonly esClient: ElasticsearchClient; + private readonly productDocClient: ProductDocInstallClient; + private readonly endpointManager: InferenceEndpointManager; + private readonly artifactRepositoryUrl: string; + private readonly currentVersion: string; + + constructor({ + artifactsFolder, + logger, + esClient, + productDocClient, + endpointManager, + artifactRepositoryUrl, + kibanaVersion, + }: PackageInstallerOpts) { + this.esClient = esClient; + this.productDocClient = productDocClient; + this.artifactsFolder = artifactsFolder; + this.endpointManager = endpointManager; + this.artifactRepositoryUrl = artifactRepositoryUrl; + this.currentVersion = majorMinor(kibanaVersion); + this.log = logger; + } + + /** + * Make sure that the currently installed doc packages are up to date. + * Will not upgrade products that are not already installed + */ + async ensureUpToDate({}: {}) { + const [repositoryVersions, installStatuses] = await Promise.all([ + fetchArtifactVersions({ + artifactRepositoryUrl: this.artifactRepositoryUrl, + }), + this.productDocClient.getInstallationStatus(), + ]); + + const toUpdate: Array<{ + productName: ProductName; + productVersion: string; + }> = []; + Object.entries(installStatuses).forEach(([productName, productState]) => { + if (productState.status === 'uninstalled') { + return; + } + const availableVersions = repositoryVersions[productName as ProductName]; + if (!availableVersions || !availableVersions.length) { + return; + } + const selectedVersion = selectVersion(this.currentVersion, availableVersions); + if (productState.version !== selectedVersion) { + toUpdate.push({ + productName: productName as ProductName, + productVersion: selectedVersion, + }); + } + }); + + for (const { productName, productVersion } of toUpdate) { + await this.installPackage({ + productName, + productVersion, + }); + } + } + + async installAll({}: {}) { + const repositoryVersions = await fetchArtifactVersions({ + artifactRepositoryUrl: this.artifactRepositoryUrl, + }); + const allProducts = Object.values(DocumentationProduct) as ProductName[]; + for (const productName of allProducts) { + const availableVersions = repositoryVersions[productName]; + if (!availableVersions || !availableVersions.length) { + this.log.warn(`No version found for product [${productName}]`); + continue; + } + const selectedVersion = selectVersion(this.currentVersion, availableVersions); + + await this.installPackage({ + productName, + productVersion: selectedVersion, + }); + } + } + + async installPackage({ + productName, + productVersion, + }: { + productName: ProductName; + productVersion: string; + }) { + this.log.info( + `Starting installing documentation for product [${productName}] and version [${productVersion}]` + ); + + productVersion = majorMinor(productVersion); + + await this.uninstallPackage({ productName }); + + let zipArchive: ZipArchive | undefined; + try { + await this.productDocClient.setInstallationStarted({ + productName, + productVersion, + }); + + await this.endpointManager.ensureInternalElserInstalled(); + + const artifactFileName = getArtifactName({ productName, productVersion }); + const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`; + const artifactPath = `${this.artifactsFolder}/${artifactFileName}`; + + this.log.debug(`Downloading from [${artifactUrl}] to [${artifactPath}]`); + await downloadToDisk(artifactUrl, artifactPath); + + zipArchive = await openZipArchive(artifactPath); + + validateArtifactArchive(zipArchive); + + const mappings = await loadMappingFile(zipArchive); + + const indexName = getProductDocIndexName(productName); + + await createIndex({ + indexName, + mappings, + esClient: this.esClient, + log: this.log, + }); + + await populateIndex({ + indexName, + archive: zipArchive, + esClient: this.esClient, + log: this.log, + }); + await this.productDocClient.setInstallationSuccessful(productName, indexName); + + this.log.info( + `Documentation installation successful for product [${productName}] and version [${productVersion}]` + ); + } catch (e) { + this.log.error( + `Error during documentation installation of product [${productName}]/[${productVersion}] : ${e.message}` + ); + + await this.productDocClient.setInstallationFailed(productName, e.message); + throw e; + } finally { + zipArchive?.close(); + } + } + + async uninstallPackage({ productName }: { productName: ProductName }) { + const indexName = getProductDocIndexName(productName); + await this.esClient.indices.delete( + { + index: indexName, + }, + { ignore: [404] } + ); + + await this.productDocClient.setUninstalled(productName); + } + + async uninstallAll() { + const allProducts = Object.values(DocumentationProduct); + for (const productName of allProducts) { + await this.uninstallPackage({ productName }); + } + } +} + +const selectVersion = (currentVersion: string, availableVersions: string[]): string => { + return availableVersions.includes(currentVersion) + ? currentVersion + : latestVersion(availableVersions); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts new file mode 100644 index 0000000000000..fca8b5283c300 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { createIndex } from './create_index'; +import { internalElserInferenceId } from '../../../../common/consts'; + +describe('createIndex', () => { + let log: MockedLogger; + let esClient: ElasticsearchClient; + + beforeEach(() => { + log = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('calls esClient.indices.create with the right parameters', async () => { + const mappings: MappingTypeMapping = { + properties: {}, + }; + const indexName = '.some-index'; + + await createIndex({ + indexName, + mappings, + log, + esClient, + }); + + expect(esClient.indices.create).toHaveBeenCalledTimes(1); + expect(esClient.indices.create).toHaveBeenCalledWith({ + index: indexName, + mappings, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }, + }); + }); + + it('rewrites the inference_id attribute of semantic_text fields in the mapping', async () => { + const mappings: MappingTypeMapping = { + properties: { + semantic: { + type: 'semantic_text', + inference_id: '.elser', + }, + bool: { + type: 'boolean', + }, + }, + }; + + await createIndex({ + indexName: '.some-index', + mappings, + log, + esClient, + }); + + expect(esClient.indices.create).toHaveBeenCalledWith( + expect.objectContaining({ + mappings: { + properties: { + semantic: { + type: 'semantic_text', + inference_id: internalElserInferenceId, + }, + bool: { + type: 'boolean', + }, + }, + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts new file mode 100644 index 0000000000000..decd62e556ba5 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/create_index.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { MappingTypeMapping, MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import { internalElserInferenceId } from '../../../../common/consts'; + +export const createIndex = async ({ + esClient, + indexName, + mappings, + log, +}: { + esClient: ElasticsearchClient; + indexName: string; + mappings: MappingTypeMapping; + log: Logger; +}) => { + log.debug(`Creating index ${indexName}`); + + overrideInferenceId(mappings, internalElserInferenceId); + + await esClient.indices.create({ + index: indexName, + mappings, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }, + }); +}; + +const overrideInferenceId = (mappings: MappingTypeMapping, inferenceId: string) => { + const recursiveOverride = (current: MappingTypeMapping | MappingProperty) => { + if ('type' in current && current.type === 'semantic_text') { + current.inference_id = inferenceId; + } + if ('properties' in current && current.properties) { + for (const prop of Object.values(current.properties)) { + recursiveOverride(prop); + } + } + }; + recursiveOverride(mappings); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts new file mode 100644 index 0000000000000..805008ccab698 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fetch, { Response } from 'node-fetch'; +import { fetchArtifactVersions } from './fetch_artifact_versions'; +import { getArtifactName, DocumentationProduct, ProductName } from '@kbn/product-doc-common'; + +jest.mock('node-fetch'); +const fetchMock = fetch as jest.MockedFn; + +const createResponse = ({ + artifactNames, + truncated = false, +}: { + artifactNames: string[]; + truncated?: boolean; +}) => { + return ` + + kibana-ai-assistant-kb-artifacts + + + ${truncated} + ${artifactNames.map( + (artifactName) => ` + + ${artifactName} + 1728486063097626 + 1 + 2024-10-09T15:01:03.137Z + "e0584955969eccf2a16b8829f768cb1f" + 36781438 + ` + )} + + `; +}; + +const artifactRepositoryUrl = 'https://lost.com'; + +const expectVersions = ( + versions: Partial> +): Record => { + const response = {} as Record; + Object.values(DocumentationProduct).forEach((productName) => { + response[productName] = []; + }); + return { + ...response, + ...versions, + }; +}; + +describe('fetchArtifactVersions', () => { + beforeEach(() => { + fetchMock.mockReset(); + }); + + const mockResponse = (responseText: string) => { + const response = { + text: () => Promise.resolve(responseText), + }; + fetchMock.mockResolvedValue(response as Response); + }; + + it('calls fetch with the right parameters', async () => { + mockResponse(createResponse({ artifactNames: [] })); + + await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith(`${artifactRepositoryUrl}?max-keys=1000`); + }); + + it('returns the list of versions from the repository', async () => { + const artifactNames = [ + getArtifactName({ productName: 'kibana', productVersion: '8.16' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }), + ]; + mockResponse(createResponse({ artifactNames })); + + const versions = await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(versions).toEqual( + expectVersions({ + kibana: ['8.16'], + elasticsearch: ['8.16'], + }) + ); + }); + + it('retrieve all versions for each product', async () => { + const artifactNames = [ + getArtifactName({ productName: 'kibana', productVersion: '8.15' }), + getArtifactName({ productName: 'kibana', productVersion: '8.16' }), + getArtifactName({ productName: 'kibana', productVersion: '8.17' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }), + getArtifactName({ productName: 'elasticsearch', productVersion: '9.0' }), + ]; + mockResponse(createResponse({ artifactNames })); + + const versions = await fetchArtifactVersions({ artifactRepositoryUrl }); + + expect(versions).toEqual( + expectVersions({ + kibana: ['8.15', '8.16', '8.17'], + elasticsearch: ['8.16', '9.0'], + }) + ); + }); + + it('throws an error if the response is truncated', async () => { + mockResponse(createResponse({ artifactNames: [], truncated: true })); + + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects.toThrowError( + /bucket content is truncated/ + ); + }); + + it('throws an error if the response is not valid xml', async () => { + mockResponse('some plain text'); + + await expect(fetchArtifactVersions({ artifactRepositoryUrl })).rejects.toThrowError(); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts new file mode 100644 index 0000000000000..69c6db2d5d8ae --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fetch from 'node-fetch'; +import { parseString } from 'xml2js'; +import { type ProductName, DocumentationProduct, parseArtifactName } from '@kbn/product-doc-common'; + +type ArtifactAvailableVersions = Record; + +export const fetchArtifactVersions = async ({ + artifactRepositoryUrl, +}: { + artifactRepositoryUrl: string; +}): Promise => { + const res = await fetch(`${artifactRepositoryUrl}?max-keys=1000`); + const xml = await res.text(); + return new Promise((resolve, reject) => { + parseString(xml, (err, result: ListBucketResponse) => { + if (err) { + reject(err); + } + + // 6 artifacts per minor stack version means we have a few decades before facing this problem + if (result.ListBucketResult.IsTruncated?.includes('true')) { + throw new Error('bucket content is truncated, cannot retrieve all versions'); + } + + const allowedProductNames: ProductName[] = Object.values(DocumentationProduct); + + const record: ArtifactAvailableVersions = {} as ArtifactAvailableVersions; + allowedProductNames.forEach((product) => { + record[product] = []; + }); + + result.ListBucketResult.Contents?.forEach((contentEntry) => { + const artifactName = contentEntry.Key[0]; + const parsed = parseArtifactName(artifactName); + if (parsed) { + const { productName, productVersion } = parsed; + record[productName]!.push(productVersion); + } + }); + + resolve(record); + }); + }); +}; + +interface ListBucketResponse { + ListBucketResult: { + Name?: string[]; + IsTruncated?: string[]; + Contents?: Array<{ Key: string[] }>; + }; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts new file mode 100644 index 0000000000000..3c84fc9cccf1a --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { createIndex } from './create_index'; +export { populateIndex } from './populate_index'; +export { validateArtifactArchive } from './validate_artifact_archive'; +export { fetchArtifactVersions } from './fetch_artifact_versions'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts new file mode 100644 index 0000000000000..2f301f9928e9a --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { times } from 'lodash'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { internalElserInferenceId } from '../../../../common/consts'; +import type { ZipArchive } from '../utils/zip_archive'; +import { populateIndex } from './populate_index'; + +const createMockArchive = (entries: Record): ZipArchive => { + return { + hasEntry: (entryPath) => Object.keys(entries).includes(entryPath), + getEntryPaths: () => Object.keys(entries), + getEntryContent: async (entryPath) => Buffer.from(entries[entryPath]), + close: () => undefined, + }; +}; + +const createContentFile = (count: number, offset: number = 0): string => { + return times(count) + .map((i) => JSON.stringify({ idx: offset + i })) + .join('\n'); +}; + +describe('populateIndex', () => { + let log: MockedLogger; + let esClient: ReturnType; + + beforeEach(() => { + log = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('calls `esClient.bulk` once per content file', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': createContentFile(2), + 'content/content-1.ndjson': createContentFile(2), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(2); + }); + + it('calls `esClient.bulk` with the right payload', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': createContentFile(2), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(1); + expect(esClient.bulk).toHaveBeenCalledWith({ + refresh: false, + operations: [ + { index: { _index: '.foo' } }, + { idx: 0 }, + { index: { _index: '.foo' } }, + { idx: 1 }, + ], + }); + }); + + it('rewrites the inference_id of semantic fields', async () => { + const archive = createMockArchive({ + 'content/content-0.ndjson': JSON.stringify({ + semantic: { text: 'foo', inference: { inference_id: '.some-inference' } }, + }), + }); + + await populateIndex({ + indexName: '.foo', + archive, + log, + esClient, + }); + + expect(esClient.bulk).toHaveBeenCalledTimes(1); + expect(esClient.bulk).toHaveBeenCalledWith({ + refresh: false, + operations: [ + { index: { _index: '.foo' } }, + { + semantic: { + inference: { + inference_id: internalElserInferenceId, + }, + text: 'foo', + }, + }, + ], + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts new file mode 100644 index 0000000000000..017757ca90b99 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/populate_index.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { isArtifactContentFilePath } from '@kbn/product-doc-common'; +import { internalElserInferenceId } from '../../../../common/consts'; +import type { ZipArchive } from '../utils/zip_archive'; + +export const populateIndex = async ({ + esClient, + indexName, + archive, + log, +}: { + esClient: ElasticsearchClient; + indexName: string; + archive: ZipArchive; + log: Logger; +}) => { + log.debug(`Starting populating index ${indexName}`); + + const contentEntries = archive.getEntryPaths().filter(isArtifactContentFilePath); + + for (let i = 0; i < contentEntries.length; i++) { + const entryPath = contentEntries[i]; + log.debug(`Indexing content for entry ${entryPath}`); + const contentBuffer = await archive.getEntryContent(entryPath); + await indexContentFile({ indexName, esClient, contentBuffer }); + } + + log.debug(`Done populating index ${indexName}`); +}; + +const indexContentFile = async ({ + indexName, + contentBuffer, + esClient, +}: { + indexName: string; + contentBuffer: Buffer; + esClient: ElasticsearchClient; +}) => { + const fileContent = contentBuffer.toString('utf-8'); + const lines = fileContent.split('\n'); + + const documents = lines + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + return JSON.parse(line); + }) + .map((doc) => rewriteInferenceId(doc, internalElserInferenceId)); + + const operations = documents.reduce((ops, document) => { + ops!.push(...[{ index: { _index: indexName } }, document]); + return ops; + }, [] as BulkRequest['operations']); + + const response = await esClient.bulk({ + refresh: false, + operations, + }); + + if (response.errors) { + const error = response.items.find((item) => item.index?.error)?.index?.error ?? 'unknown error'; + throw new Error(`Error indexing documents: ${JSON.stringify(error)}`); + } +}; + +const rewriteInferenceId = (document: Record, inferenceId: string) => { + // we don't need to handle nested fields, we don't have any and won't. + Object.values(document).forEach((field) => { + if (field.inference) { + field.inference.inference_id = inferenceId; + } + }); + return document; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts new file mode 100644 index 0000000000000..607277aaf3466 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ZipArchive } from '../utils/zip_archive'; +import { validateArtifactArchive } from './validate_artifact_archive'; + +const createMockArchive = (entryPaths: string[]): ZipArchive => { + return { + hasEntry: (entryPath) => entryPaths.includes(entryPath), + getEntryPaths: () => entryPaths, + getEntryContent: () => { + throw new Error('non implemented'); + }, + close: () => undefined, + }; +}; + +describe('validateArtifactArchive', () => { + it('validates that the archive contains all the mandatory files', () => { + const archive = createMockArchive([ + 'manifest.json', + 'mappings.json', + 'content/content-1.ndjson', + ]); + + const validation = validateArtifactArchive(archive); + + expect(validation).toEqual({ valid: true }); + }); + + it('does not validate if the archive does not contain a manifest', () => { + const archive = createMockArchive(['something.txt']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "Manifest file not found", + "valid": false, + } + `); + }); + + it('does not validate if the archive does not contain mappings', () => { + const archive = createMockArchive(['manifest.json']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "Mapping file not found", + "valid": false, + } + `); + }); + + it('does not validate if the archive does not contain content files', () => { + const archive = createMockArchive(['manifest.json', 'mappings.json']); + + const validation = validateArtifactArchive(archive); + + expect(validation).toMatchInlineSnapshot(` + Object { + "error": "No content files were found", + "valid": false, + } + `); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts new file mode 100644 index 0000000000000..471d7c080c481 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/steps/validate_artifact_archive.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArtifactContentFilePath } from '@kbn/product-doc-common'; +import type { ZipArchive } from '../utils/zip_archive'; + +type ValidationResult = { valid: true } | { valid: false; error: string }; + +export const validateArtifactArchive = (archive: ZipArchive): ValidationResult => { + if (!archive.hasEntry('manifest.json')) { + return { valid: false, error: 'Manifest file not found' }; + } + if (!archive.hasEntry('mappings.json')) { + return { valid: false, error: 'Mapping file not found' }; + } + if (!archive.getEntryPaths().some(isArtifactContentFilePath)) { + return { valid: false, error: 'No content files were found' }; + } + return { valid: true }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts new file mode 100644 index 0000000000000..9d42be652d74d --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ArtifactManifest } from '@kbn/product-doc-common'; +import type { ZipArchive } from './zip_archive'; +import { loadManifestFile, loadMappingFile } from './archive_accessors'; + +const createMockArchive = (entries: Record): ZipArchive => { + return { + hasEntry: (entryPath) => Object.keys(entries).includes(entryPath), + getEntryPaths: () => Object.keys(entries), + getEntryContent: async (entryPath) => Buffer.from(entries[entryPath]), + close: () => undefined, + }; +}; + +describe('loadManifestFile', () => { + it('parses the manifest from the archive', async () => { + const manifest: ArtifactManifest = { + formatVersion: '1.0.0', + productName: 'kibana', + productVersion: '8.16', + }; + const archive = createMockArchive({ 'manifest.json': JSON.stringify(manifest) }); + + const parsedManifest = await loadManifestFile(archive); + + expect(parsedManifest).toEqual(manifest); + }); + + it('throws if the archive does not contain the manifest', async () => { + const archive = createMockArchive({}); + + await expect(loadManifestFile(archive)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Could not load archive file: \\"manifest.json\\" not found in archive"` + ); + }); + + it('throws if the manifest cannot be parsed', async () => { + const archive = createMockArchive({ 'manifest.json': '{}}}{' }); + + await expect(loadManifestFile(archive)).rejects.toThrowError(); + }); +}); + +describe('loadMappingFile', () => { + it('parses the manifest from the archive', async () => { + const mappings: MappingTypeMapping = { + properties: { + foo: { type: 'text' }, + }, + }; + const archive = createMockArchive({ 'mappings.json': JSON.stringify(mappings) }); + + const parsedMappings = await loadMappingFile(archive); + + expect(parsedMappings).toEqual(mappings); + }); + + it('throws if the archive does not contain the manifest', async () => { + const archive = createMockArchive({}); + + await expect(loadMappingFile(archive)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Could not load archive file: \\"mappings.json\\" not found in archive"` + ); + }); + + it('throws if the manifest cannot be parsed', async () => { + const archive = createMockArchive({ 'mappings.json': '{}}}{' }); + + await expect(loadMappingFile(archive)).rejects.toThrowError(); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts new file mode 100644 index 0000000000000..a4ec4f4418f3c --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/archive_accessors.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ArtifactManifest } from '@kbn/product-doc-common'; +import type { ZipArchive } from './zip_archive'; + +const manifestEntryPath = 'manifest.json'; +const mappingsEntryPath = 'mappings.json'; + +export const loadManifestFile = async (archive: ZipArchive): Promise => { + return await parseEntryContent(manifestEntryPath, archive); +}; + +export const loadMappingFile = async (archive: ZipArchive): Promise => { + return await parseEntryContent(mappingsEntryPath, archive); +}; + +const parseEntryContent = async (entryPath: string, archive: ZipArchive): Promise => { + if (!archive.hasEntry(entryPath)) { + throw new Error(`Could not load archive file: "${entryPath}" not found in archive`); + } + try { + const buffer = await archive.getEntryContent(entryPath); + return JSON.parse(buffer.toString('utf-8')); + } catch (e) { + throw new Error(`Could not parse archive file: ${e}`); + } +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts new file mode 100644 index 0000000000000..ea5357792ef5f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createWriteStream } from 'fs'; +import { mkdir } from 'fs/promises'; +import Path from 'path'; +import fetch from 'node-fetch'; + +export const downloadToDisk = async (fileUrl: string, filePath: string) => { + const dirPath = Path.dirname(filePath); + await mkdir(dirPath, { recursive: true }); + const res = await fetch(fileUrl); + const fileStream = createWriteStream(filePath); + await new Promise((resolve, reject) => { + res.body.pipe(fileStream); + res.body.on('error', reject); + fileStream.on('finish', resolve); + }); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts new file mode 100644 index 0000000000000..a612a8c6e9f46 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { downloadToDisk } from './download'; +export { openZipArchive, type ZipArchive } from './zip_archive'; +export { loadManifestFile, loadMappingFile } from './archive_accessors'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts new file mode 100644 index 0000000000000..9bc20f2eecdbd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { latestVersion, majorMinor } from './semver'; + +describe('majorMinor', () => { + it('returns the version in a {major.minor} format', () => { + expect(majorMinor('9.17.5')).toEqual('9.17'); + }); + it('ignores qualifiers', () => { + expect(majorMinor('10.42.9000-snap')).toEqual('10.42'); + }); + it('accepts {major.minor} format as input', () => { + expect(majorMinor('8.16')).toEqual('8.16'); + }); +}); + +describe('latestVersion', () => { + it('returns the highest version from the list', () => { + expect(latestVersion(['7.16.3', '8.1.4', '6.14.2'])).toEqual('8.1.4'); + }); + it('accepts versions in a {major.minor} format', () => { + expect(latestVersion(['9.16', '9.3'])).toEqual('9.16'); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts new file mode 100644 index 0000000000000..b4e38215af90e --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/semver.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Semver from 'semver'; + +export const latestVersion = (versions: string[]): string => { + let latest: string = versions[0]; + for (let i = 1; i < versions.length; i++) { + const current = versions[i]; + if (Semver.gt(Semver.coerce(current)!, Semver.coerce(latest)!)) { + latest = current; + } + } + return latest; +}; + +export const majorMinor = (version: string): string => { + const parsed = Semver.coerce(version); + if (!parsed) { + throw new Error(`Not a valid semver version: [${version}]`); + } + return `${parsed.major}.${parsed.minor}`; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip new file mode 100644 index 0000000000000..fce195d2c4db2 Binary files /dev/null and b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/test_data/test_archive_1.zip differ diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts new file mode 100644 index 0000000000000..71cd5891c5e5d --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; +import { openZipArchive, ZipArchive } from './zip_archive'; + +const ZIP_PATH = Path.resolve(__dirname, './test_data/test_archive_1.zip'); + +describe('ZipArchive', () => { + let archive: ZipArchive; + + beforeAll(async () => { + archive = await openZipArchive(ZIP_PATH); + }); + + afterAll(() => { + archive?.close(); + }); + + test('#getEntryPaths returns the path of all entries', () => { + expect(archive.getEntryPaths().sort()).toEqual([ + 'nested/', + 'nested/nested_1.txt', + 'text_1.txt', + 'text_2.txt', + 'text_3.txt', + ]); + }); + + test('#hasEntry returns true if the entry exists, false otherwise', () => { + expect(archive.hasEntry('nested/nested_1.txt')).toBe(true); + expect(archive.hasEntry('not_an_entry')).toBe(false); + }); + + test('#getEntryContent returns the content of the entry', async () => { + const buffer = await archive.getEntryContent('text_1.txt'); + expect(buffer.toString('utf-8')).toEqual('text_1'); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts new file mode 100644 index 0000000000000..dbc4ec1b3e41f --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/package_installer/utils/zip_archive.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import yauzl from 'yauzl'; + +export interface ZipArchive { + hasEntry(entryPath: string): boolean; + getEntryPaths(): string[]; + getEntryContent(entryPath: string): Promise; + close(): void; +} + +export const openZipArchive = async (archivePath: string): Promise => { + return new Promise((resolve, reject) => { + const entries: yauzl.Entry[] = []; + yauzl.open(archivePath, { lazyEntries: true, autoClose: false }, (err, zipFile) => { + if (err || !zipFile) { + return reject(err ?? 'No zip file'); + } + + zipFile!.on('entry', (entry) => { + entries.push(entry); + zipFile.readEntry(); + }); + + zipFile.on('end', () => { + const archive = new ZipArchiveImpl(entries, zipFile); + resolve(archive); + }); + + zipFile.on('close', () => {}); + + zipFile.readEntry(); + }); + }); +}; + +class ZipArchiveImpl implements ZipArchive { + private readonly zipFile: yauzl.ZipFile; + private readonly entries: Map; + + constructor(entries: yauzl.Entry[], zipFile: yauzl.ZipFile) { + this.zipFile = zipFile; + this.entries = new Map(entries.map((entry) => [entry.fileName, entry])); + } + + hasEntry(entryPath: string) { + return this.entries.has(entryPath); + } + + getEntryPaths() { + return [...this.entries.keys()]; + } + + getEntryContent(entryPath: string) { + const foundEntry = this.entries.get(entryPath); + if (!foundEntry) { + throw new Error(`Entry ${entryPath} not found in archive`); + } + return getZipEntryContent(this.zipFile, foundEntry); + } + + close() { + this.zipFile.close(); + } +} + +const getZipEntryContent = async (zipFile: yauzl.ZipFile, entry: yauzl.Entry): Promise => { + return new Promise((resolve, reject) => { + zipFile.openReadStream(entry, (err, readStream) => { + if (err) { + return reject(err); + } else { + const chunks: Buffer[] = []; + readStream!.on('data', (chunk: Buffer) => { + chunks.push(chunk); + }); + readStream!.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + readStream!.on('error', () => { + reject(); + }); + } + }); + }); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts new file mode 100644 index 0000000000000..3e5ac95ae4edf --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { SearchService } from './search_service'; +export type { DocSearchOptions, DocSearchResult, DocSearchResponse, SearchApi } from './types'; diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts similarity index 78% rename from x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts rename to x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts index 373a6b8755429..03c3b72f86f92 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/perform_semantic_search.ts +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/perform_search.ts @@ -5,29 +5,27 @@ * 2.0. */ -import type { Client } from '@elastic/elasticsearch'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; // https://search-labs.elastic.co/search-labs/blog/elser-rag-search-for-relevance -export const performSemanticSearch = async ({ +export const performSearch = async ({ searchQuery, + size, index, client, }: { searchQuery: string; - index: string; - client: Client; + size: number; + index: string | string[]; + client: ElasticsearchClient; }) => { - const results = await client.search({ + const results = await client.search({ index, - size: 3, + size, query: { bool: { - filter: { - bool: { - must: [{ term: { version: '8.15' } }], - }, - }, should: [ { multi_match: { @@ -37,7 +35,7 @@ export const performSemanticSearch = async ({ fields: [ 'content_title', 'content_body.text', - 'ai_subtitle.text', + 'ai_subtitle', 'ai_summary.text', 'ai_questions_answered.text', 'ai_tags', @@ -65,12 +63,6 @@ export const performSemanticSearch = async ({ query: searchQuery, }, }, - { - semantic: { - field: 'ai_subtitle', - query: searchQuery, - }, - }, { semantic: { field: 'ai_summary', diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts new file mode 100644 index 0000000000000..c8053ca981e71 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { SearchService } from './search_service'; +import { getIndicesForProductNames } from './utils'; + +import { performSearch } from './perform_search'; +jest.mock('./perform_search'); +const performSearchMock = performSearch as jest.MockedFn; + +describe('SearchService', () => { + let logger: MockedLogger; + let esClient: ReturnType; + let service: SearchService; + + beforeEach(() => { + logger = loggerMock.create(); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + service = new SearchService({ logger, esClient }); + + performSearchMock.mockResolvedValue([]); + }); + + afterEach(() => { + performSearchMock.mockReset(); + }); + + describe('#search', () => { + it('calls `performSearch` with the right parameters', async () => { + await service.search({ + query: 'What is Kibana?', + products: ['kibana'], + max: 42, + }); + + expect(performSearchMock).toHaveBeenCalledTimes(1); + expect(performSearchMock).toHaveBeenCalledWith({ + searchQuery: 'What is Kibana?', + size: 42, + index: getIndicesForProductNames(['kibana']), + client: esClient, + }); + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts new file mode 100644 index 0000000000000..a0b1e4fd4a836 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/search_service.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import { getIndicesForProductNames, mapResult } from './utils'; +import { performSearch } from './perform_search'; +import type { DocSearchOptions, DocSearchResponse } from './types'; + +export class SearchService { + private readonly log: Logger; + private readonly esClient: ElasticsearchClient; + + constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) { + this.log = logger; + this.esClient = esClient; + } + + async search(options: DocSearchOptions): Promise { + const { query, max = 3, products } = options; + this.log.debug(`performing search - query=[${query}]`); + const results = await performSearch({ + searchQuery: query, + size: max, + index: getIndicesForProductNames(products), + client: this.esClient, + }); + + return { + results: results.map(mapResult), + }; + } +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts new file mode 100644 index 0000000000000..fb474bbf4deab --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/types.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ProductName } from '@kbn/product-doc-common'; + +export interface DocSearchOptions { + query: string; + max?: number; + products?: ProductName[]; +} + +export interface DocSearchResult { + title: string; + content: string; + url: string; + productName: ProductName; +} + +export interface DocSearchResponse { + results: DocSearchResult[]; +} + +export type SearchApi = (options: DocSearchOptions) => Promise; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts new file mode 100644 index 0000000000000..0293d086d4f13 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { productDocIndexPattern, getProductDocIndexName } from '@kbn/product-doc-common'; +import { getIndicesForProductNames } from './get_indices_for_product_names'; + +describe('getIndicesForProductNames', () => { + it('returns the index pattern when product names are not specified', () => { + expect(getIndicesForProductNames(undefined)).toEqual(productDocIndexPattern); + expect(getIndicesForProductNames([])).toEqual(productDocIndexPattern); + }); + it('returns individual index names when product names are specified', () => { + expect(getIndicesForProductNames(['kibana', 'elasticsearch'])).toEqual([ + getProductDocIndexName('kibana'), + getProductDocIndexName('elasticsearch'), + ]); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts new file mode 100644 index 0000000000000..e97ed9cea3611 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/get_indices_for_product_names.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + productDocIndexPattern, + getProductDocIndexName, + type ProductName, +} from '@kbn/product-doc-common'; + +export const getIndicesForProductNames = ( + productNames: ProductName[] | undefined +): string | string[] => { + if (!productNames || !productNames.length) { + return productDocIndexPattern; + } + return productNames.map(getProductDocIndexName); +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts new file mode 100644 index 0000000000000..1a6a2eaa24a99 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getIndicesForProductNames } from './get_indices_for_product_names'; +export { mapResult } from './map_result'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts new file mode 100644 index 0000000000000..56e8ce4875cc5 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; +import { mapResult } from './map_result'; + +const createHit = ( + attrs: ProductDocumentationAttributes +): SearchHit => { + return { + _index: '.foo', + _source: attrs, + }; +}; + +describe('mapResult', () => { + it('returns the expected shape', () => { + const input = createHit({ + content_title: 'content_title', + content_body: { text: 'content_body' }, + product_name: 'kibana', + root_type: 'documentation', + slug: 'foo.html', + url: 'http://lost.com/foo.html', + version: '8.16', + ai_subtitle: 'ai_subtitle', + ai_summary: { text: 'ai_summary' }, + ai_questions_answered: { text: ['question A'] }, + ai_tags: ['foo', 'bar', 'test'], + }); + + const output = mapResult(input); + + expect(output).toEqual({ + content: 'content_body', + productName: 'kibana', + title: 'content_title', + url: 'http://lost.com/foo.html', + }); + }); +}); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts new file mode 100644 index 0000000000000..f4f66b2111827 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/services/search/utils/map_result.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { ProductDocumentationAttributes } from '@kbn/product-doc-common'; +import type { DocSearchResult } from '../types'; + +export const mapResult = (docHit: SearchHit): DocSearchResult => { + return { + title: docHit._source!.content_title, + content: docHit._source!.content_body.text, + url: docHit._source!.url, + productName: docHit._source!.product_name, + }; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts new file mode 100644 index 0000000000000..d971561914ff1 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/ensure_up_to_date.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const ENSURE_DOC_UP_TO_DATE_TASK_TYPE = 'ProductDocBase:EnsureUpToDate'; +export const ENSURE_DOC_UP_TO_DATE_TASK_ID = 'ProductDocBase:EnsureUpToDate'; + +export const registerEnsureUpToDateTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [ENSURE_DOC_UP_TO_DATE_TASK_TYPE]: { + title: 'Ensure product documentation up to date task', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.ensureUpToDate({}); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleEnsureUpToDateTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: ENSURE_DOC_UP_TO_DATE_TASK_ID, + taskType: ENSURE_DOC_UP_TO_DATE_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(ENSURE_DOC_UP_TO_DATE_TASK_ID); + + logger.info(`Task ${ENSURE_DOC_UP_TO_DATE_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return ENSURE_DOC_UP_TO_DATE_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts new file mode 100644 index 0000000000000..0b5833055fd8b --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { registerEnsureUpToDateTaskDefinition } from './ensure_up_to_date'; +import { registerInstallAllTaskDefinition } from './install_all'; +import { registerUninstallAllTaskDefinition } from './uninstall_all'; + +export const registerTaskDefinitions = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + registerEnsureUpToDateTaskDefinition({ getServices, taskManager }); + registerInstallAllTaskDefinition({ getServices, taskManager }); + registerUninstallAllTaskDefinition({ getServices, taskManager }); +}; + +export { scheduleEnsureUpToDateTask, ENSURE_DOC_UP_TO_DATE_TASK_ID } from './ensure_up_to_date'; +export { scheduleInstallAllTask, INSTALL_ALL_TASK_ID } from './install_all'; +export { scheduleUninstallAllTask, UNINSTALL_ALL_TASK_ID } from './uninstall_all'; +export { waitUntilTaskCompleted, getTaskStatus } from './utils'; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts new file mode 100644 index 0000000000000..0d2cc48fb06bb --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/install_all.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const INSTALL_ALL_TASK_TYPE = 'ProductDocBase:InstallAll'; +export const INSTALL_ALL_TASK_ID = 'ProductDocBase:InstallAll'; + +export const registerInstallAllTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [INSTALL_ALL_TASK_TYPE]: { + title: 'Install all product documentation artifacts', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.installAll({}); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleInstallAllTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: INSTALL_ALL_TASK_ID, + taskType: INSTALL_ALL_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(INSTALL_ALL_TASK_ID); + + logger.info(`Task ${INSTALL_ALL_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return INSTALL_ALL_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts new file mode 100644 index 0000000000000..6a88fec205ddd --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/uninstall_all.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { InternalServices } from '../types'; +import { isTaskCurrentlyRunningError } from './utils'; + +export const UNINSTALL_ALL_TASK_TYPE = 'ProductDocBase:UninstallAll'; +export const UNINSTALL_ALL_TASK_ID = 'ProductDocBase:UninstallAll'; + +export const registerUninstallAllTaskDefinition = ({ + getServices, + taskManager, +}: { + getServices: () => InternalServices; + taskManager: TaskManagerSetupContract; +}) => { + taskManager.registerTaskDefinitions({ + [UNINSTALL_ALL_TASK_TYPE]: { + title: 'Uninstall all product documentation artifacts', + timeout: '10m', + maxAttempts: 3, + createTaskRunner: (context) => { + return { + async run() { + const { packageInstaller } = getServices(); + return packageInstaller.uninstallAll(); + }, + }; + }, + stateSchemaByVersion: {}, + }, + }); +}; + +export const scheduleUninstallAllTask = async ({ + taskManager, + logger, +}: { + taskManager: TaskManagerStartContract; + logger: Logger; +}) => { + try { + await taskManager.ensureScheduled({ + id: UNINSTALL_ALL_TASK_ID, + taskType: UNINSTALL_ALL_TASK_TYPE, + params: {}, + state: {}, + scope: ['productDoc'], + }); + + await taskManager.runSoon(UNINSTALL_ALL_TASK_ID); + + logger.info(`Task ${UNINSTALL_ALL_TASK_ID} scheduled to run soon`); + } catch (e) { + if (!isTaskCurrentlyRunningError(e)) { + throw e; + } + } + + return UNINSTALL_ALL_TASK_ID; +}; diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts new file mode 100644 index 0000000000000..e32ea02a11b0c --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/tasks/utils.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; + +export const getTaskStatus = async ({ + taskManager, + taskId, +}: { + taskManager: TaskManagerStartContract; + taskId: string; +}) => { + try { + const taskInstance = await taskManager.get(taskId); + return taskInstance.status; + } catch (e) { + // not found means the task was completed and the entry removed + if (SavedObjectsErrorHelpers.isNotFoundError(e)) { + return 'not_scheduled'; + } + throw e; + } +}; + +export const isTaskCurrentlyRunningError = (err: Error): boolean => { + return err.message?.includes('currently running'); +}; + +export const waitUntilTaskCompleted = async ({ + taskManager, + taskId, + timeout = 120_000, + interval = 5_000, +}: { + taskManager: TaskManagerStartContract; + taskId: string; + timeout?: number; + interval?: number; +}): Promise => { + const start = Date.now(); + const max = start + timeout; + let now = start; + while (now < max) { + try { + const taskInstance = await taskManager.get(taskId); + const { status } = taskInstance; + if (status === 'idle' || status === 'claiming' || status === 'running') { + await sleep(interval); + now = Date.now(); + } else { + return; + } + } catch (e) { + if (SavedObjectsErrorHelpers.isNotFoundError(e)) { + // not found means the task was completed and the entry removed + return; + } + } + } + + throw new Error(`Timeout waiting for task ${taskId} to complete.`); +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/x-pack/plugins/ai_infra/product_doc_base/server/types.ts b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts new file mode 100644 index 0000000000000..f00943b696708 --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/server/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/logging'; +import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import type { SearchApi } from './services/search'; +import type { ProductDocInstallClient } from './services/doc_install_status'; +import type { PackageInstaller } from './services/package_installer'; +import type { DocumentationManager, DocumentationManagerAPI } from './services/doc_manager'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface ProductDocBaseSetupDependencies { + taskManager: TaskManagerSetupContract; +} + +export interface ProductDocBaseStartDependencies { + licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; +} + +export interface ProductDocBaseSetupContract {} + +export interface ProductDocBaseStartContract { + search: SearchApi; + management: DocumentationManagerAPI; +} + +export interface InternalServices { + logger: Logger; + installClient: ProductDocInstallClient; + packageInstaller: PackageInstaller; + documentationManager: DocumentationManager; + licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; +} diff --git a/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json new file mode 100644 index 0000000000000..9a2d1969556bf --- /dev/null +++ b/x-pack/plugins/ai_infra/product_doc_base/tsconfig.json @@ -0,0 +1,29 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + "scripts/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [ + "@kbn/core", + "@kbn/logging", + "@kbn/config-schema", + "@kbn/product-doc-common", + "@kbn/core-saved-objects-server", + "@kbn/utils", + "@kbn/core-http-browser", + "@kbn/logging-mocks", + "@kbn/licensing-plugin", + "@kbn/task-manager-plugin", + ] +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts index 361d13e6d77f2..f693fa53c06cc 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts @@ -77,7 +77,7 @@ export class ObservabilityAIAssistantPlugin privileges: { all: { app: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'kibana'], - api: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'ai_assistant'], + api: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID, 'ai_assistant', 'manage_llm_product_doc'], catalogue: [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID], savedObject: { all: [ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc index efc948503b0c0..957ca0272c087 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc @@ -32,7 +32,8 @@ "alerting", "features", "inference", - "logsDataAccess" + "logsDataAccess", + "llmTasks" ], "optionalPlugins": [ "cloud" diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts new file mode 100644 index 0000000000000..00072e0c79c48 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/documentation.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DocumentationProduct } from '@kbn/product-doc-common'; +import { FunctionVisibility } from '@kbn/observability-ai-assistant-plugin/common'; +import type { FunctionRegistrationParameters } from '.'; + +export const RETRIEVE_DOCUMENTATION_NAME = 'retrieve_elastic_doc'; + +export async function registerDocumentationFunction({ + functions, + resources, + pluginsStart: { llmTasks }, +}: FunctionRegistrationParameters) { + const isProductDocAvailable = (await llmTasks.retrieveDocumentationAvailable()) ?? false; + + functions.registerInstruction(({ availableFunctionNames }) => { + return availableFunctionNames.includes(RETRIEVE_DOCUMENTATION_NAME) + ? `When asked questions about the Elastic stack or products, You should use the ${RETRIEVE_DOCUMENTATION_NAME} function before answering, + to retrieve documentation related to the question. Consider that the documentation returned by the function + is always more up to date and accurate than any own internal knowledge you might have.` + : undefined; + }); + + functions.registerFunction( + { + name: RETRIEVE_DOCUMENTATION_NAME, + visibility: isProductDocAvailable + ? FunctionVisibility.AssistantOnly + : FunctionVisibility.Internal, + description: `Use this function to retrieve documentation about Elastic products. + You can retrieve documentation about the Elastic stack, such as Kibana and Elasticsearch, + or for Elastic solutions, such as Elastic Security, Elastic Observability or Elastic Enterprise Search + `, + parameters: { + type: 'object', + properties: { + query: { + description: `The query to use to retrieve documentation + Examples: + - "How to enable TLS for Elasticsearch?" + - "What is Kibana Lens?"`, + type: 'string' as const, + }, + product: { + description: `If specified, will filter the products to retrieve documentation for + Possible options are: + - "kibana": Kibana product + - "elasticsearch": Elasticsearch product + - "observability": Elastic Observability solution + - "security": Elastic Security solution + If not specified, will search against all products + `, + type: 'string' as const, + enum: Object.values(DocumentationProduct), + }, + }, + required: ['query'], + } as const, + }, + async ({ arguments: { query, product }, connectorId, useSimulatedFunctionCalling }) => { + const response = await llmTasks!.retrieveDocumentation({ + searchTerm: query, + products: product ? [product] : undefined, + max: 3, + connectorId, + request: resources.request, + functionCalling: useSimulatedFunctionCalling ? 'simulated' : 'native', + }); + + return { + content: { + documents: response.documents, + }, + }; + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts index 7554164a55a69..ba876ad9457bc 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/index.ts @@ -12,6 +12,7 @@ import { registerLensFunction } from './lens'; import { registerVisualizeESQLFunction } from './visualize_esql'; import { ObservabilityAIAssistantAppPluginStartDependencies } from '../types'; import { registerChangesFunction } from './changes'; +import { registerDocumentationFunction } from './documentation'; export type FunctionRegistrationParameters = Omit< Parameters[0], @@ -24,4 +25,5 @@ export const registerFunctions = async (registrationParameters: FunctionRegistra registerVisualizeESQLFunction(registrationParameters); registerAlertsFunction(registrationParameters); registerChangesFunction(registrationParameters); + await registerDocumentationFunction(registrationParameters); }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts index fc39e0b7fb24e..a1196be6a829a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts @@ -37,6 +37,7 @@ import type { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plu import type { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import type { InferenceServerStart, InferenceServerSetup } from '@kbn/inference-plugin/server'; import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/server'; +import type { LlmTasksPluginStart } from '@kbn/llm-tasks-plugin/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ObservabilityAIAssistantAppServerStart {} @@ -57,6 +58,7 @@ export interface ObservabilityAIAssistantAppPluginStartDependencies { serverless?: ServerlessPluginStart; inference: InferenceServerStart; logsDataAccess: LogsDataAccessPluginStart; + llmTasks: LlmTasksPluginStart; } export interface ObservabilityAIAssistantAppPluginSetupDependencies { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json index 6608799caaf61..e0a520fb574c7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json @@ -70,6 +70,8 @@ "@kbn/logs-data-access-plugin", "@kbn/ai-assistant-common", "@kbn/inference-common", + "@kbn/llm-tasks-plugin", + "@kbn/product-doc-common", ], "exclude": [ "target/**/*" diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc index cda6fdf0192fa..c228f147dbfc3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc @@ -21,10 +21,11 @@ "optionalPlugins": [ "home", "serverless", + "productDocBase" ], "requiredBundles": [ "kibanaReact", - "logsDataAccess", + "logsDataAccess" ] } } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts index a680da5ed3f93..3bfe3dff3f9f4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/constants.ts @@ -9,6 +9,9 @@ export const REACT_QUERY_KEYS = { GET_GENAI_CONNECTORS: 'get_genai_connectors', GET_KB_ENTRIES: 'get_kb_entries', GET_KB_USER_INSTRUCTIONS: 'get_kb_user_instructions', + GET_PRODUCT_DOC_STATUS: 'get_product_doc_status', + INSTALL_PRODUCT_DOC: 'install_product_doc', + UNINSTALL_PRODUCT_DOC: 'uninstall_product_doc', CREATE_KB_ENTRIES: 'create_kb_entry', IMPORT_KB_ENTRIES: 'import_kb_entry', }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts new file mode 100644 index 0000000000000..ef95d51f78d49 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_get_product_doc_status.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from '@tanstack/react-query'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +export function useGetProductDocStatus() { + const { productDocBase } = useKibana().services; + + const { isLoading, isError, isSuccess, isRefetching, data, refetch } = useQuery({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + queryFn: async () => { + return productDocBase!.installation.getStatus(); + }, + keepPreviousData: false, + refetchOnWindowFocus: false, + }); + + return { + status: data, + refetch, + isLoading, + isRefetching, + isSuccess, + isError, + }; +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts new file mode 100644 index 0000000000000..cb32efa7e3908 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_install_product_doc.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; +import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; +import type { PerformInstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +type ServerError = IHttpFetchError; + +export function useInstallProductDoc() { + const { + productDocBase, + notifications: { toasts }, + } = useKibana().services; + const queryClient = useQueryClient(); + + return useMutation( + [REACT_QUERY_KEYS.INSTALL_PRODUCT_DOC], + () => { + return productDocBase!.installation.install(); + }, + { + onSuccess: () => { + toasts.addSuccess( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.installProductDoc.successNotification', + { + defaultMessage: 'The Elastic documentation was successfully installed', + } + ) + ); + + queryClient.invalidateQueries({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + refetchType: 'all', + }); + }, + onError: (error) => { + toasts.addError(new Error(error.body?.message ?? error.message), { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.installProductDoc.errorNotification', + { + defaultMessage: 'Something went wrong while installing the Elastic documentation', + } + ), + }); + }, + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts new file mode 100644 index 0000000000000..4aa3b5423faa1 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_uninstall_product_doc.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { i18n } from '@kbn/i18n'; +import type { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public'; +import type { UninstallResponse } from '@kbn/product-doc-base-plugin/common/http_api/installation'; +import { REACT_QUERY_KEYS } from '../constants'; +import { useKibana } from './use_kibana'; + +type ServerError = IHttpFetchError; + +export function useUninstallProductDoc() { + const { + productDocBase, + notifications: { toasts }, + } = useKibana().services; + const queryClient = useQueryClient(); + + return useMutation( + [REACT_QUERY_KEYS.UNINSTALL_PRODUCT_DOC], + () => { + return productDocBase!.installation.uninstall(); + }, + { + onSuccess: () => { + toasts.addSuccess( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.uninstallProductDoc.successNotification', + { + defaultMessage: 'The Elastic documentation was successfully uninstalled', + } + ) + ); + + queryClient.invalidateQueries({ + queryKey: [REACT_QUERY_KEYS.GET_PRODUCT_DOC_STATUS], + refetchType: 'all', + }); + }, + onError: (error) => { + toasts.addError(new Error(error.body?.message ?? error.message), { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.kb.uninstallProductDoc.errorNotification', + { + defaultMessage: 'Something went wrong while uninstalling the Elastic documentation', + } + ), + }); + }, + } + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts index b7c6bb089663a..67b294a5fef36 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/plugin.ts @@ -10,6 +10,7 @@ import type { CoreSetup, Plugin, PluginInitializerContext } from '@kbn/core/publ import type { ManagementSetup } from '@kbn/management-plugin/public'; import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; import type { ServerlessPluginStart } from '@kbn/serverless/public'; +import type { ProductDocBasePluginStart } from '@kbn/product-doc-base-plugin/public'; import type { ObservabilityAIAssistantPublicSetup, @@ -31,6 +32,7 @@ export interface SetupDependencies { export interface StartDependencies { observabilityAIAssistant: ObservabilityAIAssistantPublicStart; serverless?: ServerlessPluginStart; + productDocBase?: ProductDocBasePluginStart; } export interface ConfigSchema { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx new file mode 100644 index 0000000000000..668e363d071ee --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/product_doc_entry.tsx @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useState, useCallback, useMemo } from 'react'; +import { + EuiButton, + EuiDescribedFormGroup, + EuiFormRow, + EuiText, + EuiFlexGroup, + EuiFlexItem, + EuiHealth, + EuiLoadingSpinner, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { useKibana } from '../../../hooks/use_kibana'; +import { useGetProductDocStatus } from '../../../hooks/use_get_product_doc_status'; +import { useInstallProductDoc } from '../../../hooks/use_install_product_doc'; +import { useUninstallProductDoc } from '../../../hooks/use_uninstall_product_doc'; + +export function ProductDocEntry() { + const { overlays } = useKibana().services; + + const [isInstalled, setInstalled] = useState(true); + const [isInstalling, setInstalling] = useState(false); + + const { mutateAsync: installProductDoc } = useInstallProductDoc(); + const { mutateAsync: uninstallProductDoc } = useUninstallProductDoc(); + const { status, isLoading: isStatusLoading } = useGetProductDocStatus(); + + useEffect(() => { + if (status) { + setInstalled(status.overall === 'installed'); + } + }, [status]); + + const onClickInstall = useCallback(() => { + setInstalling(true); + installProductDoc().then( + () => { + setInstalling(false); + setInstalled(true); + }, + () => { + setInstalling(false); + setInstalled(false); + } + ); + }, [installProductDoc]); + + const onClickUninstall = useCallback(() => { + overlays + .openConfirm( + i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocUninstallConfirmText', + { + defaultMessage: `Are you sure you want to uninstall the Elastic documentation?`, + } + ), + { + title: i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocUninstallConfirmTitle', + { + defaultMessage: `Uninstalling Elastic documentation`, + } + ), + } + ) + .then((confirmed) => { + if (confirmed) { + uninstallProductDoc().then(() => { + setInstalling(false); + setInstalled(false); + }); + } + }); + }, [overlays, uninstallProductDoc]); + + const content = useMemo(() => { + if (isStatusLoading) { + return <>; + } + if (isInstalling) { + return ( + + + + + + + ); + } + if (isInstalled) { + return ( + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.installProductDocInstalledLabel', + { defaultMessage: 'Installed' } + )} + + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.uninstallProductDocButtonLabel', + { defaultMessage: 'Uninstall' } + )} + + + + ); + } + return ( + + + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.installProductDocButtonLabel', + { defaultMessage: 'Install' } + )} + + + + ); + }, [isInstalled, isInstalling, isStatusLoading, onClickInstall, onClickUninstall]); + + return ( + + {i18n.translate('xpack.observabilityAiAssistantManagement.settingsPage.productDocLabel', { + defaultMessage: 'Elastic documentation', + })} + + } + description={ +

+ + {i18n.translate('xpack.observabilityAiAssistantManagement.settingsPage.techPreview', { + defaultMessage: '[technical preview] ', + })} + + {i18n.translate( + 'xpack.observabilityAiAssistantManagement.settingsPage.productDocDescription', + { + defaultMessage: + "Install Elastic documentation to improve the assistant's efficiency.", + } + )} +

+ } + > + {content} + + ); +} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx index 831ba9ff58054..00c3fb76ae66a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.tsx @@ -11,10 +11,12 @@ import { i18n } from '@kbn/i18n'; import { useAppContext } from '../../../hooks/use_app_context'; import { useKibana } from '../../../hooks/use_kibana'; import { UISettings } from './ui_settings'; +import { ProductDocEntry } from './product_doc_entry'; export function SettingsTab() { const { application: { navigateToApp }, + productDocBase, } = useKibana().services; const { config } = useAppContext(); @@ -108,6 +110,7 @@ export function SettingsTab() { + {productDocBase ? : undefined} ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json index bc5cf69357dce..7b78d52c64806 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/tsconfig.json @@ -26,7 +26,8 @@ "@kbn/logs-data-access-plugin", "@kbn/core-plugins-browser", "@kbn/ai-assistant", - "@kbn/core-plugins-server" + "@kbn/core-plugins-server", + "@kbn/product-doc-base-plugin" ], "exclude": [ "target/**/*" diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 88ef256b353e6..a6bf7e7e9d5f2 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -49,6 +49,9 @@ export default function ({ getService }: FtrProviderContext) { 'Fleet-Usage-Logger', 'Fleet-Usage-Sender', 'ML:saved-objects-sync', + 'ProductDocBase:EnsureUpToDate', + 'ProductDocBase:InstallAll', + 'ProductDocBase:UninstallAll', 'SLO:ORPHAN_SUMMARIES-CLEANUP-TASK', 'Synthetics:Clean-Up-Package-Policies', 'UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects', diff --git a/yarn.lock b/yarn.lock index 79da7243edd61..18b5fed51c272 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5550,6 +5550,10 @@ version "0.0.0" uid "" +"@kbn/llm-tasks-plugin@link:x-pack/plugins/ai_infra/llm_tasks": + version "0.0.0" + uid "" + "@kbn/locator-examples-plugin@link:examples/locator_examples": version "0.0.0" uid "" @@ -6034,6 +6038,14 @@ version "0.0.0" uid "" +"@kbn/product-doc-base-plugin@link:x-pack/plugins/ai_infra/product_doc_base": + version "0.0.0" + uid "" + +"@kbn/product-doc-common@link:x-pack/packages/ai-infra/product-doc-common": + version "0.0.0" + uid "" + "@kbn/profiling-data-access-plugin@link:x-pack/plugins/observability_solution/profiling_data_access": version "0.0.0" uid ""