[8.x] [LLM tasks] Add product documentation retrieval task (#194379) (#…

…200754) # Backport This will backport the following commits from `main` to `8.x`: - [[LLM tasks] Add product documentation retrieval task (#194379)](#194379)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)
elastic · Nov 19, 2024 · b6fe362 · b6fe362
1 parent d2f38b9
commit b6fe362
Show file tree

Hide file tree

Showing 149 changed files with 5,660 additions and 64 deletions.
diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc
@@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana.
 using the CURL scripts in the scripts folder.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks]
+|This plugin contains various LLM tasks.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess]
 |Exposes services to access logs data.
 
@@ -767,6 +771,10 @@ Elastic.
 |This plugin helps users learn how to use the Painless scripting language.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase]
+|This plugin contains the product documentation base service.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling]
 |Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views.
 

diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc
@@ -148,6 +148,9 @@ Refer to the corresponding {es} logs for potential write errors.
 | `success` | Creating trained model.
 | `failure` | Failed to create trained model.
 
+.1+| `product_documentation_create`
+| `unknown` | User requested to install the product documentation for use in AI Assistants.
+
 3+a|
 ====== Type: change
 
@@ -334,6 +337,9 @@ Refer to the corresponding {es} logs for potential write errors.
 | `success` | Updating trained model deployment.
 | `failure` | Failed to update trained model deployment.
 
+.1+| `product_documentation_update`
+| `unknown` | User requested to update the product documentation for use in AI Assistants.
+
 3+a|
 ====== Type: deletion
 
@@ -425,6 +431,9 @@ Refer to the corresponding {es} logs for potential write errors.
 | `success` | Deleting trained model.
 | `failure` | Failed to delete trained model.
 
+.1+| `product_documentation_delete`
+| `unknown` | User requested to delete the product documentation for use in AI Assistants.
+
 3+a|
 ====== Type: access
 

diff --git a/package.json b/package.json
@@ -616,6 +616,7 @@
     "@kbn/licensing-plugin": "link:x-pack/plugins/licensing",
     "@kbn/links-plugin": "link:src/plugins/links",
     "@kbn/lists-plugin": "link:x-pack/plugins/lists",
+    "@kbn/llm-tasks-plugin": "link:x-pack/plugins/ai_infra/llm_tasks",
     "@kbn/locator-examples-plugin": "link:examples/locator_examples",
     "@kbn/locator-explorer-plugin": "link:examples/locator_explorer",
     "@kbn/logging": "link:packages/kbn-logging",
@@ -718,6 +719,8 @@
     "@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel",
     "@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing",
     "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util",
+    "@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base",
+    "@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common",
     "@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access",
     "@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling",
     "@kbn/profiling-utils": "link:packages/kbn-profiling-utils",

diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json
@@ -855,6 +855,13 @@
   "policy-settings-protection-updates-note": [
     "note"
   ],
+  "product-doc-install-status": [
+    "index_name",
+    "installation_status",
+    "last_installation_date",
+    "product_name",
+    "product_version"
+  ],
   "query": [
     "description",
     "title",

diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -2841,6 +2841,26 @@
       }
     }
   },
+  "product-doc-install-status": {
+    "dynamic": false,
+    "properties": {
+      "index_name": {
+        "type": "keyword"
+      },
+      "installation_status": {
+        "type": "keyword"
+      },
+      "last_installation_date": {
+        "type": "date"
+      },
+      "product_name": {
+        "type": "keyword"
+      },
+      "product_version": {
+        "type": "keyword"
+      }
+    }
+  },
   "query": {
     "dynamic": false,
     "properties": {

diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
@@ -124,6 +124,7 @@ pageLoadAssetSize:
   painlessLab: 179748
   presentationPanel: 55463
   presentationUtil: 58834
+  productDocBase: 22500
   profiling: 36694
   remoteClusters: 51327
   reporting: 58600

diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -145,6 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75",
         "osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4",
         "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352",
+        "product-doc-install-status": "ca6e96840228e4cc2f11bae24a0797f4f7238c8c",
         "query": "501bece68f26fe561286a488eabb1a8ab12f1137",
         "risk-engine-configuration": "aea0c371a462e6d07c3ceb3aff11891b47feb09d",
         "rules-settings": "ba57ef1881b3dcbf48fbfb28902d8f74442190b2",

diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts
@@ -115,6 +115,7 @@ const previouslyRegisteredTypes = [
   'osquery-usage-metric',
   'osquery-manager-usage-metric',
   'policy-settings-protection-updates-note',
+  'product-doc-install-status',
   'query',
   'rules-settings',
   'sample-data-telemetry',

diff --git a/tsconfig.base.json b/tsconfig.base.json
@@ -1146,6 +1146,8 @@
       "@kbn/lint-ts-projects-cli/*": ["packages/kbn-lint-ts-projects-cli/*"],
       "@kbn/lists-plugin": ["x-pack/plugins/lists"],
       "@kbn/lists-plugin/*": ["x-pack/plugins/lists/*"],
+      "@kbn/llm-tasks-plugin": ["x-pack/plugins/ai_infra/llm_tasks"],
+      "@kbn/llm-tasks-plugin/*": ["x-pack/plugins/ai_infra/llm_tasks/*"],
       "@kbn/locator-examples-plugin": ["examples/locator_examples"],
       "@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"],
       "@kbn/locator-explorer-plugin": ["examples/locator_explorer"],
@@ -1384,6 +1386,10 @@
       "@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"],
       "@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"],
       "@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"],
+      "@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"],
+      "@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"],
+      "@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"],
+      "@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"],
       "@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"],
       "@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/observability_solution/profiling_data_access/*"],
       "@kbn/profiling-plugin": ["x-pack/plugins/observability_solution/profiling"],

diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md b/x-pack/packages/ai-infra/product-doc-artifact-builder/README.md
@@ -1,3 +1,49 @@
 # @kbn/product-doc-artifact-builder
 
-Script to build the knowledge base artifacts
+Script to build the knowledge base artifacts.
+
+## How to run
+
+```
+node scripts/build_product_doc_artifacts.js --stack-version {version} --product-name {product}
+```
+
+### parameters
+
+#### `stack-version`: 
+
+the stack version to generate the artifacts for.
+
+#### `product-name`: 
+
+(multi-value) the list of products to generate artifacts for.
+
+possible values: 
+- "kibana"
+- "elasticsearch"
+- "observability"
+- "security"
+
+#### `target-folder`:
+
+The folder to generate the artifacts in.
+
+Defaults to `{REPO_ROOT}/build-kb-artifacts`.
+
+#### `build-folder`:
+
+The folder to use for temporary files.
+
+Defaults to `{REPO_ROOT}/build/temp-kb-artifacts`
+
+#### Cluster infos
+
+- params for the source cluster:
+`sourceClusterUrl` / env.KIBANA_SOURCE_CLUSTER_URL
+`sourceClusterUsername` / env.KIBANA_SOURCE_CLUSTER_USERNAME
+`sourceClusterPassword` / env.KIBANA_SOURCE_CLUSTER_PASSWORD
+
+- params for the embedding cluster:
+`embeddingClusterUrl` / env.KIBANA_EMBEDDING_CLUSTER_URL
+`embeddingClusterUsername` / env.KIBANA_EMBEDDING_CLUSTER_USERNAME
+`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD
diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts
@@ -5,17 +5,13 @@
  * 2.0.
  */
 
-export interface ArtifactManifest {
-  formatVersion: string;
-  productName: string;
-  productVersion: string;
-}
+import type { ArtifactManifest, ProductName } from '@kbn/product-doc-common';
 
 export const getArtifactManifest = ({
   productName,
   stackVersion,
 }: {
-  productName: string;
+  productName: ProductName;
   stackVersion: string;
 }): ArtifactManifest => {
   return {

diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/mappings.ts
@@ -21,10 +21,7 @@ export const getArtifactMappings = (inferenceEndpoint: string): MappingTypeMappi
       slug: { type: 'keyword' },
       url: { type: 'keyword' },
       version: { type: 'version' },
-      ai_subtitle: {
-        type: 'semantic_text',
-        inference_id: inferenceEndpoint,
-      },
+      ai_subtitle: { type: 'text' },
       ai_summary: {
         type: 'semantic_text',
         inference_id: inferenceEndpoint,

diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts
@@ -5,7 +5,34 @@
  * 2.0.
  */
 
-/**
- * The allowed product names, as found in the source's cluster
- */
-export const sourceProductNames = ['Kibana', 'Elasticsearch', 'Security', 'Observability'];
+import type { ProductName } from '@kbn/product-doc-common';
+
+const productNameToSourceNamesMap: Record<ProductName, string[]> = {
+  kibana: ['Kibana'],
+  elasticsearch: ['Elasticsearch'],
+  security: ['Security'],
+  observability: ['Observability'],
+};
+
+const sourceNameToProductName = Object.entries(productNameToSourceNamesMap).reduce<
+  Record<string, ProductName>
+>((map, [productName, sourceNames]) => {
+  sourceNames.forEach((sourceName) => {
+    map[sourceName] = productName as ProductName;
+  });
+  return map;
+}, {});
+
+export const getSourceNamesFromProductName = (productName: ProductName): string[] => {
+  if (!productNameToSourceNamesMap[productName]) {
+    throw new Error(`Unknown product name: ${productName}`);
+  }
+  return productNameToSourceNamesMap[productName];
+};
+
+export const getProductNameFromSource = (source: string): ProductName => {
+  if (!sourceNameToProductName[source]) {
+    throw new Error(`Unknown source name: ${source}`);
+  }
+  return sourceNameToProductName[source];
+};
diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts
@@ -8,6 +8,7 @@
 import Path from 'path';
 import { Client } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/tooling-log';
+import type { ProductName } from '@kbn/product-doc-common';
 import {
   // checkConnectivity,
   createTargetIndex,
@@ -18,6 +19,7 @@ import {
   createArtifact,
   cleanupFolders,
   deleteIndex,
+  processDocuments,
 } from './tasks';
 import type { TaskConfig } from './types';
 
@@ -93,7 +95,7 @@ const buildArtifact = async ({
   sourceClient,
   log,
 }: {
-  productName: string;
+  productName: ProductName;
   stackVersion: string;
   buildFolder: string;
   targetFolder: string;
@@ -105,14 +107,16 @@ const buildArtifact = async ({
 
   const targetIndex = getTargetIndexName({ productName, stackVersion });
 
-  const documents = await extractDocumentation({
+  let documents = await extractDocumentation({
     client: sourceClient,
     index: 'search-docs-1',
     log,
     productName,
     stackVersion,
   });
 
+  documents = await processDocuments({ documents, log });
+
   await createTargetIndex({
     client: embeddingClient,
     indexName: targetIndex,

diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts
@@ -6,19 +6,19 @@
  */
 
 import Path from 'path';
-import { REPO_ROOT } from '@kbn/repo-info';
 import yargs from 'yargs';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { DocumentationProduct } from '@kbn/product-doc-common';
 import type { TaskConfig } from './types';
 import { buildArtifacts } from './build_artifacts';
-import { sourceProductNames } from './artifact/product_name';
 
 function options(y: yargs.Argv) {
   return y
     .option('productName', {
       describe: 'name of products to generate documentation for',
       array: true,
-      choices: sourceProductNames,
-      default: ['Kibana'],
+      choices: Object.values(DocumentationProduct),
+      default: [DocumentationProduct.kibana],
     })
     .option('stackVersion', {
       describe: 'The stack version to generate documentation for',

diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts
@@ -8,9 +8,9 @@
 import Path from 'path';
 import AdmZip from 'adm-zip';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { getArtifactName, type ProductName } from '@kbn/product-doc-common';
 import { getArtifactMappings } from '../artifact/mappings';
 import { getArtifactManifest } from '../artifact/manifest';
-import { getArtifactName } from '../artifact/artifact_name';
 
 export const createArtifact = async ({
   productName,
@@ -21,7 +21,7 @@ export const createArtifact = async ({
 }: {
   buildFolder: string;
   targetFolder: string;
-  productName: string;
+  productName: ProductName;
   stackVersion: string;
   log: ToolingLog;
 }) => {

diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts
@@ -10,7 +10,7 @@ import Fs from 'fs/promises';
 import type { Client } from '@elastic/elasticsearch';
 import type { ToolingLog } from '@kbn/tooling-log';
 
-const fileSizeLimit = 250_000;
+const fileSizeLimit = 500_000;
 
 export const createChunkFiles = async ({
   index,

diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_index.ts
@@ -21,10 +21,7 @@ const mappings: MappingTypeMapping = {
     slug: { type: 'keyword' },
     url: { type: 'keyword' },
     version: { type: 'version' },
-    ai_subtitle: {
-      type: 'semantic_text',
-      inference_id: 'kibana-elser2',
-    },
+    ai_subtitle: { type: 'text' },
     ai_summary: {
       type: 'semantic_text',
       inference_id: 'kibana-elser2',