[LLM tasks] Add product documentation retrieval task

.buildkite/scripts/steps/checks/i18n.sh

@@ -5,4 +5,4 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Check i18n
-node scripts/i18n_check
+node scripts/i18n_check --quiet

.github/CODEOWNERS

@@ -570,6 +570,7 @@ src/plugins/links @elastic/kibana-presentation
 packages/kbn-lint-packages-cli @elastic/kibana-operations
 packages/kbn-lint-ts-projects-cli @elastic/kibana-operations
 x-pack/plugins/lists @elastic/security-detection-engine
+x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra
 examples/locator_examples @elastic/appex-sharedux
 examples/locator_explorer @elastic/appex-sharedux
 packages/kbn-logging @elastic/kibana-core
@@ -688,6 +689,8 @@ src/plugins/presentation_panel @elastic/kibana-presentation
 packages/presentation/presentation_publishing @elastic/kibana-presentation
 src/plugins/presentation_util @elastic/kibana-presentation
 x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra
+x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra
+x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra
 x-pack/plugins/observability_solution/profiling_data_access @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/profiling @elastic/obs-ux-infra_services-team
 packages/kbn-profiling-utils @elastic/obs-ux-infra_services-team

docs/developer/plugin-list.asciidoc

@@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana.
 using the CURL scripts in the scripts folder.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks]
+|This plugin contains various LLM tasks
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess]
 |Exposes services to access logs data.
 
@@ -767,6 +771,10 @@ Elastic.
 |This plugin helps users learn how to use the Painless scripting language.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase]
+|This plugin contains the registry for the knowledge base.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling]
 |Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views.
 

package.json

@@ -612,6 +612,7 @@
     "@kbn/licensing-plugin": "link:x-pack/plugins/licensing",
     "@kbn/links-plugin": "link:src/plugins/links",
     "@kbn/lists-plugin": "link:x-pack/plugins/lists",
+    "@kbn/llm-tasks-plugin": "link:x-pack/plugins/ai_infra/llm_tasks",
     "@kbn/locator-examples-plugin": "link:examples/locator_examples",
     "@kbn/locator-explorer-plugin": "link:examples/locator_explorer",
     "@kbn/logging": "link:packages/kbn-logging",
@@ -713,6 +714,8 @@
     "@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel",
     "@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing",
     "@kbn/presentation-util-plugin": "link:src/plugins/presentation_util",
+    "@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base",
+    "@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common",
     "@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access",
     "@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling",
     "@kbn/profiling-utils": "link:packages/kbn-profiling-utils",

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -854,6 +854,13 @@
   "policy-settings-protection-updates-note": [
     "note"
   ],
+  "product-doc-install-status": [
+    "index_name",
+    "installation_status",
+    "last_installation_date",
+    "product_name",
+    "product_version"
+  ],
   "query": [
     "description",
     "title",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -2837,6 +2837,26 @@
       }
     }
   },
+  "product-doc-install-status": {
+    "dynamic": false,
+    "properties": {
+      "index_name": {
+        "type": "keyword"
+      },
+      "installation_status": {
+        "type": "keyword"
+      },
+      "last_installation_date": {
+        "type": "integer"
+      },
+      "product_name": {
+        "type": "keyword"
+      },
+      "product_version": {
+        "type": "keyword"
+      }
+    }
+  },
   "query": {
     "dynamic": false,
     "properties": {

packages/kbn-optimizer/limits.yml

@@ -99,6 +99,7 @@ pageLoadAssetSize:
   licensing: 29004
   links: 44490
   lists: 22900
+  llmTasks: 22500
   logsDataAccess: 16759
   logsExplorer: 60000
   logsShared: 281060
@@ -124,6 +125,7 @@ pageLoadAssetSize:
   painlessLab: 179748
   presentationPanel: 55463
   presentationUtil: 58834
+  productDocBase: 22500
   profiling: 36694
   remoteClusters: 51327
   reporting: 58600

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -145,6 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75",
         "osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4",
         "policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352",
+        "product-doc-install-status": "bf0f1cb29850b2b5c45351d93fa7abfea3a39907",
         "query": "501bece68f26fe561286a488eabb1a8ab12f1137",
         "risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb",
         "rules-settings": "892a2918ebaeba809a612b8d97cec0b07c800b5f",

src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts

@@ -115,6 +115,7 @@ const previouslyRegisteredTypes = [
   'osquery-usage-metric',
   'osquery-manager-usage-metric',
   'policy-settings-protection-updates-note',
+  'product-doc-install-status',
   'query',
   'rules-settings',
   'sample-data-telemetry',

src/dev/i18n_tools/bin/run_i18n_check.ts

@@ -9,7 +9,6 @@
 
 import { Listr } from 'listr2';
 import { run } from '@kbn/dev-cli-runner';
-import { ToolingLog } from '@kbn/tooling-log';
 import { getTimeReporter } from '@kbn/ci-stats-reporter';
 import { isFailError } from '@kbn/dev-cli-errors';
 import { I18nCheckTaskContext, MessageDescriptor } from '../types';
@@ -24,13 +23,7 @@ import {
 import { TaskReporter } from '../utils/task_reporter';
 import { flagFailError, isDefined, undefinedOrBoolean } from '../utils/verify_bin_flags';
 
-const toolingLog = new ToolingLog({
-  level: 'info',
-  writeTo: process.stdout,
-});
-
 const runStartTime = Date.now();
-const reportTime = getTimeReporter(toolingLog, 'scripts/i18n_check');
 
 const skipOnNoTranslations = ({ config }: I18nCheckTaskContext) =>
   !config?.translations.length && 'No translations found.';
@@ -50,9 +43,13 @@ run(
       namespace: namespace,
       fix = false,
       path,
+      silent,
+      quiet,
     },
     log,
   }) => {
+    const reportTime = getTimeReporter(log, 'scripts/i18n_check');
+
     if (
       fix &&
       (isDefined(ignoreIncompatible) ||
@@ -131,13 +128,15 @@ run(
       {
         concurrent: false,
         exitOnError: true,
-        renderer: process.env.CI ? 'verbose' : ('default' as any),
+        forceTTY: false,
+        renderer:
+          ((silent || quiet) && 'silent') || (process.env.CI ? 'verbose' : ('default' as any)),
       }
     );
 
     try {
       const messages: Map<string, MessageDescriptor[]> = new Map();
-      const taskReporter = new TaskReporter({ toolingLog });
+      const taskReporter = new TaskReporter({ toolingLog: log });
       await list.run({ messages, taskReporter });
 
       reportTime(runStartTime, 'total', {
@@ -150,6 +149,7 @@ run(
         reportTime(runStartTime, 'error', {
           success: false,
         });
+        log.error(error);
       } else {
         log.error('Unhandled exception!');
         log.error(error);

tsconfig.base.json

@@ -1134,6 +1134,8 @@
       "@kbn/lint-ts-projects-cli/*": ["packages/kbn-lint-ts-projects-cli/*"],
       "@kbn/lists-plugin": ["x-pack/plugins/lists"],
       "@kbn/lists-plugin/*": ["x-pack/plugins/lists/*"],
+      "@kbn/llm-tasks-plugin": ["x-pack/plugins/ai_infra/llm_tasks"],
+      "@kbn/llm-tasks-plugin/*": ["x-pack/plugins/ai_infra/llm_tasks/*"],
       "@kbn/locator-examples-plugin": ["examples/locator_examples"],
       "@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"],
       "@kbn/locator-explorer-plugin": ["examples/locator_explorer"],
@@ -1370,6 +1372,10 @@
       "@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"],
       "@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"],
       "@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"],
+      "@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"],
+      "@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"],
+      "@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"],
+      "@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"],
       "@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"],
       "@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/observability_solution/profiling_data_access/*"],
       "@kbn/profiling-plugin": ["x-pack/plugins/observability_solution/profiling"],

x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/manifest.ts

@@ -5,11 +5,7 @@
  * 2.0.
  */
 
-export interface ArtifactManifest {
-  formatVersion: string;
-  productName: string;
-  productVersion: string;
-}
+import type { ArtifactManifest } from '@kbn/product-doc-common';
 
 export const getArtifactManifest = ({
   productName,

x-pack/packages/ai-infra/product-doc-artifact-builder/src/artifact/product_name.ts

@@ -5,7 +5,34 @@
  * 2.0.
  */
 
-/**
- * The allowed product names, as found in the source's cluster
- */
-export const sourceProductNames = ['Kibana', 'Elasticsearch', 'Security', 'Observability'];
+import type { ProductName } from '@kbn/product-doc-common';
+
+export const getSourceProductName = (productName: ProductName) => {
+  switch (productName) {
+    case 'elasticsearch':
+      return 'Elasticsearch';
+    case 'observability':
+      return 'Observability';
+    case 'security':
+      return 'Security';
+    case 'kibana':
+      return 'Kibana';
+    default:
+      throw new Error(`Unknown product name: ${productName}`);
+  }
+};
+
+export const getProductNameFromSource = (source: string): ProductName => {
+  switch (source) {
+    case 'Elasticsearch':
+      return 'elasticsearch';
+    case 'Observability':
+      return 'observability';
+    case 'Security':
+      return 'security';
+    case 'Kibana':
+      return 'kibana';
+    default:
+      throw new Error(`Unknown source product name: ${source}`);
+  }
+};

x-pack/packages/ai-infra/product-doc-artifact-builder/src/build_artifacts.ts

@@ -8,6 +8,7 @@
 import Path from 'path';
 import { Client } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/tooling-log';
+import type { ProductName } from '@kbn/product-doc-common';
 import {
   // checkConnectivity,
   createTargetIndex,
@@ -93,7 +94,7 @@ const buildArtifact = async ({
   sourceClient,
   log,
 }: {
-  productName: string;
+  productName: ProductName;
   stackVersion: string;
   buildFolder: string;
   targetFolder: string;

x-pack/packages/ai-infra/product-doc-artifact-builder/src/command.ts

@@ -6,19 +6,19 @@
  */
 
 import Path from 'path';
-import { REPO_ROOT } from '@kbn/repo-info';
 import yargs from 'yargs';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { DocumentationProduct } from '@kbn/product-doc-common';
 import type { TaskConfig } from './types';
 import { buildArtifacts } from './build_artifacts';
-import { sourceProductNames } from './artifact/product_name';
 
 function options(y: yargs.Argv) {
   return y
     .option('productName', {
       describe: 'name of products to generate documentation for',
       array: true,
-      choices: sourceProductNames,
-      default: ['Kibana'],
+      choices: Object.values(DocumentationProduct),
+      default: [DocumentationProduct.kibana],
     })
     .option('stackVersion', {
       describe: 'The stack version to generate documentation for',

x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_artifact.ts

@@ -8,9 +8,9 @@
 import Path from 'path';
 import AdmZip from 'adm-zip';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { getArtifactName } from '@kbn/product-doc-common';
 import { getArtifactMappings } from '../artifact/mappings';
 import { getArtifactManifest } from '../artifact/manifest';
-import { getArtifactName } from '../artifact/artifact_name';
 
 export const createArtifact = async ({
   productName,

x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/create_chunk_files.ts

@@ -10,7 +10,7 @@ import Fs from 'fs/promises';
 import type { Client } from '@elastic/elasticsearch';
 import type { ToolingLog } from '@kbn/tooling-log';
 
-const fileSizeLimit = 250_000;
+const fileSizeLimit = 500_000;
 
 export const createChunkFiles = async ({
   index,

x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/extract_documentation.ts

@@ -8,6 +8,8 @@
 import type { Client } from '@elastic/elasticsearch';
 import type { SearchHit } from '@elastic/elasticsearch/lib/api/types';
 import type { ToolingLog } from '@kbn/tooling-log';
+import type { ProductName } from '@kbn/product-doc-common';
+import { getSourceProductName, getProductNameFromSource } from '../artifact/product_name';
 
 /** the list of fields to import from the source cluster */
 const fields = [
@@ -27,7 +29,7 @@ const fields = [
 export interface ExtractedDocument {
   content_title: string;
   content_body: string;
-  product_name: string;
+  product_name: ProductName;
   root_type: string;
   slug: string;
   url: string;
@@ -43,7 +45,7 @@ const convertHit = (hit: SearchHit<any>): ExtractedDocument => {
   return {
     content_title: source.content_title,
     content_body: source.content_body,
-    product_name: source.product_name,
+    product_name: getProductNameFromSource(source.product_name),
     root_type: 'documentation',
     slug: source.slug,
     url: source.url,
@@ -65,7 +67,7 @@ export const extractDocumentation = async ({
   client: Client;
   index: string;
   stackVersion: string;
-  productName: string;
+  productName: ProductName;
   log: ToolingLog;
 }) => {
   log.info(`Starting to extract documents from source cluster`);
@@ -76,7 +78,7 @@ export const extractDocumentation = async ({
     query: {
       bool: {
         must: [
-          { term: { product_name: productName } },
+          { term: { product_name: getSourceProductName(productName) } },
           { term: { version: stackVersion } },
           { exists: { field: 'ai_fields.ai_summary' } },
         ],

x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/index.ts

@@ -10,7 +10,6 @@ export { indexDocuments } from './index_documents';
 export { createTargetIndex } from './create_index';
 export { installElser } from './install_elser';
 export { createChunkFiles } from './create_chunk_files';
-export { performSemanticSearch } from './perform_semantic_search';
 export { checkConnectivity } from './check_connectivity';
 export { createArtifact } from './create_artifact';
 export { cleanupFolders } from './cleanup_folders';

x-pack/packages/ai-infra/product-doc-artifact-builder/src/types.ts

@@ -5,8 +5,10 @@
  * 2.0.
  */
 
+import type { ProductName } from '@kbn/product-doc-common';
+
 export interface TaskConfig {
-  productNames: string[];
+  productNames: ProductName[];
   stackVersion: string;
   buildFolder: string;
   targetFolder: string;